8e80217e59
Former-commit-id:a02aeb236c[formerly9f19e3f712] [formerly06a8b51d6d[formerly 64fa9254b946eae7e61bbc3f513b7c3696c4f54f]] Former-commit-id:06a8b51d6dFormer-commit-id:3360eb6c5f
473 lines
No EOL
20 KiB
Python
Executable file
473 lines
No EOL
20 KiB
Python
Executable file
#!/common/bphillip/awips/bin/python
|
|
|
|
##
|
|
# This software was developed and / or modified by Raytheon Company,
|
|
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
|
|
#
|
|
# U.S. EXPORT CONTROLLED TECHNICAL DATA
|
|
# This software product contains export-restricted data whose
|
|
# export/transfer/disclosure is restricted by U.S. law. Dissemination
|
|
# to non-U.S. persons whether in the United States or abroad requires
|
|
# an export license or other authorization.
|
|
#
|
|
# Contractor Name: Raytheon Company
|
|
# Contractor Address: 6825 Pine Street, Suite 340
|
|
# Mail Stop B8
|
|
# Omaha, NE 68106
|
|
# 402.291.0100
|
|
#
|
|
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
|
|
# further licensing information.
|
|
##
|
|
|
|
|
|
import JUtil
|
|
import ifpnetCDF, iscUtil
|
|
import numpy
|
|
import tempfile, os, stat, getopt, sys, cPickle, siteConfig
|
|
import LogStream, time, traceback, string, IrtAccess, urllib, urllib2
|
|
import xml, copy, string
|
|
from xml.etree import ElementTree
|
|
from xml.etree.ElementTree import Element, SubElement
|
|
from java.io import File
|
|
from java.awt import Point
|
|
from com.vividsolutions.jts.geom import Coordinate
|
|
from java.util import ArrayList
|
|
from com.raytheon.edex.plugin.gfe.config import IFPServerConfig
|
|
from com.raytheon.edex.plugin.gfe.config import IFPServerConfigManager
|
|
from com.raytheon.edex.plugin.gfe.util import CartDomain2D
|
|
from com.raytheon.uf.common.dataplugin.gfe.db.objects import DatabaseID
|
|
from com.raytheon.uf.common.dataplugin.gfe.db.objects import GridLocation
|
|
from com.raytheon.uf.common.dataplugin.gfe.reference import ReferenceData
|
|
from com.raytheon.uf.common.dataplugin.gfe.reference import ReferenceID
|
|
from com.raytheon.uf.common.dataplugin.gfe.reference import ReferenceData_CoordinateType as CoordinateType
|
|
from com.raytheon.uf.common.dataplugin.gfe.util import GfeUtil
|
|
|
|
from com.vividsolutions.jts.geom import Coordinate
|
|
|
|
#
|
|
# Port of iscExtract.py
|
|
#
|
|
#
|
|
# SOFTWARE HISTORY
|
|
#
|
|
# Date Ticket# Engineer Description
|
|
# ------------ ---------- ----------- --------------------------
|
|
# 07/06/09 1995 bphillip Initial Creation.
|
|
#
|
|
#
|
|
#
|
|
|
|
parms = [] #parm names
|
|
dbid = None #database identifier
|
|
startTR = None #ifpnetCDF start time
|
|
endTR = None #ifpnetCDF start time
|
|
xmlDestinations = None #XML destinations information
|
|
ancf = None #IRT web address
|
|
bncf = None
|
|
xmtScript = None #transmit script
|
|
serverHost = None #server host
|
|
serverPort = None #server port
|
|
serverProtocol = None #serverProtocol
|
|
mhsid = None #MHS site identifier
|
|
siteID = None #our site id
|
|
startT = None
|
|
|
|
iscExtractLogger=None
|
|
|
|
def logEvent(*msg):
|
|
iscUtil.getLogger("iscExtract").info(iscUtil.tupleToString(*msg))
|
|
|
|
def logProblem(*msg):
|
|
iscUtil.getLogger("iscExtract").error(iscUtil.tupleToString(*msg))
|
|
|
|
def logException(*msg):
|
|
iscUtil.getLogger("iscExtract").exception(iscUtil.tupleToString(*msg))
|
|
|
|
def logVerbose(*msg):
|
|
iscUtil.getLogger("iscExtract").debug(iscUtil.tupleToString(*msg))
|
|
|
|
def logDebug(*msg):
|
|
logVerbose(iscUtil.tupleToString(*msg))
|
|
|
|
|
|
def executeIscExtract(parmNames, databaseName, startTime, endTime,
|
|
irtTableAddressA, irtTableAddressB, transmitScript, ourServerHost,
|
|
ourServerPort, ourServerProtocol, ourMHSid, ourSiteID, destinations=None):
|
|
|
|
startT = time.time()
|
|
parms = parmNames
|
|
dbid = databaseName
|
|
startTR = startTime
|
|
endTR = endTime
|
|
xmlDestinations = destinations
|
|
ancf = irtTableAddressA
|
|
bncf = irtTableAddressB
|
|
xmtScript = transmitScript
|
|
serverHost = ourServerHost
|
|
serverPort = ourServerPort
|
|
serverProtocol = ourServerProtocol
|
|
mhsid = ourMHSid
|
|
siteID = ourSiteID
|
|
|
|
myOfficeType = IFPServerConfigManager.getServerConfig(siteID).officeType()
|
|
|
|
|
|
#--------------------------------------------------------------------
|
|
# determine the list of destination servers
|
|
#--------------------------------------------------------------------
|
|
try:
|
|
nowT = time.time() #current time
|
|
useUntilTime = None #cached use until time
|
|
cacheFilename = "/tmp/" + serverHost + serverPort + ".iscExtract"
|
|
cachedXmlDestinations = None
|
|
#if xmlDestinations is None: #destinations not on command line
|
|
# # check the cache
|
|
# try:
|
|
# fd = open(cacheFilename, 'rb')
|
|
# buf = fd.read()
|
|
# fd.close()
|
|
# useUntilTime, cachedXmlDestinations = cPickle.loads(buf)
|
|
# nowT = time.time() #current time
|
|
# if nowT > useUntilTime:
|
|
# xmlDestinations = None #cache is too old
|
|
# useUntilTime = None
|
|
# else:
|
|
# logEvent('Using xmlDestinations cache')
|
|
# xmlDestinations = cachedXmlDestinations
|
|
# except:
|
|
# pass
|
|
|
|
|
|
# need to contact IRT to get destinations
|
|
irt = IrtAccess.IrtAccess(ancf, bncf)
|
|
if xmlDestinations is None:
|
|
logEvent('contacting IRT to get destinations')
|
|
count = 1
|
|
while True:
|
|
status, xmlDestinations = irt.getSendAddrs(siteID)
|
|
logEvent('IRT getSendAddrs status:', status)
|
|
if status:
|
|
# if we obtained XML destinations from IRT, then decode
|
|
# the useUntilTime field
|
|
try:
|
|
d = ElementTree.ElementTree(ElementTree.XML(xmlDestinations))
|
|
dE = d.getroot()
|
|
for e in dE:
|
|
if e.tag == "useuntil":
|
|
isoTimeStr = e.text
|
|
idx = isoTimeStr.find(".")
|
|
if idx != - 1:
|
|
isoTimeStr = isoTimeStr[0:idx] #eliminate subseconds
|
|
useUntilTime = time.mktime(time.strptime(isoTimeStr,
|
|
"%Y-%m-%dT%H:%M:%S"))
|
|
logEvent("Use Until: ", isoTimeStr)
|
|
except:
|
|
logProblem("Malformed XML on getSendAddrs()")
|
|
logProblem("XML=", xmlDestinations)
|
|
return
|
|
if useUntilTime is None:
|
|
useUntilTime = time.time() + 180.0 #3 minutes default
|
|
logEvent("Using default 180 second useUntilTime")
|
|
|
|
# write the cache
|
|
fd = open(cacheFilename, 'wb')
|
|
buf = cPickle.dumps((useUntilTime, xmlDestinations))
|
|
fd.write(buf)
|
|
fd.close()
|
|
break #success from the irt
|
|
else:
|
|
# try again and again for 10 minutes, then use cache
|
|
# if available and alert GFE users
|
|
if time.time() - nowT > 600.00:
|
|
logProblem("Unable to access IRT for send addrs")
|
|
if cachedXmlDestinations is None:
|
|
s = "Unable to access IRT for send addrs. Previous" + \
|
|
" cache not available."
|
|
logProblem(s)
|
|
return
|
|
# use cached value, even if out of date
|
|
else:
|
|
xmlDestinations = cachedXmlDestinations
|
|
if useUntilTime is not None:
|
|
s = time.asctime(time.gmtime(useUntilTime))
|
|
else:
|
|
s = "Unknown"
|
|
logProblem("Using expired cache. Date=", s)
|
|
|
|
#determine when we issued our last GFE alert
|
|
#we alert every 30 minutes.
|
|
try:
|
|
fd = open(cacheFilename + "-warn", 'rb')
|
|
buf = fd.read()
|
|
fd.close()
|
|
lastAlertTime = cPickle.loads(buf)
|
|
except:
|
|
lastAlertTime = 0 #for way long ago
|
|
if time.time() - lastAlertTime > 1800.0:
|
|
logProblem("Sending GFE notification")
|
|
msg = """
|
|
Contact NCF. ifpServer is unable to contact IRT central server. ISC
|
|
traffic routing information is old and possibly incorrect."""
|
|
os.system("sendGfeMessage -u -c GFE -m '" + \
|
|
msg + "'")
|
|
fd = open(cacheFilename + "-warn", 'wb')
|
|
fd.write(cPickle.dumps(time.time()))
|
|
fd.close()
|
|
break
|
|
|
|
time.sleep(15.0) #sleep awhile and then try again
|
|
count = count + 1
|
|
logProblem("Retrying to getSendAddrs()", count)
|
|
# qc the XML
|
|
try:
|
|
destTree = ElementTree.ElementTree(ElementTree.XML(xmlDestinations))
|
|
destE = destTree.getroot()
|
|
except:
|
|
logProblem("Malformed XML on getSendAddrs() or provided xmlDest")
|
|
logProblem("XML=", xmlDestinations)
|
|
return
|
|
#--------------------------------------------------------------------
|
|
# determine how many transmissions are necessary
|
|
#--------------------------------------------------------------------
|
|
xmt = []
|
|
|
|
logEvent("XML dest:", xmlDestinations)
|
|
if destE.tag != "destinations":
|
|
logProblem("Destinations packet missing from web service")
|
|
return
|
|
|
|
# create list of individual transmissions (before attempting to combine
|
|
doClip = 1 #0 to send entire domain, 1 to do destination clipping (default)
|
|
destStr = "Destination Servers:\n"
|
|
for addressE in destE:
|
|
if addressE.tag == "doclip":
|
|
for name, value in addressE.items():
|
|
if name == "clip":
|
|
if value == "1":
|
|
doClip = 1
|
|
elif value == "0":
|
|
doClip = 0
|
|
|
|
logEvent("Clipping State: ", doClip)
|
|
|
|
for addressE in destE:
|
|
|
|
if addressE.tag != "address":
|
|
continue
|
|
|
|
# find destination server info and domain information
|
|
serverInfo = irt.decodeXMLAddress(addressE)
|
|
|
|
if doClip == 0:
|
|
serverInfo['domain'] = None
|
|
keycheckfail = False
|
|
for key in ['mhsid', 'host', 'port', 'protocol', 'site']:
|
|
if not serverInfo.has_key(key):
|
|
logProblem("Fail to decode XML. Skipping serverInfo:",
|
|
serverInfo)
|
|
keycheckfail = True
|
|
continue
|
|
if keycheckfail:
|
|
continue #skipping this destination due to insufficient info
|
|
|
|
# get the destination office type
|
|
try:
|
|
siteIndex = IFPServerConfigManager.getServerConfig(siteID).allSites().indexOf(serverInfo['site'])
|
|
destOfficeType = str(IFPServerConfigManager.getServerConfig(siteID).officeTypes().get(siteIndex))
|
|
except:
|
|
logProblem("Unknown site id to get office type. ",
|
|
"Skipping serverInfo:", serverInfo)
|
|
continue #skipping this destination due to unknown site id
|
|
|
|
# find weather elements that remote ifpServer wants
|
|
# that is available in our server and in the -p parm switches
|
|
any = False
|
|
for parm in serverInfo['parms']:
|
|
p1 = string.replace(parm, "_SFC", "") #remove _SFC if exists
|
|
|
|
# translation of parm name needed, also if no office type, then
|
|
# not wanted from this office.
|
|
# example: changes QPFwfo to QPF if we are wfo
|
|
# example: discards T if we are wfo and site is non-wfo
|
|
if myOfficeType != destOfficeType:
|
|
if p1.find(myOfficeType) != - 1:
|
|
p1 = string.replace(p1, myOfficeType, "") #remove type
|
|
else:
|
|
continue #no type, so not intended for our type
|
|
# see if parm was listed in the command line switches
|
|
if parms.contains(p1):
|
|
xmt.append({'serverInfo':[serverInfo], 'parms':[p1],
|
|
'domain': serverInfo['domain'], 'area': serverInfo['area']})
|
|
if not any:
|
|
destStr += irt.printServerInfo(serverInfo) + "\n"
|
|
any = True
|
|
|
|
logEvent(destStr)
|
|
|
|
# now combine transmissions
|
|
# find same domains, same parms, to combine servers/destinations
|
|
i = 0
|
|
while i < len(xmt):
|
|
j = i + 1
|
|
while j < len(xmt):
|
|
if xmt[i]['domain'] == xmt[j]['domain'] and \
|
|
xmt[i]['area'] == xmt[j]['area'] and \
|
|
xmt[i]['parms'] == xmt[j]['parms']:
|
|
for si in xmt[j]['serverInfo']:
|
|
if si not in xmt[i]['serverInfo']:
|
|
dests = xmt[i]['serverInfo']
|
|
dests.append(si)
|
|
xmt[j]['serverInfo'] = dests
|
|
del xmt[j] #delete the entry
|
|
j = j - 1 #redo this entry index next loop
|
|
j = j + 1
|
|
i = i + 1
|
|
|
|
# now try to combine common parm lists (same domain, same servers/destinations)
|
|
i = 0
|
|
while i < len(xmt):
|
|
j = i + 1
|
|
while j < len(xmt):
|
|
if xmt[i]['domain'] == xmt[j]['domain'] and \
|
|
xmt[i]['area'] == xmt[j]['area'] and \
|
|
xmt[i]['serverInfo'] == xmt[j]['serverInfo'] :
|
|
iparms = xmt[i]['parms']
|
|
for p in xmt[j]['parms']:
|
|
if p not in iparms:
|
|
iparms.append(p)
|
|
xmt[i]['parms'] = iparms
|
|
del xmt[j] #delete the entry
|
|
j = j - 1 #redo this entry index for next loop
|
|
j = j + 1
|
|
i = i + 1
|
|
|
|
# if doClip, gather some required information
|
|
if doClip:
|
|
#get the isc send area and grid domain from the ifpServer
|
|
iscSendAreaGrid = iscUtil.getEditArea("ISC_Send_Area",siteID)
|
|
sourceDomain = IFPServerConfigManager.getServerConfig(siteID).dbDomain()
|
|
|
|
iscSendAreaGrid.setGloc(sourceDomain)
|
|
iscSendAreaGrid = iscSendAreaGrid.getGrid()
|
|
|
|
|
|
#--------------------------------------------------------------------
|
|
# prepare output files
|
|
#--------------------------------------------------------------------
|
|
for dest in xmt:
|
|
s = "Processing Xmt Pass:\n"
|
|
for sv in dest['serverInfo']:
|
|
s += irt.printServerInfo(sv) + '\n'
|
|
s += "Domain:" + `dest['domain']` + '\n'
|
|
s += "Area:" + `dest['area']` + '\n'
|
|
s += "Parms:" + `dest['parms']` + '\n\n'
|
|
logEvent(s)
|
|
# extract the data using ifpnetCDF
|
|
if os.path.exists(siteConfig.GFESUITE_HOME + "/products/ISC") == False:
|
|
os.makedirs(siteConfig.GFESUITE_HOME + "/products/ISC")
|
|
|
|
tempfile.tempdir = siteConfig.GFESUITE_HOME + "/products/ISC"
|
|
fname = tempfile.mktemp(".isc")
|
|
|
|
# Determine domain edit area.
|
|
|
|
if doClip == 1 and dest['domain'] is not None and \
|
|
dest['domain']['proj'] == sourceDomain.getProjection().getProjectionID():
|
|
#make a GridLocation for our domain
|
|
gridSize = Coordinate(float(str(sourceDomain.getNx())), float(str(sourceDomain.getNy())))
|
|
origin = sourceDomain.getOrigin()
|
|
extent = sourceDomain.getExtent()
|
|
domain = CartDomain2D(origin, extent)
|
|
gloc = sourceDomain
|
|
|
|
#make a GridLocation covering the area for the destination, expanded
|
|
#by 1/2 grid cell
|
|
dd = dest['domain']
|
|
da = dest['area']
|
|
cellsizeX = float(dd['extx']) / (float(da['xdim']) - 1.0)
|
|
cellsizeY = float(dd['exty']) / (float(da['ydim']) - 1.0)
|
|
originD = Coordinate(float(dd['origx']) - cellsizeX / 2.0,
|
|
float(dd['origy']) - cellsizeY / 2.0)
|
|
extentD = Coordinate(float(dd['extx']) + cellsizeX,
|
|
float(dd['exty']) + cellsizeY)
|
|
domainD = CartDomain2D(originD, extentD)
|
|
|
|
#check for overlap
|
|
if not domainD.overlaps(domain):
|
|
logEvent("No intersection of domain box, skipping....")
|
|
continue #no bits set in the resulting mask, no intersect
|
|
|
|
domainD.trim(domain) #trim it to just the overlapping section
|
|
|
|
|
|
gridSize = Point(int(da['xdim']),int(da['ydim']))
|
|
destGridLocation = GridLocation("Dest",sourceDomain.getProjection(),
|
|
gridSize,domainD.getOrigin(),domainD.getExtent(),"GMT")
|
|
|
|
# make a Reference Set
|
|
refid = ReferenceID("jibberish")
|
|
refSet = ReferenceData(gloc, refid, destGridLocation.getGeometry(), CoordinateType.LATLON)
|
|
|
|
# convert destination site's domain to gridpoints
|
|
iscMask = refSet.getGrid()
|
|
|
|
# "and" it with our ISC_Send_Area
|
|
iscMask.andEquals(iscSendAreaGrid)
|
|
|
|
if not iscMask.isAnyBitsSet():
|
|
logEvent("No intersection of domain points, skipping....")
|
|
continue #no bits set in the resulting mask, no intersect
|
|
|
|
# store the grid back into the ifpServer
|
|
maskName = "iscExtract" + `time.time()`
|
|
refSet.setGrid(iscMask)
|
|
iscUtil.saveEditAreaGrid(maskName, refSet, siteID)
|
|
|
|
else: #no clipping, or different projection
|
|
maskName = "ISC_Send_Area"
|
|
|
|
# Run ifpnetCDF for the data
|
|
ifpnetCDF.executeIfpNetCDF(siteConfig.GFESUITE_SERVER, siteConfig.GFESUITE_PORT,
|
|
fname, JUtil.pyValToJavaObj(dest['parms']), dbid, startTR, endTR, maskName, 0, 1,
|
|
None, 6, 1, 1, "iscExtract")
|
|
|
|
fname = fname + '.gz'
|
|
size = os.stat(fname)[stat.ST_SIZE]
|
|
endT = time.time()
|
|
logEvent('File Size: ', size)
|
|
logEvent('After ifpnetCDF, ,wctime:', "%-6.2f" % (endT - startT),
|
|
',cputime:', "%-6.2f" % time.clock())
|
|
|
|
# create XML destinations file for this output
|
|
iscE = Element('isc') #create the XML tree root
|
|
sourceServer = {'mhsid': mhsid, 'host': serverHost, 'port': serverPort,
|
|
'protocol': serverProtocol, 'site': siteID}
|
|
irt.addSourceXML(iscE, sourceServer)
|
|
irt.addDestinationXML(iscE, dest['serverInfo'])
|
|
|
|
#get the unique list of mhs sites
|
|
mhsSites = []
|
|
for si in dest['serverInfo']:
|
|
if si['mhsid'] not in mhsSites:
|
|
mhsSites.append(si['mhsid'])
|
|
|
|
# create the XML file
|
|
fnameXML = tempfile.mktemp(".xml")
|
|
fd = open(fnameXML, 'wb')
|
|
fd.write(ElementTree.tostring(iscE))
|
|
fd.close()
|
|
|
|
# Transmit files - do string substitution
|
|
irt.transmitFiles("ISCGRIDS2", mhsSites, mhsid, [fname,fnameXML], xmtScript)
|
|
# Delete temporary files
|
|
if maskName != "ISC_Send_Area":
|
|
iscUtil.deleteEditArea(maskName,siteID)
|
|
|
|
endT = time.time()
|
|
logEvent('After transmission pass, ,wctime:',
|
|
"%-6.2f" % (endT - startT), ',cputime:', "%-6.2f" % time.clock())
|
|
|
|
except:
|
|
logProblem("Failure", traceback.format_exc())
|
|
|