407 lines
14 KiB
Python
407 lines
14 KiB
Python
|
##
|
||
|
|
# This software was developed and / or modified by Raytheon Company,
|
||
|
|
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
|
||
|
|
#
|
||
|
|
# U.S. EXPORT CONTROLLED TECHNICAL DATA
|
||
|
|
# This software product contains export-restricted data whose
|
||
|
|
# export/transfer/disclosure is restricted by U.S. law. Dissemination
|
||
|
|
# to non-U.S. persons whether in the United States or abroad requires
|
||
|
|
# an export license or other authorization.
|
||
|
|
#
|
||
|
|
# Contractor Name: Raytheon Company
|
||
|
|
# Contractor Address: 6825 Pine Street, Suite 340
|
||
|
|
# Mail Stop B8
|
||
|
|
# Omaha, NE 68106
|
||
|
|
# 402.291.0100
|
||
|
|
#
|
||
|
|
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
|
||
|
|
# further licensing information.
|
||
|
|
##
|
||
|
|
# Utility classes for the VTEC util table
|
||
|
|
#
|
||
|
|
# SOFTWARE HISTORY
|
||
|
|
#
|
||
|
|
# Date Ticket# Engineer Description
|
||
|
|
# ------------- -------- --------- ---------------------------------------------
|
||
|
|
# Jun 17, 2013 3296 randerso Moved active table backup and purging to a
|
||
|
|
# separate thread in java.
|
||
|
|
# Apr 23, 2015 4259 njensen Updated for new JEP API
|
||
|
|
# Feb 01, 2017 6107 dgilling Improve file name uniqueness when writing
|
||
|
|
# active table backups.
|
||
|
|
# Apr 29, 2020 8151 randerso Use SiteMap.getSite4LetterId()
|
||
|
|
#
|
||
|
|
##
|
||
|
|
|
||
|
|
##
|
||
|
|
# This is a base file that is not intended to be overridden.
|
||
|
|
##
|
||
|
|
|
||
|
|
import copy
|
||
|
|
import pickle
|
||
|
|
import datetime
|
||
|
|
import errno
|
||
|
|
import glob
|
||
|
|
import gzip
|
||
|
|
import os
|
||
|
|
import stat
|
||
|
|
import time
|
||
|
|
|
||
|
|
import LogStream
|
||
|
|
import JUtil
|
||
|
|
|
||
|
|
from java.util import ArrayList
|
||
|
|
from com.raytheon.uf.common.activetable import ActiveTableUtil
|
||
|
|
|
||
|
|
|
||
|
|
class VTECTableUtil:
|
||
|
|
|
||
|
|
def __init__(self, activeTableFileName=None):
|
||
|
|
self._activeTableFilename = activeTableFileName
|
||
|
|
self._activeTableLockFD = None
|
||
|
|
if activeTableFileName is not None:
|
||
|
|
self._activeTableLockFilename = self._activeTableFilename + "lock"
|
||
|
|
else:
|
||
|
|
self._activeTableLockFilename = None
|
||
|
|
|
||
|
|
#------------------------------------------------------------------
|
||
|
|
# Consolidate Utilities
|
||
|
|
#------------------------------------------------------------------
|
||
|
|
|
||
|
|
# given the table, will consolidate like records and return a table
|
||
|
|
# with identical hazards, but with multiple id entries
|
||
|
|
def consolidateByID(self, ptable, compare=None):
|
||
|
|
if compare is None:
|
||
|
|
compare = ['etn', 'vtecstr', 'ufn', 'areaPoints', 'valuePoints', 'hdln',
|
||
|
|
'phensig', 'previousStart', 'previousEnd', 'purgeTime', 'issueTime',
|
||
|
|
'downgradeFrom', 'upgradeFrom', 'startTime', 'endTime', 'act', 'pil', 'phen',
|
||
|
|
'sig', 'officeid', 'seg', 'state']
|
||
|
|
ctable = []
|
||
|
|
for a in ptable:
|
||
|
|
found = 0
|
||
|
|
for c in ctable:
|
||
|
|
if self.hazardCompare(a, c, compare):
|
||
|
|
found = 1
|
||
|
|
if isinstance(a['id'], list):
|
||
|
|
zones = a['id']
|
||
|
|
else:
|
||
|
|
zones = [a['id']]
|
||
|
|
|
||
|
|
allzones = c['id']
|
||
|
|
for z in zones:
|
||
|
|
allzones.append(z)
|
||
|
|
c['id'] = allzones
|
||
|
|
break
|
||
|
|
if found == 0:
|
||
|
|
newc = copy.deepcopy(a)
|
||
|
|
if newc['id'] is not list:
|
||
|
|
newc['id'] = [newc['id']]
|
||
|
|
ctable.append(newc)
|
||
|
|
|
||
|
|
return ctable
|
||
|
|
|
||
|
|
#------------------------------------------------------------------
|
||
|
|
# Printing Utilities
|
||
|
|
#------------------------------------------------------------------
|
||
|
|
|
||
|
|
# Pretty-prints the given table or table entry, returns the string
|
||
|
|
def printActiveTable(self, table, combine=False, idType='zone'):
|
||
|
|
if table is None:
|
||
|
|
s = "Table is None"
|
||
|
|
return s
|
||
|
|
elif hasattr(table, "java_name"):
|
||
|
|
table = JUtil.javaObjToPyVal(table)
|
||
|
|
print(type(table))
|
||
|
|
|
||
|
|
# dictionary, single record
|
||
|
|
if isinstance(table, dict):
|
||
|
|
ptable = []
|
||
|
|
ptable.append(table)
|
||
|
|
# list of dictionaries
|
||
|
|
else:
|
||
|
|
ptable = table
|
||
|
|
|
||
|
|
s = '\n' # Return value
|
||
|
|
|
||
|
|
# Use the zone or city printEntry method
|
||
|
|
if idType == 'zone':
|
||
|
|
printEntry = self.printEntry
|
||
|
|
compare = None
|
||
|
|
elif idType == 'city':
|
||
|
|
printEntry = self.printCityEntry
|
||
|
|
compare = ['start', 'end', 'phen', 'sig', 'seg']
|
||
|
|
|
||
|
|
# combine mode, attempt to combine records
|
||
|
|
if combine:
|
||
|
|
ctable = self.consolidateByID(ptable, compare=compare)
|
||
|
|
for c in ctable:
|
||
|
|
s = s + printEntry(c)
|
||
|
|
# non-combine mode
|
||
|
|
else:
|
||
|
|
for p in ptable:
|
||
|
|
s = s + printEntry(p)
|
||
|
|
|
||
|
|
return s
|
||
|
|
|
||
|
|
# Pretty-prints an entry in the table
|
||
|
|
def printEntry(self, entry):
|
||
|
|
# formatting for the etn
|
||
|
|
if isinstance(entry['etn'], int):
|
||
|
|
etn = "%04i" % entry['etn']
|
||
|
|
else:
|
||
|
|
etn = entry['etn']
|
||
|
|
|
||
|
|
# formatting for the vstr
|
||
|
|
if 'vtecstr' in entry:
|
||
|
|
vstr = entry['vtecstr']
|
||
|
|
else:
|
||
|
|
vstr = "????"
|
||
|
|
|
||
|
|
# formatting for the id depending upon the type of table
|
||
|
|
zones = ''
|
||
|
|
if isinstance(type(entry['id']), list):
|
||
|
|
zones = sorted(entry['id'])
|
||
|
|
zones = repr(zones)
|
||
|
|
else:
|
||
|
|
zones = entry['id']
|
||
|
|
|
||
|
|
# until further notice
|
||
|
|
ufn = entry.get('ufn', 0)
|
||
|
|
|
||
|
|
# formatting for areaPoints, valuePoints if available
|
||
|
|
apvp = ''
|
||
|
|
if 'areaPoints' in entry and 'valuePoints' in entry:
|
||
|
|
ratio = (entry['valuePoints'] / float(entry['areaPoints'])) * 100.0
|
||
|
|
apvp = 'AreaPoints: ' + '%6i' % entry['areaPoints'] +\
|
||
|
|
' ValuePoints: ' + '%6i' % entry['valuePoints'] + \
|
||
|
|
' % Area: ' + '%6.1f' % ratio + '%\n'
|
||
|
|
|
||
|
|
# formatting for hdln
|
||
|
|
if 'hdln' in entry:
|
||
|
|
hdln = entry['hdln']
|
||
|
|
else:
|
||
|
|
hdln = "????"
|
||
|
|
|
||
|
|
# formatting for key
|
||
|
|
if 'phensig' in entry:
|
||
|
|
key = entry['phensig']
|
||
|
|
else:
|
||
|
|
key = "????"
|
||
|
|
|
||
|
|
# formatting for previousStart/End
|
||
|
|
if 'previousStart' in entry and 'previousEnd' in entry:
|
||
|
|
prev = '\nPrevStart: ' + \
|
||
|
|
time.asctime(time.gmtime(entry['previousStart'])) + \
|
||
|
|
' ' + repr(entry['previousStart']) + '\nPrevEnd: ' + \
|
||
|
|
time.asctime(time.gmtime(entry['previousEnd'])) + \
|
||
|
|
' ' + repr(entry['previousEnd'])
|
||
|
|
else:
|
||
|
|
prev = ''
|
||
|
|
|
||
|
|
# formatting for purgeTime
|
||
|
|
if 'purgeTime' in entry:
|
||
|
|
expireT = '\nPurge: ' + \
|
||
|
|
time.asctime(time.gmtime(entry['purgeTime'])) + \
|
||
|
|
' ' + repr(entry['purgeTime'])
|
||
|
|
else:
|
||
|
|
expireT = ''
|
||
|
|
|
||
|
|
# formatting for issueTime
|
||
|
|
if 'issueTime' in entry:
|
||
|
|
issueT = '\nIssue: ' + \
|
||
|
|
time.asctime(time.gmtime(entry['issueTime'])) + \
|
||
|
|
' ' + repr(entry['issueTime'])
|
||
|
|
else:
|
||
|
|
issueT = ''
|
||
|
|
|
||
|
|
# formatting for recState
|
||
|
|
if 'state' in entry:
|
||
|
|
recState = " RecState: " + entry['state']
|
||
|
|
else:
|
||
|
|
recState = ''
|
||
|
|
|
||
|
|
# formatting for upgrades/downgrades special records
|
||
|
|
if 'downgradeFrom' in entry:
|
||
|
|
related = entry['downgradeFrom']
|
||
|
|
duType = "DowngradeFrom: "
|
||
|
|
elif 'upgradeFrom' in entry:
|
||
|
|
related = entry['upgradeFrom']
|
||
|
|
duType = "UpgradeFrom: "
|
||
|
|
else:
|
||
|
|
related = None
|
||
|
|
if related is not None:
|
||
|
|
if 'phensig' in related:
|
||
|
|
rkey = related['phensig']
|
||
|
|
else:
|
||
|
|
rkey = "????"
|
||
|
|
if isinstance(related['etn'], int):
|
||
|
|
retn = "%04i" % related['etn']
|
||
|
|
else:
|
||
|
|
retn = related['etn']
|
||
|
|
relatedText = (
|
||
|
|
f"{duType}Action: {related['act']} Phen: {related['phen']} Sig: {related['sig']} Key: {rkey} Etn:{retn}\n"
|
||
|
|
f"{duType}Start: {time.asctime(time.gmtime(related['startTime']))} {related['startTime']!r}\n"
|
||
|
|
f"{duType}End: {time.asctime(time.gmtime(related['endTime']))} {related['endTime']!r}\n"
|
||
|
|
)
|
||
|
|
else:
|
||
|
|
relatedText = ""
|
||
|
|
|
||
|
|
t = (
|
||
|
|
f"Vtec: {vstr}\n"
|
||
|
|
f"Hdln: {hdln}\n"
|
||
|
|
f"Start: {time.asctime(time.gmtime(entry['startTime']))} {entry['startTime']!r} Action: {entry['act']} Pil: {entry['pil']}\n"
|
||
|
|
f"End: {time.asctime(time.gmtime(entry['endTime']))} {entry['endTime']!r} UFN: {ufn!r}{recState}{expireT}{issueT}{prev}\n"
|
||
|
|
f"Phen: {entry['phen']} Sig: {entry['sig']} Office: {entry['officeid']} Etn: {etn} Seg: {entry['seg']!r} Key: {key}\n"
|
||
|
|
f"{apvp}Zone: {zones}\n"
|
||
|
|
f"{relatedText}\n"
|
||
|
|
)
|
||
|
|
return t
|
||
|
|
|
||
|
|
# Pretty-prints an entry in the table
|
||
|
|
def printCityEntry(self, entry):
|
||
|
|
# formatting for the id depending upon the type of table
|
||
|
|
zones = ''
|
||
|
|
if isinstance(entry['id'], list):
|
||
|
|
zones = sorted(entry['id'])
|
||
|
|
zones = repr(zones)
|
||
|
|
else:
|
||
|
|
zones = entry['id']
|
||
|
|
|
||
|
|
# formatting for key
|
||
|
|
if 'key' in entry:
|
||
|
|
key = entry['key']
|
||
|
|
else:
|
||
|
|
key = "????"
|
||
|
|
|
||
|
|
# until further notice
|
||
|
|
ufn = entry.get('ufn', 0)
|
||
|
|
|
||
|
|
start = entry['startTime']
|
||
|
|
end = entry['endTime']
|
||
|
|
t = 'Phen: ' + entry['phen'] + ' Sig: ' + entry['sig'] + \
|
||
|
|
' Seg: ' + repr(entry['seg']) + ' Key: ' + key + \
|
||
|
|
' UFN: ' + repr(ufn) + \
|
||
|
|
'\nStart: ' + time.asctime(time.gmtime(start)) + ' ' + repr(start) +\
|
||
|
|
'\nEnd: ' + time.asctime(time.gmtime(end)) + ' ' + repr(end) +\
|
||
|
|
'\nCities: ' + zones + '\n\n'
|
||
|
|
return t
|
||
|
|
|
||
|
|
# Comparison routine for two hazard entries. Returns 1 if the
|
||
|
|
# two records are equal. The fields to compare are provided.
|
||
|
|
def hazardCompare(self, rec1, rec2, fields):
|
||
|
|
for f in fields:
|
||
|
|
inRec1 = f in rec1
|
||
|
|
inRec2 = f in rec2
|
||
|
|
if inRec1 and inRec2:
|
||
|
|
if rec1[f] != rec2[f]:
|
||
|
|
return 0
|
||
|
|
elif inRec1 or inRec2:
|
||
|
|
return 0 # one record has the field, set it not equal
|
||
|
|
else:
|
||
|
|
continue # neither record has this field, that is ok
|
||
|
|
return 1
|
||
|
|
|
||
|
|
def _get4ID(self, id):
|
||
|
|
from com.raytheon.uf.common.site import SiteMap
|
||
|
|
return SiteMap.getInstance().getSite4LetterId(id)
|
||
|
|
|
||
|
|
#------------------------------------------------------------------
|
||
|
|
# Table lock/unlock read/write utility
|
||
|
|
#------------------------------------------------------------------
|
||
|
|
|
||
|
|
def saveOldActiveTable(self, oldActiveTable):
|
||
|
|
# saves off the specified table and time stamps it
|
||
|
|
|
||
|
|
if self._activeTableFilename is None:
|
||
|
|
raise Exception("saveOldActiveTable without filename")
|
||
|
|
|
||
|
|
# determine filename
|
||
|
|
directory = os.path.join(
|
||
|
|
os.path.dirname(
|
||
|
|
self._activeTableFilename),
|
||
|
|
"backup")
|
||
|
|
try:
|
||
|
|
os.makedirs(directory)
|
||
|
|
except OSError as e:
|
||
|
|
if e.errno != errno.EEXIST:
|
||
|
|
LogStream.logProblem("Could not create active table backup directory:",
|
||
|
|
directory, LogStream.exc())
|
||
|
|
raise e
|
||
|
|
|
||
|
|
baseN = os.path.basename(self._activeTableFilename)
|
||
|
|
fn = datetime.datetime.utcnow().strftime("%Y%m%d_%H%M%S_%f") + "_" + baseN
|
||
|
|
filename = os.path.join(directory, fn + ".gz")
|
||
|
|
|
||
|
|
t = time.time()
|
||
|
|
try:
|
||
|
|
os.chmod(filename, 0o666)
|
||
|
|
os.remove(filename)
|
||
|
|
except Exception:
|
||
|
|
pass
|
||
|
|
|
||
|
|
# output file
|
||
|
|
# gzip it to save space
|
||
|
|
with gzip.GzipFile(filename, 'wb', 9) as fd:
|
||
|
|
buf = pickle.dumps(oldActiveTable)
|
||
|
|
fd.write(buf)
|
||
|
|
os.chmod(filename, 0o664)
|
||
|
|
|
||
|
|
t1 = time.time()
|
||
|
|
tstr = "%.3f" % (t1 - t)
|
||
|
|
LogStream.logVerbose("Saved Previous Active Table: ", fn, "t=",
|
||
|
|
tstr, "sec.")
|
||
|
|
|
||
|
|
def purgeOldSavedTables(self, purgeTime):
|
||
|
|
# purges old saved tables
|
||
|
|
|
||
|
|
if self._activeTableFilename is None:
|
||
|
|
raise Exception("purgeOldSavedTables without filename")
|
||
|
|
|
||
|
|
# calculate purge time
|
||
|
|
purgeTime = time.time() - (purgeTime * 3600)
|
||
|
|
|
||
|
|
#directory and files
|
||
|
|
directory = os.path.join(
|
||
|
|
os.path.dirname(
|
||
|
|
self._activeTableFilename),
|
||
|
|
"backup")
|
||
|
|
baseN = os.path.basename(self._activeTableFilename)
|
||
|
|
idx = baseN.find(".")
|
||
|
|
if idx != -1:
|
||
|
|
baseN = baseN[0:idx]
|
||
|
|
files = glob.glob(directory + "/*" + baseN + "*.gz")
|
||
|
|
|
||
|
|
# delete files older than purgeTime
|
||
|
|
for f in files:
|
||
|
|
try:
|
||
|
|
modTime = os.stat(f)[stat.ST_MTIME]
|
||
|
|
if modTime < purgeTime:
|
||
|
|
os.remove(f)
|
||
|
|
LogStream.logDebug("Removing old file: ", f)
|
||
|
|
except Exception:
|
||
|
|
LogStream.logProblem("Problem Removing old backup file: ", f,
|
||
|
|
LogStream.exc())
|
||
|
|
|
||
|
|
def _convertTableToPurePython(self, table, siteId):
|
||
|
|
javaRecList = ArrayList()
|
||
|
|
for rec in table:
|
||
|
|
javaRecList.add(rec.javaRecord())
|
||
|
|
|
||
|
|
javaDictFormat = ActiveTableUtil.convertToDict(javaRecList, siteId)
|
||
|
|
|
||
|
|
return JUtil.javaObjToPyVal(javaDictFormat)
|
||
|
|
|
||
|
|
|
||
|
|
def backupActiveTable(activeTable, activeTableMode, filePath, siteId):
|
||
|
|
import ActiveTableRecord
|
||
|
|
pyActive = []
|
||
|
|
szActive = activeTable.size()
|
||
|
|
for i in range(szActive):
|
||
|
|
pyActive.append(
|
||
|
|
ActiveTableRecord.ActiveTableRecord(
|
||
|
|
activeTable.get(i)))
|
||
|
|
|
||
|
|
# create a dummy name to simplify the file access code in VTECTableUtil
|
||
|
|
util = VTECTableUtil(os.path.join(filePath, activeTableMode + ".tbl"))
|
||
|
|
oldActiveTable = util._convertTableToPurePython(pyActive, siteId)
|
||
|
|
util.saveOldActiveTable(oldActiveTable)
|