xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge "Omaha #4129 Log active table changes from ingestAT" into omaha_14.4.1

Browse source 
Former-commit-id: eaecbaa0b66153b43bfe59f96c43a1876b9e235f
This commit is contained in:
Ron Anderson 2015-03-04 16:06:41 -06:00 • committed by Gerrit Code Review
commit e08d469768
5 changed files with 68 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
edexOsgi/build.edex/esb/conf/logback.xml
View file

@ -22,6 +22,30 @@
</encoder>
</appender>
<!-- activeTableChange log -->
<!-- When migrating this change to 15.1.1 it will need to be split up properly.
See Roger Ferrel for instruction
-->
<appender name="activeTableChangeLog" class="ch.qos.logback.core.rolling.RollingFileAppender">
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${edex.home}/logs/edex-${edex.run.mode}-activeTableChange-%d{yyyyMMdd}.log</fileNamePattern>
<maxHistory>30</maxHistory>
</rollingPolicy>
<encoder>
<pattern>%-5p %d [%t] %c{0}: %m%n</pattern>
</encoder>
</appender>
<appender name="activeTableChangeLogAsync" class="ch.qos.logback.classic.AsyncAppender">
<appender-ref ref="activeTableChangeLog" />
</appender>
<logger name="ActiveTableChange" additivity="false">
<level value="DEBUG"/>
<appender-ref ref="activeTableChangeLogAsync" />
</logger>
<!-- ProductSrvRequest log -->
<appender name="ProductSrvRequestLog" class="ch.qos.logback.core.rolling.RollingFileAppender">
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">

2
edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/gfe/isc/iscDataRec.py
View file

@ -196,7 +196,7 @@ def execIscDataRec(MSGID,SUBJECT,FILES):
elif SUBJECT == 'GET_ACTIVE_TABLE':
IrtServer.getVTECActiveTable(dataFile, None)
elif SUBJECT == 'GET_ACTIVE_TABLE2':
IrtServer.getVTECActiveTable(dateFile, xmlFileBuf)
IrtServer.getVTECActiveTable(dataFile, xmlFileBuf)
elif SUBJECT in ['ISCGRIDS', 'ISCGRIDS2']:
args = {"siteID": siteConfig.GFESUITE_SITEID,
"userID": 'SITE',

5
edexOsgi/com.raytheon.uf.edex.activetable/src/com/raytheon/uf/edex/activetable/ActiveTable.java
View file

@ -100,6 +100,7 @@ import com.raytheon.uf.edex.database.query.DatabaseQuery;
* Pass issuance site id to getActiveTable()
* in updateActiveTable() so records will
* be updated correctly.
* Mar 04, 2015 4129 randerso Pass active table change logger to ingestAt and/or MergeVTEC
*
* </pre>
*
@ -300,7 +301,8 @@ public class ActiveTable {
.getHandler("ActiveTable");
ITimer timer = TimeUtil.getTimer();
timer.start();
List<ActiveTableRecord> activeTable = getActiveTable(issueSiteId, mode);
List<ActiveTableRecord> activeTable = getActiveTable(issueSiteId,
mode);
timer.stop();
perfStat.logDuration("getActiveTable", timer.getElapsedTime());
@ -589,6 +591,7 @@ public class ActiveTable {
args.put("newRecords", newRecords);
args.put("drt", timeOffset);
args.put("makeBackups", makeBackup);
args.put("atChangeLog", changeLog);
if (runIngestAT) {
args.put("xmlIncoming", xmlSource);
}

63
edexOsgi/com.raytheon.uf.edex.activetable/utility/common_static/base/vtec/MergeVTEC.py
View file

@ -35,8 +35,10 @@
# 01/24/14 #2504 randerso change to use iscUtil.getLogger for consistency
# 03/25/14 #2884 randerso Added xxxid to VTECChange
# 05/15/14 #3157 dgilling Support multiple TPC and SPC sites.
# 03/04/2015 #4129 randerso Log the active table changes at info level
# in the active table change log
#
##
import copy
import cPickle
@ -64,7 +66,7 @@ from com.raytheon.uf.common.activetable import VTECPartners as JavaVTECPartners
class MergeVTEC(VTECTableUtil.VTECTableUtil):
def __init__(self, activeTable, activeTableMode, newRecords, offsetSecs=0.0,
makeBackups=True, logger=None):
makeBackups=True, logger=None, atChangeLog=None):
# activeTable - current activeTable records
# activeTableMode - which table is being modified--OPERATIONAL or PRACTICE
# newRecords - records to merge in to activeTable
@ -125,7 +127,7 @@ class MergeVTEC(VTECTableUtil.VTECTableUtil):
self._log.info("Other Table squeezed size: %d", len(otherTable))
#merge the tables
updatedTable, toDelete, changes = self._mergeTable(activeTable, otherTable)
updatedTable, toDelete, changes = self._mergeTable(activeTable, otherTable, atChangeLog)
self._log.info("Updated Active Table size: %d", len(updatedTable))
updatedTable, tossRecordsMerged = vts.squeeze(updatedTable)
self._log.info("Updated Active Table squeeze size: %d",
@ -161,7 +163,7 @@ class MergeVTEC(VTECTableUtil.VTECTableUtil):
# merges the active and other table together and returns the merged
# table along with the list of changes that occurred.
def _mergeTable(self, activeTable, otherTable):
def _mergeTable(self, activeTable, otherTable, atChangeLog):
changes = []
purges = []
compare = ['id', 'phen', 'sig', 'officeid', 'etn', 'pil']
@ -342,30 +344,31 @@ class MergeVTEC(VTECTableUtil.VTECTableUtil):
changes.append(chgRec)
# log the changes
if len(missingEntriesAct):
self._log.debug("Active Missing entries added: %s",
self.printActiveTable(missingEntriesAct, 1))
if len(newReplaceEntriesAct):
self._log.debug("Active Replacement entries (new): %s",
self.printActiveTable(newReplaceEntriesAct, 1))
if len(oldReplaceEntriesAct):
self._log.debug("Active Entries Replaced (old): %s",
self.printActiveTable(oldReplaceEntriesAct, 1))
if len(missingEntriesPast):
self._log.debug("Past Missing entries added %s",
self.printActiveTable(missingEntriesPast, 1))
if len(newReplaceEntriesPast):
self._log.debug("Past Replacement entries (new): %s",
self.printActiveTable(newReplaceEntriesPast, 1))
if len(oldReplaceEntriesPast):
self._log.debug("Past Entries Replaced (old): %s",
self.printActiveTable(oldReplaceEntriesPast, 1))
if len(ignoredNewReplaceAct):
self._log.debug("Ignored Different Year Issuance (new): %s",
self.printActiveTable(ignoredNewReplaceAct, 1))
self._log.debug("Ignored Different Year Issuance (old): %s",
self.printActiveTable(ignoredOldReplaceAct, 1))
self._log.debug("Table Changes: %s", changes)
if atChangeLog is not None:
if len(missingEntriesAct):
atChangeLog.info("Active Missing entries added: " +
self.printActiveTable(missingEntriesAct, 1))
if len(newReplaceEntriesAct):
atChangeLog.info("Active Replacement entries (new): " +
self.printActiveTable(newReplaceEntriesAct, 1))
if len(oldReplaceEntriesAct):
atChangeLog.info("Active Entries Replaced (old): " +
self.printActiveTable(oldReplaceEntriesAct, 1))
if len(missingEntriesPast):
atChangeLog.info("Past Missing entries added " +
self.printActiveTable(missingEntriesPast, 1))
if len(newReplaceEntriesPast):
atChangeLog.info("Past Replacement entries (new): " +
self.printActiveTable(newReplaceEntriesPast, 1))
if len(oldReplaceEntriesPast):
atChangeLog.info("Past Entries Replaced (old): " +
self.printActiveTable(oldReplaceEntriesPast, 1))
if len(ignoredNewReplaceAct):
atChangeLog.info("Ignored Different Year Issuance (new): " +
self.printActiveTable(ignoredNewReplaceAct, 1))
atChangeLog.info("Ignored Different Year Issuance (old): " +
self.printActiveTable(ignoredOldReplaceAct, 1))
atChangeLog.info("Table Changes: " + str(changes))
purges.extend(oldReplaceEntriesAct)
purges.extend(oldReplaceEntriesPast)
@ -411,7 +414,7 @@ class MergeVTEC(VTECTableUtil.VTECTableUtil):
return iscUtil.getLogger("MergeVTEC", logLevel=logging.INFO)
def merge(activeTable, activeTableMode, newRecords, drt=0.0, makeBackups=True,
logger=None):
logger=None, atChangeLog=None):
pyActive = []
for i in range(activeTable.size()):
pyActive.append(ActiveTableRecord.ActiveTableRecord(activeTable.get(i)))
@ -420,7 +423,7 @@ def merge(activeTable, activeTableMode, newRecords, drt=0.0, makeBackups=True,
for i in range(newRecords.size()):
pyNew.append(ActiveTableRecord.ActiveTableRecord(newRecords.get(i)))
decoder = MergeVTEC(pyActive, activeTableMode, pyNew, drt, makeBackups, logger)
decoder = MergeVTEC(pyActive, activeTableMode, pyNew, drt, makeBackups, logger, atChangeLog)
mergeResults = decoder.getMergeResults()
decoder = None

10
edexOsgi/com.raytheon.uf.edex.activetable/utility/common_static/base/vtec/ingestAT.py
View file

@ -28,7 +28,9 @@
# ------------ ---------- ----------- --------------------------
# 02/13/13 1447 dgilling Initial Creation.
# 01/24/14 2504 randerso change to use iscUtil.getLogger for consistency
# 03/04/2015 4129 randerso Pass active table change logger through to MergeVtec.merge
#
##
import os
@ -48,7 +50,7 @@ def init_logging():
logger = iscUtil.getLogger("ingestAT", logLevel=logging.INFO)
def execute_ingest_at(incomingRecords, activeTable, atName, ztime, makeBackups, xmlIncoming):
def execute_ingest_at(incomingRecords, activeTable, atName, ztime, makeBackups, xmlIncoming, atChangeLog):
# log the source of this data
if xmlIncoming is not None:
irt = IrtAccess.IrtAccess("")
@ -63,20 +65,20 @@ def execute_ingest_at(incomingRecords, activeTable, atName, ztime, makeBackups,
results = None
try:
results = MergeVTEC.merge(activeTable, atName, incomingRecords, ztime, makeBackups,
logger)
logger, atChangeLog)
except:
logger.exception("MergeVTEC fail:")
return results
def runFromJava(activeTable, activeTableMode, newRecords, drt, makeBackups,
xmlIncoming):
xmlIncoming, atChangeLog=None):
init_logging()
logger.info('************* ingestAT ************************')
startT = time.time()
results = execute_ingest_at(newRecords, activeTable, activeTableMode, drt,
makeBackups, xmlIncoming)
makeBackups, xmlIncoming, atChangeLog=atChangeLog)
#--------------------------------------------------------------------
# Finish