diff --git a/cave/build/cave/memorySettings.xml b/cave/build/cave/memorySettings.xml index b37876845e..65a6c0c1e1 100644 --- a/cave/build/cave/memorySettings.xml +++ b/cave/build/cave/memorySettings.xml @@ -107,6 +107,40 @@ + + + + -component + servicebackup + + + + + 256M + + + + 128m + + + + + + + -perspective + National Centers + + + + + 1536M + + + + DEFAULT + + + @@ -182,6 +216,40 @@ - + + + + -component + servicebackup + + + + + 256M + + + + 128m + + + + + + -perspective + National Centers + + + + + 2048M + + + + DEFAULT + + + + + diff --git a/cave/build/static/linux/alertviz/alertviz.sh b/cave/build/static/linux/alertviz/alertviz.sh index c5ad28154e..e9ee6f88e7 100644 --- a/cave/build/static/linux/alertviz/alertviz.sh +++ b/cave/build/static/linux/alertviz/alertviz.sh @@ -85,8 +85,10 @@ if [ -f /awips2/java/jre/lib/amd64/server/libjvm.so ]; then fi #run a loop for alertviz -while [ $exitVal -ne 0 ] +count=0 +while [ $exitVal -ne 0 -a $count -lt 10 ] do + count=`expr $count + 1` curTime=`date +%Y%m%d_%H%M%S` LOGFILE=${LOGDIR}/alertviz_${curTime}_console.log export LOGFILE_ALERTVIZ=${LOGDIR}/alertviz_${curTime}_admin.log diff --git a/cave/build/static/linux/cave/cave.sh b/cave/build/static/linux/cave/cave.sh index d2d2908f16..7dcd826280 100644 --- a/cave/build/static/linux/cave/cave.sh +++ b/cave/build/static/linux/cave/cave.sh @@ -28,6 +28,8 @@ # cave sessions. # Dec 05, 2013 #2590 dgilling Modified so gfeclient.sh can be wrapped # around this script. +# Jan 24, 2014 #2739 bsteffen Log exit status +# Jan 30, 2014 #2593 bclement warns based on memory usage, fixed for INI files with spaces # # @@ -45,6 +47,8 @@ JAVA_INSTALL="/awips2/java" PYTHON_INSTALL="/awips2/python" export AWIPS_INSTALL_DIR="${CAVE_INSTALL}" +MAX_MEM_PROPORTION="0.9" + source ${CAVE_INSTALL}/caveUtil.sh RC=$? if [ ${RC} -ne 0 ]; then @@ -61,6 +65,9 @@ copyVizShutdownUtilIfNecessary # delete any old disk caches in the background deleteOldCaveDiskCaches & +# Enable core dumps +ulimit -c unlimited + export LD_LIBRARY_PATH=${JAVA_INSTALL}/lib:${PYTHON_INSTALL}/lib:$LD_LIBRARY_PATH export LD_PRELOAD=libpython.so if [[ -z "$CALLED_EXTEND_LIB_PATH" ]]; then @@ -109,20 +116,39 @@ export TEXTWS=`hostname | sed -e 's/lx/xt/g'` hostName=`hostname -s` -if [[ $hostName =~ xt.* ]]; then - export IGNORE_NUM_CAVES=1 -fi - # check number of running caves if [[ -z $IGNORE_NUM_CAVES ]]; then - # free usually reports below on G threshold (11 instead of 12G), giving the 3 cave recommended in field - mem=( `free -g | grep "Mem:"` ) + # get total memory on system in bytes + mem=( `free -b | grep "Mem:"` ) mem=${mem[1]} - let _maxCaves=mem/3 - - getPidsOfMyRunningCaves - if [[ "$_numPids" -ge "$_maxCaves" ]]; then - zenity --question --title "Max CAVE sessions already running" --text "$_numPids CAVE sessions already running. Starting more may impact system performance and stability.\n\nProceed?" + # get max amount of system memory used before we warn + memThreshold=$(echo "$mem * $MAX_MEM_PROPORTION" | bc) + # remove decimal + printf -v memThreshold "%.0f" "$memThreshold" + # get launcher.ini argument determined by user arguments + lookupINI "$@" + launcherRegex='--launcher.ini\s(.+\.ini)' + # default to cave.ini + targetIni="/awips2/cave/cave.ini" + if [[ $CAVE_INI_ARG =~ $launcherRegex ]] + then + targetIni="${BASH_REMATCH[1]}" + fi + # read max memory that could be used by this instance + memOfLaunchingCave=$(readMemFromIni "$targetIni") + # read total max memory of caves already running + getTotalMemOfRunningCaves + # add them together + _totalAfterStart=$(($memOfLaunchingCave + $_totalRunningMem)) + if [[ "$_totalAfterStart" -ge "$memThreshold" ]]; then + # convert to megs for display + memOfLaunchingCave=$(($memOfLaunchingCave / $BYTES_IN_MB)) + _totalRunningMem=$(($_totalRunningMem / $BYTES_IN_MB)) + getPidsOfMyRunningCaves + memMsg="$_numPids CAVE applications already running with a combined max memory of ${_totalRunningMem}MB. " + memMsg+="The requested application has a max memory requirement of ${memOfLaunchingCave}MB. " + memMsg+="Starting may impact system performance and stability.\n\nProceed?" + zenity --question --title "Low Available Memory for Application" --text "$memMsg" cancel="$?" if [[ "$cancel" == "1" ]]; then @@ -172,49 +198,59 @@ if [ ! -d $LOGDIR ]; then mkdir -p $LOGDIR fi -export pid=$$ - curTime=`date +%Y%m%d_%H%M%S` -LOGFILE="${LOGDIR}/${PROGRAM_NAME}_${curTime}_pid_${pid}_console.log" -export LOGFILE_CAVE="${LOGDIR}/${PROGRAM_NAME}_${curTime}_pid_${pid}_alertviz.log" -export LOGFILE_PERFORMANCE="${LOGDIR}/${PROGRAM_NAME}_${curTime}_pid_${pid}_perf.log" -# can we write to log directory -if [ -w ${LOGDIR} ]; then - touch ${LOGFILE} -fi +# At this point fork so that log files can be set up with the process pid and +# this process can log the exit status of cave. +( + export pid=`/bin/bash -c 'echo $PPID'` -# remove "-noredirect" flag from command-line if set so it doesn't confuse any -# commands we call later. -redirect="true" -USER_ARGS=() -while [[ $1 ]] -do + LOGFILE="${LOGDIR}/${PROGRAM_NAME}_${curTime}_pid_${pid}_console.log" + export LOGFILE_CAVE="${LOGDIR}/${PROGRAM_NAME}_${curTime}_pid_${pid}_alertviz.log" + export LOGFILE_PERFORMANCE="${LOGDIR}/${PROGRAM_NAME}_${curTime}_pid_${pid}_perf.log" + + # can we write to log directory + if [ -w ${LOGDIR} ]; then + touch ${LOGFILE} + fi + + # remove "-noredirect" flag from command-line if set so it doesn't confuse any + # commands we call later. + redirect="true" + USER_ARGS=() + while [[ $1 ]] + do if [[ "$1" == "-noredirect" ]] then - redirect="false" + redirect="false" else - USER_ARGS+=("$1") + USER_ARGS+=("$1") fi shift -done + done -# Special instructions for the 64-bit jvm. -ARCH_ARGS="" -if [ -f /awips2/java/jre/lib/amd64/server/libjvm.so ]; then - ARCH_ARGS="-vm /awips2/java/jre/lib/amd64/server/libjvm.so" -fi + # Special instructions for the 64-bit jvm. + ARCH_ARGS="" + if [ -f /awips2/java/jre/lib/amd64/server/libjvm.so ]; then + ARCH_ARGS="-vm /awips2/java/jre/lib/amd64/server/libjvm.so" + fi -lookupINI "${USER_ARGS[@]}" + lookupINI "${USER_ARGS[@]}" -if [[ "${runMonitorThreads}" == "true" ]] ; then - # nohup to allow tar process to continue after user has logged out - nohup ${CAVE_INSTALL}/monitorThreads.sh $pid >> /dev/null 2>&1 & -fi + if [[ "${runMonitorThreads}" == "true" ]] ; then + # nohup to allow tar process to continue after user has logged out + nohup ${CAVE_INSTALL}/monitorThreads.sh $pid >> /dev/null 2>&1 & + fi + + if [[ "${redirect}" == "true" ]] ; then + exec ${CAVE_INSTALL}/cave ${ARCH_ARGS} ${SWITCHES} "${CAVE_INI_ARG}" "${USER_ARGS[@]}" > ${LOGFILE} 2>&1 + else + exec ${CAVE_INSTALL}/cave ${ARCH_ARGS} ${SWITCHES} "${CAVE_INI_ARG}" "${USER_ARGS[@]}" 2>&1 | tee ${LOGFILE} + fi +) & + +pid=$! +LOGFILE="${LOGDIR}/${PROGRAM_NAME}_${curTime}_pid_${pid}_console.log" +logExitStatus $pid $LOGFILE -if [[ "${redirect}" == "true" ]] ; then - exec ${CAVE_INSTALL}/cave ${ARCH_ARGS} ${SWITCHES} ${CAVE_INI_ARG} "${USER_ARGS[@]}" > ${LOGFILE} 2>&1 -else - exec ${CAVE_INSTALL}/cave ${ARCH_ARGS} ${SWITCHES} ${CAVE_INI_ARG} "${USER_ARGS[@]}" 2>&1 | tee ${LOGFILE} -fi diff --git a/cave/build/static/linux/cave/caveUtil.sh b/cave/build/static/linux/cave/caveUtil.sh index a0bcf5ec78..0930433efe 100644 --- a/cave/build/static/linux/cave/caveUtil.sh +++ b/cave/build/static/linux/cave/caveUtil.sh @@ -25,6 +25,10 @@ # Dec 05, 2013 #2593 rjpeter Fix getPidsOfMyRunningCaves # Dec 05, 2013 #2590 dgilling Modified extendLibraryPath() to export a # var if it's already been run. +# Jan 24, 2014 #2739 bsteffen Add method to log exit status of process. +# Jan 30, 2014 #2593 bclement extracted generic part of getPidsOfMyRunningCaves into forEachRunningCave +# added methods for reading max memory from .ini files +# fixes for INI files with spaces # # @@ -39,6 +43,10 @@ fi # This script will be sourced by cave.sh. export CAVE_INI_ARG= +BYTES_IN_KB=1024 +BYTES_IN_MB=1048576 +BYTES_IN_GB=1073741824 + function lookupINI() { # Arguments: @@ -55,7 +63,7 @@ function lookupINI() position=$(( $position + 1 )) nextArg=${!position} - retrieveAssociatedINI ${arg} ${nextArg} + retrieveAssociatedINI ${arg} "${nextArg}" RC=$? if [ ${RC} -eq 0 ]; then export CAVE_INI_ARG="--launcher.ini /awips2/cave/${ASSOCIATED_INI}" @@ -116,8 +124,8 @@ function copyVizShutdownUtilIfNecessary() chmod a+x ${HOME}/.kde/shutdown/${VIZ_UTILITY_SCRIPT} } -# returns _numPids and array _pids containing the pids of the currently running cave sessions. -function getPidsOfMyRunningCaves() +# takes a function as an argument and calls the function passing in the ps string of the process +function forEachRunningCave() { local user=`whoami` local caveProcs=`ps -ef | grep -E "(/awips2/cave|/usr/local/viz)/cave " | grep -v "grep" | grep $user` @@ -125,17 +133,89 @@ function getPidsOfMyRunningCaves() # preserve IFS and set it to line feed only local PREV_IFS=$IFS IFS=$'\n' - _numPids=0 - # grab the pids for future use for caveProc in $caveProcs do - _pids[$_numPids]=`echo $caveProc | awk '{print $2}'` - let "_numPids+=1" + "$@" $caveProc done IFS=$PREV_IFS } +# takes in ps string of cave process, stores pid in _pids and increments _numPids +function processPidOfCave() +{ + _pids[$_numPids]=`echo $1 | awk '{print $2}'` + let "_numPids+=1" +} + +# returns _numPids and array _pids containing the pids of the currently running cave sessions. +function getPidsOfMyRunningCaves() +{ + _numPids=0 + forEachRunningCave processPidOfCave +} + +# takes a name of an ini file as an argument, echos the memory (in bytes) from file (or default) +function readMemFromIni() +{ + local inifile="$1" + local mem + local unit + local regex='^[^#]*-Xmx([0-9]+)([bBkKmMgG])?' + # read ini file line by line looking for Xmx arg + while read -r line + do + if [[ $line =~ $regex ]] + then + mem=${BASH_REMATCH[1]} + unit=${BASH_REMATCH[2]} + break + fi + done < "$inifile" + # convert to bytes + case "$unit" in + [kK]) + mem=$(($mem * $BYTES_IN_KB)) + ;; + [mM]) + mem=$(($mem * $BYTES_IN_MB)) + ;; + [gG]) + mem=$(($mem * $BYTES_IN_GB)) + ;; + esac + regex='^[0-9]+$' + if [[ ! $mem =~ $regex ]] + then + # we couldn't find a valid Xmx value + # java default is usually 1G + mem=1073741824 + fi + echo $mem +} + +# takes in ps string of cave process, reads Xmx from ini and adds bytes to _totalRunninMem +function addMemOfCave() +{ + local inifile + # get ini file from process string + local regex='--launcher.ini\s(.+\.ini)' + if [[ $1 =~ $regex ]] + then + inifile="${BASH_REMATCH[1]}" + else + inifile="/awips2/cave/cave.ini" + fi + let "_totalRunningMem+=$(readMemFromIni "$inifile")" +} + +# finds total max memory of running caves in bytes and places it in _totalRunningMem +function getTotalMemOfRunningCaves() +{ + _totalRunningMem=0 + forEachRunningCave addMemOfCave +} + function deleteOldCaveDiskCaches() { local curDir=`pwd` @@ -193,3 +273,28 @@ function deleteOldCaveDiskCaches() cd $curDir } + +# log the exit status and time to a log file, requires 2 args pid and log file +function logExitStatus() +{ + pid=$1 + logFile=$2 + + trap 'kill $pid' SIGHUP SIGINT SIGQUIT SIGTERM + wait $pid + exitCode=$? + curTime=`date --rfc-3339=seconds` + echo Exited at $curTime with an exit status of $exitCode >> $logFile + + # If a core file was generated attempt to save it to a better place + coreFile=core.$pid + if [ -f "$coreFile" ]; then + basePath="/data/fxa/cave" + hostName=`hostname -s` + hostPath="$basePath/$hostName/" + mkdir -p $hostPath + if [ -d "$hostPath" ]; then + cp $coreFile $hostPath + fi + fi +} diff --git a/cave/com.raytheon.uf.viz.archive/src/com/raytheon/uf/viz/archive/ui/GenerateCaseDlg.java b/cave/com.raytheon.uf.viz.archive/src/com/raytheon/uf/viz/archive/ui/GenerateCaseDlg.java index dc398ba589..2dc0b715b2 100644 --- a/cave/com.raytheon.uf.viz.archive/src/com/raytheon/uf/viz/archive/ui/GenerateCaseDlg.java +++ b/cave/com.raytheon.uf.viz.archive/src/com/raytheon/uf/viz/archive/ui/GenerateCaseDlg.java @@ -81,6 +81,7 @@ import com.raytheon.viz.ui.dialogs.CaveSWTDialog; * archive and category directory and * implementation of compression. * Oct 08, 2013 2442 rferrel Remove category directory. + * Feb 04, 2013 2270 rferrel Move HDF files to parent's directory. * * * @@ -89,6 +90,10 @@ import com.raytheon.viz.ui.dialogs.CaveSWTDialog; */ public class GenerateCaseDlg extends CaveSWTDialog { + + /** Extension for HDF files. */ + private static final String hdfExt = ".h5"; + private final IUFStatusHandler statusHandler = UFStatus .getHandler(GenerateCaseDlg.class); @@ -539,6 +544,11 @@ public class GenerateCaseDlg extends CaveSWTDialog { new File(destination, file)); } } else { + // DR 2270 bump HDF files up a directory. + if (destination.getName().endsWith(hdfExt)) { + destination = new File(destination.getParentFile() + .getParentFile(), destination.getName()); + } FileUtil.copyFile(source, destination); destination.setLastModified(source.lastModified()); } @@ -652,6 +662,13 @@ public class GenerateCaseDlg extends CaveSWTDialog { addTarFiles(file.listFiles()); } } else { + // DR 2270 bump HDF files up a directory. + if (name.endsWith(hdfExt)) { + File destination = new File(file.getParentFile() + .getParentFile(), file.getName()); + name = destination.getAbsolutePath().substring( + startRelativePath); + } TarArchiveEntry entry = new TarArchiveEntry(file, name); entry.setSize(file.length()); FileInputStream fileStream = null; diff --git a/cave/com.raytheon.uf.viz.personalities.cave/src/com/raytheon/uf/viz/personalities/cave/workbench/VizWorkbenchAdvisor.java b/cave/com.raytheon.uf.viz.personalities.cave/src/com/raytheon/uf/viz/personalities/cave/workbench/VizWorkbenchAdvisor.java index 1032a6b61d..f45b2e448a 100644 --- a/cave/com.raytheon.uf.viz.personalities.cave/src/com/raytheon/uf/viz/personalities/cave/workbench/VizWorkbenchAdvisor.java +++ b/cave/com.raytheon.uf.viz.personalities.cave/src/com/raytheon/uf/viz/personalities/cave/workbench/VizWorkbenchAdvisor.java @@ -57,6 +57,7 @@ import com.raytheon.viz.ui.perspectives.VizPerspectiveListener; * Jul 16, 2013 2158 bsteffen Allow VizGlobalsManager to work without * accessing UI thread. * Oct 15, 2013 2361 njensen Added startupTimer + * Jan 27, 2014 2744 njensen Add Local History pref back in * * * @@ -192,8 +193,17 @@ public class VizWorkbenchAdvisor extends WorkbenchAdvisor { for (IPreferenceNode root : topNodes) { String rootId = root.getId(); if (rootId.equals("org.eclipse.ui.preferencePages.Workbench")) { + IPreferenceNode node = root + .findSubNode("org.eclipse.ui.preferencePages.Workspace"); + if (node != null) { + node.remove("org.eclipse.ui.preferencePages.LinkedResources"); + node.remove("org.eclipse.ui.preferencePages.BuildOrder"); + IPreferenceNode localHistoryNode = node + .findSubNode("org.eclipse.ui.preferencePages.FileStates"); + root.add(localHistoryNode); + root.remove("org.eclipse.ui.preferencePages.Workspace"); + } root.remove("org.eclipse.search.preferences.SearchPreferencePage"); - root.remove("org.eclipse.ui.preferencePages.Workspace"); } else if (rootId.equals("org.python.pydev.prefs")) { root.remove("org.python.pydev.ui.pythonpathconf.interpreterPreferencesPageJython"); root.remove("org.python.pydev.ui.pythonpathconf.interpreterPreferencesPageIronpython"); @@ -325,7 +335,7 @@ public class VizWorkbenchAdvisor extends WorkbenchAdvisor { startupTimer.stop(); System.out.println("Workbench startup time: " + startupTimer.getElapsedTime() + " ms"); - } + } } diff --git a/cave/com.raytheon.viz.gfe/GFESuite/gfeclient.sh b/cave/com.raytheon.viz.gfe/GFESuite/gfeclient.sh index affb32aa57..7540a923f2 100644 --- a/cave/com.raytheon.viz.gfe/GFESuite/gfeclient.sh +++ b/cave/com.raytheon.viz.gfe/GFESuite/gfeclient.sh @@ -1,5 +1,4 @@ #!/bin/bash - # CAVE startup script # Note: CAVE will not run as 'root' @@ -25,8 +24,6 @@ # SOFTWARE HISTORY # Date Ticket# Engineer Description # ------------ ---------- ----------- -------------------------- -# Dec 04, 2013 #2589 dgilling Create command-line arg that controls -# xvfb initialization. # Dec 05, 2013 #2593 rjpeter set IGNORE_NUM_CAVES # Dec 05, 2013 #2590 dgilling Remove duplicated code and call to # cave.sh. @@ -50,28 +47,12 @@ fi PROGRAM_NAME="gfeclient" -# remove "-enablegl" flag from command-line if set so it doesn't confuse any -# commands we call later. -USER_ARGS=() -while [[ $1 ]] -do - if [ "$1" == "-enablegl" ] - then - ENABLEGL="true" - else - USER_ARGS+=("$1") - fi - shift -done - -if [ -n "$ENABLEGL" ] +# if display not set +if [ -n "$DISPLAY" ] then - # if display not set - if [ -n "$DISPLAY" ] - then echo "Using Display set to $DISPLAY" extendLibraryPath - else +else echo "Display not set, creating offscreen x on port $$" extendLibraryPath "-noX" Xvfb :$$ -screen 0 1280x1024x24 & @@ -79,13 +60,12 @@ then export DISPLAY="localhost:$$.0" #don't use shader when no display set SWITCHES="${SWITCHES} -no_shader" - fi fi export IGNORE_NUM_CAVES=1 -source /awips2/cave/cave.sh -nosplash -noredirect -component gfeclient "${USER_ARGS[@]}" & -wait +source /awips2/cave/cave.sh -nosplash -noredirect -component gfeclient "$@" & +wait $! if [ -n "$xvfb" ] then diff --git a/cave/com.raytheon.viz.gfe/GFESuite/ifpIMAGE b/cave/com.raytheon.viz.gfe/GFESuite/ifpIMAGE index 5b27b0f1b4..a9f5522e6d 100755 --- a/cave/com.raytheon.viz.gfe/GFESuite/ifpIMAGE +++ b/cave/com.raytheon.viz.gfe/GFESuite/ifpIMAGE @@ -1,33 +1,5 @@ #!/bin/sh -# This software was developed and / or modified by Raytheon Company, -# pursuant to Contract DG133W-05-CQ-1067 with the US Government. -# -# U.S. EXPORT CONTROLLED TECHNICAL DATA -# This software product contains export-restricted data whose -# export/transfer/disclosure is restricted by U.S. law. Dissemination -# to non-U.S. persons whether in the United States or abroad requires -# an export license or other authorization. -# -# Contractor Name: Raytheon Company -# Contractor Address: 6825 Pine Street, Suite 340 -# Mail Stop B8 -# Omaha, NE 68106 -# 402.291.0100 -# -# See the AWIPS II Master Rights File ("Master Rights File.pdf") for -# further licensing information. -# -# -# SOFTWARE HISTORY -# Date Ticket# Engineer Description -# ------------ ---------- ----------- -------------------------- -# Dec 04, 2013 #2589 dgilling Create command-line arg that controls -# xvfb initialization. -# -# - - # get path to cave path_to_script=`readlink -f $0` RUN_FROM_DIR=`dirname $path_to_script` @@ -37,7 +9,6 @@ CAVE_DIR=/awips2/cave # execute the runProcedure module _GFECLI="${RUN_FROM_DIR}/gfeclient.sh" -_GFECLI_ARGS="-enablegl" _MODULE="${CAVE_DIR}/etc/gfe/utility/PngWriter.py" # quoting of '$@' is used to prevent command line interpretation @@ -46,6 +17,6 @@ then echo "CAVE and/or gfeclient not installed on this workstation ..exiting" exit 1 else - $_GFECLI $_GFECLI_ARGS $_MODULE "$@" + $_GFECLI $_MODULE "$@" fi diff --git a/cave/com.raytheon.viz.gfe/localization/gfe/userPython/textUtilities/headline/HazardsTable.py b/cave/com.raytheon.viz.gfe/localization/gfe/userPython/textUtilities/headline/HazardsTable.py index ab34845b77..4fa223d733 100644 --- a/cave/com.raytheon.viz.gfe/localization/gfe/userPython/textUtilities/headline/HazardsTable.py +++ b/cave/com.raytheon.viz.gfe/localization/gfe/userPython/textUtilities/headline/HazardsTable.py @@ -1,19 +1,19 @@ ## # This software was developed and / or modified by Raytheon Company, # pursuant to Contract DG133W-05-CQ-1067 with the US Government. -# +# # U.S. EXPORT CONTROLLED TECHNICAL DATA # This software product contains export-restricted data whose # export/transfer/disclosure is restricted by U.S. law. Dissemination # to non-U.S. persons whether in the United States or abroad requires # an export license or other authorization. -# +# # Contractor Name: Raytheon Company # Contractor Address: 6825 Pine Street, Suite 340 # Mail Stop B8 # Omaha, NE 68106 # 402.291.0100 -# +# # See the AWIPS II Master Rights File ("Master Rights File.pdf") for # further licensing information. ## @@ -21,9 +21,9 @@ # # Port of A1 HazardsTable.py. # -# +# # SOFTWARE HISTORY -# +# # Date Ticket# Engineer Description # ------------ ---------- ----------- -------------------------- # ??/??/?? ???????? Initial Creation. @@ -31,11 +31,15 @@ # ETN assignment. # 09/24/13 1843 dgilling Handle GetNextEtnResponse. # 11/20/13 2490 randerso Corrected error handling in __getActiveTable -# -# +# +# 02/05/14 2774 dgilling Fix error logging statements in +# __warnETNduplication() and +# __highestETNActiveTable. +# import time, getopt, sys, copy, string, logging +import os import VTECTableUtil, VTECTable import TimeRange, AbsTime, ActiveTableVtec import JUtil @@ -53,9 +57,9 @@ import cPickle # hazard strings and VTEC strings for formatters. Alternate active tables # may be defined for test purposes. class HazardsTable(VTECTableUtil.VTECTableUtil): - def __init__(self, ifpClient, editAreas, productCategory, - filterMethod, databaseID, siteID4, activeTableName = "", - vtecMode = None, samplingThreshold = (10, None), hazardEndTime=None, + def __init__(self, ifpClient, editAreas, productCategory, + filterMethod, databaseID, siteID4, activeTableName="", + vtecMode=None, samplingThreshold=(10, None), hazardEndTime=None, creationTime=None, dataMgr=None, accurateCities=False, cityEditAreas=[]): self.log = logging.getLogger("FormatterRunner.HazardsTable.HazardsTable") # self.log.setLevel(logging.DEBUG) @@ -90,17 +94,17 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): self.__tpcKeys = self.__processJavaCollection(GFEVtecUtil.TROPICAL_PHENSIGS, self.__convertPhensig) self.__tpcBaseETN = '1001' self.__ncKeys = self.__processJavaCollection(GFEVtecUtil.NATIONAL_PHENSIGS, self.__convertPhensig) - self.__ufnKeys = [('HU','A'), ('HU','S'), ('HU','W'), ('TR','A'), ('TR','W'), - ('TY','A'), ('TY','W')] - + self.__ufnKeys = [('HU', 'A'), ('HU', 'S'), ('HU', 'W'), ('TR', 'A'), ('TR', 'W'), + ('TY', 'A'), ('TY', 'W')] + self.__sitesIgnoreNatlEtn = self.__processJavaCollection(GFEVtecUtil.IGNORE_NATIONAL_ETN, str) - + self.__marineZonesPrefix = ["AM", "GM", "PZ", "PK", "PH", "PM", "AN", "PS", "SL"] #list of zone name prefix that are marine zones - + # tuple of (% area coverage, numberGridCells) self.__samplingThreshold = \ - (samplingThreshold[0]/100.0, samplingThreshold[1]) + (samplingThreshold[0] / 100.0, samplingThreshold[1]) #determine creation time if creationTime is not None: @@ -117,7 +121,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): self.__zoneList = self.__singleZoneList(editAreas) #sample, and merge vtec codes - self.__rawAnalyzedTable = self.__analyzedTable(self.__zoneList, + self.__rawAnalyzedTable = self.__analyzedTable(self.__zoneList, self.filterMethod) #reorganize raw analyzed table into hazards by zone, might cause @@ -127,7 +131,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): # organize by id self.__hazardsByZoneDict = self.__organizeByZone( self.__rawAnalyzedTable) - + self.__hazardCombinations = self.__recombineZoneGroups( self.__hazardsByZoneDict, editAreas) @@ -162,7 +166,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): # Find the hazards that apply to the area and timeRange, and returns # a list of dictionaries. This function can take a single string or # a list. Restriction: only looks at the first element in the list. - # The returned list's 'id' field is a list of zones with that + # The returned list's 'id' field is a list of zones with that # hazard. if type(editAreaList) is list and len(editAreaList): @@ -179,7 +183,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): if self.__hazardsByZoneDict.has_key(ea): haz = self.__hazardsByZoneDict[ea] for h in haz: - # if a segment number is present copy while removing seg + # if a segment number is present copy while removing seg # from the key if h.has_key('seg') and h['seg'] != "": # make a copy and change the key if we need to @@ -198,7 +202,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): # Now consolidate this list of hazards with segment numbers removed. hazards = self.__consolidateTime(hazards) - + return hazards def getVTECString(self, fcstArea): @@ -215,7 +219,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): hazards.sort(self.__hazardsSort) # hazards need upgrade records to be paired up hazards = self.__pairUpgradeRecords(hazards) - + # get VTEC strings and VTEC records vtecStrings = [] for h in hazards: @@ -234,7 +238,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): def getCities(self, cityList, zoneHazards): if self.__cityHazards is None: return - + relevant = [] compare = ('phen', 'sig', 'endTime') for p in self.__cityHazards: @@ -244,7 +248,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): break return self.__getCities(cityList, relevant) - + # Get cities associated with a VTEC with an EXP action # returns None if the grid is deleted def getCitiesForEXP(self, cityList, zone, phen, sig, expTime): @@ -320,7 +324,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): unmatched = {} for rec in checkedVTEC: unmatched[event(rec)] = ugcList[:] - + cities = [] certain = 1 compare = ('phen', 'sig', 'etn') @@ -348,7 +352,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): self.log.error("Too many matches for %s.%s:%04d"\ % event(active)\ + " in zone %s" % active['id']) - + if active.get('cities') is not None: for city in active['cities']: if city not in cities: @@ -385,7 +389,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): self.log.error("The following hazard/zones are not found" " in active table:\n" + str(msg)) - return cities, certain + return cities, certain def __hazardsSort(self, a, b): @@ -440,7 +444,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): self.log.error("Hazards are identical in __hazardsSort %s %s", a, b) return 0 - + def __marineHazardsSort(self, a, b): # Returns 1, 0, or -1 depending on whether the first MARINE hazard # is considered higher, equal, or lower priority when compared to @@ -514,7 +518,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): # Hunt down their counterparts and add the record in the correct slot for upDown in upDownList: # get the fields from the up/downgradeFrom record - oldRec = {} + oldRec = {} if upDown.has_key('upgradeFrom'): oldRec = upDown['upgradeFrom'] elif upDown.has_key('downgradeFrom'): @@ -534,7 +538,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): self.log.error("Match not found for upgrade/downgrade.") return hazardsList - + #----------------------------------------------------------------- # The following set of functions are utility functions. #----------------------------------------------------------------- @@ -549,7 +553,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): ',' + time.asctime(time.gmtime(e[1])) + '),' s = s + ']' return s - else: + else: s = '(' + time.asctime(time.gmtime(t[0])) + \ ',' + time.asctime(time.gmtime(t[1])) + ')' return s @@ -560,7 +564,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): for id in hazardsByZone.keys(): s = s + " Hazards for " + `id` + \ self.printActiveTable(hazardsByZone[id]) - return s + return s #provides intersection of two time ranges def __timeIntersection(self, tr1, tr2): #tr1, tr2 tuples (startT, endT) @@ -576,7 +580,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): return None # no intersection else: return (startTime, endTime) - + #provides the time ranges of non-intersection in tr1, based on #the time range tr2. Returns a list of 0, 1, or 2 items. def __nonTimeIntersection(self, tr1, tr2): @@ -589,7 +593,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): return [] #startT same elif tr1[0] == intersect[0]: - return [(intersect[1], tr1[1])] + return [(intersect[1], tr1[1])] #endT same elif tr1[1] == intersect[1]: return [(tr1[0], intersect[0])] @@ -642,19 +646,19 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): times.append((proposedRecord['startTime'], proposedRecord['endTime'], 0, [id])) else: - self.__etnCache[phensig] = [(proposedRecord['startTime'], + self.__etnCache[phensig] = [(proposedRecord['startTime'], proposedRecord['endTime'], 0, [id])] - + # assign new etns to the etn cache. This is done after all requests # for new etns have been made def __assignNewETNs(self, activeTable): - + # go through each new phen,sig for phen, sig in self.__etnCache.keys(): #determine the first new ETN to use if we need a new one - etn_base = self.__highestETNActiveTable(phen, sig, - self.__allGEOActiveTable) + etn_base = self.__highestETNActiveTable(phen, sig, + self.__allGEOActiveTable) etn_base = int(etn_base) + 1 #the next one in sequence #sort the etn cache by (start, end, etn, ids) @@ -666,7 +670,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): #process sequentially each (phen, sig). Entries in cache #are list of startT (0), endT (1), etn# (2), [id] (3). times = self.__etnCache[(phen, sig)] - for x in xrange(len(times)): + for x in xrange(len(times)): s1, e1, etn1, ids = times[x] #if no etn, then use a new one if etn1 == 0: #etn == 0? @@ -679,7 +683,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): assigned = coverage[etn1] #search for all adjacent or overlapping, give it the same etn - for y in xrange(x+1, len(times)): + for y in xrange(x + 1, len(times)): s2, e2, etn2, ids2 = times[y] if etn2 == 0 and \ (self.__isAdjacent((s1, e1), (s2, e2)) or\ @@ -702,11 +706,11 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): # find highest etn in active table for phen/sig, returns it. # This method has been dramatically re-written for A2 to use # GFEVtecUtil to do preliminary ETN assignment instead of scrubbing - # the whole set of ActiveTableRecords to calculate it. + # the whole set of ActiveTableRecords to calculate it. def __highestETNActiveTable(self, phen, sig, activeTable): etn_base = 0 phensig = (phen, sig) - + # find the max ETN... # 1. highest ETN period for non-tropical and all GUM products (tpcKeys) # or @@ -720,11 +724,11 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): presentyear = time.gmtime(self.__time)[0] for active in activeTable: activeyear = time.gmtime(active['issueTime'])[0] - activephensig = (active['phen'],active['sig']) + activephensig = (active['phen'], active['sig']) if phensig == activephensig and presentyear == activeyear: # causes failure if tropical hazards are less than 1001 if active['etn'] < int(self.__tpcBaseETN): - LogStream.logProblem("Incorrect ETN for tropical hazard.") + self.log.error("Incorrect ETN for tropical hazard.") return etn_base #determine the new etn to use, using the etn cache @@ -775,7 +779,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): key = (h['phen'], h['sig']) hazards.setdefault(key, []).append(h) - self.log.debug("HazardByPhenSig:"+ self.__printHBZ(hazards)) + self.log.debug("HazardByPhenSig:" + self.__printHBZ(hazards)) return hazards @@ -804,7 +808,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): return 1 else: return 0 - + #analyzes the hazardsByZone and the list of desired editArea combinations, #and ensures that the hazards are the same for every zone in each @@ -817,7 +821,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): for i in range(1, len(combo)): found = 0 for j in range(len(newCombo)): - if self.__comboCompare(hazardsByZone, newCombo[j][0], + if self.__comboCompare(hazardsByZone, newCombo[j][0], combo[i]): newCombo[j].append(combo[i]) found = 1 @@ -831,14 +835,14 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): #-------------------------------------------------------------- # The following methods sample Hazard grids, obtain the active - # table, and create the analyzed table (including injecting - # the vtec strings into the table. + # table, and create the analyzed table (including injecting + # the vtec strings into the table. #-------------------------------------------------------------- def __analyzedTable(self, areas, filter): # main routine to obtain the analyzed table. Analyzed table # is the composite between the proposed and active tables. - # filter is the function that filters out the hazards that + # filter is the function that filters out the hazards that # should be considered. # Sample the Hazards Grid @@ -855,10 +859,10 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): # Get the active table from the IFPServer rawactTable = self.__getActiveTable() - self.log.info("Raw Active Table: " + + self.log.info("Raw Active Table: " + self.printActiveTable(rawactTable, combine=True)) if rawactTable is None: - self.log.error("Unable to retrieve VTEC active table. " + + self.log.error("Unable to retrieve VTEC active table. " + "Product VTEC codes may be suspect.") rawactTable = [] self.log.info("Raw Active Table length: " + str(len(rawactTable))) @@ -868,9 +872,9 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): self.printActiveTable(atable, combine=True)) atable = filter(atable, allowedHazardsOnly=False) self.log.info(\ - "Filtered Analyzed Table length, prior to VTEC injection: " + + "Filtered Analyzed Table length, prior to VTEC injection: " + str(len(atable))) - + # Perform site filtering on the active table. We keep # our site and SPC. allGEOTable = [] @@ -896,17 +900,17 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): if not testEntry: actTable.append(a) actTable = filter(actTable, allowedHazardsOnly=True) #also filter the active table - - self.log.info("Filtered Active Table length: " + str(len(actTable))) - self.log.info("Filtered Active Table:" + + + self.log.info("Filtered Active Table length: " + str(len(actTable))) + self.log.info("Filtered Active Table:" + self.printActiveTable(actTable, combine=True)) self.__activeTable = copy.deepcopy(actTable) # Merge the proposed and active tables, to arrive at the analyzed table - atable = self.__mergeActiveProposed(atable, actTable, self.__pil, + atable = self.__mergeActiveProposed(atable, actTable, self.__pil, areas) - self.log.info("Analyzed Table length: " + str(len(atable))) - + self.log.info("Analyzed Table length: " + str(len(atable))) + # Finished self.log.info("Analyzed Table: " + self.printActiveTable(atable, combine=True)) @@ -917,25 +921,23 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): #Uses the IFPClient interface to get the VTEC active table from #the server. Returns None on failure. from com.raytheon.uf.common.activetable import ActiveTableMode - + try: if self.__activeTableName != "PRACTICE": table = self.__ifpClient.getVTECActiveTable(self.__dataMgr.getSiteID()) else: table = self.__ifpClient.getVTECActiveTable(self.__dataMgr.getSiteID(), ActiveTableMode.PRACTICE) - table = ActiveTableVtec.transformActiveTableToPython(table) + table = ActiveTableVtec.transformActiveTableToPython(table) return table - - except: - import traceback - s = "Unable to access VTEC Active Table: " - self.log.exception(s) - raise Exception(s + traceback.format_exc()) + + except: + self.log.exception("Unable to access VTEC Active Table: ") + raise Exception, s def __createCityHazards(self): if not self.__accurateCities: return None - + self.log.info("Evaluating hazards for cities.") # set up sample requests and get the ParmHistos @@ -956,17 +958,17 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): pTable = self.__consolidateTime(pTable) # remove old - keep those ended within 30min - cutoff = self.__time - 30*60 + cutoff = self.__time - 30 * 60 pTable = filter(lambda x: x['endTime'] > cutoff, pTable) # handle UFN events - convert ending time to max for proposed in pTable: if (proposed['phen'], proposed['sig']) in self.__ufnKeys: proposed['startTime'] = self.__time #now - proposed['endTime'] = float(2**31-1) #forever + proposed['endTime'] = float(2 ** 31 - 1) #forever proposed['ufn'] = 1 #until further notice - self.log.info("Hazards afflicting cities:"+ + self.log.info("Hazards afflicting cities:" + self.printActiveTable(pTable, combine=True, idType='city')) return pTable @@ -977,7 +979,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): phIter = parmHistos.iterator() while phIter.hasNext(): - ph = phIter.next() + ph = phIter.next() areaID = ph.area().getId().getName() areaPoints = ph.numberOfGridPoints() samples = ph.histoSamples() @@ -1000,7 +1002,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): d['seg'] = 0 #normally zero, except if aux data d['startTime'] = float(areaTime.startTime().unixTime()) - # possibly shorten the endTime based on + # possibly shorten the endTime based on # self.__hazardEndTime if self.__hazardEndTime is not None and \ areaTime.endTime().unixTime() > self.__hazardEndTime: @@ -1025,7 +1027,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): # of ParmHistos. # Determine the ParmID for Hazards out of the given database - dbid = JavaDatabaseID(self.__databaseID) + dbid = JavaDatabaseID(self.__databaseID) # pid = filter(lambda x: str(x).find("Hazards") != -1, # self.__ifpClient.getParmList(self.__databaseID))[0] parmList = self.__ifpClient.getParmList(dbid) @@ -1039,7 +1041,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): # TimeRange to sample # Use hazardEndTime if present if self.__hazardEndTime is not None: - tr = TimeRange.TimeRange(AbsTime.AbsTime.current(), + tr = TimeRange.TimeRange(AbsTime.AbsTime.current(), AbsTime.AbsTime(self.__hazardEndTime)) else: #(everything) tr = TimeRange.allTimes() @@ -1071,9 +1073,9 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): areaPoints = ph.numberOfGridPoints() samples = ph.histoSamples() - for s in samples: + for s in samples: areaTime = TimeRange.TimeRange(s.validTime()) # timerange - histpairs = s.histogram() + histpairs = s.histogram() for p in histpairs: subkeys = p.value().discrete().getSubKeys() sksize = subkeys.size() @@ -1087,7 +1089,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): d['seg'] = 0 #normally zero, except if aux data d['startTime'] = float(areaTime.startTime().unixTime()) - # possibly shorten the endTime based on + # possibly shorten the endTime based on # self.__hazardEndTime if self.__hazardEndTime is not None and \ areaTime.endTime().unixTime() > self.__hazardEndTime: @@ -1108,13 +1110,13 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): d['sig'] = "" # empty significance desc = \ DiscreteKey.discreteDefinition(self.__dataMgr.getSiteID()).keyDesc( - "Hazards_SFC", sk) + "Hazards_SFC", sk) d['hdln'] = desc #special checks for aux data auxindex = sk.find(':') if auxindex != -1: - auxData = sk[auxindex+1:] + auxData = sk[auxindex + 1:] #national center uses: aux data is the etn number if (d['phen'], d['sig']) in self.__ncKeys: try: @@ -1129,7 +1131,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): d['etn'] = number except: self.log.error("Bad auxData for ", - "National Center:"+ auxData+ d) + "National Center:" + auxData + d) #other aux data interpreted as segment number else: @@ -1137,8 +1139,8 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): segment = int(auxData) d['seg'] = segment except: - self.log.error("Bad auxData for seg:"+ - auxData+ d) + self.log.error("Bad auxData for seg:" + + auxData + d) rval.append(d) return rval @@ -1159,12 +1161,12 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): for proposed in pTable: if (proposed['phen'], proposed['sig']) in self.__ufnKeys: proposed['startTime'] = self.__time #now - proposed['endTime'] = float(2**31-1) #forever + proposed['endTime'] = float(2 ** 31 - 1) #forever proposed['ufn'] = 1 #until further notice return pTable - # Utility function to combine + # Utility function to combine def __timeReduce(self, atable, index): if index >= len(atable) - 1: return @@ -1189,9 +1191,9 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): nowHour = int(self.__time / 3600) * 3600 for a in atable: if a['startTime'] < nowHour: - a['startTime'] = nowHour + a['startTime'] = nowHour return atable - + # Remove any entries that occupy less than the sampling threshold # of the area. Threshold is met for a given % of the area covered # or a number of grid points covered. If None is given, then that @@ -1260,7 +1262,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): atable = self.__coverageFilter(atable) atable = self.__consolidateTime(atable) - # for cities list - keep these records to check for existence of grid + # for cities list - keep these records to check for existence of grid self.__oldZoneTable = filter(lambda x: 0 <= self.__time - x['endTime'] < 1800, atable) @@ -1279,8 +1281,8 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): return d #------------------------------------------------------------- - # The following functions handle the merging of the - # proposed and active tables. VTEC strings are calculated + # The following functions handle the merging of the + # proposed and active tables. VTEC strings are calculated # in these routines. #------------------------------------------------------------- @@ -1331,7 +1333,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): activeStart = spcActive['startTime'] activeEnd = spcActive['endTime'] else: - self.log.error("Unable to match SPC watch for "+ + self.log.error("Unable to match SPC watch for " + self.printActiveTable(proposed)) activeStart = proposed['startTime'] activeEnd = proposed['endTime'] #failsafe code @@ -1360,17 +1362,17 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): # additional CAN events. def __checkForMergedEvents(self, proposedTable, activeTable): - compare = ['id','phen','sig','pil'] + compare = ['id', 'phen', 'sig', 'pil'] createdCANEntries = [] for proposed in proposedTable: matches = [] - + #record match and time overlaps for real events for active in activeTable: if self.hazardCompare(proposed, active, compare) and \ - active['act'] not in ['CAN','UPG','EXP'] and \ + active['act'] not in ['CAN', 'UPG', 'EXP'] and \ active['endTime'] > self.__time and \ proposed['startTime'] <= active['endTime'] and \ proposed['endTime'] >= active['startTime']: @@ -1379,9 +1381,9 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): #if multiple records match, we have a merged event #we need to find the highest etn for the event matches if len(matches) > 1: - self.log.debug("MERGE event: proposed="+ - self.printActiveTable(proposed)+ - " matches="+ self.printActiveTable(matches)) + self.log.debug("MERGE event: proposed=" + + self.printActiveTable(proposed) + + " matches=" + self.printActiveTable(matches)) highestETN = 0 for m in matches: highestETN = max(highestETN, m['etn']) @@ -1393,7 +1395,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): canEntry = copy.deepcopy(m) canEntry['act'] = 'CAN' createdCANEntries.append(canEntry) - self.log.debug("CAN event: %s%s%s", + self.log.debug("CAN event: %s%s%s", self.printActiveTable(canEntry), " remEntry: ", self.printActiveTable(m)) del activeTable[activeTable.index(m)] @@ -1423,7 +1425,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): if len(proposed['sig']): #is VTEC, must compare with active for active in activeTable: if self.hazardCompare(proposed, active, compare) and \ - active['act'] not in ['CAN', 'UPG', 'EXP']: + active['act'] not in ['CAN', 'UPG', 'EXP']: # and not self.__separateETNtrack(proposed, active): #convective watch (special case, also compare etn) @@ -1438,7 +1440,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): proposed['act'] = 'CON' proposed['etn'] = active['etn'] self.__copyTextFields(proposed, active) - + # start times both before current time, end # times the same, CON state elif self.__time >= proposed['startTime'] and \ @@ -1457,7 +1459,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): # except when user changed the start time # of an event has gone into effect. elif self.__hazardsOverlap(proposed, active): - + if active['startTime'] <= self.__time: if proposed['startTime'] <= self.__time or \ active.has_key('conexted'): @@ -1483,7 +1485,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): del active['conexted'] return proposedTable - + # Checks for CAN, EXP, UPG def __checkForCANEXPUPG(self, pTable, activeTable): compare1 = ['id', 'phen', 'sig'] @@ -1528,15 +1530,15 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): break # CAN's have three special forms. CAN when a product is no longer - # in the proposed table, EXP when the product is no longer + # in the proposed table, EXP when the product is no longer # in the proposed table, and the end was within 30 min of now, - # and UPG when the phen is the same, but + # and UPG when the phen is the same, but # sig is upgraded, and the VTEC is still in effect. # if cancel_needed == 1: # Case One - UPG - # Area matches, phen matches, and we are going from an + # Area matches, phen matches, and we are going from an # advisory to a watch, a watch to a warning, or an # advisory to a warning. @@ -1553,7 +1555,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): if active not in newEntries: newEntries.append(active) cancel_needed = 0 - + # Case Two - EXP # If it wasn't an UPG, then check for EXP. EXP if entry # not in the proposed table, and current time is after @@ -1569,7 +1571,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): cancel_needed = 0 # Final Case - CAN - # Only Allow "CAN" entries if the event is still ongoing, + # Only Allow "CAN" entries if the event is still ongoing, # otherwise ignore the entry. if cancel_needed == 1: if self.__time < active['endTime']: @@ -1585,13 +1587,13 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): for entry in newEntries: if entry.has_key('segText'): entry['prevText'] = entry['segText'] - del entry['segText'] + del entry['segText'] if entry.has_key('overviewText'): entry['prevOverviewText'] = entry['overviewText'] - del entry['overviewText'] + del entry['overviewText'] if entry.has_key('vtec'): entry['vtecstr'] = "" #erase the VTEC string. - del entry['overviewText'] + del entry['overviewText'] pTable.append(entry) return pTable @@ -1617,16 +1619,16 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): # Assume first that this is EXA or EXB exaexb_flag = 1 - #if we find a match, and it overlaps in time, + #if we find a match, and it overlaps in time, #then it isn't an EXA, EXB for active in activeTable: if self.hazardCompare(proposed, active, compare1): - #if proposed['startTime'] <= active['endTime'] and - # proposed['endTime'] >= active['startTime'] and + #if proposed['startTime'] <= active['endTime'] and + # proposed['endTime'] >= active['startTime'] and if self.__hazardsOverlap(proposed, active) and \ - active['act'] not in ['CAN','EXP','UPG']: + active['act'] not in ['CAN', 'EXP', 'UPG']: exaexb_flag = 0 - + # no match was found, thus this is either a EXA, or EXB, # match records with phen and sig the same if exaexb_flag == 1: @@ -1637,7 +1639,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): # and not self.__separateETNtrack(proposed, active): if active['act'] not in ['CAN', 'UPG', 'EXP']: - #if times are identical, then we extended in area + #if times are identical, then we extended in area if proposed['startTime'] == active['startTime'] and \ proposed['endTime'] == active['endTime']: if proposed['etn'] == "???" or \ @@ -1715,12 +1717,12 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): self.__copyTextFields(proposed, active) if act == 'EXB': - #save original time so we can later - #determine whether it is EXTENDED + #save original time so we can later + #determine whether it is EXTENDED #or SHORTENED proposed['previousStart'] = active['startTime'] proposed['previousEnd'] = active['endTime'] - + return pTable @@ -1736,7 +1738,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): # #Already identified are basic TO.A, SV.A using aux data fields, - allowedActions = ['NEW','CON','EXT','EXA','EXB'] + allowedActions = ['NEW', 'CON', 'EXT', 'EXA', 'EXB'] for proposed in pTable: if proposed['act'] == '???': @@ -1781,7 +1783,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): rTable.append(h) #Convert EXP into CON codes for non-yet expired events (30min) - #since marine does not permit EXP codes + #since marine does not permit EXP codes elif h['endTime'] > self.__time: h['act'] = 'CON' #convert to CON code rTable.append(h) @@ -1795,18 +1797,18 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): # add in EXP codes (for events just about ready to expire) def __addEXPCodes(self, pTable): #looks for events that have "CON", but are within 30 minutes of - #event ending time and converts those events to EXP. + #event ending time and converts those events to EXP. for each_hazard in pTable: if each_hazard['act'] == 'CON': timeFromEnd = self.__time - each_hazard['endTime'] # +after - if timeFromEnd >= -30*60 and timeFromEnd <= 0: + if timeFromEnd >= -30 * 60 and timeFromEnd <= 0: each_hazard['act'] = 'EXP' #convert to expired return pTable - + # remove EXP (actual EXP codes) when another event of same phen/sig is # now ongoing, but only if same issuance year def __removeEXPWithOngoingCodes(self, pTable): - compare = ['phen','sig','etn','id'] + compare = ['phen', 'sig', 'etn', 'id'] tmp = [] for h in pTable: #events with EXP, and after ending time @@ -1818,7 +1820,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): #active event with same phen/sig/etn h1IssueT = h1.get('issueTime', self.__time) h1IssueYear = time.gmtime(h1IssueT)[0] - if h1['act'] in ['CON','EXA','EXB','EXT'] and \ + if h1['act'] in ['CON', 'EXA', 'EXB', 'EXT'] and \ self.hazardCompare(h, h1, compare) and \ h1IssueYear == hIssueYear: removeIt = 1 @@ -1826,8 +1828,8 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): if removeIt == 0: tmp.append(h) return tmp - - + + # generate VTEC strings for hazards def __addVTECStrings(self, pTable): for h in pTable: @@ -1847,7 +1849,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): sig = h['sig'] if len(sig) == 0: #local headline, non-VTEC h['vtecstr'] = "" - continue + continue # get the office ID if h.has_key('officeid'): @@ -1869,7 +1871,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): action = h['act'] else: action = "???" - + # adjust time of NEW events to ensure they don't start # earlier than now if h['startTime'] < self.__time: @@ -1881,7 +1883,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): (action == 'EXT' and h['previousStart'] > self.__time) or \ (action == 'EXB' and h['previousStart'] > self.__time) or \ (h['startTime'] > self.__time): - startStr = time.strftime("%y%m%dT%H%MZ-", + startStr = time.strftime("%y%m%dT%H%MZ-", time.gmtime(h['startTime'])) else: startStr = "000000T0000Z-" #ongoing @@ -1893,7 +1895,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): endStr = time.strftime("%y%m%dT%H%MZ/", time.gmtime(h['endTime'])) # format the beastly string - vtec = '/' + self.__vtecMode + "." + action + "." +\ + vtec = '/' + self.__vtecMode + "." + action + "." + \ siteID + '.' + phen + '.' + sig + '.' + ETN + '.' + \ startStr + endStr h['vtecstr'] = vtec @@ -1905,8 +1907,8 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): # Leaving 'hdln' blank indicates no headline and no mention in hazard # products. def __addHeadlinesIfMissing(self, pTable): - compare = ['id','phen','sig','pil'] - ongoingAct = ['EXT','EXB','CON','NEW','EXA'] + compare = ['id', 'phen', 'sig', 'pil'] + ongoingAct = ['EXT', 'EXB', 'CON', 'NEW', 'EXA'] for h in pTable: if h.has_key('hdln'): continue @@ -1929,9 +1931,9 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): else: h['hdln'] = VTECTable.VTECTable[phensig]['hdln'] else: - h['hdln'] = "" + h['hdln'] = "" + - # isUpgrade(), indicates whether rec2 upgrades rec1, only looks # at act, phen and sig. Proposed gets NEW, EXA or EXB active gets UPG def __isUpgrade(self, proposed, active): @@ -1941,16 +1943,16 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): if proposed['act'] in ['CON', 'EXT']: return 0 #not an upgrade else: - if VTECTable.checkForUpgrade(proposed['phen'], proposed['sig'], + if VTECTable.checkForUpgrade(proposed['phen'], proposed['sig'], active['phen'], active['sig']): return 1 else: return 0 #not an upgrade - + # isDowngrade(), indicates whether rec2 downgrades rec1, only looks - # at phen and sig. Proposed gets NEW, active gets CAN. + # at phen and sig. Proposed gets NEW, active gets CAN. def __isDowngrade(self, proposed, active): - if VTECTable.checkForDowngrade(proposed['phen'], proposed['sig'], + if VTECTable.checkForDowngrade(proposed['phen'], proposed['sig'], active['phen'], active['sig']): return 1 else: @@ -1958,7 +1960,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): # Checks for records with the same phen/sig for the same geographical # area (id). Eliminates the records with the lower segment number with - # same times. Combines records with multiple segment numbers with + # same times. Combines records with multiple segment numbers with # different times. Result is only to have 1 record per ID for phen/sig. def __checkForMultipleSegsInSameID(self, pTable): @@ -1986,7 +1988,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): orgMaxSeg[phensig] = max(p['seg'], orgMaxSeg[phensig]) else: orgMaxSeg[phensig] = p['seg'] - + #step 2: Check for multiple records for phensig and zone. #Mark records that can be combined (adjacent/overlap). @@ -2005,7 +2007,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): xtr = (records[x]['startTime'], records[x]['endTime']) #search for adjacent/overlapping - for y in xrange(x+1, len(records)): + for y in xrange(x + 1, len(records)): ytr = (records[y]['startTime'], records[y]['endTime']) rny = records[y].get('rn', None) if rny is None and (self.__isAdjacent(xtr, ytr) or \ @@ -2014,9 +2016,9 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): records[y]['rn'] = rnx #overlaps/adjacent,reuse rn records[x]['rn'] = rnx #assign to orig to match if trDict.has_key(rnx): - trDict[rnx] = self.__combineTR(ytr,trDict[rnx]) + trDict[rnx] = self.__combineTR(ytr, trDict[rnx]) else: - trDict[rnx] = self.__combineTR(xtr,ytr) + trDict[rnx] = self.__combineTR(xtr, ytr) maxSN = self.__maxSegNumber(orgHaz, phensig) #max seg num @@ -2050,7 +2052,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): #step 4: Combine new segments if possible. We can tell we have #generated new segments based on the orgMaxSeg dictionary. We assign #them the same segments. - compare = ['pil','startTime','endTime','phen','sig'] + compare = ['pil', 'startTime', 'endTime', 'phen', 'sig'] for x in xrange(len(updatedList)): p = updatedList[x] phensig = (p['phen'], p['sig']) @@ -2060,7 +2062,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): #find matching records and assign all the same seg# #and key - for y in xrange(x+1, len(updatedList)): + for y in xrange(x + 1, len(updatedList)): p1 = updatedList[y] if self.hazardCompare(p, p1, compare) and \ p1['seg'] > orgMax: @@ -2104,37 +2106,37 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): def __checkValidETNcw(self, pTable): errorLine = '**************************************************\n' for p in pTable: - if (p['phen'],p['sig']) in self.__ncKeys and p['officeid'] != 'PGUM': + if (p['phen'], p['sig']) in self.__ncKeys and p['officeid'] != 'PGUM': try: a = int(p['etn']) except: - raise Exception, "\n\n" + errorLine + "\n" +\ + raise Exception, "\n\n" + errorLine + "\n" + \ "ABORTING: Found National Hazard " + \ "with no ETN in grids. \n" + self.printActiveTable(p) + \ " Fix your grids by adding watch/storm number." + \ - "\nFor tropical hazards, an override to MakeHazard" +\ + "\nFor tropical hazards, an override to MakeHazard" + \ "\n is likely to blame.\n" + errorLine # check for valid ETN/Actions in the analyzed table. Cannot have # a split ETN where one part of ongoing/NEW, and the other part # is being dropped (e.g., CAN, UPG). pTable is the analyzed active table. def __checkValidETNsActions(self, pTable): - byZones = self.__organizeByZone(pTable) - compare = ['etn','phen','sig'] + byZones = self.__organizeByZone(pTable) + compare = ['etn', 'phen', 'sig'] errorLine = '**************************************************\n' currentYear = time.gmtime(self.__time)[0] for key in byZones: for h in byZones[key]: if (h['phen'], h['sig']) not in self.__ncKeys: continue #only interested in checking national keys - if h['act'] in ['EXP','UPG','CAN']: + if h['act'] in ['EXP', 'UPG', 'CAN']: hissueTime = h.get('issueTime', 0) hissueYear = time.gmtime(hissueTime)[0] #issueYear for h1 in byZones[key]: if self.hazardCompare(h, h1, compare) and \ - h1['act'] in ['NEW','CON','EXA','EXT','EXB'] and \ + h1['act'] in ['NEW', 'CON', 'EXA', 'EXT', 'EXB'] and \ currentYear == hissueYear: - raise Exception, "\n\n" + errorLine + "\n" +\ + raise Exception, "\n\n" + errorLine + "\n" + \ "ABORTING: Found VTEC Error"\ " with same ETN, same hazard, conflicting "\ "actions.\n" + self.printActiveTable(h) + \ @@ -2151,11 +2153,11 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): newTable = [] for p in pTable: if p['act'] == 'EXP' and \ - (self.__time - p['endTime']) >= 30*60: + (self.__time - p['endTime']) >= 30 * 60: pass else: newTable.append(p) - + return newTable #ensure that we don't have two vtecs with same action code, same etns. @@ -2173,7 +2175,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): if self.hazardCompare(p, e, compare2) and \ e['etn'] > etn_max: etn_max = e['etn'] - keyetnmax[vteckey]= etn_max + keyetnmax[vteckey] = etn_max assigned = {} for p in pTable: @@ -2184,7 +2186,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): for p1 in pTable: #check for matching id,etn,phen,sig,act combinations, these - #are the ones that need to be reassigned. + #are the ones that need to be reassigned. if self.hazardCompare(p, p1, compare) and \ p['startTime'] > p1['endTime']: #found a newer record that needs to be reassigned @@ -2215,7 +2217,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): p['etn'] = int(keyetnmax[vteckey]) + 1 trs[tr] = p['etn'] #new etn assigned assigned[akey] = trs #put back into dictionary - keyetnmax[vteckey]= p['etn'] #updated for new assign + keyetnmax[vteckey] = p['etn'] #updated for new assign def __warnETNduplication(self, pTable): # Check should only operate on applicable VTEC products. @@ -2224,7 +2226,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): return dups = [] - byZones = self.__organizeByZone(pTable) + byZones = self.__organizeByZone(pTable) for id, hazards in byZones.iteritems(): visited = [] for p in hazards: @@ -2238,8 +2240,8 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): if len(dups) > 0: errorLine = '\n******************************************************\n' - LogStream.logProblem("Illegal ETN duplication is found for:\n", \ - dups, errorLine) + self.log.error("Illegal ETN duplication is found for:\n" + \ + str(dups) + errorLine) # send message to GFE msg = "The formatted %s product contains a duplicate ETN.\n"\ @@ -2254,7 +2256,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): if active.has_key("overviewText"): proposed['prevOverviewText'] = active['overviewText'] - + # add upgrade/downgrade records from the active table def __addUpgradeDowngradeRec(self, proposedTable): compare = ['id', 'pil', 'officeid'] @@ -2270,12 +2272,12 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): ################### if self.__isDowngrade(rec, checkR): rec['downgradeFrom'] = self.__copyFields(checkR, fields) - elif self.__isUpgrade(rec, checkR): + elif self.__isUpgrade(rec, checkR): rec['upgradeFrom'] = self.__copyFields(checkR, fields) return proposedTable - + ############################################ # 'inject' is the main function in vtec.py # ############################################ @@ -2284,59 +2286,59 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): # convert active table EXP still in effect to CON activeTable = self.__convertEXPtoCON(activeTable) - self.log.debug("After convertEXPtoCON: " + + self.log.debug("After convertEXPtoCON: " + self.printActiveTable(pTable, combine=True)) # Special handling for the SPC watches (TO.A, SV.A) pTable = self.__handleSPCWatches(pTable, activeTable) - self.log.debug("After handleSPCWatches: " + + self.log.debug("After handleSPCWatches: " + self.printActiveTable(pTable, combine=True)) # Drop multiple segments for same phen/sig in same "id" pTable = self.__checkForMultipleSegsInSameID(pTable) - self.log.debug("After checkForMultipleSegsInSameID: " + + self.log.debug("After checkForMultipleSegsInSameID: " + self.printActiveTable(pTable, combine=True)) - + # Check for Merged Events pTable, activeTable = self.__checkForMergedEvents(pTable, activeTable) - self.log.debug("After checkForMergedEvents: " + + self.log.debug("After checkForMergedEvents: " + self.printActiveTable(pTable, combine=True)) # Check for CON and EXT actions pTable = self.__checkForCONEXT(pTable, activeTable) - self.log.debug("After checkForCONEXT: " + + self.log.debug("After checkForCONEXT: " + self.printActiveTable(pTable, combine=True)) # Check for CAN, EXP, and UPG pTable = self.__checkForCANEXPUPG(pTable, activeTable) - self.log.debug("After checkForCANEXPUPG: " + + self.log.debug("After checkForCANEXPUPG: " + self.printActiveTable(pTable, combine=True)) # Check for EXA/EXB pTable = self.__checkForEXAEXB(pTable, activeTable) - self.log.debug("After checkForEXAEXB: " + + self.log.debug("After checkForEXAEXB: " + self.printActiveTable(pTable, combine=True)) # Assign NEW to remaining records pTable = self.__checkForNEW(pTable, activeTable) - self.log.debug("After checkForNEW: " + + self.log.debug("After checkForNEW: " + self.printActiveTable(pTable, combine=True)) # Check for upgrades and downgrades, add records if needed pTable = self.__addUpgradeDowngradeRec(pTable) - self.log.debug("After addUpgradeDowngradeRec: " + + self.log.debug("After addUpgradeDowngradeRec: " + self.printActiveTable(pTable, combine=True)) # Convert ongoing events about ready to expire (still in the # proposed grids) to switch from CON to EXP pTable = self.__addEXPCodes(pTable) - self.log.debug("After addEXPCodes: " + + self.log.debug("After addEXPCodes: " + self.printActiveTable(pTable, combine=True)) # Eliminate any EXPs if other events (same phen/sig) in effect # at present time. pTable = self.__removeEXPWithOngoingCodes(pTable) - self.log.debug("After removeEXPWithOngoingCodes: " + + self.log.debug("After removeEXPWithOngoingCodes: " + self.printActiveTable(pTable, combine=True)) # Ensure valid ETN/Actions - no EXP/CAN with valid same ETN @@ -2361,13 +2363,13 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): # Complete the VTEC Strings self.__addVTECStrings(pTable) - self.log.debug("After addVTECStrings: " + + self.log.debug("After addVTECStrings: " + self.printActiveTable(pTable, combine=True)) #add in hdln entries if they are missing self.__addHeadlinesIfMissing(pTable) - self.log.debug("After addHeadlinesIfMissing: " + - self.printActiveTable(pTable, combine=True)) + self.log.debug("After addHeadlinesIfMissing: " + + self.printActiveTable(pTable, combine=True)) # Ensure that all SV.A and TO.A have valid ETNs self.__checkValidETNcw(pTable) @@ -2385,7 +2387,7 @@ class HazardsTable(VTECTableUtil.VTECTableUtil): # # # marine zones and non-marine zones for tpc phen/sigs follow their own # # sequence of ETNs and actions. This routine determines if separate -# # ETNs/actions should occur between id1 and id2. Returns true if +# # ETNs/actions should occur between id1 and id2. Returns true if # # separate ETN tracks are required - basically if id1 and id2 are one # # marine and the other not, and the phen/sigs are identical and are tpc # # phen/sigs. Also returns true if phen/sigs are not identical. Otherwise diff --git a/cave/com.raytheon.viz.gfe/localization/gfe/userPython/utilities/CombinationsInterface.py b/cave/com.raytheon.viz.gfe/localization/gfe/userPython/utilities/CombinationsInterface.py index f7cebeb983..9c92ee47d3 100644 --- a/cave/com.raytheon.viz.gfe/localization/gfe/userPython/utilities/CombinationsInterface.py +++ b/cave/com.raytheon.viz.gfe/localization/gfe/userPython/utilities/CombinationsInterface.py @@ -29,7 +29,7 @@ # ------------ ---------- ----------- -------------------------- # 07/25/08 njensen Initial Creation. # 09/05/13 #2329 randerso Added error handling -# +# 02/06/2014 #2591 randerso Changed log level to debug # import sys, traceback, os, time, LogStream @@ -55,7 +55,7 @@ def getCombinations(comboName): with open(filename,'r') as fd: filecontents = fd.read() - LogStream.logProblem("\nERROR loading combinations file: "+ comboName + + LogStream.logDebug("ERROR loading combinations file: "+ comboName + "\nmd.__file__: " + md.__file__ + "\ndir(md): " + str(dir(md)) + "\n" + md.__file__ + " last modified: " + time.strftime("%Y-%m-%d %H:%M:%S",time.gmtime(os.path.getmtime(md.__file__))) + diff --git a/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/dialogs/formatterlauncher/ProductAreaComp.java b/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/dialogs/formatterlauncher/ProductAreaComp.java index c7c33d3408..ac377cd260 100644 --- a/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/dialogs/formatterlauncher/ProductAreaComp.java +++ b/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/dialogs/formatterlauncher/ProductAreaComp.java @@ -63,6 +63,7 @@ import com.raytheon.viz.gfe.textformatter.TextProductManager; * DataManager instance. * 05 SEP 2013 2329 randerso Added call to ZoneCombinerComp.applyZoneCombo when * when run formatter button is clicked. + * 05 FEB 2014 2591 randerso Added dataManager to ZoneCombinerComp constructor * * * @@ -592,7 +593,7 @@ public class ProductAreaComp extends Composite implements */ private void createZoneCombinerComp() { zoneCombiner = new ZoneCombinerComp(stackGridComp, productTabCB, - productName, getTextProductManager()); + productName, getTextProductManager(), this.dataMgr); } /** diff --git a/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/dialogs/formatterlauncher/ProductEditorComp.java b/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/dialogs/formatterlauncher/ProductEditorComp.java index a268e9f543..98e82708bc 100644 --- a/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/dialogs/formatterlauncher/ProductEditorComp.java +++ b/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/dialogs/formatterlauncher/ProductEditorComp.java @@ -152,6 +152,10 @@ import com.raytheon.viz.ui.dialogs.ICloseCallback; * 05/08/2013 #1842 dgilling Add alternate setProductText(), fix * warnings. * 09/03/2013 16534 ryu Refactor; sneak in a change for Ron (RM #1597). + * 01/06/2014 2649 dgilling Pass flag to StoreTransmitDlg to only + * update VTEC lines on products that + * aren't being corrected. + * 02/05/2014 17022 ryu Modified loadDraft() to fix merging of WMO heading and AWIPS ID. * * * @@ -1139,7 +1143,7 @@ public class ProductEditorComp extends Composite implements // prevent the launching of another dialog until the modal dialog is // closed. StoreTransmitDlg storeDlg = new StoreTransmitDlg(parent.getShell(), - showStore, this, transmissionCB, pid); + showStore, this, transmissionCB, pid, !textComp.isCorMode()); storeDlg.open(); } } @@ -2386,10 +2390,13 @@ public class ProductEditorComp extends Composite implements } DraftProduct draft = DraftProduct.load(lf); - setTabColorFunc(productStateEnum.Finished); productDefinition = draft.getProductDefinition(); + + clearProductText(); setProductText(draft.getProductText()); + setTabColorFunc(productStateEnum.Finished); setStatusText('R', productId + " draft loaded."); + if (productDefinition.get("brain") != null) { brain(); String msg = "Your saved draft was loaded, but the draft is invalid " @@ -2877,7 +2884,7 @@ public class ProductEditorComp extends Composite implements mb2.open(); return; } - + // Word-wrap the whole selection. int curLine = styledText.getLineAtOffset(selectionRange.x); int lastSelIdx = selectionRange.x + selectionRange.y - 1; diff --git a/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/dialogs/formatterlauncher/StoreTransmitDlg.java b/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/dialogs/formatterlauncher/StoreTransmitDlg.java index faceaf7403..535ddd2f7b 100644 --- a/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/dialogs/formatterlauncher/StoreTransmitDlg.java +++ b/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/dialogs/formatterlauncher/StoreTransmitDlg.java @@ -67,19 +67,24 @@ import com.raytheon.viz.ui.dialogs.CaveSWTDialog; * SOFTWARE HISTORY * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- - * 21 APR 2008 ### lvenable Initial creation - * 19 FEB 2010 4132 ryu Product correction. - * 28May2010 2187 cjeanbap Added StdTextProductFactory - * functionality. - * 09 NOV 2012 1298 rferrel Changes for non-blocking dialog. - * 02apr2013 15564 mgamazaychikov Ensured awipsWanPil to be 10 characters space-padded long - * 08 MAY 2013 1842 dgilling Use VtecUtil to set product ETNs, fix + * Apr 21, 2008 ### lvenable Initial creation + * Feb 19, 2010 4132 ryu Product correction. + * May 28, 2010 2187 cjeanbap Added StdTextProductFactory + * functionality. + * Nov 09, 2012 1298 rferrel Changes for non-blocking dialog. + * Apr 02, 2013 15564 mgamazaychikov Ensured awipsWanPil to be 10 characters + * space-padded long + * May 08, 2013 1842 dgilling Use VtecUtil to set product ETNs, fix * warnings. - * 07 Jun 2013 1981 mpduff Set user's id in OUPRequest as it is now a protected operation. - * 23 Oct 2013 1843 dgilling Ensure that dialog is always closed, + * Jun 07, 2013 1981 mduff Set user's id in OUPRequest as it is + * now a protected operation. + * Oct 23, 2013 1843 dgilling Ensure that dialog is always closed, * even on failure, changes for error handling * of intersite ETN assignment. - * 18 Dec 2013 2641 dgilling Support changes to GFEVtecUtil.getVtecLinesThatNeedEtn(). + * Dec 18, 2013 2641 dgilling Support changes to GFEVtecUtil.getVtecLinesThatNeedEtn(). + * Jan 06, 2014 2649 dgilling Make ETN assignment process optional. + * Feb 17, 2014 2774 dgilling Merge changes from 14.1 baseline to 14.2. + * * * * @author lvenable @@ -142,17 +147,26 @@ public class StoreTransmitDlg extends CaveSWTDialog implements private final String pid; + private final boolean updateVtec; + /** - * Constructor. - * * @param parent * Parent shell. * @param storeDialog * Store flag. True is store, false is transmit. + * @param editor + * Parent editor. Product will be updated in this editor after + * transmission. + * @param transmissionCB + * @param pid + * @param updateVtec + * Whether or not to update the ETNs of any VTEC lines in the + * product to be transmitted. Recommend setting this to false + * when correcting a previously transmitted product. */ public StoreTransmitDlg(Shell parent, boolean storeDialog, ProductEditorComp editor, ITransmissionState transmissionCB, - String pid) { + String pid, boolean updateVtec) { super(parent, SWT.DIALOG_TRIM | SWT.APPLICATION_MODAL, CAVE.DO_NOT_BLOCK); @@ -161,6 +175,7 @@ public class StoreTransmitDlg extends CaveSWTDialog implements parentEditor = editor; this.productText = editor.getProductText(); this.pid = pid; + this.updateVtec = updateVtec; } @Override @@ -344,114 +359,78 @@ public class StoreTransmitDlg extends CaveSWTDialog implements // Store/Transmit the product... if (!countdownThread.threadCancelled()) { - boolean retrieveEtnFailed = false; + try { + if (updateVtec) { + // With GFE VTEC products, it's possible to have multiple + // segments with NEW vtec action codes and the same phensig. + // For + // this reason, HazardsTable.py implemented a "cache" that + // would + // ensure all NEWs for the same phensig would be assigned + // the + // same ETN. This Map replicates that legacy behavior. + // + // This "cache" has two levels: + // 1. The first level is keyed by the hazard's phensig. + // 2. The second level is keyed by the valid period of the + // hazard. + // Effectively, making this a Map>. + Map> vtecsToAssignEtn = GFEVtecUtil + .initETNCache(productText); + Map> etnCache = new HashMap>(); - List vtecsToAssignEtn = GFEVtecUtil - .getVtecLinesThatNeedEtn(productText); - // With GFE VTEC products, it's possible to have multiple segments - // with - // NEW vtec action codes and the same phensig. For this reason, - // HazardsTable.py implemented a "cache" that would ensure all NEWs - // for - // the same phensig would be assigned the same ETN. This Map - // replicates - // that legacy behavior. - // - // This "cache" has two levels: - // 1. The first level is keyed by the hazard's phensig. - // 2. The second level is keyed by the valid period of the hazard. - // Effectively, making this a Map>. - Map> etnCache = new HashMap>(); + for (String phensig : vtecsToAssignEtn.keySet()) { + Map l2EtnCache = new HashMap(); + List vtecs = vtecsToAssignEtn.get(phensig); - for (VtecObject vtec : vtecsToAssignEtn) { - try { - GetNextEtnResponse serverResponse = GFEVtecUtil.getNextEtn( - vtec.getOffice(), vtec.getPhensig(), true, true); - if (!serverResponse.isOkay()) { - final VtecObject vtecToFix = vtec; - final boolean[] exitLoopContainer = { false }; - final Exception[] exceptionContainer = { null }; - final GetNextEtnResponse[] responseContainer = { serverResponse }; + for (int i = 0; i < vtecs.size(); i++) { + VtecObject vtec = vtecs.get(i); + TimeRange validPeriod = new TimeRange( + vtec.getStartTime(), vtec.getEndTime()); + Integer currentEtn = vtec.getSequence(); - do { - getDisplay().syncExec(new Runnable() { - @Override - public void run() { - GetNextEtnResponse serverResponse = responseContainer[0]; - ETNConfirmationDialog dlg = new ETNConfirmationDialog( - getShell(), serverResponse); - if (dlg.open() == ETNConfirmationDialog.OK) { - int etn = dlg.getProposedEtn(); - statusHandler.info(String - .format("User confirmed ETN for %s: %04d", - serverResponse - .getPhensig(), - etn)); - try { - GetNextEtnResponse followupResp = GFEVtecUtil.getNextEtn( - vtecToFix.getOffice(), - vtecToFix.getPhensig(), - true, true, true, etn); - responseContainer[0] = followupResp; - } catch (VizException e) { - exceptionContainer[0] = e; - exitLoopContainer[0] = true; - } - } else { - statusHandler - .info("User declined to fix ETN for %s", - serverResponse - .getPhensig()); - exitLoopContainer[0] = true; - } - } - }); - } while (!responseContainer[0].isOkay() - && !exitLoopContainer[0]); - - if (!responseContainer[0].isOkay()) { - String msg = "Unable to set ETN for phensig " - + responseContainer[0].getPhensig() - + "\nStatus: " - + responseContainer[0].toString(); - Exception e = exceptionContainer[0]; - if (e == null) { - throw new VizException(msg); + // the first time we select a new, unique ETN, any + // other + // VTEC lines in the product that have the same + // phensig + // and an adjacent TimeRange can also re-use this + // ETN + if (currentEtn == 0) { + currentEtn = getNextEtn(vtec); + l2EtnCache.put(validPeriod, currentEtn); } else { - throw new VizException(msg, e); + // BUT...once we've made our one pass through + // the + // product and re-used the ETN where + // appropriate, we + // should not check again + continue; + } + + for (int j = i + 1; j < vtecs.size(); j++) { + VtecObject vtec2 = vtecs.get(j); + TimeRange validPeriod2 = new TimeRange( + vtec2.getStartTime(), + vtec2.getEndTime()); + Integer currentEtn2 = vtec2.getSequence(); + + if ((currentEtn2 == 0) + && (validPeriod2 + .isAdjacentTo(validPeriod) || validPeriod2 + .overlaps(validPeriod))) { + l2EtnCache.put(validPeriod2, currentEtn); + vtec2.setSequence(currentEtn); + } } - } else { - serverResponse = responseContainer[0]; } + + etnCache.put(phensig, l2EtnCache); } - TimeRange validPeriod = new TimeRange(vtec.getStartTime() - .getTime(), vtec.getEndTime().getTime()); - String phensig = vtec.getPhensig(); - Map etnsByTR = etnCache.get(phensig); - if (etnsByTR == null) { - etnsByTR = new HashMap(); - etnCache.put(phensig, etnsByTR); - } - etnsByTR.put(validPeriod, serverResponse.getNextEtn()); - } catch (VizException e) { - statusHandler.handle(Priority.CRITICAL, - "Error setting ETNs for product", e); - retrieveEtnFailed = true; - VizApp.runAsync(new Runnable() { - - @Override - public void run() { - sendTransmissionStatus(ConfigData.productStateEnum.Failed); - StoreTransmitDlg.this.parentEditor.revive(); - } - }); - break; + productText = GFEVtecUtil.finalizeETNs(productText, + etnCache); } - } - - if (!retrieveEtnFailed) { - productText = GFEVtecUtil.finalizeETNs(productText, etnCache); VizApp.runSync(new Runnable() { @@ -476,6 +455,17 @@ public class StoreTransmitDlg extends CaveSWTDialog implements } }); + } catch (VizException e) { + statusHandler.handle(Priority.CRITICAL, + "Error preparing product for transmission.", e); + VizApp.runAsync(new Runnable() { + + @Override + public void run() { + sendTransmissionStatus(ConfigData.productStateEnum.Failed); + StoreTransmitDlg.this.parentEditor.revive(); + } + }); } } @@ -490,6 +480,65 @@ public class StoreTransmitDlg extends CaveSWTDialog implements }); } + private Integer getNextEtn(VtecObject vtec) throws VizException { + GetNextEtnResponse serverResponse = GFEVtecUtil.getNextEtn( + vtec.getOffice(), vtec.getPhensig(), true, true); + if (!serverResponse.isOkay()) { + final VtecObject vtecToFix = vtec; + final boolean[] exitLoopContainer = { false }; + final Exception[] exceptionContainer = { null }; + final GetNextEtnResponse[] responseContainer = { serverResponse }; + + do { + getDisplay().syncExec(new Runnable() { + @Override + public void run() { + GetNextEtnResponse serverResponse = responseContainer[0]; + ETNConfirmationDialog dlg = new ETNConfirmationDialog( + getShell(), serverResponse); + if (dlg.open() == ETNConfirmationDialog.OK) { + int etn = dlg.getProposedEtn(); + statusHandler.info(String.format( + "User confirmed ETN for %s: %04d", + serverResponse.getPhensig(), etn)); + try { + GetNextEtnResponse followupResp = GFEVtecUtil + .getNextEtn(vtecToFix.getOffice(), + vtecToFix.getPhensig(), true, + true, true, etn); + responseContainer[0] = followupResp; + } catch (VizException e) { + exceptionContainer[0] = e; + exitLoopContainer[0] = true; + } + } else { + statusHandler.info( + "User declined to fix ETN for %s", + serverResponse.getPhensig()); + exitLoopContainer[0] = true; + } + } + }); + } while (!responseContainer[0].isOkay() && !exitLoopContainer[0]); + + if (!responseContainer[0].isOkay()) { + String msg = "Unable to set ETN for phensig " + + responseContainer[0].getPhensig() + "\nStatus: " + + responseContainer[0].toString(); + Exception e = exceptionContainer[0]; + if (e == null) { + throw new VizException(msg); + } else { + throw new VizException(msg, e); + } + } else { + serverResponse = responseContainer[0]; + } + } + + return serverResponse.getNextEtn(); + } + /** * Method to transmit the product. */ @@ -597,4 +646,4 @@ public class StoreTransmitDlg extends CaveSWTDialog implements transmissionCB.setTransmissionState(status); } } -} +} \ No newline at end of file diff --git a/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/dialogs/formatterlauncher/ZoneCombinerComp.java b/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/dialogs/formatterlauncher/ZoneCombinerComp.java index 85e155ce35..972f4c582a 100644 --- a/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/dialogs/formatterlauncher/ZoneCombinerComp.java +++ b/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/dialogs/formatterlauncher/ZoneCombinerComp.java @@ -58,8 +58,7 @@ import org.opengis.referencing.operation.TransformException; import com.raytheon.uf.common.dataplugin.gfe.db.objects.GridLocation; import com.raytheon.uf.common.dataplugin.gfe.exception.GfeException; -import com.raytheon.uf.common.localization.FileUpdatedMessage; -import com.raytheon.uf.common.localization.ILocalizationFileObserver; +import com.raytheon.uf.common.dataplugin.gfe.server.notify.CombinationsFileChangedNotification; import com.raytheon.uf.common.localization.IPathManager; import com.raytheon.uf.common.localization.LocalizationContext; import com.raytheon.uf.common.localization.LocalizationContext.LocalizationLevel; @@ -74,7 +73,9 @@ import com.raytheon.uf.viz.core.RGBColors; import com.raytheon.uf.viz.core.VizApp; import com.raytheon.uf.viz.core.exception.VizException; import com.raytheon.viz.gfe.Activator; +import com.raytheon.viz.gfe.core.DataManager; import com.raytheon.viz.gfe.core.DataManagerUIFactory; +import com.raytheon.viz.gfe.core.internal.NotificationRouter.AbstractGFENotificationObserver; import com.raytheon.viz.gfe.textformatter.CombinationsFileUtil; import com.raytheon.viz.gfe.textformatter.TextProductManager; import com.raytheon.viz.gfe.ui.zoneselector.ZoneSelector; @@ -99,6 +100,12 @@ import com.raytheon.viz.gfe.ui.zoneselector.ZoneSelector; * not found to match A1. * Dec 03, 2013 #2591 dgilling Ensure all change states to the combo * file are handled. + * Jan 07, 2014 #2662 randerso Disabled zone combiner if no maps are selected + * Feb 04, 2014 #2591 randerso Forced reload of combinations file after save to ensure + * file is updated prior to running formatter + * Changed to use CombinationsFileChangedNotification instead of + * FileUpdatedMessage so we can ignore our own changes + * Moved retrieval of combinations file to CombinationsFileUtil.init * * * @@ -106,8 +113,7 @@ import com.raytheon.viz.gfe.ui.zoneselector.ZoneSelector; * @version 1.0 * */ -public class ZoneCombinerComp extends Composite implements - ILocalizationFileObserver, IZoneCombiner { +public class ZoneCombinerComp extends Composite implements IZoneCombiner { private final transient IUFStatusHandler statusHandler = UFStatus .getHandler(ZoneCombinerComp.class); @@ -190,6 +196,8 @@ public class ZoneCombinerComp extends Composite implements protected TextProductManager textProductMgr; + protected DataManager dataManager; + protected String product; protected boolean mapRequired; @@ -223,6 +231,8 @@ public class ZoneCombinerComp extends Composite implements private List mapNames; + private AbstractGFENotificationObserver comboChangeListener; + private void initPreferences() { IPreferenceStore prefs = Activator.getDefault().getPreferenceStore(); @@ -246,7 +256,8 @@ public class ZoneCombinerComp extends Composite implements * Product name. */ public ZoneCombinerComp(Composite parent, IProductTab callBack, - String productName, TextProductManager textProductMgr) { + String productName, TextProductManager textProductMgr, + DataManager dataManager) { super(parent, SWT.NONE); this.parent = parent; @@ -257,7 +268,13 @@ public class ZoneCombinerComp extends Composite implements this.textProductMgr = textProductMgr; - mapRequired = textProductMgr.mapRequired(productName); + this.dataManager = dataManager; + + mapRequired = this.textProductMgr.mapRequired(productName); + this.mapNames = getMapNames(productName); + if (mapNames.isEmpty()) { + mapRequired = false; + } initPreferences(); init(); @@ -267,15 +284,47 @@ public class ZoneCombinerComp extends Composite implements LocalizationType.CAVE_STATIC, LocalizationLevel.BASE); comboDir = pathMgr.getLocalizationFile(baseCtx, CombinationsFileUtil.COMBO_DIR_PATH); - comboDir.addFileUpdatedObserver(this); + + this.comboChangeListener = new AbstractGFENotificationObserver( + CombinationsFileChangedNotification.class) { + + @Override + public void notify( + CombinationsFileChangedNotification notificationMessage) { + comboFileChanged(notificationMessage); + } + + }; this.addDisposeListener(new DisposeListener() { @Override public void widgetDisposed(DisposeEvent e) { - comboDir.removeFileUpdatedObserver(ZoneCombinerComp.this); + ZoneCombinerComp.this.dataManager.getNotificationRouter() + .removeObserver( + ZoneCombinerComp.this.comboChangeListener); } }); + + dataManager.getNotificationRouter().addObserver( + this.comboChangeListener); + } + + private List getMapNames(String productName) { + Object obj = this.textProductMgr.getMapNameForCombinations(productName); + List mapNames = new ArrayList(); + if (obj instanceof String) { + String s = (String) obj; + if (!s.isEmpty()) { + mapNames.add(s); + } + } else if (obj instanceof List) { + @SuppressWarnings("unchecked") + List list = (List) obj; + mapNames.addAll(list); + } + + return mapNames; } /** @@ -757,9 +806,14 @@ public class ZoneCombinerComp extends Composite implements } try { + String comboName = getCombinationsFileName(); CombinationsFileUtil.generateAutoCombinationsFile( - zoneSelector.getZoneGroupings(), getCombinationsFileName() - + ".py"); + zoneSelector.getZoneGroupings(), comboName); + + // reload here forces file to be retrieved from server before + // allowing formatter to run + loadCombinationsFile(comboName); + applyButtonState(false); } catch (Exception e) { statusHandler.handle(Priority.PROBLEM, "Unable to save " @@ -790,21 +844,9 @@ public class ZoneCombinerComp extends Composite implements } Map comboDict = loadCombinationsFile(comboName); - Object obj = textProductMgr.getMapNameForCombinations(productName); - mapNames = new ArrayList(); - if (obj instanceof String) { - String s = (String) obj; - if (!s.isEmpty()) { - mapNames.add(s); - } - } else if (obj instanceof List) { - @SuppressWarnings("unchecked") - List list = (List) obj; - mapNames.addAll(list); - } - boolean singleComboOnly = false; - obj = textProductMgr.getDefinitionValue(productName, "singleComboOnly"); + Object obj = textProductMgr.getDefinitionValue(productName, + "singleComboOnly"); if (obj != null) { if (obj instanceof Integer) { singleComboOnly = ((Integer) obj) != 0; @@ -915,17 +957,7 @@ public class ZoneCombinerComp extends Composite implements public Map loadCombinationsFile(String comboName) { Map dict = new HashMap(); try { - File localFile = PathManagerFactory.getPathManager().getStaticFile( - FileUtil.join(CombinationsFileUtil.COMBO_DIR_PATH, - comboName + ".py")); - - List> combolist = new ArrayList>(); - if ((localFile != null) && localFile.exists()) { - combolist = CombinationsFileUtil.init(comboName); - } else { - // statusHandler - // .error("Combinations file not found: " + comboName); - } + List> combolist = CombinationsFileUtil.init(comboName); // reformat combinations into combo dictionary int group = 1; @@ -941,8 +973,7 @@ public class ZoneCombinerComp extends Composite implements return new HashMap(); } - currentComboFile = FileUtil.join(CombinationsFileUtil.COMBO_DIR_PATH, - comboName + ".py"); + currentComboFile = comboName; return dict; } @@ -977,20 +1008,14 @@ public class ZoneCombinerComp extends Composite implements return colors; } - /* - * (non-Javadoc) - * - * @see - * com.raytheon.uf.common.localization.ILocalizationFileObserver#fileUpdated - * (com.raytheon.uf.common.localization.FileUpdatedMessage) - */ - @Override - public void fileUpdated(FileUpdatedMessage message) { - if (message.getFileName().equalsIgnoreCase(currentComboFile)) { - File file = new File(message.getFileName()); - String comboName = file.getName().replace(".py", ""); + private void comboFileChanged(CombinationsFileChangedNotification notif) { + String comboName = notif.getCombinationsFileName(); + + // if it's the same file and not changed by me update the combos + if (comboName.equalsIgnoreCase(currentComboFile) + && !VizApp.getWsId().equals(notif.getWhoChanged())) { statusHandler - .info("Received FileUpdatedMessage for combinations file: " + .info("Received CombinationsFileChangedNotification for combinations file: " + comboName); Map comboDict = loadCombinationsFile(comboName); this.zoneSelector.updateCombos(comboDict); diff --git a/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/textformatter/CombinationsFileUtil.java b/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/textformatter/CombinationsFileUtil.java index 255356c1a8..bb281db254 100644 --- a/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/textformatter/CombinationsFileUtil.java +++ b/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/textformatter/CombinationsFileUtil.java @@ -51,6 +51,7 @@ import com.raytheon.uf.common.serialization.SerializationException; import com.raytheon.uf.common.serialization.SingleTypeJAXBManager; import com.raytheon.uf.common.status.IUFStatusHandler; import com.raytheon.uf.common.status.UFStatus; +import com.raytheon.uf.common.status.UFStatus.Priority; import com.raytheon.uf.common.util.FileUtil; import com.raytheon.viz.gfe.core.DataManagerUIFactory; import com.raytheon.viz.gfe.core.internal.IFPClient; @@ -68,6 +69,8 @@ import com.raytheon.viz.gfe.textformatter.CombinationsFileUtil.ComboData.Entry; * Sep 05, 2013 #2329 randerso Moved genereateAutoCombinationsFile here * Cleaned up error handling * Sep 30, 2013 2361 njensen Use JAXBManager for XML + * Feb 05, 2014 #2591 randerso Forced retrieval of combinations file + * Implemented retry on error * * * @@ -79,6 +82,8 @@ public class CombinationsFileUtil { private static final transient IUFStatusHandler statusHandler = UFStatus .getHandler(CombinationsFileUtil.class); + private static final int MAX_TRIES = 2; + public static String COMBO_DIR_PATH = FileUtil.join("gfe", "combinations"); public static String SAVED_COMBO_DIR = FileUtil.join("gfe", "comboData"); @@ -215,13 +220,20 @@ public class CombinationsFileUtil { IPathManager pm = PathManagerFactory.getPathManager(); + // retrieve combinations file if it's changed + LocalizationFile lf = pm.getStaticLocalizationFile(FileUtil.join( + COMBO_DIR_PATH, comboName + ".py")); + File pyFile; + try { + pyFile = lf.getFile(true); + } catch (LocalizationException e) { + throw new GfeException("Error retrieving combinations file: " + + comboName, e); + } + LocalizationContext baseContext = pm.getContext( LocalizationType.CAVE_STATIC, LocalizationLevel.BASE); - // Strip any path components from comboName - File comboFile = new File(comboName); - comboName = comboFile.getName(); - String scriptPath = FileUtil.join( GfePyIncludeUtil.getUtilitiesLF(baseContext).getFile() .getPath(), "CombinationsInterface.py"); @@ -230,21 +242,41 @@ public class CombinationsFileUtil { HashMap map = new HashMap(); map.put("comboName", comboName); PythonScript python = null; + for (int retryCount = 0; retryCount < MAX_TRIES; retryCount++) { try { - python = new PythonScript(scriptPath, PyUtil.buildJepIncludePath( + python = new PythonScript(scriptPath, + PyUtil.buildJepIncludePath( GfePyIncludeUtil.getCombinationsIncludePath(), GfePyIncludeUtil.getCommonPythonIncludePath()), CombinationsFileUtil.class.getClassLoader()); + Object com = python.execute("getCombinations", map); combos = (List>) com; + + // if successfully retrieved break out of the loop + break; } catch (JepException e) { + // remove the .pyo file + new File(pyFile.getAbsolutePath() + "o").delete(); + + // if not last try, log and try again + if (retryCount < MAX_TRIES - 1) { + // log but don't pop up + statusHandler.handle(Priority.EVENTB, + "Error loading combinations file: " + comboName + + ", retrying ", e); + } + // else throw exception + else { throw new GfeException("Error loading combinations file: " + comboName, e); + } } finally { if (python != null) { python.dispose(); } } + } return combos; } diff --git a/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/ui/zoneselector/AbstractZoneSelector.java b/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/ui/zoneselector/AbstractZoneSelector.java index 7ddbc5c471..186980baa2 100644 --- a/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/ui/zoneselector/AbstractZoneSelector.java +++ b/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/ui/zoneselector/AbstractZoneSelector.java @@ -74,6 +74,7 @@ import com.vividsolutions.jts.geom.Envelope; * ------------ ---------- ----------- -------------------------- * Aug 23, 2011 randerso Initial creation * May 30, 2013 #2028 randerso Fixed date line issue with map display + * Jan 07, 2014 #2662 randerso Fixed limitZones (subDomainUGCs) support * * * @@ -158,7 +159,6 @@ public abstract class AbstractZoneSelector extends PaneManager { return this.zoomLevel; } - // command to label the zones, calls setMap() to update the display public void setLabelZones(boolean labelZones) { if (labelZones == this.labelZones) { return; @@ -182,7 +182,9 @@ public abstract class AbstractZoneSelector extends PaneManager { return; } this.limitZoneNames = limitZones; - setMapInternal(this.mapRscList); + for (ZoneSelectorResource mapRsc : this.mapRscList) { + mapRsc.setLimitZones(limitZones); + } } /* @@ -287,7 +289,7 @@ public abstract class AbstractZoneSelector extends PaneManager { .retrieveMap(bundlePath).getResourceData(); ZoneSelectorResource rsc = new ZoneSelectorResource(rscData, - new LoadProperties(), gloc); + new LoadProperties(), gloc, this.limitZoneNames); mapRscList.add(rsc); } catch (VizException e) { statusHandler.handle(Priority.PROBLEM, "Error loading map: " diff --git a/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/ui/zoneselector/ZoneSelectorResource.java b/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/ui/zoneselector/ZoneSelectorResource.java index a1cfec80d5..76a738821c 100644 --- a/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/ui/zoneselector/ZoneSelectorResource.java +++ b/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/ui/zoneselector/ZoneSelectorResource.java @@ -102,6 +102,7 @@ import com.vividsolutions.jts.io.WKBReader; * Apr 10, 2013 #1854 randerso Fix for compatibility with PostGIS 2.0 * May 30, 2013 #2028 randerso Fixed date line issue with map display * Jul 31, 2013 #2239 randerso Fixed scaling of maps that cross the date line + * Jan 07, 2014 #2662 randerso Fixed limitZones (subDomainUGCs) support * * * @@ -230,6 +231,7 @@ public class ZoneSelectorResource extends DbMapResource { int numPoints = 0; int wfoPoints = 0; + List limitZones = req.rsc.getLimitZones(); WKBReader wkbReader = new WKBReader(); for (int i = 0; i < mappedResult.getResultCount(); i++) { if (canceled) { @@ -259,6 +261,12 @@ public class ZoneSelectorResource extends DbMapResource { // TODO: what do we do with this? // zoneName = ""; } + + if (limitZones != null + && !limitZones.contains(zoneName)) { + continue; + } + String wfo = (String) mappedResult .getRowColumnValue(i, "wfo"); @@ -532,6 +540,8 @@ public class ZoneSelectorResource extends DbMapResource { private Map zoneData; + private List limitZones; + private RGB defaultFillColor; private RGB outlineColor; @@ -559,9 +569,12 @@ public class ZoneSelectorResource extends DbMapResource { /** * @param data * @param loadProperties + * @param gloc + * @param limitZones */ public ZoneSelectorResource(DbMapResourceData data, - LoadProperties loadProperties, GridLocation gloc) { + LoadProperties loadProperties, GridLocation gloc, + List limitZones) { super(data, loadProperties); this.zoneData = new HashMap(); this.geomFactory = new GeometryFactory(); @@ -570,6 +583,7 @@ public class ZoneSelectorResource extends DbMapResource { this.outlineColor = RGBColors.getRGBColor("white"); this.wfoOutlineColor = RGBColors.getRGBColor("yellow"); this.gloc = gloc; + this.limitZones = limitZones; GeneralEnvelope env = new GeneralEnvelope(MapUtil.LATLON_PROJECTION); env.setEnvelope(-180.0, -90.0, 180.0, 90.0); @@ -848,13 +862,10 @@ public class ZoneSelectorResource extends DbMapResource { query.append(geometryField); // add any additional columns - List additionalColumns = new ArrayList(); if (resourceData.getColumns() != null) { for (ColumnDefinition column : resourceData.getColumns()) { query.append(", "); query.append(column); - - additionalColumns.add(column.getName()); } } @@ -915,6 +926,22 @@ public class ZoneSelectorResource extends DbMapResource { return newShadedShape; } + /** + * @return the limitZones + */ + public List getLimitZones() { + return limitZones; + } + + /** + * @param limitZones + * the limitZones to set + */ + public void setLimitZones(List limitZones) { + this.limitZones = limitZones; + issueRefresh(); + } + /** * @param labelZones */ @@ -1023,6 +1050,16 @@ public class ZoneSelectorResource extends DbMapResource { query.append(resourceData.getGeomField()); query.append(")) as extent"); + // add editarea column + if (resourceData.getColumns() != null) { + for (ColumnDefinition column : resourceData.getColumns()) { + if (column.getName().equalsIgnoreCase("editarea")) { + query.append(", "); + query.append(column); + } + } + } + // add the geometry table query.append(" FROM "); query.append(resourceData.getTable()); @@ -1047,6 +1084,14 @@ public class ZoneSelectorResource extends DbMapResource { WKBReader wkbReader = new WKBReader(); for (int i = 0; i < mappedResult.getResultCount(); i++) { + String zoneName = (String) mappedResult.getRowColumnValue( + i, "editarea"); + + if (this.limitZones != null + && !this.limitZones.contains(zoneName)) { + continue; + } + byte[] b = (byte[]) mappedResult.getRowColumnValue(i, "extent"); if (b != null) { diff --git a/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/vtec/GFEVtecUtil.java b/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/vtec/GFEVtecUtil.java index b1ee8cdd46..69d0cc4d77 100644 --- a/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/vtec/GFEVtecUtil.java +++ b/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/vtec/GFEVtecUtil.java @@ -22,6 +22,8 @@ package com.raytheon.viz.gfe.vtec; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.regex.Matcher; @@ -60,6 +62,7 @@ import com.raytheon.viz.texteditor.util.VtecUtil; * phensig but disjoint TimeRanges. * Dec 18, 2013 #2641 dgilling Force ordering of items returned by * getVtecLinesThatNeedEtn(). + * Feb 05, 2014 #2774 dgilling Additional correction to previous fix. * * * @@ -78,6 +81,24 @@ public class GFEVtecUtil { public static final Collection IGNORE_NATIONAL_ETN = ImmutableSet .copyOf(GFEVtecConfig.getInstance().getSitesIgnoreNationalEtn()); + private static final Comparator VTEC_COMPARATOR = new Comparator() { + + @Override + public int compare(VtecObject vtec1, VtecObject vtec2) { + TimeRange tr1 = new TimeRange(vtec1.getStartTime(), + vtec1.getEndTime()); + TimeRange tr2 = new TimeRange(vtec2.getStartTime(), + vtec2.getEndTime()); + + int retVal = tr1.getStart().compareTo(tr2.getStart()); + if (retVal == 0) { + retVal = tr1.getEnd().compareTo(tr2.getEnd()); + } + + return retVal; + } + }; + /** * A private constructor so that Java does not attempt to create one for us. * As this class should not be instantiated, do not attempt to ever call @@ -194,12 +215,12 @@ public class GFEVtecUtil { * @return A Set of VtecObjects that need to have * a new ETN assigned to them. */ - public static List getVtecLinesThatNeedEtn(String product) { + public static Map> initETNCache(String product) { if (StringUtil.isEmptyString(product)) { - return Collections.emptyList(); + return Collections.emptyMap(); } - List phensigs = new ArrayList(); + Map> cache = new HashMap>(); Matcher vtecMatcher = VtecUtil.VTEC_REGEX.matcher(product); while (vtecMatcher.find()) { @@ -208,11 +229,24 @@ public class GFEVtecUtil { && ((!NATIONAL_PHENSIGS.contains(vtec.getPhensig())) || (IGNORE_NATIONAL_ETN .contains(vtec.getOffice()) && TROPICAL_PHENSIGS .contains(vtec.getPhensig())))) { - phensigs.add(vtec); + List vtecsForPhensig = cache.get(vtec.getPhensig()); + if (vtecsForPhensig == null) { + vtecsForPhensig = new ArrayList(); + } + vtec.setSequence(0); + vtecsForPhensig.add(vtec); + cache.put(vtec.getPhensig(), vtecsForPhensig); } } - return phensigs; + for (String phensig : cache.keySet()) { + List vtecsByTimeOrder = new ArrayList( + cache.get(phensig)); + Collections.sort(vtecsByTimeOrder, VTEC_COMPARATOR); + cache.put(phensig, vtecsByTimeOrder); + } + + return cache; } /** @@ -244,46 +278,12 @@ public class GFEVtecUtil { && ((!NATIONAL_PHENSIGS.contains(vtec.getPhensig())) || (IGNORE_NATIONAL_ETN .contains(vtec.getOffice()) && TROPICAL_PHENSIGS .contains(vtec.getPhensig())))) { - // With GFE VTEC products, it's possible to have multiple - // segments with - // NEW vtec action codes and the same phensig. For this reason, - // HazardsTable.py implemented a "cache" that would ensure all - // NEWs for - // the same phensig would be assigned the same ETN. This Map - // replicates - // that legacy behavior. - // - // This "cache" has two levels: - // 1. The first level is keyed by the hazard's phensig. - // 2. The second level is keyed by the valid period of the - // hazard. - // Effectively, making this a Map>. - - // Some more clarification on the ETN assignment behavior: all - // NEW VTEC lines with the same phensig should be assigned the - // same ETN if the hazards' valid periods are adjacent or - // overlapping. - // If there's a discontinuity in TimeRanges we increment to the - // next ETN. - Integer newEtn = null; - String cacheKey = vtec.getPhensig(); + String phensig = vtec.getPhensig(); TimeRange validPeriod = new TimeRange(vtec.getStartTime() .getTime(), vtec.getEndTime().getTime()); - - Map etnsByTR = etnCache.get(cacheKey); - if (etnsByTR != null) { - newEtn = etnsByTR.get(validPeriod); - for (TimeRange tr : etnsByTR.keySet()) { - if ((validPeriod.isAdjacentTo(tr)) - || (validPeriod.overlaps(tr))) { - vtec.setSequence(newEtn); - break; - } - } - } + Integer newEtn = etnCache.get(phensig).get(validPeriod); + vtec.setSequence(newEtn); } - vtecMatcher .appendReplacement( finalOutput, @@ -293,4 +293,4 @@ public class GFEVtecUtil { vtecMatcher.appendTail(finalOutput); return finalOutput.toString(); } -} +} \ No newline at end of file diff --git a/cave/com.raytheon.viz.hydro/src/com/raytheon/viz/hydro/pointdatacontrol/engine/PDCFindBasis.java b/cave/com.raytheon.viz.hydro/src/com/raytheon/viz/hydro/pointdatacontrol/engine/PDCFindBasis.java index da37cc5866..2a7bbae813 100644 --- a/cave/com.raytheon.viz.hydro/src/com/raytheon/viz/hydro/pointdatacontrol/engine/PDCFindBasis.java +++ b/cave/com.raytheon.viz.hydro/src/com/raytheon/viz/hydro/pointdatacontrol/engine/PDCFindBasis.java @@ -34,6 +34,7 @@ import com.raytheon.viz.hydrocommon.data.GageData.ThreatIndex; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Nov 21, 2008 mpduff Initial creation + * Feb 03, 2014 16843 lbousaidi add a check for index OutOfBoundsException * * * @@ -101,8 +102,9 @@ public class PDCFindBasis { } else { currentLid = fcstReportList.get(fcstIndex).getLid(); } - } else if (obsReportList.get(obsIndex) != null) { - currentLid = obsReportList.get(obsIndex).getLid(); + } else if ((obsIndex< obsReportList.size()) && (obsReportList + .get(obsIndex) != null)) { + currentLid = obsReportList.get(obsIndex).getLid(); } else { currentLid = fcstReportList.get(fcstIndex).getLid(); } diff --git a/cave/com.raytheon.viz.hydro/src/com/raytheon/viz/hydro/timeseries/TimeSeriesDisplayCanvas.java b/cave/com.raytheon.viz.hydro/src/com/raytheon/viz/hydro/timeseries/TimeSeriesDisplayCanvas.java index a556addad8..db009a60c5 100644 --- a/cave/com.raytheon.viz.hydro/src/com/raytheon/viz/hydro/timeseries/TimeSeriesDisplayCanvas.java +++ b/cave/com.raytheon.viz.hydro/src/com/raytheon/viz/hydro/timeseries/TimeSeriesDisplayCanvas.java @@ -138,8 +138,9 @@ import com.raytheon.viz.hydrocommon.util.DbUtils; * 16 Jan 2013 15695 wkwock Fix popup menu * 24 Apr 2013 1921 mpduff Fix zoom reset to only reset the "active" graph * 06 May 2013 1976 mpduff Refactored Hydro time series data access. - * 29 May 2013 2016 mpduff Fix TS Toggle Traces. + * 29 May 2013 2016 mpduff Fix TS Toggle Traces. * 05 Sep 2013 #2332 lvenable Fixed memory leaks. + * 24 Jan 2013 15959 lbousaidi Swap the corner points of the bounding box when zooming. * @author lvenable * @version 1.0 * @@ -1241,7 +1242,13 @@ public class TimeSeriesDisplayCanvas extends TimeSeriesGraphCanvas implements } Date xMin = pixel2x(gd, rubberBandX1 - GRAPHBORDER_LEFT); Date xMax = pixel2x(gd, rubberBandX2 - GRAPHBORDER_LEFT); - + //Swap the corner points of the bounding box when zooming + if (xMin.after(xMax)) { + Date xtmp; + xtmp= xMin; + xMin=xMax; + xMax=xtmp; + } gd.setXMin(xMin); gd.setXMax(xMax); gd.setX(gd.getXMax().getTime() - gd.getXMin().getTime()); @@ -1256,7 +1263,13 @@ public class TimeSeriesDisplayCanvas extends TimeSeriesGraphCanvas implements if (ymin < gd.getYmin()) { ymin = gd.getYmin(); } - + //Swap the corner points of the bounding box when zooming + if (ymin > ymax) { + double ytmp; + ytmp= ymin; + ymin=ymax; + ymax=ytmp; + } gd.setYmin(ymin); gd.setYmax(ymax); gd.setY2(gd.getYmax2() - gd.getYmin2()); diff --git a/cave/com.raytheon.viz.mpe/src/com/raytheon/viz/mpe/util/RenderPcp.java b/cave/com.raytheon.viz.mpe/src/com/raytheon/viz/mpe/util/RenderPcp.java index 49bd21c60a..a6c0bcc9f8 100644 --- a/cave/com.raytheon.viz.mpe/src/com/raytheon/viz/mpe/util/RenderPcp.java +++ b/cave/com.raytheon.viz.mpe/src/com/raytheon/viz/mpe/util/RenderPcp.java @@ -38,7 +38,8 @@ import com.raytheon.viz.mpe.util.DailyQcUtils.Station; * ------------ ---------- ----------- -------------------------- * Mar 11, 2009 snaples Initial creation * May 02, 2011 8962 snaples Added render24hrPcpUsingFour6hr() method - * + * Jan 10, 2014 16976 cgobs Fixed issue on line 153. + * Changed pcp.value[row][col] to pcp.value[col][row] * * * @author snaples @@ -149,7 +150,7 @@ public class RenderPcp { for (int col = 0; col < hrap_grid.maxi; col++) { for (int row = 0; row < hrap_grid.maxj; row++) { - value = pcp.value[row][col]; + value = pcp.value[col][row]; if (value > maxValue) { maxValue = value; diff --git a/cave/com.raytheon.viz.mpe/src/com/raytheon/viz/mpe/util/RenderPcpBlocking.java b/cave/com.raytheon.viz.mpe/src/com/raytheon/viz/mpe/util/RenderPcpBlocking.java index c45d46ace5..f55adad8e4 100644 --- a/cave/com.raytheon.viz.mpe/src/com/raytheon/viz/mpe/util/RenderPcpBlocking.java +++ b/cave/com.raytheon.viz.mpe/src/com/raytheon/viz/mpe/util/RenderPcpBlocking.java @@ -38,7 +38,8 @@ import com.raytheon.viz.mpe.util.DailyQcUtils.Station; * ------------ ---------- ----------- -------------------------- * Mar 11, 2009 snaples Initial creation * May 02, 2011 8962 snaples Added render24hrPcpUsingFour6hr() method - * + * Jan 10, 2014 16976 cgobs Fixed issue on line 290. + * Changed pcp.value[row][col] to pcp.value[col][row] * * * @author snaples @@ -286,7 +287,7 @@ public class RenderPcpBlocking { * not estimate precipitation for it. */ if (hrap_grid.owner[col][row] == -1) { - pcp.value[row][col] = 0; + pcp.value[col][row] = 0; continue; } @@ -308,7 +309,7 @@ public class RenderPcpBlocking { resultingPrecipValue = 0.0; } - // pcp.value[row][col] is the value of grid, + // pcp.value[col][row] is the value of grid, // so we don't need to estimate a value for [row][col] } diff --git a/cave/com.raytheon.viz.texteditor/src/com/raytheon/viz/texteditor/util/VtecObject.java b/cave/com.raytheon.viz.texteditor/src/com/raytheon/viz/texteditor/util/VtecObject.java index 8508d5799a..4e6a87a18c 100644 --- a/cave/com.raytheon.viz.texteditor/src/com/raytheon/viz/texteditor/util/VtecObject.java +++ b/cave/com.raytheon.viz.texteditor/src/com/raytheon/viz/texteditor/util/VtecObject.java @@ -34,6 +34,7 @@ import java.util.regex.Pattern; * ------------ ---------- ----------- -------------------------- * Oct 14, 2009 bwoodle Initial creation * May 08, 2013 #1842 dgilling Add getPhensig() method. + * Feb 17, 2014 #2774 dgilling Added toString() for debugging * * * @@ -315,6 +316,11 @@ public class VtecObject { VtecUtil.formatVtecTime(this.endTime)); } + @Override + public String toString() { + return generateVtec(); + } + /* * (non-Javadoc) */ diff --git a/cave/com.raytheon.viz.warngen/src/com/raytheon/viz/warngen/gis/PolygonUtil.java b/cave/com.raytheon.viz.warngen/src/com/raytheon/viz/warngen/gis/PolygonUtil.java index 13c686c89a..a70ead9915 100644 --- a/cave/com.raytheon.viz.warngen/src/com/raytheon/viz/warngen/gis/PolygonUtil.java +++ b/cave/com.raytheon.viz.warngen/src/com/raytheon/viz/warngen/gis/PolygonUtil.java @@ -34,6 +34,9 @@ import org.opengis.metadata.spatial.PixelOrientation; import org.opengis.referencing.operation.MathTransform; import com.raytheon.uf.common.dataplugin.warning.util.GeometryUtil; +import com.raytheon.uf.common.status.IUFStatusHandler; +import com.raytheon.uf.common.status.UFStatus; +import com.raytheon.uf.common.status.UFStatus.Priority; import com.raytheon.uf.viz.core.IExtent; import com.raytheon.uf.viz.core.exception.VizException; import com.raytheon.viz.core.contours.util.ContourContainer; @@ -76,6 +79,7 @@ import com.vividsolutions.jts.geom.prep.PreparedGeometryFactory; * 10/01/2013 DR 16632 Qinglu Lin Fixed the bug in for loop range. * 10/17/2013 DR 16632 Qinglu Lin Updated removeOverlaidLinesegments(). * 10/18/2013 DR 16632 Qinglu Lin Catch exception thrown when coords length is less than 4 and doing createLinearRing(coords). + * 01/09/2014 DR 16974 D. Friedman Improve followup redraw-from-hatched-area polygons. * * * @author mschenke @@ -83,6 +87,8 @@ import com.vividsolutions.jts.geom.prep.PreparedGeometryFactory; */ public class PolygonUtil { + private static final transient IUFStatusHandler statusHandler = UFStatus + .getHandler(PolygonUtil.class); private WarngenLayer layer; @@ -114,6 +120,39 @@ public class PolygonUtil { Geometry origWarningArea, Polygon oldWarningPolygon) throws VizException { float[][] contourAreaData = toFloatData(origWarningArea); + /* If we have an oldWarningPolygon, we can take a shortcut and see + * if the intersection of the current polygon and the old polygon + * generates the same input to the contouring algorithm as the current + * hatched area. If it does, that intersection can be used instead of + * generating a new contour. + */ + if (oldWarningPolygon != null) { + try { + Geometry intersection = origPolygon.intersection(oldWarningPolygon); + if (intersection instanceof Polygon) { + Polygon polygonIntersection = (Polygon) intersection; + if (polygonIntersection.isValid() && + polygonIntersection.getNumInteriorRing() == 0 && + polygonIntersection.getNumPoints() - 1 <= maxVertices) { + /* + * Use buildIdealArea to clip the current polygon against the old + * polygon (actually oldWarningArea) and the CWA, using the same + * coordinate transformations that are used to generate + * origWarningArea. + */ + Geometry comparableIntersection = layer.buildIdealArea(origPolygon); + float[][] interAreaData = toFloatData(comparableIntersection); + if (areasEqual(interAreaData, contourAreaData)) { + return polygonIntersection; + } + } + } + } catch (RuntimeException e) { + statusHandler.handle(Priority.WARN, + "Error while using simple polygon redraw method. Will continue using contouring method.", e); + } + } + // Create contouring configuration FortConConfig config = new FortConConfig(); config.generateMaxes = false; diff --git a/cave/com.raytheon.viz.warngen/src/com/raytheon/viz/warngen/gui/WarngenLayer.java b/cave/com.raytheon.viz.warngen/src/com/raytheon/viz/warngen/gui/WarngenLayer.java index 13ccbf53d4..dbbc1303e5 100644 --- a/cave/com.raytheon.viz.warngen/src/com/raytheon/viz/warngen/gui/WarngenLayer.java +++ b/cave/com.raytheon.viz.warngen/src/com/raytheon/viz/warngen/gui/WarngenLayer.java @@ -194,6 +194,7 @@ import com.vividsolutions.jts.io.WKTReader; * 10/21/2013 DR 16632 D. Friedman Modify areaPercent exception handling. Fix an NPE. * Use A1 hatching behavior when no county passes the inclusion filter. * 10/29/2013 DR 16734 D. Friedman If redraw-from-hatched-area fails, don't allow the pollygon the be used. + * 01/09/2014 DR 16974 D. Friedman Improve followup redraw-from-hatched-area polygons. * * * @author mschenke @@ -2934,41 +2935,11 @@ public class WarngenLayer extends AbstractStormTrackResource { state.snappedToArea = false; if (areaHatcher != null) { Polygon polygon = state.getWarningPolygon(); - polygon = tryToIntersectWithOriginalPolygon(polygon); areaHatcher.hatchArea(polygon, state.getWarningArea(), state.getOldWarningPolygon()); } } - /** - * Try to determine the intersection of the given polygon with the original - * warning polygon. If there is no original polygon, if the result of the - * intersection is not a single polygon, or if a problem occurs, just return - * the original polygon. The purpose of this is to pass the polygon that - * best represents the user's intent to the polygon redrawing algorithm. - */ - private Polygon tryToIntersectWithOriginalPolygon(Polygon polygon) { - if (state.getOldWarningPolygon() != null) { - try { - Geometry g = polygon.intersection(state.getOldWarningPolygon()); - Polygon newPolygon = null; - if (g instanceof Polygon) { - newPolygon = (Polygon) g; - } else if (g instanceof GeometryCollection - && g.getNumGeometries() == 1 - && g.getGeometryN(0) instanceof Polygon) { - newPolygon = (Polygon) g.getGeometryN(0); - } - if (newPolygon != null && newPolygon.isValid()) { - polygon = newPolygon; - } - } catch (TopologyException e) { - // ignore - } - } - return polygon; - } - private Collection getDataWithFips(String fips) { List data = new ArrayList(); for (GeospatialData d : geoData.features) { @@ -3252,4 +3223,40 @@ public class WarngenLayer extends AbstractStormTrackResource { } } } + + /** + * Like buildArea, but does not take inclusion filters into account. Also + * returns a Geometry in lat/lon space. + * @param inputArea + * @return + */ + public Geometry buildIdealArea(Geometry inputArea) { + Geometry localHatchedArea = latLonToLocal(inputArea); + Geometry oldWarningArea = latLonToLocal(state.getOldWarningArea()); + Geometry newHatchedArea = null; + + for (GeospatialData f : geoData.features) { + // get the geometry of the county and make sure it intersects + // with our hatched area + PreparedGeometry prepGeom = (PreparedGeometry) f.attributes + .get(GeospatialDataList.LOCAL_PREP_GEOM); + Geometry intersection = null; + try { + // Get intersection between county and hatched boundary + intersection = GeometryUtil.intersection(localHatchedArea, prepGeom); + if (oldWarningArea != null) { + intersection = GeometryUtil.intersection(intersection, + oldWarningArea); + } + } catch (RuntimeException e) { + continue; + // This is a workaround for JTS 1.7.1 + } + + newHatchedArea = union(newHatchedArea, intersection); + } + Geometry result = newHatchedArea != null ? newHatchedArea : new GeometryFactory() + .createGeometryCollection(new Geometry[0]); + return localToLatLon(result); + } } diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/SaveCombinationsFileHandler.java b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/SaveCombinationsFileHandler.java index 3869df8221..1454e700e5 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/SaveCombinationsFileHandler.java +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/SaveCombinationsFileHandler.java @@ -20,21 +20,23 @@ package com.raytheon.edex.plugin.gfe.server.handler; import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; +import java.io.OutputStreamWriter; import java.io.Writer; import java.text.DecimalFormat; import java.text.NumberFormat; import java.util.ArrayList; import java.util.List; +import com.raytheon.edex.plugin.gfe.util.SendNotifications; import com.raytheon.uf.common.dataplugin.gfe.request.SaveCombinationsFileRequest; import com.raytheon.uf.common.dataplugin.gfe.server.message.ServerResponse; +import com.raytheon.uf.common.dataplugin.gfe.server.notify.CombinationsFileChangedNotification; import com.raytheon.uf.common.localization.FileUpdatedMessage; import com.raytheon.uf.common.localization.FileUpdatedMessage.FileChangeType; import com.raytheon.uf.common.localization.IPathManager; import com.raytheon.uf.common.localization.LocalizationContext; import com.raytheon.uf.common.localization.LocalizationContext.LocalizationType; +import com.raytheon.uf.common.localization.LocalizationFile; import com.raytheon.uf.common.localization.PathManagerFactory; import com.raytheon.uf.common.serialization.comm.IRequestHandler; import com.raytheon.uf.common.util.FileUtil; @@ -55,6 +57,7 @@ import com.raytheon.uf.edex.core.EDEXUtil; * May 16, 2011 dgilling Initial creation * Dec 02, 2013 #2591 dgilling Only send notification after Writer is * flushed/closed. + * Feb 05, 2014 #2591 Added CombinationFileChangedNotification * * * @@ -78,19 +81,22 @@ public class SaveCombinationsFileHandler implements @Override public ServerResponse handleRequest( SaveCombinationsFileRequest request) throws Exception { + String siteID = request.getSiteID(); IPathManager pm = PathManagerFactory.getPathManager(); LocalizationContext localization = pm.getContextForSite( - LocalizationType.CAVE_STATIC, request.getSiteID()); + LocalizationType.CAVE_STATIC, siteID); - File localFile = pm.getFile(localization, - FileUtil.join(COMBO_FILE_DIR, request.getFileName())); - boolean isAdded = (!localFile.exists()); + String comboName = request.getFileName(); + String fileName = FileUtil.join(COMBO_FILE_DIR, comboName) + ".py"; + LocalizationFile lf = pm.getLocalizationFile(localization, fileName); + boolean isAdded = (!lf.exists()); Writer outWriter = null; try { - outWriter = new BufferedWriter(new FileWriter(localFile)); + outWriter = new BufferedWriter(new OutputStreamWriter( + lf.openOutputStream())); String zoneComments = "\n# Automatically generated combinations file\n# " - + request.getFileName() + "\n\nCombinations = [\n"; + + comboName + "\n\nCombinations = [\n"; outWriter.write(zoneComments); NumberFormat df = new DecimalFormat("00"); @@ -115,20 +121,25 @@ public class SaveCombinationsFileHandler implements outWriter.close(); } } + lf.save(); // placing the notification code here ensures we only send the // notification on a successful file write operation. Otherwise we would // have thrown an IOException and never gotten to this portion of the // request handler. + + // TODO: remove sending of FileUpdateMessage after DR #2768 is fixed FileChangeType changeType = isAdded ? FileChangeType.ADDED : FileChangeType.UPDATED; EDEXUtil.getMessageProducer().sendAsync( "utilityNotify", - new FileUpdatedMessage(localization, FileUtil.join( - COMBO_FILE_DIR, request.getFileName()), changeType, - localFile.lastModified())); + new FileUpdatedMessage(localization, fileName, changeType, lf + .getTimeStamp().getTime())); + + CombinationsFileChangedNotification notif = new CombinationsFileChangedNotification( + comboName, request.getWorkstationID(), siteID); + SendNotifications.send(notif); return new ServerResponse(); } - } diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/gfe/isc/iscMosaic.py b/edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/gfe/isc/iscMosaic.py index 9cd47ed306..0a647dd556 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/gfe/isc/iscMosaic.py +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/gfe/isc/iscMosaic.py @@ -85,7 +85,7 @@ from com.raytheon.uf.edex.database.cluster import ClusterTask # 11/05/13 2517 randerso Restructured logging so it coulde be used by WECache # Changed WECache to limit the number of cached grids kept in memory # 01/09/14 16952 randerso Fix regression made in #2517 which caused errors with overlapping grids -# +# 02/04/14 17042 ryu Check in changes for randerso. # BATCH_DELAY = 0.0 @@ -1144,7 +1144,7 @@ class IscMosaic: self.__storeGrid(m[1], oldGrid) self.__dbGrid = None - #------------------------------------------------------------------------- + #------------------------------------------------------------------------- # Get Incoming netCDF file grid valid times # netCDFfile, var is the netCDF variable #------------------------------------------------------------------------- @@ -1526,7 +1526,7 @@ class IscMosaic: smsg, oldEntry, keyentry, ",".join(changedReasons)) msg = "%s %s %s [%s] -> [%s] (%s)" % \ - self.__siteID, parmName, printShortTR(tr), oldEntry, keyentry, ",".join(changedReasons) + (self.__siteID, parmName, printShortTR(tr), oldEntry, keyentry, ",".join(changedReasons)) self.__adjDataMsg.append(msg) key[idx] = keyentry #store back into list @@ -1626,7 +1626,7 @@ class IscMosaic: smsg, oldEntry, keyentry, ",".join(changedReasons)) msg = "%s %s %s [%s] -> [%s] (%s)" % \ - self.__siteID, parmName, printShortTR(tr), oldEntry, keyentry, ",".join(changedReasons) + (self.__siteID, parmName, printShortTR(tr), oldEntry, keyentry, ",".join(changedReasons)) self.__adjDataMsg.append(msg) diff --git a/edexOsgi/com.raytheon.edex.plugin.textlightning/src/com/raytheon/edex/plugin/textlightning/impl/TextLightningParser.java b/edexOsgi/com.raytheon.edex.plugin.textlightning/src/com/raytheon/edex/plugin/textlightning/impl/TextLightningParser.java index 67c5f4389b..bee05af58c 100644 --- a/edexOsgi/com.raytheon.edex.plugin.textlightning/src/com/raytheon/edex/plugin/textlightning/impl/TextLightningParser.java +++ b/edexOsgi/com.raytheon.edex.plugin.textlightning/src/com/raytheon/edex/plugin/textlightning/impl/TextLightningParser.java @@ -45,6 +45,10 @@ import com.raytheon.uf.common.dataplugin.binlightning.impl.LtgStrikeType; * ------------ ---------- ----------- -------------------------- * Dec 15, 2009 3983 jsanchez Initial creation * Feb 27, 2013 DCS 152 jgerth/elau Support for WWLLN + * Jan 27, 2014 DR 16080 M.Porricelli Changed LIGHTNING_PTRN_A + * to accommodate AK BLM + * lgtng intensities -999 to + * 999 * * * @@ -61,7 +65,10 @@ public class TextLightningParser { private List reports; // 03/23/2010 13:35:01 72.00 -157.00 -14 1 - private static final String LIGHTNING_PTRN_A = "(\\d{2,2}/\\d{2,2}/\\d{4,4}) (\\d{2,2}:\\d{2,2}:\\d{2,2})\\s{1,}(\\d{1,2}.\\d{2,2})\\s{1,}( |-\\d{1,3}.\\d{2,2})\\s{1,}( |-\\d{1,2})\\s{1,}(\\d{1,2})"; + // 03/23/2010 13:35:01 72.00 -157.00 14 1 + // 03/23/2010 13:35:01 72.00 -157.00 -142 1 + // 03/23/2010 13:35:01 72.00 -157.00 142 1 + private static final String LIGHTNING_PTRN_A = "(\\d{2,2}/\\d{2,2}/\\d{4,4}) (\\d{2,2}:\\d{2,2}:\\d{2,2})\\s{1,}(\\d{1,2}.\\d{2,2})\\s{1,}( |-\\d{1,3}.\\d{2,2})\\s{1,}(-?\\d{1,3})\\s{1,}(\\d{1,2})"; private static final Pattern LTG_PTRN_A = Pattern.compile(LIGHTNING_PTRN_A); // 10:03:24:13:35:00.68 72.000 157.000 -14.2 1 diff --git a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfigManager.java b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfigManager.java index 0943772df9..1c6166dae0 100644 --- a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfigManager.java +++ b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfigManager.java @@ -189,6 +189,15 @@ public class ArchiveConfigManager { public Collection getArchives() { String fileName = ArchiveConstants.selectFileName(Type.Retention, null); SelectConfig selections = loadSelection(fileName); + List emptySelection = new ArrayList(0); + + // Clear old selections. + for (ArchiveConfig archive : archiveMap.values()) { + for (CategoryConfig category : archive.getCategoryList()) { + category.setSelectedDisplayNames(emptySelection); + } + } + if ((selections != null) && !selections.isEmpty()) { for (ArchiveSelect archiveSelect : selections.getArchiveList()) { String archiveName = archiveSelect.getName(); @@ -325,7 +334,8 @@ public class ArchiveConfigManager { Map helperMap = new HashMap(); for (CategoryConfig category : archive.getCategoryList()) { - CategoryFileDateHelper helper = new CategoryFileDateHelper(category); + CategoryFileDateHelper helper = new CategoryFileDateHelper( + archiveRootDirPath, category); helperMap.put(category, helper); } @@ -470,13 +480,8 @@ public class ArchiveConfigManager { if (file.isDirectory()) { purgeCount += purgeDir(file, FileFilterUtils.trueFileFilter()); - if (file.list().length == 0) { - purgeCount += purgeDir(file, - FileFilterUtils.trueFileFilter()); - } - } else { - purgeCount += deleteFile(file); } + purgeCount += deleteFile(file); } } else if (file.isDirectory()) { purgeCount += purgeDir(file, defaultTimeFilter, diff --git a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryConfig.java b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryConfig.java index 42f5ba8d9a..b22fadc77b 100644 --- a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryConfig.java +++ b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryConfig.java @@ -72,6 +72,7 @@ import javax.xml.bind.annotation.XmlRootElement; * ------------ ---------- ----------- -------------------------- * May 1, 2013 1966 rferrel Initial creation * Aug 03, 2013 2224 rferrel Changes to include DataSet. + * Jan 09, 2014 2603 rferrel Fix bug in setSelectedDisplayNames * * * @@ -155,7 +156,7 @@ public class CategoryConfig implements Comparable { public void setSelectedDisplayNames( Collection selectedDisplayNameList) { selectedDisplayNames.clear(); - selectedDisplayNameList.addAll(selectedDisplayNameList); + selectedDisplayNames.addAll(selectedDisplayNameList); } public void addSelectedDisplayName(String displayName) { diff --git a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryFileDateHelper.java b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryFileDateHelper.java index 441eb00c7d..acf2ea6d8a 100644 --- a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryFileDateHelper.java +++ b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryFileDateHelper.java @@ -125,15 +125,19 @@ public class CategoryFileDateHelper implements IFileDateHelper { private final List dateInfoList; + private final String archiveRootDirPath; + /** * Initialization constructor. * + * @param archiveRootDirPath + * - Assumes path ends with file separator. * @param config - * @param rootDirPattern - * categoryTopLevelDirPattern */ - public CategoryFileDateHelper(CategoryConfig config) { + public CategoryFileDateHelper(String archiveRootDirPath, + CategoryConfig config) { List categoryDataSetList = config.getDataSetList(); + this.archiveRootDirPath = archiveRootDirPath; int size = 0; for (CategoryDataSet dataSet : categoryDataSetList) { size += dataSet.getDirPatterns().size(); @@ -176,6 +180,9 @@ public class CategoryFileDateHelper implements IFileDateHelper { @Override public DataSetStatus getFileDate(File file) { String filenamePath = file.getAbsolutePath(); + if (filenamePath.indexOf(archiveRootDirPath) == 0) { + filenamePath = filenamePath.substring(archiveRootDirPath.length()); + } Long timestamp = null; DataSetStatus result = new DataSetStatus(file); diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/server/notify/CombinationsFileChangedNotification.java b/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/server/notify/CombinationsFileChangedNotification.java new file mode 100644 index 0000000000..7884b433a0 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/server/notify/CombinationsFileChangedNotification.java @@ -0,0 +1,83 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.common.dataplugin.gfe.server.notify; + +import com.raytheon.uf.common.message.WsId; +import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; +import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; + +/** + * Combinations File Changed Notification + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Feb 5, 2014   #2591     randerso     Initial creation
+ * 
+ *
+ * + * @author randerso + * @version 1.0 + */ + +@DynamicSerialize +public class CombinationsFileChangedNotification extends GfeNotification { + + @DynamicSerializeElement + private String combinationsFileName; + + @DynamicSerializeElement + private WsId whoChanged; + + /** + * default constructor for serialization + */ + public CombinationsFileChangedNotification() { + super(); + } + + public CombinationsFileChangedNotification(String combinationsFileName, + WsId whoChanged, String siteID) { + super(); + this.siteID = siteID; + this.combinationsFileName = combinationsFileName; + this.whoChanged = whoChanged; + } + + public String getCombinationsFileName() { + return combinationsFileName; + } + + public void setCombinationsFileName(String combinationsFileName) { + this.combinationsFileName = combinationsFileName; + } + + public WsId getWhoChanged() { + return whoChanged; + } + + public void setWhoChanged(WsId whoChanged) { + this.whoChanged = whoChanged; + } + +} diff --git a/edexOsgi/com.raytheon.uf.edex.archive/src/com/raytheon/uf/edex/archive/DatabaseArchiveProcessor.java b/edexOsgi/com.raytheon.uf.edex.archive/src/com/raytheon/uf/edex/archive/DatabaseArchiveProcessor.java index 4bbb77c4d1..8c81d43e9c 100644 --- a/edexOsgi/com.raytheon.uf.edex.archive/src/com/raytheon/uf/edex/archive/DatabaseArchiveProcessor.java +++ b/edexOsgi/com.raytheon.uf.edex.archive/src/com/raytheon/uf/edex/archive/DatabaseArchiveProcessor.java @@ -74,15 +74,17 @@ import com.raytheon.uf.edex.database.processor.IDatabaseProcessor; * * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- - * Dec 10, 2013 2555 rjpeter Initial creation - * + * Dec 10, 2013 2555 rjpeter Initial creation. + * Jan 23, 2014 2555 rjpeter Updated to be a row at a time using ScrollableResults. + * Feb 04, 2014 2770 rferrel The dumpPdos now dumps all PluginDataObjects. * * * @author rjpeter * @version 1.0 */ -public class DatabaseArchiveProcessor implements IDatabaseProcessor { +public class DatabaseArchiveProcessor> + implements IDatabaseProcessor { private static final transient IUFStatusHandler statusHandler = UFStatus .getHandler(DatabaseArchiveProcessor.class); @@ -110,9 +112,11 @@ public class DatabaseArchiveProcessor implements IDatabaseProcessor { protected int fetchSize = 1000; + protected int entriesInMemory = 0; + protected Set datastoreFilesToArchive = new HashSet(); - protected Set filesCreatedThisSession = new HashSet(); + protected Map filesCreatedThisSession = new HashMap(); protected Set dirsToCheckNumbering = new HashSet(); @@ -120,6 +124,8 @@ public class DatabaseArchiveProcessor implements IDatabaseProcessor { protected boolean failed = false; + protected Map>> pdosByFile; + public DatabaseArchiveProcessor(String archivePath, String pluginName, PluginDao dao, IPluginArchiveFileNameFormatter nameFormatter) { this.archivePath = archivePath; @@ -136,46 +142,43 @@ public class DatabaseArchiveProcessor implements IDatabaseProcessor { * .util.List) */ @Override - public boolean process(List objects) { - if ((objects != null) && !objects.isEmpty()) { - Set datastoreFiles = new HashSet(); - statusHandler.info(pluginName + ": Processing rows " + recordsSaved - + " to " + (recordsSaved + objects.size())); - - @SuppressWarnings("unchecked") - List> pdos = (List>) objects; - Map>> pdosByFile = new HashMap>>(); - for (PersistableDataObject pdo : pdos) { - String path = nameFormatter.getFilename(pluginName, dao, pdo); - if (path.endsWith(".h5")) { - datastoreFiles.add(path); - path = path.substring(0, path.length() - 3); - } - - List> list = pdosByFile.get(path); - if (list == null) { - list = new LinkedList>(); - pdosByFile.put(path, list); - } - - list.add(pdo); + public boolean process(T object) { + if (object != null) { + if (pdosByFile == null) { + pdosByFile = new HashMap>>( + (int) (fetchSize * 1.3)); } - if (statusHandler.isPriorityEnabled(Priority.DEBUG)) { - statusHandler.debug(pluginName + ": Processed " - + objects.size() + " rows into " + pdosByFile.size() - + " files"); + String path = nameFormatter.getFilename(pluginName, dao, object); + if (path.endsWith(".h5")) { + datastoreFilesToArchive.add(path); + path = path.substring(0, path.length() - 3); } - try { - savePdoMap(pdosByFile); - datastoreFilesToArchive.addAll(datastoreFiles); - recordsSaved += pdos.size(); - } catch (Exception e) { - statusHandler.error(pluginName - + ": Error occurred saving data to archive", e); - failed = true; - return false; + List> list = pdosByFile.get(path); + if (list == null) { + list = new LinkedList>(); + pdosByFile.put(path, list); + } + + list.add(object); + + entriesInMemory++; + if (entriesInMemory >= fetchSize) { + try { + savePdoMap(pdosByFile); + pdosByFile.clear(); + int prev = recordsSaved; + recordsSaved += entriesInMemory; + entriesInMemory = 0; + statusHandler.info(pluginName + ": Processed rows " + prev + + " to " + recordsSaved); + } catch (Exception e) { + statusHandler.error(pluginName + + ": Error occurred saving data to archive", e); + failed = true; + return false; + } } } @@ -188,6 +191,20 @@ public class DatabaseArchiveProcessor implements IDatabaseProcessor { */ @Override public void finish() { + if (entriesInMemory > 0) { + try { + savePdoMap(pdosByFile); + int prev = recordsSaved; + recordsSaved += entriesInMemory; + statusHandler.info(pluginName + ": Processed rows " + prev + + " to " + recordsSaved); + } catch (Exception e) { + statusHandler.error(pluginName + + ": Error occurred saving data to archive", e); + failed = true; + } + } + for (File dir : dirsToCheckNumbering) { checkFileNumbering(dir); } @@ -370,7 +387,10 @@ public class DatabaseArchiveProcessor implements IDatabaseProcessor { + fileCount); fileMap.put(fileCount, newFile); writeDataToDisk(newFile, pdos); - filesCreatedThisSession.add(newFile.getAbsolutePath()); + FileStatus status = new FileStatus(); + status.dupElimUntilIndex = 0; + status.fileFull = pdos.size() >= fetchSize; + filesCreatedThisSession.put(newFile.getAbsolutePath(), status); // check if we have added another digit and should add a 0 to // previous numbers @@ -404,14 +424,15 @@ public class DatabaseArchiveProcessor implements IDatabaseProcessor { Iterator fileIter = fileMap.values().iterator(); while (fileIter.hasNext()) { File dataFile = fileIter.next(); + int dupElimUntil = Integer.MAX_VALUE; + FileStatus prevFileStatus = filesCreatedThisSession + .get(dataFile.getAbsolutePath()); - if (filesCreatedThisSession - .contains(dataFile.getAbsolutePath())) { - statusHandler - .debug(pluginName - + ": Skipping dup check on data file created this session: " - + dataFile.getName()); - continue; + if (prevFileStatus != null) { + dupElimUntil = prevFileStatus.dupElimUntilIndex; + if ((dupElimUntil <= 0) && prevFileStatus.fileFull) { + continue; + } } List> pdosFromDisk = readDataFromDisk(dataFile); @@ -424,13 +445,17 @@ public class DatabaseArchiveProcessor implements IDatabaseProcessor { .iterator(); boolean needsUpdate = false; int dupsRemoved = 0; - while (pdoIter.hasNext()) { + int index = 0; + while (pdoIter.hasNext() && (index < dupElimUntil)) { PersistableDataObject pdo = pdoIter.next(); + if (identifierSet.contains(pdo.getIdentifier())) { pdoIter.remove(); needsUpdate = true; dupsRemoved++; } + + index++; } if (statusHandler.isPriorityEnabled(Priority.DEBUG) @@ -443,6 +468,15 @@ public class DatabaseArchiveProcessor implements IDatabaseProcessor { if (!fileIter.hasNext() && (pdosFromDisk.size() < fetchSize)) { // last file, add more data to it needsUpdate = true; + + if (prevFileStatus == null) { + prevFileStatus = new FileStatus(); + prevFileStatus.dupElimUntilIndex = pdosFromDisk.size(); + prevFileStatus.fileFull = pdos.size() >= fetchSize; + filesCreatedThisSession.put(dataFile.getAbsolutePath(), + prevFileStatus); + } + int numToAdd = fetchSize - pdosFromDisk.size(); numToAdd = Math.min(numToAdd, pdos.size()); @@ -463,6 +497,9 @@ public class DatabaseArchiveProcessor implements IDatabaseProcessor { if (needsUpdate) { if (!pdosFromDisk.isEmpty()) { writeDataToDisk(dataFile, pdosFromDisk); + if (prevFileStatus != null) { + prevFileStatus.fileFull = pdosFromDisk.size() >= fetchSize; + } } else { dirsToCheckNumbering.add(dataFile.getParentFile()); dataFile.delete(); @@ -629,8 +666,11 @@ public class DatabaseArchiveProcessor implements IDatabaseProcessor { Iterator> pdoIter = pdos.iterator(); writer = new BufferedWriter(new FileWriter(dumpFile)); - statusHandler.info(String.format("%s: Dumping records to: %s", - pluginName, dumpFile.getAbsolutePath())); + + if (statusHandler.isPriorityEnabled(Priority.INFO)) { + statusHandler.info(String.format("%s: Dumping records to: %s", + pluginName, dumpFile.getAbsolutePath())); + } while (pdoIter.hasNext()) { PersistableDataObject pdo = pdoIter.next(); @@ -640,9 +680,11 @@ public class DatabaseArchiveProcessor implements IDatabaseProcessor { // otherwise was read from file and will be recorded in // a previous entry writer.write("" + pluginDataObject.getId() + ":"); - writer.write(pluginDataObject.getDataURI()); - writer.write("\n"); + } else { + writer.write("-:"); } + writer.write(pluginDataObject.getDataURI()); + writer.write("\n"); } else { writer.write(pdo.getIdentifier().toString()); writer.write("\n"); @@ -736,4 +778,22 @@ public class DatabaseArchiveProcessor implements IDatabaseProcessor { } } } + + /** + * Inner class for tracking status of files that have been written out this + * session. + */ + private static class FileStatus { + /** + * Apply dup elim logic until this index is reached. + */ + private int dupElimUntilIndex; + + /** + * Way of tracking if file is considered full. Tracked so that if the + * file doesn't need to be dup elim'd due to being written this session + * and the file is full then there is no reason to deserialize it. + */ + private boolean fileFull; + } } diff --git a/edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/dao/CoreDao.java b/edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/dao/CoreDao.java index c4bf5da325..9318c58209 100644 --- a/edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/dao/CoreDao.java +++ b/edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/dao/CoreDao.java @@ -43,6 +43,8 @@ import net.sf.ehcache.management.ManagementService; import org.hibernate.Criteria; import org.hibernate.Query; +import org.hibernate.ScrollMode; +import org.hibernate.ScrollableResults; import org.hibernate.Session; import org.hibernate.Transaction; import org.hibernate.criterion.Criterion; @@ -95,10 +97,9 @@ import com.raytheon.uf.edex.database.query.DatabaseQuery; * 5/14/08 1076 brockwoo Fix for distinct with multiple properties * Oct 10, 2012 1261 djohnson Incorporate changes to DaoConfig, add generic to {@link IPersistableDataObject}. * Apr 15, 2013 1868 bsteffen Rewrite mergeAll in PluginDao. - * * Nov 08, 2013 2361 njensen Changed method signature of saveOrUpdate to take Objects, not PersistableDataObjects * Dec 13, 2013 2555 rjpeter Added processByCriteria and fixed Generics warnings. - * + * Jan 23, 2014 2555 rjpeter Updated processByCriteriato be a row at a time using ScrollableResults. * * * @author bphillip @@ -460,8 +461,9 @@ public class CoreDao extends HibernateDaoSupport { * @throws DataAccessLayerException * If the query fails */ - public int processByCriteria(final DatabaseQuery query, - final IDatabaseProcessor processor) throws DataAccessLayerException { + public int processByCriteria(final DatabaseQuery query, + final IDatabaseProcessor processor) + throws DataAccessLayerException { int rowsProcessed = 0; try { // Get a session and create a new criteria instance @@ -480,24 +482,29 @@ public class CoreDao extends HibernateDaoSupport { "Error populating query", e); } - if (processor.getBatchSize() > 0) { - hibQuery.setMaxResults(processor.getBatchSize()); - } else if (query.getMaxResults() != null) { - hibQuery.setMaxResults(query.getMaxResults()); + int batchSize = processor.getBatchSize(); + if (batchSize <= 0) { + batchSize = 1000; } - List results = null; - boolean continueProcessing = false; - int count = 0; + hibQuery.setFetchSize(processor.getBatchSize()); - do { - hibQuery.setFirstResult(count); - results = hibQuery.list(); - continueProcessing = processor.process(results); - count += results.size(); + int count = 0; + ScrollableResults rs = hibQuery + .scroll(ScrollMode.FORWARD_ONLY); + boolean continueProcessing = true; + + while (rs.next() && continueProcessing) { + Object[] row = rs.get(); + if (row.length > 0) { + continueProcessing = processor + .process((T) row[0]); + } + count++; + if ((count % batchSize) == 0) { getSession().clear(); - } while (continueProcessing && (results != null) - && (results.size() > 0)); + } + } processor.finish(); return count; } diff --git a/edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/processor/IDatabaseProcessor.java b/edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/processor/IDatabaseProcessor.java index 9fd67b00cc..0a3a4e7e73 100644 --- a/edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/processor/IDatabaseProcessor.java +++ b/edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/processor/IDatabaseProcessor.java @@ -19,12 +19,9 @@ **/ package com.raytheon.uf.edex.database.processor; -import java.util.List; - /** * Interface for working with a batched set of results inside a database - * session. Process can be called multiple times based on the batchSize of the - * processor. + * session. Process will be called for each row. * * 
  * 
@@ -32,21 +29,22 @@ import java.util.List;
  * 
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
- * Dec 9, 2013  2555      rjpeter     Initial creation
+ * Dec 9, 2013  2555       rjpeter     Initial creation.
+ * Jan 23, 2014 2555       rjpeter     Updated to be a row at a time using ScrollableResults.
  *
* * @author rjpeter * @version 1.0 */ -public interface IDatabaseProcessor { +public interface IDatabaseProcessor { /** - * Perform any processing on this batch of objects. + * Perform any processing on this row. * - * @param objects + * @param row * @return True if should continue processing, false otherwise. */ - public boolean process(List objects); + public boolean process(T row); /** * Perform any post processing if necessary. diff --git a/pythonPackages/numpy/numpy/core/numeric.py b/pythonPackages/numpy/numpy/core/numeric.py old mode 100644 new mode 100755 index e7d2261722..8c9f507d21 --- a/pythonPackages/numpy/numpy/core/numeric.py +++ b/pythonPackages/numpy/numpy/core/numeric.py @@ -1493,10 +1493,8 @@ def set_string_function(f, repr=True): else: return multiarray.set_string_function(f, repr) -# randerso DR #2513 remove calls to non-threadsafe set_string_function -# https://github.com/numpy/numpy/issues/3961 -# set_string_function(array_str, 0) -# set_string_function(array_repr, 1) +set_string_function(array_str, 0) +set_string_function(array_repr, 1) little_endian = (sys.byteorder == 'little') diff --git a/rpms/awips2.core/Installer.ant/scripts/profile.d/awips2Ant.sh b/rpms/awips2.core/Installer.ant/scripts/profile.d/awips2Ant.sh index e4f87f6538..4b3d02ebc3 100644 --- a/rpms/awips2.core/Installer.ant/scripts/profile.d/awips2Ant.sh +++ b/rpms/awips2.core/Installer.ant/scripts/profile.d/awips2Ant.sh @@ -8,10 +8,7 @@ if [ ${RC} -ne 0 ]; then fi # Determine Where Ant Has Been Installed. -ANT_INSTALL=`rpm -q --queryformat '%{INSTPREFIXES}' awips2-ant` -if [ "${ANT_INSTALL}" = "" ]; then - return -fi +ANT_INSTALL=/awips2/ant # Update The Environment. export ANT_HOME="${ANT_INSTALL}" diff --git a/rpms/awips2.core/Installer.python/src/x86_64/hdf5-1.8.4-patch1-linux-x86_64-shared.tar.gz b/rpms/awips2.core/Installer.python/src/x86_64/hdf5-1.8.4-patch1-linux-x86_64-shared.tar.gz index 23dfbc4ee6..995031055b 100644 Binary files a/rpms/awips2.core/Installer.python/src/x86_64/hdf5-1.8.4-patch1-linux-x86_64-shared.tar.gz and b/rpms/awips2.core/Installer.python/src/x86_64/hdf5-1.8.4-patch1-linux-x86_64-shared.tar.gz differ diff --git a/rpms/awips2.core/Installer.tools/source/lzf.tar.gz b/rpms/awips2.core/Installer.tools/source/lzf.tar.gz index 22de2de72c..39534f2eb4 100644 Binary files a/rpms/awips2.core/Installer.tools/source/lzf.tar.gz and b/rpms/awips2.core/Installer.tools/source/lzf.tar.gz differ diff --git a/rpms/build/x86_64/build.sh b/rpms/build/x86_64/build.sh index 2224eb264a..e15fbc57ca 100644 --- a/rpms/build/x86_64/build.sh +++ b/rpms/build/x86_64/build.sh @@ -409,17 +409,21 @@ fi if [ "${1}" = "-viz" ]; then buildRPM "awips2" - #buildRPM "awips2-common-base" + buildRPM "awips2-common-base" + #buildRPM "awips2-python-numpy" + buildRPM "awips2-ant" #buildRPM "awips2-python-dynamicserialize" #buildRPM "awips2-python" - buildRPM "awips2-adapt-native" + #buildRPM "awips2-adapt-native" #unpackHttpdPypies #if [ $? -ne 0 ]; then # exit 1 #fi #buildRPM "awips2-httpd-pypies" - buildRPM "awips2-hydroapps-shared" + #buildRPM "awips2-hydroapps-shared" #buildRPM "awips2-rcm" + #buildRPM "awips2-gfesuite-client" + #buildRPM "awips2-gfesuite-server" #buildRPM "awips2-tools" #buildRPM "awips2-cli" buildCAVE @@ -447,12 +451,13 @@ if [ "${1}" = "-custom" ]; then #if [ $? -ne 0 ]; then # exit 1 #fi - buildRPM "awips2-adapt-native" - buildRPM "awips2-hydroapps-shared" + #buildRPM "awips2-adapt-native" + #buildRPM "awips2-hydroapps-shared" #buildRPM "awips2-alertviz" #buildRPM "awips2-python" #buildRPM "awips2-alertviz" #buildRPM "awips2-eclipse" + buildRPM "awips2-python" exit 0 fi