Merge branch 'master_14.4.1'(OB_14.4.1-36) into emc_14.4.1

Change-Id: Ie153b2a36947fa999b47cde069c1915e371c3f98

Former-commit-id: 5abb5644ef50d32fe131941ea46c70f371ed9a3b

cave/com.raytheon.viz.gfe/localization/gfe/userPython/textUtilities/headline/FormatterRunner.py

@@ -26,7 +26,6 @@ import JUtil, VarDictGroker
 import RedirectLogging
 import UFStatusHandler
 
-from java.io import File
 #
 # Runs the text formatter to generate text products
 #   
@@ -40,6 +39,9 @@ from java.io import File
 #    12/10/14        #14946        ryu           Add getTimeZones() function.
 #    04/16/15        #14946        ryu           Fix getTimeZones to return the office TZ if timezone
 #                                                is not set for any zone in a segment.
+#    05/06/2015      #4467         randerso      Convert to upper case before writing to files if
+#                                                mixed case is not enabled for the product.
+#                                                Cleaned up file writing code
 # 
 #
 
@@ -104,6 +106,33 @@ def executeFromJava(databaseID, site, username, dataMgr, forecastList, logFile,
     RedirectLogging.restore()
     return forecasts
     
+def getPid(forecast):
+    # taken from ProductParser.py
+    import re
+    
+    sl = r'^'                            # start of line
+    el = r'\s*?\n'                       # end of line
+    id3 = r'[A-Za-z]{3}'                 # 3 charater word
+    empty = r'^\s*' + el                 # empty line
+    
+    wmoid = r'(?P<wmoid>[A-Z]{4}\d{2})' # wmoid
+    fsid  = r'(?P<fsid>[A-Z]{4})'       # full station id
+    pit   = r'(?P<pit>\d{6})'           # product issuance time UTC
+    ff    = r'(?P<funnyfield> ' + id3 + ')?'          # "funny" field
+    
+    # CI block
+    ci_start = sl + wmoid + ' ' + fsid + ' ' + pit + ff + el
+    awipsid = r'(?P<pil>(?P<cat>[A-Z0-9]{3})(?P<lid>[A-Z0-9]{1,3}))' + el
+    ci_block = r'(?P<ciblock>' + ci_start + awipsid + '\n?)' 
+    
+    ci_re = re.compile(ci_block)
+
+    pid = None
+    m = ci_re.search(forecast)
+    if m is not None:
+        pid = m.group('cat')
+
+    return pid
 
 def runFormatter(databaseID, site, forecastList, cmdLineVarDict, vtecMode,
                     username, dataMgr, serverFile=None,
@@ -234,15 +263,31 @@ def runFormatter(databaseID, site, forecastList, cmdLineVarDict, vtecMode,
 
     # For each Forecast Type,
     #   Create generate forecast
-    forecasts = ""
+    forecasts = ""      # returned value
+    outForecasts = ""   # written to output files
     for forecastType in forecastList:           
         forecast = formatter.getForecast(forecastType, argDict)        
         forecasts = forecasts + forecast
         
+        # Convert data written to files to upper case if required
+        mixedCase = False
+        pid = getPid(forecast)
+        if pid is None:
+            logger.warning("Unable to determine PID: defaulting to upper case")
+        else:
+            from com.raytheon.uf.common.dataplugin.text.db import MixedCaseProductSupport
+            mixedCase = MixedCaseProductSupport.isMixedCase(pid)
+        
+        if mixedCase:
+            outForecasts = outForecasts + forecast
+        else:
+            outForecasts = outForecasts + forecast.upper()
+
     logger.info("Text:\n" + str(forecasts))
+    
     try:
         outputFile = argDict["outputFile"]
-        success = writeToFile(forecasts, outputFile, "w")
+        success = writeToFile(outForecasts, outputFile, "w")
         if success == 0:
             print "Couldn't open output file", outputFile
             logger.error("Couldn't open output file: ", outputFile)
@@ -252,7 +297,7 @@ def runFormatter(databaseID, site, forecastList, cmdLineVarDict, vtecMode,
 
     try:
         outputFile = argDict["serverOutputFile"]
-        success = writeToFile(forecasts, outputFile, "w")
+        success = writeToFile(outForecasts, outputFile, "w")
         if success == 0:
             print "Couldn't open output file", outputFile
             logger.error("Couldn't open output file: ", outputFile)
@@ -263,7 +308,7 @@ def runFormatter(databaseID, site, forecastList, cmdLineVarDict, vtecMode,
 
     try:
         appendFile = argDict["appendFile"]
-        success = writeToFile(forecasts, appendFile, "a")
+        success = writeToFile(outForecasts, appendFile, "a")
         if success == 0:
             print "Couldn't open append file", appendFile
             logger.error("Couldn't write to append file: ", appendFile)
@@ -274,7 +319,7 @@ def runFormatter(databaseID, site, forecastList, cmdLineVarDict, vtecMode,
     try:
         serverFile = argDict["serverFile"]
         writeToSite = (username == "SITE")
-        success = writeToServerFile(forecasts, serverFile, writeToSite)
+        success = writeToServerFile(outForecasts, serverFile, writeToSite)
         if success == 0:
             print "Couldn't open server output file", serverFile
             logger.error("Couldn't open server output file: ", serverFile)
@@ -282,6 +327,8 @@ def runFormatter(databaseID, site, forecastList, cmdLineVarDict, vtecMode,
     except:
         pass
 
+    del outForecasts
+    
     # Remove any lat/lon areas created temporarily
     #global LatLonIds
     #argDict["ifpClient"].deleteReferenceData(LatLonIds)
@@ -308,29 +355,37 @@ def getAbsTime(timeStr):
     return AbsTime.absTimeYMD(year, month, day, hour, minute)
     
 def writeToFile(forecasts, outputFile, mode):
-    if not outputFile is None and outputFile != "":
+    if outputFile:
-        outfile = open(outputFile, mode)
+        logger.info("Writing forecast to " + outputFile)
-        os.chmod(outputFile, 0644)
+        try:
-        if outfile is None:
+            with open(outputFile, mode) as outfile:
-            return 0
-        else:
                 outfile.write(forecasts)
-            outfile.close()
+                
+            os.chmod(outputFile, 0644)
+        except:
+            logger.exception("Error writing forecast to "+outputFile)
+            return 0
     return 1
 
 def writeToServerFile(forecasts, outputFile, writeToSite):
-    if not outputFile is None and outputFile != "":
+    if outputFile:
+        try:
             if writeToSite:
                 ctx = PATH_MGR.getContext(LocalizationType.COMMON_STATIC, LocalizationLevel.SITE)
             else:
                 ctx = PATH_MGR.getContext(LocalizationType.COMMON_STATIC, LocalizationLevel.USER)
-        filePath = File.separatorChar.join(["gfe", "text", "PRODGEN", outputFile + ".PRODGEN"])
+            filePath = PATH_MGR.SEPARATOR.join(["gfe", "text", "PRODGEN", outputFile + ".PRODGEN"])
             lFile = PATH_MGR.getLocalizationFile(ctx, filePath)
-        javaFile = lFile.getFile()        
+            logger.info("Writing forecast to " + str(lFile))
-        outfile = open(javaFile.getAbsolutePath(), 'w')
+    
+            from LockingFile import File
+            with File(lFile.getFile(), "", 'w') as outfile:
                 outfile.write(forecasts)
-        outfile.close()
+    
             return lFile.save()
+        except:
+            logger.exception("Error writing forecast to " + str(lFile))
+            return 0
     return 1
 
 def getScripts(paths, nameMap, definitionMap):

deltaScripts/14.4.1/DR4462/updateStdTextProducts.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+# Issue: #4462 Update script to update stdtextprodcts entries with blank site values.
+#
+# This script will modify table stdtextproducts in the fxatext.public schema.
+# 
+PSQL="/awips2/psql/bin/psql"
+
+if [ ${#1} != 4 ] ; then
+	echo "ERROR: First argument must be local site to use in the stdtextproducts table. Example: KOAX"
+	exit 1
+else
+	siteId=${1}
+	siteLtr=${siteId:0:1}
+fi
+
+if [ ! -f ${PSQL} ];
+then
+	echo "ERROR: The PSQL executable does not exist - ${PSQL}."
+	echo "FATAL: Update Failed!"
+	exit 1
+fi
+
+UPDATE_BY_XXX="update public.stdtextproducts set site = '${siteLtr}' || xxxid where site='' and xxxid not like '% ';"
+UPDATE_LOC_SITE="update public.stdtextproducts set site='${siteId}' where site='' and xxxid like '% ';"
+
+function updateXXXentries
+{
+	echo "INFO: Updating stdtextproducts using xxxid"
+${PSQL} -U awips -d fxatext -a -c "${UPDATE_BY_XXX}"
+	if [ $? -ne 0 ];
+	then
+		echo "FATAL: Update by xxxid Failed!"
+		exit 1	
+	fi
+	echo "INFO: Completed updating stdtextproducts using xxxid."
+}
+
+function updateSITEentries
+{
+	echo "INFO: Updating stdtextproduct table's site using local site."
+${PSQL} -U awips -d fxatext -a -c "${UPDATE_LOC_SITE}" 
+	if [ $? -ne 0 ];
+	then
+		echo "FATAL: unable to update stdtextproducts using local site."
+		exit 1
+	fi
+	echo "INFO: Finish updating stdtextproducts using local site."
+}
+
+echo "INFO: start update stdtextproducts"
+updateXXXentries
+updateSITEentries
+echo "INFO: finish update stdtextproducts"
+exit 0
+
+

edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/textproducts/templates/product/AreaFcst.py

@@ -17,6 +17,14 @@
 # See the AWIPS II Master Rights File ("Master Rights File.pdf") for
 # further licensing information.
 ##
+#
+# SOFTWARE HISTORY
+# Date            Ticket#        Engineer    Description
+# ------------    ----------     ----------- --------------------------
+# May 01, 2015    17421          ryu         Changed analysis methods for StormTotalSnow
+#
+##
+
 #-------------------------------------------------------------------------
 # Description: This product creates a ZFP-type series of text phrases 
 # for consecutive time periods for a list of edit areas. It can be 
@@ -475,7 +483,7 @@ class TextProduct(TextRules.TextRules, SampleAnalysis.SampleAnalysis):
             ("PoP", self._PoP_analysisMethod("Period_1"), [3]),
             ("PoP", self.binnedPercent, [3]),
             ("SnowAmt", self.accumMinMax),
-            ("StormTotalSnow", self.accumMinMax),
+            ("StormTotalSnow", self.minMax),
             ("IceAccum", self.accumMinMax),
             ("SnowLevel", self.avg),
             ("Wind", self.vectorMedianRange, [6]),
@@ -564,7 +572,7 @@ class TextProduct(TextRules.TextRules, SampleAnalysis.SampleAnalysis):
                        ("PoP", self._PoP_analysisMethod("Period_2_3"), [6]),
                        ("PoP", self.binnedPercent, [6]),
                        ("SnowAmt", self.accumMinMax),
-                       ("StormTotalSnow", self.accumMinMax),
+                       ("StormTotalSnow", self.minMax),
                        ("IceAccum", self.accumMinMax),
                        ("SnowLevel", self.avg),
                        ("Wind", self.vectorMedianRange, [6]),
@@ -791,7 +799,7 @@ class TextProduct(TextRules.TextRules, SampleAnalysis.SampleAnalysis):
                        ("PoP", self._PoP_analysisMethod("FirstFcstPeriod"), [6]),
                        ("PoP", self.binnedPercent, [6]),
                        ("SnowAmt", self.accumMinMax),
-                       ("StormTotalSnow", self.accumMinMax),
+                       ("StormTotalSnow", self.minMax),
                        ("IceAccum", self.accumMinMax),
                        ("SnowLevel", self.avg),
                        ("Wind", self.vectorMedianRange, [6]),
@@ -852,7 +860,7 @@ class TextProduct(TextRules.TextRules, SampleAnalysis.SampleAnalysis):
                        ("PoP", self._PoP_analysisMethod("AreaFcstPeriod"), [6]),
                        ("PoP", self.binnedPercent, [6]),
                        ("SnowAmt", self.accumMinMax),
-                       ("StormTotalSnow", self.accumMinMax),
+                       ("StormTotalSnow", self.minMax),
                        ("IceAccum", self.accumMinMax),
                        ("SnowLevel", self.avg),
                        ("Wind", self.vectorMedianRange, [6]),

edexOsgi/com.raytheon.edex.plugin.grib/utility/common_static/base/grid/datasetInfo/gribDatasets_NCEP-7.xml

@@ -77,12 +77,12 @@
     </info>
     <info>
         <title>GFS20-PAC</title>
-        <dataset>GFS-PAC-20KM</dataset>
+        <datasetId>GFS-PAC-20KM</datasetId>
         <dt>3</dt>
     </info>
     <info>
         <title>GFS20-PRICO</title>
-        <dataset>GFS-PRICO-20KM</dataset>
+        <datasetId>GFS-PRICO-20KM</datasetId>
         <dt>3</dt>
     </info>
     <info>

edexOsgi/com.raytheon.uf.edex.plugin.text/src/com/raytheon/uf/edex/plugin/text/dao/StdTextProductDao.java

@@ -100,6 +100,7 @@ import com.raytheon.uf.edex.database.purge.PurgeLogger;
  * 10/16/2014   3454       bphillip    Upgrading to Hibernate 4
  * 10/28/2014   3454        bphillip    Fix usage of getSession()
  * Jan 27, 2015 4031        rferrel     Resolve AFOS PILs site conflict using preferredAfosFirstLetter.
+ * May 05, 2015 4462        rferrel     {@link #write(StdTextProduct)} when missing set the textProduct's site.
  * </pre>
  * 
  * @author garmendariz
@@ -220,6 +221,22 @@ public class StdTextProductDao extends CoreDao {
         prodId.setNnnid(nnn);
         prodId.setXxxid(xxx);
         Session session = this.getSession();
+        String site = textProduct.getProdId().getSite();
+        if ((site == null) || site.trim().isEmpty()) {
+            // Determine product site.
+            if (xxx.trim().length() == MAX_FIELD_LENGTH) {
+                site = SiteMap.getInstance().getSite4LetterId(xxx);
+            } else {
+                site = SiteMap.getInstance().getSite4LetterId(
+                        SiteUtil.getSite());
+            }
+            if (logger.isInfoEnabled()) {
+                logger.info("Write \"" + ccc + nnn + xxx
+                        + "\" setting site to " + site);
+            }
+            textProduct.getProdId().setSite(site);
+        }
+
         try {
             try {
                 Query query = session.createQuery("SELECT refTime from "

rpms/awips2.core/Installer.ldm/patch/etc/pqact.conf.template

@@ -783,5 +783,5 @@ NGRID	^(Y.C[A-MZ][05789][0-9]) (KWBY) (..)(..)(..)[^!]*!(grib|grib2)/[^/]*/([^/]
 	FILE	-overwrite -log -close -edex	/data_store/\6/(\3:yyyy)(\3:mm)\3/\4/\7/GRID\8/\(10)Z_\(11)_\(12)-\1_\2_\3\4\5_(seq).\6.%Y%m%d%H
 
 # MRMS
-NGRID	^(YAU[CDLMPQS][0-2][0-9]) (KWNR) (..)(..)(..)[^!]*!(grib|grib2)/[^/]*/([^/]*)/#([^/]*)/([0-9]{8})([0-9]{4})(F[0-9]{3})/([^/]*)
+#NGRID	^(YAU[CDLMPQS][0-2][0-9]) (KWNR) (..)(..)(..)[^!]*!(grib|grib2)/[^/]*/([^/]*)/#([^/]*)/([0-9]{8})([0-9]{4})(F[0-9]{3})/([^/]*)
-	FILE	-overwrite -log -close -edex	/data_store/\6/(\3:yyyy)(\3:mm)\3/\4/\7/GRID\8/\(10)Z_\(11)_\(12)-\1_\2_\3\4\5_(seq).\6.%Y%m%d%H
+#	FILE	-overwrite -log -close -edex	/data_store/\6/(\3:yyyy)(\3:mm)\3/\4/\7/GRID\8/\(10)Z_\(11)_\(12)-\1_\2_\3\4\5_(seq).\6.%Y%m%d%H