Merge branch '9-Wes2Bridge' into development

Former-commit-id: 522a35f61b [formerly 732d7b77d80e7647bc79de5df072f37a2449b0cf]
Former-commit-id: e1addfad12

.gitignore

@@ -9,3 +9,4 @@ bin-test/
 *.pyo
 *.o
 *.orig
+

cave/com.raytheon.uf.viz.coopprecip/src/com/raytheon/uf/viz/coopprecip/CoopPrecipDataCubeAdapter.java

@@ -42,6 +42,7 @@ import com.raytheon.uf.common.pointdata.PointDataDescription.Type;
 import com.raytheon.uf.common.pointdata.PointDataView;
 import com.raytheon.uf.common.time.BinOffset;
 import com.raytheon.uf.common.time.DataTime;
+import com.raytheon.uf.common.time.SimulatedTime;
 import com.raytheon.uf.viz.core.catalog.DirectDbQuery;
 import com.raytheon.uf.viz.core.catalog.LayerProperty;
 import com.raytheon.uf.viz.core.datastructure.IDataCubeAdapter;
@@ -127,6 +128,7 @@ public class CoopPrecipDataCubeAdapter implements IDataCubeAdapter {
         List<List<DataTime>> results = new ArrayList<List<DataTime>>(
                 requests.size());
         for (TimeQueryRequest request : requests) {
+        	request.setSimDate(SimulatedTime.getSystemTime().getTime());
             DataTime[] result = timeQuery(request.getQueryTerms(),
                     request.isMaxQuery(), request.getBinOffset());
             if (result != null) {

cave/com.raytheon.uf.viz.core/src/com/raytheon/uf/viz/core/catalog/CatalogQuery.java

@@ -36,6 +36,7 @@ import com.raytheon.uf.common.status.UFStatus;
 import com.raytheon.uf.common.status.UFStatus.Priority;
 import com.raytheon.uf.common.time.BinOffset;
 import com.raytheon.uf.common.time.DataTime;
+import com.raytheon.uf.common.time.SimulatedTime;
 import com.raytheon.uf.viz.core.comm.Connector;
 import com.raytheon.uf.viz.core.exception.VizException;
 import com.raytheon.uf.viz.core.requests.ThriftClient;
@@ -232,7 +233,9 @@ public class CatalogQuery {
         req.setMaxQuery(max);
         req.setPluginName(pluginName);
         req.setBinOffset(binOffset);
+        req.setSimDate(SimulatedTime.getSystemTime().getTime());
         req.setQueryTerms(constraintMap);
+
         return req;
     }
 
@@ -251,6 +254,7 @@ public class CatalogQuery {
      * @return the available times that meet the constraints
      * @throws VizException
      */
+    @SuppressWarnings("unchecked")
     public static DataTime[] performTimeQuery(
             Map<String, RequestConstraint> constraintMap, boolean max,
             BinOffset binOffset) throws VizException {

cave/com.raytheon.uf.viz.core/src/com/raytheon/uf/viz/core/datastructure/DefaultDataCubeAdapter.java

@@ -31,6 +31,7 @@ import com.raytheon.uf.common.datastorage.Request;
 import com.raytheon.uf.common.datastorage.records.IDataRecord;
 import com.raytheon.uf.common.pointdata.PointDataContainer;
 import com.raytheon.uf.common.time.DataTime;
+import com.raytheon.uf.common.time.SimulatedTime;
 import com.raytheon.uf.viz.core.catalog.LayerProperty;
 import com.raytheon.uf.viz.core.catalog.ScriptCreator;
 import com.raytheon.uf.viz.core.comm.Loader;
@@ -87,6 +88,11 @@ public class DefaultDataCubeAdapter implements IDataCubeAdapter {
             throws VizException {
         TimeQueryRequestSet set = new TimeQueryRequestSet();
         set.setRequests(requests.toArray(new TimeQueryRequest[0]));
+        // boundary for time queries
+        for (TimeQueryRequest req: set.getRequests()) {
+        	req.setSimDate(SimulatedTime.getSystemTime().getTime());
+        }
+        
         @SuppressWarnings("unchecked")
         List<List<DataTime>> result = (List<List<DataTime>>) ThriftClient
                 .sendRequest(set);

cave/com.raytheon.uf.viz.datastorage.remote/src/com/raytheon/uf/viz/datastorage/remote/RemoteDataStore.java

@@ -245,8 +245,14 @@ public class RemoteDataStore implements IDataStore {
     }
 
     @Override
-    public void repack(String dirName, Compression compression)
+    public void repack(Compression compression) throws StorageException {
-            throws StorageException {
+        throw new StorageException("Operation not supported", null);
+    }
+
+    @Override
+    public void copy(String outputDir, Compression compression,
+            String timestampCheck, int minMillisSinceLastChange,
+            int maxMillisSinceLastChange) throws StorageException {
         throw new StorageException("Operation not supported", null);
     }
 

cave/com.raytheon.uf.viz.derivparam/src/com/raytheon/uf/viz/derivparam/data/AbstractDataCubeAdapter.java

@@ -43,6 +43,7 @@ import com.raytheon.uf.common.datastorage.Request;
 import com.raytheon.uf.common.datastorage.records.IDataRecord;
 import com.raytheon.uf.common.pointdata.PointDataContainer;
 import com.raytheon.uf.common.time.DataTime;
+import com.raytheon.uf.common.time.SimulatedTime;
 import com.raytheon.uf.viz.core.catalog.LayerProperty;
 import com.raytheon.uf.viz.core.datastructure.IDataCubeAdapter;
 import com.raytheon.uf.viz.core.datastructure.VizDataCubeException;
@@ -108,6 +109,7 @@ public abstract class AbstractDataCubeAdapter implements IDataCubeAdapter {
         for (TimeQueryRequest request : requests) {
             List<AbstractRequestableLevelNode> requestNodes = evaluateRequestConstraints(request
                     .getQueryTerms());
+            request.setSimDate(SimulatedTime.getSystemTime().getTime());
             // pull out time queries and bulk submit
             for (AbstractRequestableLevelNode requestNode : requestNodes) {
                 getTimeQuery(request, requestNode, false, queries, cache, null);

cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPDataLoader.java

@@ -39,6 +39,7 @@ import com.raytheon.uf.common.serialization.SerializationException;
 import com.raytheon.uf.common.serialization.SerializationUtil;
 import com.raytheon.uf.common.status.IUFStatusHandler;
 import com.raytheon.uf.common.status.UFStatus;
+import com.raytheon.uf.common.time.SimulatedTime;
 import com.raytheon.uf.common.util.FileUtil;
 import com.raytheon.uf.viz.core.VizApp;
 import com.raytheon.uf.viz.monitor.ffmp.FFMPMonitor;
@@ -474,7 +475,7 @@ public class FFMPDataLoader extends Thread {
 
         FFMPBasinData basinData = null;
 
-        if (file.exists()) {
+        if (file.exists() && SimulatedTime.getSystemTime().isRealTime()) {
 
             System.out.println("File path: " + file.getName());
 

cave/com.raytheon.uf.viz.monitor.scan/src/com/raytheon/uf/viz/monitor/scan/test/ScanTestDataPopulator.java

@@ -11,6 +11,8 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.concurrent.ConcurrentHashMap;
 
+import com.raytheon.uf.common.util.FileUtil;
+
 /**
  * Utility to populate EDEX with Data for DAT tools
  * 
@@ -281,7 +283,8 @@ public class ScanTestDataPopulator {
          */
         private void copyFile(File file) {
 
-            File outfile = new File(MANUAL_INGEST_DIR + "/" + file.getName());
+            File outfile = new File(FileUtil.join(MANUAL_INGEST_DIR,
+                    file.getName()));
             InputStream in = null;
             try {
                 in = new FileInputStream(file);

cave/com.raytheon.uf.viz.monitor/src/com/raytheon/uf/viz/monitor/Monitor.java

@@ -24,6 +24,8 @@ import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
 
+import com.raytheon.uf.common.time.DataTime;
+import com.raytheon.uf.common.time.SimulatedTime;
 import com.raytheon.uf.viz.core.VizApp;
 import com.raytheon.uf.viz.core.alerts.AlertMessage;
 import com.raytheon.uf.viz.core.catalog.DirectDbQuery;
@@ -70,11 +72,15 @@ public abstract class Monitor implements IMonitor, IMonitorThresholdListener,
 	public void alertArrived(Collection<AlertMessage> alertMessages) {
 		// knock down messages we don't want
 		for (AlertMessage msg : alertMessages) {
+			DataTime dataTime = (DataTime) msg.decodedAlert.get("dataTime");
+			if (dataTime.getRefTime().before(
+					SimulatedTime.getSystemTime().getTime())) {
 				if (filterProductMessage(msg)) {
 					processProductMessage(msg);
 				}
 			}
 		}
+	}
 
     @Override
     public void notificationArrived(NotificationMessage[] notifyMessages) {

cave/com.raytheon.uf.viz.monitor/src/com/raytheon/uf/viz/monitor/trendplot/TrendPlotCanvas.java

@@ -46,6 +46,7 @@ import com.raytheon.uf.common.monitor.data.CommonConfig.AppName;
 import com.raytheon.uf.common.monitor.data.ObConst;
 import com.raytheon.uf.viz.monitor.data.ObMultiHrsReports;
 import com.raytheon.uf.viz.monitor.data.ObTrendDataSet;
+import com.raytheon.uf.viz.monitor.util.ObUtil;
 
 /**
  * The Trend Canvas
@@ -75,7 +76,7 @@ public class TrendPlotCanvas {
 
     private Canvas canvas;
 
-    Calendar curdate;
+    private Calendar curdate;
 
     private SortedMap<Date, Float> trendData;
 
@@ -200,7 +201,8 @@ public class TrendPlotCanvas {
             if (!Float.isNaN(thresholds[1])) {
                 yellowThreshold = thresholds[1];
             }
-            curdate = Calendar.getInstance();
+            curdate = ObUtil.getTimeNow(); // Calendar.getInstance();
+            
             if (trendDataSet.getYAxisMinMaxIncrement() != null) {
                 float[] minMaxIncr = trendDataSet.getYAxisMinMaxIncrement();
                 if (!Float.isNaN(minMaxIncr[0])) {

cave/com.raytheon.uf.viz.monitor/src/com/raytheon/uf/viz/monitor/ui/dialogs/ZoneTableDlg.java

@@ -68,6 +68,7 @@ import com.raytheon.uf.viz.monitor.listeners.IMonitorListener;
 import com.raytheon.uf.viz.monitor.trendplot.TrendPlotDlg;
 import com.raytheon.uf.viz.monitor.util.MonitorConfigConstants;
 import com.raytheon.viz.ui.EditorUtil;
+import com.raytheon.uf.viz.monitor.util.ObUtil;
 
 /**
  * Abstrct Zone table dialog that is the foundation for all Zone dialogs.
@@ -760,7 +761,7 @@ public abstract class ZoneTableDlg extends Dialog implements IMonitorListener,
                 HodographDlg hodographDlg = new HodographDlg(shell,
                         varName.name(), station, "METAR");
 
-                hodographDlg.setCurrentTime(Calendar.getInstance());
+                hodographDlg.setCurrentTime(ObUtil.getTimeNow()); //Calendar.getInstance());
                 if (thresholds[0] == Float.NaN || thresholds[1] == Float.NaN
                         || thresholds[2] == Float.NaN
                         || thresholds[3] == Float.NaN) {

cave/com.raytheon.uf.viz.monitor/src/com/raytheon/uf/viz/monitor/util/ObUtil.java

@@ -83,6 +83,19 @@ public final class ObUtil {
         return dropTime.getTime();
     }
     
+    /**
+     * Method that gets the current time.
+     * 
+     * @return current time.
+     */
+    public static Calendar getTimeNow() {
+        Date now = SimulatedTime.getSystemTime().getTime();
+        Calendar nowTime = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
+        nowTime.setTime(now);
+
+        return nowTime;
+    }
+
     /**
      * Checks for the existence of a localization-created flag file which
      * indicates that SAFESEAS should rank swell periods high to low. This

cave/com.raytheon.uf.viz.thinclient.cave/src/com/raytheon/uf/viz/thinclient/cave/cache/CachingDataStore.java

@@ -384,13 +384,26 @@ public class CachingDataStore implements IDataStore {
      * (non-Javadoc)
      * 
      * @see
-     * com.raytheon.uf.common.datastorage.IDataStore#repack(java.lang.String,
+     * com.raytheon.uf.common.datastorage.IDataStore#repack(com.raytheon.uf.
-     * com.raytheon.uf.common.datastorage.StorageProperties.Compression)
+     * common.datastorage.StorageProperties.Compression)
      */
     @Override
-    public void repack(String dirName, Compression compression)
+    public void repack(Compression compression) throws StorageException {
-            throws StorageException {
+        delegate.repack(compression);
-        delegate.repack(dirName, compression);
     }
 
+    /*
+     * (non-Javadoc)
+     * 
+     * @see com.raytheon.uf.common.datastorage.IDataStore#copy(java.lang.String,
+     * com.raytheon.uf.common.datastorage.StorageProperties.Compression,
+     * java.lang.String, int, int)
+     */
+    @Override
+    public void copy(String outputDir, Compression compression,
+            String timestampCheck, int minMillisSinceLastChange,
+            int maxMillisSinceLastChange) throws StorageException {
+        delegate.copy(outputDir, compression, timestampCheck,
+                minMillisSinceLastChange, maxMillisSinceLastChange);
+    }
 }

cave/com.raytheon.uf.viz.thinclient.cave/src/com/raytheon/uf/viz/thinclient/cave/refresh/ThinClientURICatalog.java

@@ -35,6 +35,7 @@ import com.raytheon.uf.common.dataquery.requests.TimeQueryRequest;
 import com.raytheon.uf.common.status.IUFStatusHandler;
 import com.raytheon.uf.common.status.UFStatus;
 import com.raytheon.uf.common.time.DataTime;
+import com.raytheon.uf.common.time.SimulatedTime;
 import com.raytheon.uf.viz.core.datastructure.DataCubeContainer;
 import com.raytheon.uf.viz.core.exception.VizException;
 import com.raytheon.uf.viz.core.rsc.URICatalog;
@@ -122,6 +123,7 @@ public class ThinClientURICatalog extends URICatalog implements
             TimeQueryRequest request = new TimeQueryRequest();
             request.setPluginName(pluginName);
             request.setMaxQuery(true);
+            request.setSimDate(SimulatedTime.getSystemTime().getTime());
             request.setQueryTerms(map);
             List<TimeQueryRequest> requestList = requests.get(pluginName);
             if (requestList == null) {

cave/com.raytheon.uf.viz.ui.menus/src/com/raytheon/uf/viz/ui/menus/DiscoverMenuContributions.java

@@ -54,7 +54,6 @@ import com.raytheon.uf.common.status.IUFStatusHandler;
 import com.raytheon.uf.common.status.UFStatus;
 import com.raytheon.uf.common.status.UFStatus.Priority;
 import com.raytheon.uf.viz.core.exception.VizException;
-import com.raytheon.uf.viz.core.localization.LocalizationConstants;
 import com.raytheon.uf.viz.core.localization.LocalizationManager;
 import com.raytheon.uf.viz.ui.menus.xml.IncludeMenuItem;
 
@@ -76,6 +75,9 @@ import com.raytheon.uf.viz.ui.menus.xml.IncludeMenuItem;
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * Mar 12, 2009            chammack     Initial creation
+ * Apr 27, 2012   #562     dgilling     Ensure call to MenuCreationJob
+ *                                      uses proper method to retrieve
+ *                                      localized site.
  * 
  * </pre>
  * 
@@ -105,13 +107,13 @@ public class DiscoverMenuContributions {
 
     public static synchronized void discoverContributions(
             final String[] menuArray, boolean scheduleJob) {
-        if (ran)
+        if (ran) {
             return;
+        }
 
         if (scheduleJob) {
             MenuCreationJob job = new MenuCreationJob(LocalizationManager
-                    .getInstance().getLocalizationStore()
+                    .getInstance().getSite());
-                    .getString(LocalizationConstants.P_LOCALIZATION_SITE_NAME));
 
             job.schedule();
             try {

cave/com.raytheon.viz.alerts/src/com/raytheon/viz/alerts/jobs/AutoUpdater.java

@@ -33,6 +33,8 @@ import com.raytheon.uf.common.dataplugin.PluginDataObject;
 import com.raytheon.uf.common.status.IUFStatusHandler;
 import com.raytheon.uf.common.status.UFStatus;
 import com.raytheon.uf.common.status.UFStatus.Priority;
+import com.raytheon.uf.common.time.DataTime;
+import com.raytheon.uf.common.time.SimulatedTime;
 import com.raytheon.uf.viz.core.IDisplayPane;
 import com.raytheon.uf.viz.core.IDisplayPaneContainer;
 import com.raytheon.uf.viz.core.RecordFactory;
@@ -113,6 +115,9 @@ public class AutoUpdater implements IAlertObserver {
 
 		for (AlertMessage message : alertMessages) {
 			Map<String, Object> attribs = message.decodedAlert;
+			DataTime dataTime = (DataTime) attribs.get("dataTime");
+			if (dataTime.getRefTime().before(
+					SimulatedTime.getSystemTime().getTime())) {
 				try {
 					// System.out.println("extract took: " + (tZ1 - tZ0));
 					java.util.List<AbstractVizResource<?, ?>> rscList = DataUpdateTree
@@ -134,8 +139,8 @@ public class AutoUpdater implements IAlertObserver {
 							if ((parserToUse = reqResourceData.getAlertParser()) == null) {
 								parserToUse = defaultParser;
 							}
-                        Object objectToSend = parserToUse.parseAlertMessage(
+							Object objectToSend = parserToUse
-                                message, reqResourceData);
+									.parseAlertMessage(message, reqResourceData);
 
 							if (objectToSend != null) {
 								if (md.getTimeMatcher() != null) {
@@ -143,7 +148,8 @@ public class AutoUpdater implements IAlertObserver {
 								}
 								displayList.add(md);
 
-                            List<Object> list = pdoSendMap.get(reqResourceData);
+								List<Object> list = pdoSendMap
+										.get(reqResourceData);
 								if (list == null) {
 									list = new ArrayList<Object>();
 									pdoSendMap.put(reqResourceData, list);
@@ -152,10 +158,12 @@ public class AutoUpdater implements IAlertObserver {
 
 								if (list.size() > 100) {
 									// update with objects
-                                Class<?> componentType = list.get(0).getClass();
+									Class<?> componentType = list.get(0)
+											.getClass();
 									reqResourceData.update(list
-                                        .toArray((Object[]) Array.newInstance(
+											.toArray((Object[]) Array
-                                                componentType, list.size())));
+													.newInstance(componentType,
+															list.size())));
 									list.clear();
 								}
 							}
@@ -171,7 +179,6 @@ public class AutoUpdater implements IAlertObserver {
 					}
 					errors++;
 				}
-
 			} 
 
 			for (AbstractRequestableResourceData arrd : pdoSendMap.keySet()) {
@@ -188,8 +195,8 @@ public class AutoUpdater implements IAlertObserver {
 			// Now do redo time matching, and refresh displays
 			for (IDescriptor disp : displayList) {
 				if (disp != null && disp.getRenderableDisplay() != null) {
-                IDisplayPaneContainer container = disp.getRenderableDisplay()
+					IDisplayPaneContainer container = disp
-                        .getContainer();
+							.getRenderableDisplay().getContainer();
 					for (IDisplayPane pane : container.getDisplayPanes()) {
 						IDescriptor desc = pane.getDescriptor();
 
@@ -202,3 +209,4 @@ public class AutoUpdater implements IAlertObserver {
 			}
 		}
 	} 
+}

cave/com.raytheon.viz.alerts/src/com/raytheon/viz/alerts/jobs/MenuUpdater.java

@@ -26,6 +26,7 @@ import java.util.Map;
 
 import com.raytheon.edex.util.Util;
 import com.raytheon.uf.common.time.DataTime;
+import com.raytheon.uf.common.time.SimulatedTime;
 import com.raytheon.uf.viz.core.alerts.AlertMessage;
 import com.raytheon.uf.viz.core.rsc.URICatalog;
 import com.raytheon.uf.viz.core.rsc.URICatalog.IURIRefreshCallback;
@@ -66,6 +67,9 @@ public class MenuUpdater implements IAlertObserver {
                     message.decodedAlert);
             DataTime dataTimeStr = (DataTime) attribs.get("dataTime");
             
+			if (dataTimeStr.getRefTime().before(
+					SimulatedTime.getSystemTime().getTime())) {
+
 				if (dataTimeStr != null) {
 					// Update the date catalog
 					List<List<IURIRefreshCallback>> dataTimeList = URICatalog
@@ -86,5 +90,6 @@ public class MenuUpdater implements IAlertObserver {
 				}
 			} 
         }
+    }
 
 }

cave/com.raytheon.viz.aviation/src/com/raytheon/viz/aviation/editor/RestoreFileSelectDlg.java

@@ -137,7 +137,7 @@ public class RestoreFileSelectDlg extends Dialog {
             StdTextProduct tmp = ((StdTextProduct) (obj));
             Calendar createTime = Calendar.getInstance(TimeZone
                     .getTimeZone("GMT"));
-            createTime.setTimeInMillis(tmp.getCreatetime());
+            createTime.setTimeInMillis(tmp.getRefTime());
             String label = tmp.getCccid() + tmp.getNnnid() + tmp.getXxxid()
                     + " - " + createTime.getTime().toString();
             fileList.add(label);

cave/com.raytheon.viz.aviation/src/com/raytheon/viz/aviation/monitor/EtaBufMonitorObserver.java

@@ -22,6 +22,8 @@ package com.raytheon.viz.aviation.monitor;
 import java.util.ArrayList;
 import java.util.Collection;
 
+import com.raytheon.uf.common.time.DataTime;
+import com.raytheon.uf.common.time.SimulatedTime;
 import com.raytheon.uf.viz.core.alerts.AlertMessage;
 import com.raytheon.viz.alerts.IAlertObserver;
 import com.raytheon.viz.aviation.guidance.EtaViewer;
@@ -65,6 +67,10 @@ public class EtaBufMonitorObserver extends MonitorObserver implements
         ArrayList<String> siteIDs = new ArrayList<String>();
 
 		for (AlertMessage alert : alertMessages) {
+			DataTime dataTime = (DataTime) alert.decodedAlert.get("dataTime");
+			if (dataTime.getRefTime().before(
+					SimulatedTime.getSystemTime().getTime())) {
+
 				String siteID = (String) alert.decodedAlert
 						.get("location.stationId");
 				if (!siteIDs.contains(siteID)) {
@@ -76,6 +82,7 @@ public class EtaBufMonitorObserver extends MonitorObserver implements
 					}
 				}
 			}
+		}
 
         if (siteIDs.size() > 0) {
             for (ViewerTab tab : dialog.getViewerTabList()) {

cave/com.raytheon.viz.aviation/src/com/raytheon/viz/aviation/monitor/EtaMonitorObserver.java

@@ -22,6 +22,8 @@ package com.raytheon.viz.aviation.monitor;
 import java.util.ArrayList;
 import java.util.Collection;
 
+import com.raytheon.uf.common.time.DataTime;
+import com.raytheon.uf.common.time.SimulatedTime;
 import com.raytheon.uf.viz.core.alerts.AlertMessage;
 import com.raytheon.viz.alerts.IAlertObserver;
 import com.raytheon.viz.aviation.guidance.MosViewer;
@@ -65,6 +67,9 @@ public class EtaMonitorObserver extends MonitorObserver implements
         ArrayList<String> siteIDs = new ArrayList<String>();
 
 		for (AlertMessage alert : alertMessages) {
+			DataTime dataTime = (DataTime) alert.decodedAlert.get("dataTime");
+			if (dataTime.getRefTime().before(
+					SimulatedTime.getSystemTime().getTime())) {
 				String siteID = (String) alert.decodedAlert
 						.get("location.stationId");
 				if (!siteIDs.contains(siteID)) {
@@ -76,6 +81,7 @@ public class EtaMonitorObserver extends MonitorObserver implements
 					}
 				}
 			}
+		}
 
         if (siteIDs.size() > 0) {
             for (ViewerTab tab : dialog.getViewerTabList()) {

cave/com.raytheon.viz.aviation/src/com/raytheon/viz/aviation/monitor/GfsLampMonitorObserver.java

@@ -22,6 +22,8 @@ package com.raytheon.viz.aviation.monitor;
 import java.util.ArrayList;
 import java.util.Collection;
 
+import com.raytheon.uf.common.time.DataTime;
+import com.raytheon.uf.common.time.SimulatedTime;
 import com.raytheon.uf.viz.core.alerts.AlertMessage;
 import com.raytheon.viz.alerts.IAlertObserver;
 import com.raytheon.viz.aviation.guidance.MosViewer;
@@ -66,6 +68,9 @@ public class GfsLampMonitorObserver extends MonitorObserver implements
         ArrayList<String> siteIDs = new ArrayList<String>();
 
 		for (AlertMessage alert : alertMessages) {
+			DataTime dataTime = (DataTime) alert.decodedAlert.get("dataTime");
+			if (dataTime.getRefTime().before(
+					SimulatedTime.getSystemTime().getTime())) {
 				String siteID = (String) alert.decodedAlert
 						.get("location.stationId");
 				if (!siteIDs.contains(siteID)) {
@@ -77,6 +82,7 @@ public class GfsLampMonitorObserver extends MonitorObserver implements
 					}
 				}
 			}
+		}
 
         if (siteIDs.size() > 0) {
             for (ViewerTab tab : dialog.getViewerTabList()) {

cave/com.raytheon.viz.aviation/src/com/raytheon/viz/aviation/monitor/GfsMonitorObserver.java

@@ -22,6 +22,8 @@ package com.raytheon.viz.aviation.monitor;
 import java.util.ArrayList;
 import java.util.Collection;
 
+import com.raytheon.uf.common.time.DataTime;
+import com.raytheon.uf.common.time.SimulatedTime;
 import com.raytheon.uf.viz.core.alerts.AlertMessage;
 import com.raytheon.viz.alerts.IAlertObserver;
 import com.raytheon.viz.aviation.guidance.MosViewer;
@@ -65,6 +67,9 @@ public class GfsMonitorObserver extends MonitorObserver implements
         ArrayList<String> siteIDs = new ArrayList<String>();
 
 		for (AlertMessage alert : alertMessages) {
+			DataTime dataTime = (DataTime) alert.decodedAlert.get("dataTime");
+			if (dataTime.getRefTime().before(
+					SimulatedTime.getSystemTime().getTime())) {
 				String siteID = (String) alert.decodedAlert
 						.get("location.stationId");
 				if (!siteIDs.contains(siteID)) {
@@ -76,6 +81,7 @@ public class GfsMonitorObserver extends MonitorObserver implements
 					}
 				}
 			}
+		}
 
         if (siteIDs.size() > 0) {
             for (ViewerTab tab : dialog.getViewerTabList()) {

cave/com.raytheon.viz.grid/src/com/raytheon/viz/grid/data/GribRequestableDataFactory.java

@@ -24,6 +24,8 @@ import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
 import com.raytheon.uf.common.dataplugin.grib.GribRecord;
+import com.raytheon.uf.common.time.DataTime;
+import com.raytheon.uf.common.time.SimulatedTime;
 import com.raytheon.uf.viz.core.alerts.AlertMessage;
 import com.raytheon.viz.alerts.IAlertObserver;
 import com.raytheon.viz.alerts.observers.ProductAlertObserver;
@@ -85,7 +87,11 @@ public class GribRequestableDataFactory implements IAlertObserver {
     @Override
     public void alertArrived(Collection<AlertMessage> alertMessages) {
         for (AlertMessage mess : alertMessages) {
+        	DataTime dataTime = (DataTime) mess.decodedAlert.get("dataTime");
+			if (dataTime.getRefTime().before(
+					SimulatedTime.getSystemTime().getTime())) {
 				requestableDataMap.remove(mess.dataURI);
 			}
 		}
 	}
+}

cave/com.raytheon.viz.grid/src/com/raytheon/viz/grid/inv/GridUpdater.java

@@ -39,6 +39,7 @@ import com.raytheon.uf.common.status.IUFStatusHandler;
 import com.raytheon.uf.common.status.UFStatus;
 import com.raytheon.uf.common.status.UFStatus.Priority;
 import com.raytheon.uf.common.time.DataTime;
+import com.raytheon.uf.common.time.SimulatedTime;
 import com.raytheon.uf.viz.core.alerts.AlertMessage;
 import com.raytheon.uf.viz.core.exception.VizException;
 import com.raytheon.uf.viz.derivparam.library.DerivParamMethod;
@@ -209,8 +210,13 @@ public class GridUpdater implements IAlertObserver {
 	public synchronized void alertArrived(Collection<AlertMessage> alertMessages) {
 		Set<String> datauris = new HashSet<String>();
 		for (AlertMessage alert : alertMessages) {
+			DataTime dataTime = (DataTime) alert.decodedAlert.get("dataTime");
+			if (dataTime.getRefTime().before(
+					SimulatedTime.getSystemTime().getTime())) {
+
 				if (myUpdates.remove(alert.dataURI)) {
-                // This updater triggered this alert, if it processes it now we
+					// This updater triggered this alert, if it processes it now
+					// we
 					// could do this forever
 					continue;
 				}
@@ -224,8 +230,8 @@ public class GridUpdater implements IAlertObserver {
 					lNode = inventory.getNode(updateKey.modelName,
 							updateKey.parameter, level);
 				} catch (CommunicationException e) {
-                statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(),
+					statusHandler.handle(Priority.PROBLEM,
-                        e);
+							e.getLocalizedMessage(), e);
 				}
 
 				if (lNode == null) {
@@ -234,14 +240,16 @@ public class GridUpdater implements IAlertObserver {
 					// System.out.println("LevelId=" + level.getId());
 				} else if (!(lNode instanceof GribRequestableLevelNode)) {
 					if (lNode instanceof OrLevelNode) {
-                    DerivParamMethod method = ((OrLevelNode) lNode).getMethod();
+						DerivParamMethod method = ((OrLevelNode) lNode)
+								.getMethod();
 						// Null means it is an alias model and supplement means
 						// there exists a true GribNode buried under the or
 						// node
 						if (method == null
 								|| !method.getName().equals("Supplement")) {
 							inventory.reinitTree();
-                        // System.out.println(((AbstractDerivedLevelNode) lNode)
+							// System.out.println(((AbstractDerivedLevelNode)
+							// lNode)
 							// .getModelName());
 						}
 					} else {
@@ -292,7 +300,9 @@ public class GridUpdater implements IAlertObserver {
 										"Unable to generate updates for derived product",
 										e);
 					}
-
+				}
+			} else {
+				System.out.println("Grid after SIMULATED TIME: "+dataTime);
 			}
 		}
 		myUpdates.addAll(datauris);

cave/com.raytheon.viz.grid/src/com/raytheon/viz/grid/inv/RadarUpdater.java

@@ -20,6 +20,7 @@ import com.raytheon.uf.common.status.IUFStatusHandler;
 import com.raytheon.uf.common.status.UFStatus;
 import com.raytheon.uf.common.status.UFStatus.Priority;
 import com.raytheon.uf.common.time.DataTime;
+import com.raytheon.uf.common.time.SimulatedTime;
 import com.raytheon.uf.viz.core.alerts.AlertMessage;
 import com.raytheon.viz.alerts.IAlertObserver;
 import com.raytheon.viz.alerts.observers.ProductAlertObserver;
@@ -129,8 +130,12 @@ public class RadarUpdater implements IAlertObserver {
         }
         Set<String> datauris = new HashSet<String>();
 		for (AlertMessage alertMessage : alertMessages) {
-            String icao = alertMessage.decodedAlert
+			DataTime dataTime = (DataTime) alertMessage.decodedAlert
-                    .get(RadarAdapter.ICAO_QUERY).toString();
+					.get("dataTime");
+			if (dataTime.getRefTime().before(
+					SimulatedTime.getSystemTime().getTime())) {
+				String icao = alertMessage.decodedAlert.get(
+						RadarAdapter.ICAO_QUERY).toString();
 				if (icao == null
 						|| !icao.equalsIgnoreCase(configuredRadar.getRdaId())) {
 					continue;
@@ -160,8 +165,9 @@ public class RadarUpdater implements IAlertObserver {
 				cache.remove(new CacheKey(productCode, elevationAngle));
 				Level level = null;
 				try {
-                level = LevelFactory.getInstance().getLevel(
+					level = LevelFactory.getInstance()
-                        RadarAdapter.CUBE_MASTER_LEVEL_NAME, elevationAngle);
+							.getLevel(RadarAdapter.CUBE_MASTER_LEVEL_NAME,
+									elevationAngle);
 				} catch (CommunicationException e1) {
 					statusHandler.handle(Priority.PROBLEM,
 							e1.getLocalizedMessage(), e1);
@@ -179,8 +185,11 @@ public class RadarUpdater implements IAlertObserver {
 					fakeRec.constructDataURI();
 					datauris.add(fakeRec.getDataURI());
 				} catch (PluginException e) {
-                statusHandler.handle(Priority.PROBLEM,
+					statusHandler
-                        "Unable to generate updates for derived product", e);
+							.handle(Priority.PROBLEM,
+									"Unable to generate updates for derived product",
+									e);
+				}
 			} 
         }
         ProductAlertObserver.processDerivedAlerts(datauris);

cave/com.raytheon.viz.pointdata/src/com/raytheon/viz/pointdata/util/PointDataCubeAdapter.java

@@ -41,6 +41,7 @@ import com.raytheon.uf.common.status.IUFStatusHandler;
 import com.raytheon.uf.common.status.UFStatus;
 import com.raytheon.uf.common.status.UFStatus.Priority;
 import com.raytheon.uf.common.time.DataTime;
+import com.raytheon.uf.common.time.SimulatedTime;
 import com.raytheon.uf.viz.core.RecordFactory;
 import com.raytheon.uf.viz.core.catalog.LayerProperty;
 import com.raytheon.uf.viz.core.catalog.ScriptCreator;
@@ -369,6 +370,10 @@ public class PointDataCubeAdapter implements IDataCubeAdapter {
             throws VizException {
         TimeQueryRequestSet set = new TimeQueryRequestSet();
         set.setRequests(requests.toArray(new TimeQueryRequest[0]));
+        // boundary for time queries
+        for (TimeQueryRequest req: set.getRequests()) {
+        	req.setSimDate(SimulatedTime.getSystemTime().getTime());
+        }
         @SuppressWarnings("unchecked")
         List<List<DataTime>> result = (List<List<DataTime>>) ThriftClient
                 .sendRequest(set);

cave/com.raytheon.viz.pointdata/src/com/raytheon/viz/pointdata/util/PointDataInventory.java

@@ -31,6 +31,8 @@ import com.raytheon.uf.common.pointdata.GetPointDataTreeRequest;
 import com.raytheon.uf.common.status.IUFStatusHandler;
 import com.raytheon.uf.common.status.UFStatus;
 import com.raytheon.uf.common.status.UFStatus.Priority;
+import com.raytheon.uf.common.time.DataTime;
+import com.raytheon.uf.common.time.SimulatedTime;
 import com.raytheon.uf.viz.core.alerts.AlertMessage;
 import com.raytheon.uf.viz.core.exception.VizException;
 import com.raytheon.uf.viz.core.requests.ThriftClient;
@@ -103,6 +105,9 @@ public class PointDataInventory extends AbstractPointDataInventory implements
     @Override
 	public void alertArrived(Collection<AlertMessage> alertMessages) {
 		for (AlertMessage message : alertMessages) {
+			DataTime dataTime = (DataTime) message.decodedAlert.get("dataTime");
+			if (dataTime.getRefTime().before(
+					SimulatedTime.getSystemTime().getTime())) {
 				String pluginName = message.decodedAlert.get(PLUGIN_NAME)
 						.toString();
 				String source = pluginName;
@@ -111,7 +116,8 @@ public class PointDataInventory extends AbstractPointDataInventory implements
 					source += message.decodedAlert.get(typeKey).toString();
 				}
 
-            if (getAllSources() != null && !getAllSources().contains(source)) {
+				if (getAllSources() != null
+						&& !getAllSources().contains(source)) {
 					try {
 						initTree(derParLibrary);
 					} catch (VizException e) {
@@ -121,5 +127,6 @@ public class PointDataInventory extends AbstractPointDataInventory implements
 				}
 			} 
 		}
+	}
 
 }

cave/com.raytheon.viz.radar/src/com/raytheon/viz/radar/util/RadarDataCubeAdapter.java

@@ -40,6 +40,7 @@ import com.raytheon.uf.common.status.UFStatus;
 import com.raytheon.uf.common.status.UFStatus.Priority;
 import com.raytheon.uf.common.time.BinOffset;
 import com.raytheon.uf.common.time.DataTime;
+import com.raytheon.uf.common.time.SimulatedTime;
 import com.raytheon.uf.viz.core.exception.VizException;
 import com.raytheon.uf.viz.core.requests.ThriftClient;
 import com.raytheon.uf.viz.derivparam.library.DerivedParameterGenerator;
@@ -132,10 +133,12 @@ public class RadarDataCubeAdapter extends PointDataCubeAdapter {
 				time = (DataTime) map.get(dataTimefield);
 				time.setLevelValue((Double) map.get(LEVEL_FIELD));
 			}
-
+			// Best res requests need this because they span a time period
+			if (time.getRefTime().before(SimulatedTime.getSystemTime().getTime())) {
 				results.add(time);
 				++i;
 			}
+		}
 
         if (binOffset != null) {
             Set<DataTime> scaledDates = new TreeSet<DataTime>();
@@ -170,6 +173,7 @@ public class RadarDataCubeAdapter extends PointDataCubeAdapter {
         List<DbQueryRequest> dbRequests = new ArrayList<DbQueryRequest>(
                 requests.size());
         for (TimeQueryRequest request : requests) {
+        	request.setSimDate(SimulatedTime.getSystemTime().getTime());
             dbRequests.add(getTimeQueryRequest(request.getQueryTerms(),
                     request.isMaxQuery()));
         }
@@ -184,6 +188,9 @@ public class RadarDataCubeAdapter extends PointDataCubeAdapter {
             TimeQueryRequest request = requests.get(i);
             Collection<DataTime> times = processTimeQueryResponse(response,
                     request.isMaxQuery(), request.getBinOffset());
+            
+            
+            
             result.add(new ArrayList<DataTime>(times));
         }
         return result;

cave/com.raytheon.viz.satellite/src/com/raytheon/viz/satellite/SatelliteDataCubeAdapter.java

@@ -58,6 +58,7 @@ import com.raytheon.uf.common.geospatial.MapUtil;
 import com.raytheon.uf.common.pointdata.PointDataContainer;
 import com.raytheon.uf.common.time.BinOffset;
 import com.raytheon.uf.common.time.DataTime;
+import com.raytheon.uf.common.time.SimulatedTime;
 import com.raytheon.uf.viz.core.HDF5Util;
 import com.raytheon.uf.viz.core.catalog.CatalogQuery;
 import com.raytheon.uf.viz.core.catalog.LayerProperty;
@@ -464,6 +465,7 @@ public class SatelliteDataCubeAdapter implements IDataCubeAdapter {
         List<TimeQueryRequest> baseRequests = new ArrayList<TimeQueryRequest>(
                 requests.size());
         for (TimeQueryRequest request : requests) {
+        	request.setSimDate(SimulatedTime.getSystemTime().getTime());
             if (!request.getQueryTerms().containsKey("DERIVED")) {
                 baseRequests.add(request);
             } else {
@@ -541,6 +543,7 @@ public class SatelliteDataCubeAdapter implements IDataCubeAdapter {
                 DerivParamField field = (DerivParamField) ifield;
                 TimeQueryRequest newRequest = new TimeQueryRequest();
                 newRequest.setBinOffset(binOffset);
+                newRequest.setSimDate(SimulatedTime.getSystemTime().getTime());
                 newRequest.setMaxQuery(false);
                 newRequest.setQueryTerms(modifyQuery(queryParams, field));
                 newRequest.setPluginName("satellite");

cave/com.raytheon.viz.text.feature/feature.xml

@@ -39,6 +39,13 @@
          version="0.0.0"
          unpack="false"/>
 
+   <plugin
+         id="com.raytheon.uf.edex.maintenance"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"
+         unpack="false"/>
+         
    <plugin
          id="com.raytheon.edex.plugin.text"
          download-size="0"

cave/com.raytheon.viz.texteditor/src/com/raytheon/viz/texteditor/TextDisplayModel.java

@@ -308,14 +308,14 @@ public final class TextDisplayModel {
      * Getter/Accessor of Create Time.
      */
     public long getCreateTime(String token) {
-        return getStdTextProduct(token, true).getCreatetime();
+        return getStdTextProduct(token, true).getRefTime();
     }
 
     /**
      * Setter/Mutator of Create Time.
      */
     public void setCreateTime(String token, long ct) {
-        getStdTextProduct(token, true).setCreatetime(ct);
+        getStdTextProduct(token, true).setRefTime(ct);
     }
 
     /**
@@ -472,7 +472,7 @@ public final class TextDisplayModel {
         tmpProduct.setXxxid(xxxid);
         tmpProduct.setHdrtime(hdrtime);
         tmpProduct.setBbbid(bbbid);
-        tmpProduct.setCreatetime(createtime);
+        tmpProduct.setRefTime(createtime);
         tmpProduct.setProduct(product);
         stdTxtProd.put(token, tmpProduct);
     }

cave/com.raytheon.viz.texteditor/src/com/raytheon/viz/texteditor/dialogs/RemoteSiteRequestDlg.java

@@ -448,7 +448,7 @@ public class RemoteSiteRequestDlg extends CaveSWTDialog {
         if (latest != null && latest.size() > 0) {
             Calendar c = new GregorianCalendar(TimeZone
                     .getTimeZone("GMT"));
-            c.setTimeInMillis(latest.get(0).getCreatetime());
+            c.setTimeInMillis(latest.get(0).getRefTime());
             req.setMostRecentTime(c.getTimeInMillis()); // default
             try {
                 WMOHeader wmo = new WMOHeader(latest.get(0).getProduct().getBytes());

cave/com.raytheon.viz.texteditor/src/com/raytheon/viz/texteditor/dialogs/TextEditorDialog.java

@@ -4636,7 +4636,7 @@ public class TextEditorDialog extends CaveSWTDialog implements VerifyListener,
                 updateTextEditor(body.toString());
             }
             storedProduct.setHdrtime(currentDate);
-            storedProduct.setCreatetime(System.currentTimeMillis());
+            storedProduct.setRefTime(System.currentTimeMillis());
             if (productText.contains(AFOSParser.DRAFT_PIL)) {
                 int start = productText.indexOf(AFOSParser.DRAFT_PIL);
                 productText = productText.replace(productText.substring(start,
@@ -4808,7 +4808,7 @@ public class TextEditorDialog extends CaveSWTDialog implements VerifyListener,
 
         request.setBbbid(product.getBbbid());
         request.setCccid(product.getCccid());
-        request.setCreatetime(product.getCreatetime());
+        request.setCreatetime(product.getRefTime());
         request.setDataCrc(product.getDataCrc());
         request.setHdrtime(product.getHdrtime());
         request.setNnnid(product.getNnnid());
@@ -4816,7 +4816,7 @@ public class TextEditorDialog extends CaveSWTDialog implements VerifyListener,
         request.setWmoid(product.getWmoid());
         request.setSite(product.getSite());
         request.setXxxid(product.getXxxid());
-        request.setCreatetime(product.getCreatetime());
+        request.setCreatetime(product.getRefTime());
         CAVEMode mode = CAVEMode.getMode();
         boolean result = (CAVEMode.OPERATIONAL.equals(mode)
                 || CAVEMode.TEST.equals(mode) ? true : false);

cave/com.raytheon.viz.texteditor/src/com/raytheon/viz/texteditor/util/AviationTextUtility.java

@@ -118,7 +118,7 @@ public class AviationTextUtility implements IAviationObserver {
         tmpProd.setXxxid("TAF");
         tmpProd.setHdrtime(currentDate);
         tmpProd.setBbbid("NOR");
-        tmpProd.setCreatetime(System.currentTimeMillis());
+        tmpProd.setRefTime(System.currentTimeMillis());
         tmpProd.setProduct(currentHeader + "\n" + tmpStr);
 
         RequestConstraint rcRow;

cave/com.raytheon.viz.texteditor/src/com/raytheon/viz/texteditor/util/RadarTextUtility.java

@@ -139,7 +139,7 @@ public class RadarTextUtility implements IRadarObserver {
             tmpProd.setXxxid(designator);
             tmpProd.setHdrtime(currentDate);
             tmpProd.setBbbid("NOR");
-            tmpProd.setCreatetime(System.currentTimeMillis());
+            tmpProd.setRefTime(System.currentTimeMillis());
             tmpProd.setProduct(currentHeader + "\n" + strBldr.toString());
 
             RequestConstraint rcRow;

deltaScripts/.project

@@ -5,7 +5,13 @@
 	<projects>
 	</projects>
 	<buildSpec>
+		<buildCommand>
+			<name>org.python.pydev.PyDevBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
 	</buildSpec>
 	<natures>
+		<nature>org.python.pydev.pythonNature</nature>
 	</natures>
 </projectDescription>

deltaScripts/.pydevproject

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?eclipse-pydev version="1.0"?>
+
+<pydev_project>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.7</pydev_property>
+</pydev_project>

deltaScripts/future/alterTextProductsTables.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+# This script should be run on dx1 as part of next delivery to update the definitions for
+# the stdtextproducts and practicestdtextproducts tables.
+
+psql -U awips -d fxa -c "ALTER TABLE stdtextproducts RENAME COLUMN createtime to reftime;"
+psql -U awips -d fxa -c "ALTER TABLE practicestdtextproducts RENAME COLUMN createtime to reftime;"
+
+psql -U awips -d fxa -c "ALTER TABLE stdtextproducts ADD COLUMN inserttime timestamp without time zone;"
+psql -U awips -d fxa -c "ALTER TABLE practicestdtextproducts ADD COLUMN inserttime timestamp without time zone;"
+
+psql -U awips -d fxa -c "CREATE INDEX stdtextproductsinserttimeindex ON stdtextproducts USING btree (inserttime);"
+psql -U awips -d fxa -c "CREATE INDEX practicestdtextproductsinserttimeindex ON practicestdtextproducts USING btree (inserttime);"
+
+psql -U awips -d fxa -c "UPDATE stdtextproducts SET inserttime = CURRENT_TIMESTAMP AT TIME ZONE 'GMT' WHERE inserttime IS NULL;"
+psql -U awips -d fxa -c "UPDATE practicestdtextproducts SET inserttime = CURRENT_TIMESTAMP AT TIME ZONE 'GMT' WHERE inserttime IS NULL;"
+
+psql -U awips -d fxa -c "ALTER TABLE stdtextproducts ALTER COLUMN inserttime SET NOT NULL;"
+psql -U awips -d fxa -c "ALTER TABLE practicestdtextproducts ALTER COLUMN inserttime SET NOT NULL;"

deltaScripts/future/combineH5Files.py

@@ -0,0 +1,82 @@
+
+import os, subprocess, re, stat, sys, time
+import h5py
+
+matchRe = re.compile('.*?(-{1,2}\\d{6,}).h5')
+
+def findEachGroup(group, datasetList):
+    if type(group) is h5py.highlevel.Group:
+        for g in group.keys():
+            findEachGroup(group[g], datasetList)
+    elif type(group) is h5py.highlevel.Dataset:
+        datasetList.append(group.name)
+        
+        
+def processFile(filename, match):    
+    startIndex = filename.find(match.groups()[0])
+    endIndex = filename.find('.h5')
+    reducedFilename = filename[0:startIndex] + filename[endIndex:]
+    if not os.path.exists(reducedFilename):
+        # this is the first one, just rename it
+        try:
+            os.rename(filename, reducedFilename)
+        except OSError, e:
+            print e            
+    else:
+        # open the file, find the datasets            
+        datasetList = []
+        hfile = None        
+        try:
+            hfile = h5py.File(filename, 'r')
+            findEachGroup(hfile['/'], datasetList)
+        finally:
+            if hfile:                
+                hfile.close()        
+    
+        fileSuccess = True
+        # for each dataset in the file, run h5copy it into the output file
+        for dataset in datasetList:
+            if not copy(filename, dataset, reducedFilename):        
+                fileSuccess = False
+                
+        # remove original file
+        if True: #if fileSuccess:
+            os.remove(filename)
+    
+def fileWalk(pth):
+    if os.path.isdir(pth):
+        innerFiles = os.listdir(pth)
+        for f in innerFiles:
+            fileWalk(pth + '/' + f)
+    else:
+        match = matchRe.match(pth)
+        if match:
+            processFile(pth, match)
+        
+def copy(filename, dataset, reducedFilename):        
+    # note that this copies links as if they were real datasets, increasing the size of the output file
+    cmd = ['h5copy', '-p', '-i', filename, '-o', reducedFilename, '-s', dataset, '-d', dataset]
+    ret = subprocess.call(cmd)
+    success = (ret == 0)
+        
+    if success:
+        os.chmod(reducedFilename, stat.S_IWUSR | stat.S_IWGRP | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)            
+        #print "Successfully copied filename:", filename, "dataset:", dataset
+        return True
+    else:
+        print "Failed to copy filename:", filename, "dataset:", dataset
+        return False 
+    
+def main():
+    if len(sys.argv) < 2:
+        print "Please provide full path to input directory"
+    else:
+        inputDir = sys.argv[1]
+        t0 = time.time()
+        fileWalk(inputDir)
+        t1 = time.time()
+        print "Total copy time for directory", inputDir, (t1-t0), "seconds"
+    
+
+if __name__ == '__main__':
+    main()

deltaScripts/future/removeHdfFileId.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+# This script should be run on dx1 as part of next delivery to remove the hdfFileId columns
+
+tablesToUpdate=`psql -U awips -d metadata -t -c "select table_name from information_schema.columns where table_catalog='metadata' and table_schema='awips' and column_name = 'hdffileid';"`
+
+for table in $tablesToUpdate
+do
+   psql -U awips -d metadata -t -c "ALTER TABLE $table DROP COLUMN hdffileid"
+done
+

edexOsgi/build.edex/.gitignore

@@ -0,0 +1 @@
+/tmp

edexOsgi/build.edex/esb/bin/linux-x86-32/wrapper.conf

@@ -118,6 +118,9 @@ wrapper.java.additional.39=-DHighMem=%HIGH_MEM_FLAG%
 wrapper.java.additional.40=-Dmanagement.port=%MGMT_PORT%
 
 wrapper.java.additional.41=-Dqpid.dest_syntax=BURL
+wrapper.java.additional.42=-Dweb.port=8080
+wrapper.java.additional.43=-Dconfidential.port=8443
+wrapper.java.additional.44=-Dhttp.port=%HTTP_PORT%
 
 # Initial Java Heap Size (in MB)
 wrapper.java.initmemory=%INIT_MEM%

edexOsgi/build.edex/esb/bin/setup.env

@@ -24,7 +24,8 @@ export PYPIES_SERVER=http://localhost:9582
 
 # moved here from environment.xml
 # these values are returned to clients that contact the localization service
-export HTTP_SERVER=http://localhost:9581/services
+export HTTP_PORT=9581
+export HTTP_SERVER=http://localhost:${HTTP_PORT}/services
 export JMS_SERVER=tcp://localhost:5672
 export RADAR_SERVER=tcp://localhost:8813
 

edexOsgi/build.edex/esb/conf/log4j-ingest.xml

@@ -63,6 +63,17 @@
       </layout> 
    </appender> 
    
+   <!-- Archive log --> 
+   <appender name="ArchiveLog" class="org.apache.log4j.rolling.RollingFileAppender">
+      <rollingPolicy class="org.apache.log4j.rolling.TimeBasedRollingPolicy">
+         <param name="FileNamePattern" value="${edex.home}/logs/edex-${edex.run.mode}-archive-%d{yyyyMMdd}.log"/>
+      </rollingPolicy> 
+
+      <layout class="org.apache.log4j.PatternLayout"> 
+         <param name="ConversionPattern" value="%-5p %d [%t] %c{1}: %m%n"/> 
+      </layout> 
+   </appender> 
+
    <!-- RouteFailedLog log --> 
    <appender name="RouteFailedLog" class="org.apache.log4j.rolling.RollingFileAppender">
       <rollingPolicy class="org.apache.log4j.rolling.TimeBasedRollingPolicy">
@@ -219,6 +230,11 @@
    		<appender-ref ref="PurgeLog"/>
    </logger>
    
+   <logger name="com.raytheon.uf.edex.maintenance.archive" additivity="false">
+   		<level value="INFO"/>
+   		<appender-ref ref="ArchiveLog"/>
+   </logger>
+
    <logger name="RouteFailedLog" additivity="false">
       <level value="WARN"/>
       <appender-ref ref="RouteFailedLog"/>

edexOsgi/build.edex/esb/conf/res/base/environment.xml

@@ -48,7 +48,7 @@
 	<svcBackupDir>${env:edex.home}/../GFESuite/</svcBackupDir>
 	<mhsData>/data/fxa/mhs</mhsData>
 	<requestTimeLogFilter>1000</requestTimeLogFilter>
-	<requestSizeLogFilter>8</requestSizeLogFilter>
+	<responseSizeLogFilter>8</responseSizeLogFilter>
 </properties>
 
 

edexOsgi/build.edex/esb/conf/spring/project.properties

@@ -22,6 +22,8 @@ qc.cron=0+2,7,12,17,22,27,32,37,42,47,52,57+*+*+*+?
 acarssounding.cron=00+10,30,50+*+*+*+?
 gfe.cron=0+15+*+*+*+?
 repack.cron=0+20+*+*+*+?
+# runs database and hdf5 archive for archive server to pull data from 
+archive.cron=0+40+*+*+*+?
 
 ###purge configuration
 # Interval at which the purge job kicks off

edexOsgi/build.edex/esb/data/utility/edex_static/base/python/ObsRequest.py

@@ -62,7 +62,8 @@ class ObsRequest(PointDataQuery.PointDataQuery):
         self._pdq.setParameters(MetarPointDataTransform.ALL_PARAMS_LIST)        
         self.queryResults = self._pdq.execute()
         if self.queryResults is None:
-            return self.makeNullResponse()
+            response = ArrayList()
+            return response
         else:
             records = MetarPointDataTransform.toMetarRecords(self.queryResults)
             size = len(records)

edexOsgi/build.edex/tmp/includes/core.includes

@@ -1,87 +0,0 @@
-## Core includes file generated on 2012-01-20 17:08:45 GMT
-com.raytheon.edex.rpgenvdata/component-deploy.xml
-com.raytheon.uf.common.jms/component-deploy.xml
-com.raytheon.uf.edex.database/component-deploy.xml
-com.raytheon.uf.common.cache/component-deploy.xml
-com.raytheon.uf.edex.pointdata/component-deploy.xml
-com.raytheon.uf.edex.esb.camel.launcher/component-deploy.xml
-com.raytheon.uf.common.dissemination/component-deploy.xml
-com.raytheon.uf.edex.bufrtools/component-deploy.xml
-com.raytheon.edex.uengine/component-deploy.xml
-com.raytheon.uf.common.pointdata/component-deploy.xml
-com.raytheon.uf.common.serialization/component-deploy.xml
-com.raytheon.uf.common.datastorage.remote/component-deploy.xml
-com.raytheon.uf.common.awipstools/component-deploy.xml
-com.raytheon.edex.ingestsrv/component-deploy.xml
-com.raytheon.uf.common.alertviz/component-deploy.xml
-com.raytheon.uf.common.serialization.comm/component-deploy.xml
-com.raytheon.uf.tools.gfesuite.servicebackup/component-deploy.xml
-com.raytheon.uf.common.auth/component-deploy.xml
-com.raytheon.uf.common.menus/component-deploy.xml
-com.raytheon.uf.edex.distribution/component-deploy.xml
-com.raytheon.uf.common.hydro/component-deploy.xml
-com.raytheon.uf.edex.core/component-deploy.xml
-com.raytheon.edex.common/component-deploy.xml
-com.raytheon.uf.edex.management/component-deploy.xml
-com.raytheon.uf.common.datastorage.hdf5/component-deploy.xml
-com.raytheon.uf.edex.cpgsrv/component-deploy.xml
-com.raytheon.uf.edex.python.decoder/component-deploy.xml
-gov.noaa.nws.ncep.edex.uengine/component-deploy.xml
-com.raytheon.uf.common.status/component-deploy.xml
-com.raytheon.uf.common.sounding/component-deploy.xml
-com.raytheon.edex.autobldsrv/component-deploy.xml
-gov.noaa.nws.ncep.edex.ingest.util/component-deploy.xml
-com.raytheon.uf.common.climo/component-deploy.xml
-com.raytheon.uf.common.localization/component-deploy.xml
-gov.noaa.nws.ncep.common/component-deploy.xml
-com.raytheon.uf.common.message/component-deploy.xml
-com.raytheon.rcm.lib/component-deploy.xml
-com.raytheon.uf.common.datastorage/component-deploy.xml
-com.raytheon.uf.common.topo/component-deploy.xml
-com.raytheon.edex.notifysrv/component-deploy.xml
-gov.noaa.nws.ncep.edex.gempak.jna/component-deploy.xml
-com.raytheon.edex.utilitysrv/component-deploy.xml
-com.raytheon.uf.common.geospatial/component-deploy.xml
-com.raytheon.uf.common.python/component-deploy.xml
-gov.noaa.nws.ncep.edex.purgeutil/component-deploy.xml
-com.raytheon.uf.edex.metartohmdbsrv/component-deploy.xml
-com.raytheon.uf.common.time/component-deploy.xml
-com.raytheon.uf.common.pypies/component-deploy.xml
-com.raytheon.uf.common.monitor/component-deploy.xml
-com.raytheon.uf.common.comm/component-deploy.xml
-com.raytheon.edex.productsrv/component-deploy.xml
-com.raytheon.uf.common.mpe/component-deploy.xml
-com.raytheon.edex.adaptersrv/component-deploy.xml
-com.raytheon.uf.edex.textdbsrv/component-deploy.xml
-com.raytheon.uf.common.util/component-deploy.xml
-com.raytheon.uf.edex.dat.utils/component-deploy.xml
-com.raytheon.uf.common.ohd/component-deploy.xml
-com.raytheon.uf.common.site/component-deploy.xml
-gov.noaa.nws.ncep.edex.ingest.grib.util/component-deploy.xml
-gov.noaa.nws.ncep.edex.common/component-deploy.xml
-com.raytheon.uf.edex.log/component-deploy.xml
-com.raytheon.uf.edex.ohd/component-deploy.xml
-com.raytheon.uf.edex.awipstools/component-deploy.xml
-com.raytheon.uf.edex.esb.camel/component-deploy.xml
-com.raytheon.uf.common.dataquery/component-deploy.xml
-com.raytheon.uf.edex.decodertools/component-deploy.xml
-com.raytheon.uf.edex.topo/component-deploy.xml
-com.raytheon.uf.common.alertmonitor/component-deploy.xml
-com.raytheon.uf.common.activetable/component-deploy.xml
-com.raytheon.uf.edex.alertviz/component-deploy.xml
-com.raytheon.uf.common.management/component-deploy.xml
-com.raytheon.uf.common.derivparam/component-deploy.xml
-com.raytheon.uf.edex.dissemination/component-deploy.xml
-com.raytheon.edex.kml/component-deploy.xml
-com.raytheon.messaging.mhs/component-deploy.xml
-com.raytheon.uf.common.colormap/component-deploy.xml
-com.raytheon.uf.common.monitor.cpg/component-deploy.xml
-gov.noaa.nws.ncep.common.log/component-deploy.xml
-com.raytheon.edex.monitorsrv/component-deploy.xml
-com.raytheon.edex.textdb/component-deploy.xml
-com.raytheon.uf.edex.purgesrv/component-deploy.xml
-com.raytheon.uf.edex.auth/component-deploy.xml
-com.raytheon.uf.edex.site/component-deploy.xml
-com.raytheon.uf.edex.maintenance/component-deploy.xml
-com.raytheon.uf.edex.activetable/component-deploy.xml
-com.raytheon.uf.common.hlstopo/component-deploy.xml

edexOsgi/build.edex/tmp/includes/cots.includes

@@ -1,51 +0,0 @@
-## Cots includes file generated on 2012-01-20 17:08:45 GMT
-com.sun.jndi.nis/*.jar
-net.sf.ehcache/*.jar
-javax.persistence/*.jar
-org.eclipse.jetty/*.jar
-net.sf.cglib/*.jar
-org.apache.commons.logging/*.jar
-org.apache.commons.codec/*.jar
-org.apache.commons.lang/*.jar
-org.apache.commons.digester/*.jar
-org.dom4j/*.jar
-org.apache.tools.bzip2/*.jar
-org.slf4j/*.jar
-org.springframework/*.jar
-uk.ltd.getahead/*.jar
-org.objectweb/*.jar
-org.apache.activemq/*.jar
-org.apache.mina/*.jar
-org.postgres/postgis.jar
-org.postgres/postgresql-8.3-603.jdbc3.jar
-javax.vecmath/*.jar
-javax.mail/*.jar
-org.apache.commons.configuration/*.jar
-org.apache.commons.beanutils/*.jar
-com.facebook.thrift/*.jar
-org.apache.camel/*.jar
-org.apache.commons.pool/*.jar
-org.apache.log4j/*.jar
-org.jep/*.jar
-ncsa.hdf5/jhdf5.jar
-com.sun.jna/*.jar
-edu.wisc.ssec.mcidas/*.jar
-org.apache.commons.management/*.jar
-com.opensymphony.ognl/*.jar
-javax.jms/*.jar
-org.apache.commons.collections/*.jar
-org.jboss.cache/*.jar
-javax.measure/*.jar
-org.apache.http/*.jar
-org.apache.commons.cli/*.jar
-ucar.nc2/*.jar
-org.apache.tomcat.nativ/*.jar
-org.hibernate/*.jar
-org.apache.velocity/velocity-1.5.jar
-org.apache.velocity/velocity-tools-generic-1.3.jar
-org.apache.commons.validator/*.jar
-com.mchange/*.jar
-org.apache.qpid/*.jar
-org.geotools/*.jar
-javax.media.opengl/gluegen-rt.jar
-javax.activation/*.jar

edexOsgi/build.edex/tmp/includes/plugins.includes

@@ -1,154 +0,0 @@
-## Plug-in includes file generated on 2012-01-20 17:08:45 GMT
-com.raytheon.uf.edex.plugin.satellite.mcidas/component-deploy.xml
-com.raytheon.edex.plugin.ldadhydro/component-deploy.xml
-com.raytheon.uf.edex.plugin.loctables/component-deploy.xml
-gov.noaa.nws.ncep.common.dataplugin.intlsigmet/component-deploy.xml
-com.raytheon.uf.edex.plugin.acars/component-deploy.xml
-com.raytheon.uf.edex.plugin.npp.viirs/component-deploy.xml
-gov.noaa.nws.ncep.edex.plugin.intlsigmet/component-deploy.xml
-com.raytheon.edex.plugin.ldadprofiler/component-deploy.xml
-com.raytheon.edex.plugin.binlightning/component-deploy.xml
-com.raytheon.edex.plugin.textlightning/component-deploy.xml
-com.raytheon.uf.edex.plugin.svrwx/component-deploy.xml
-gov.noaa.nws.ncep.edex.plugin.convsigmet/component-deploy.xml
-gov.noaa.nws.ncep.edex.plugin.bufrsgwh/component-deploy.xml
-gov.noaa.nws.ncep.common.dataplugin.scd/component-deploy.xml
-com.raytheon.edex.plugin.shef/component-deploy.xml
-com.raytheon.uf.common.dataplugin.cwa/component-deploy.xml
-com.raytheon.uf.common.dataplugin.level/component-deploy.xml
-gov.noaa.nws.ncep.common.dataplugin.airmet/component-deploy.xml
-com.raytheon.uf.edex.plugin.mesowest/component-deploy.xml
-com.raytheon.uf.common.dataplugin.ldadmesonet/component-deploy.xml
-com.raytheon.uf.common.dataplugin.gfe/component-deploy.xml
-com.raytheon.edex.plugin.radar/component-deploy.xml
-gov.noaa.nws.ncep.common.dataplugin.pafm/component-deploy.xml
-com.raytheon.uf.edex.plugin.tcs/component-deploy.xml
-com.raytheon.uf.common.dataplugin.binlightning/component-deploy.xml
-com.raytheon.edex.plugin.airep/component-deploy.xml
-gov.noaa.nws.ncep.edex.plugin.aww/component-deploy.xml
-gov.noaa.nws.ncep.edex.plugin.airmet/component-deploy.xml
-com.raytheon.uf.common.dataplugin.bufrquikscat/component-deploy.xml
-com.raytheon.uf.common.dataplugin.preciprate/component-deploy.xml
-com.raytheon.uf.edex.plugin.level.handler/component-deploy.xml
-com.raytheon.uf.common.dataplugin.bufrsigwx/component-deploy.xml
-com.raytheon.uf.common.dataplugin.ffmp/component-deploy.xml
-com.raytheon.uf.common.dataplugin.cwat/component-deploy.xml
-gov.noaa.nws.ncep.edex.plugin.atcf/component-deploy.xml
-com.raytheon.edex.plugin.ldad/component-deploy.xml
-com.raytheon.uf.edex.plugin.manualIngest/component-deploy.xml
-gov.noaa.nws.ncep.common.dataplugin.bufrsgwhv/component-deploy.xml
-gov.noaa.nws.ncep.common.dataplugin.ncccfp/component-deploy.xml
-com.raytheon.uf.common.dataplugin.radar/component-deploy.xml
-gov.noaa.nws.ncep.edex.plugin.nctext/component-deploy.xml
-com.raytheon.uf.edex.plugin.cwa/component-deploy.xml
-gov.noaa.nws.ncep.common.dataplugin.tcm/component-deploy.xml
-com.raytheon.edex.plugin.grib/component-deploy.xml
-com.raytheon.edex.plugin.warning/component-deploy.xml
-com.raytheon.edex.plugin.profiler/component-deploy.xml
-com.raytheon.uf.common.dataplugin.scan/component-deploy.xml
-gov.noaa.nws.ncep.common.dataplugin.aww/component-deploy.xml
-com.raytheon.edex.plugin.pirep/component-deploy.xml
-com.raytheon.uf.common.dataplugin.bufrua/component-deploy.xml
-com.raytheon.uf.common.dataplugin.obs/component-deploy.xml
-com.raytheon.uf.common.dataplugin.goessounding/component-deploy.xml
-com.raytheon.uf.common.dataplugin.bufrssmi/component-deploy.xml
-com.raytheon.edex.plugin.bufrua/component-deploy.xml
-com.raytheon.uf.common.plugin.nwsauth/component-deploy.xml
-com.raytheon.edex.plugin.goessounding/component-deploy.xml
-com.raytheon.uf.common.dataplugin.fog/component-deploy.xml
-gov.noaa.nws.ncep.common.dataplugin.wcp/component-deploy.xml
-com.raytheon.edex.plugin.sfcobs/component-deploy.xml
-gov.noaa.nws.ncep.common.dataplugin.bufrsgwh/component-deploy.xml
-com.raytheon.edex.plugin.poessounding/component-deploy.xml
-gov.noaa.nws.ncep.edex.plugin.ncscat/component-deploy.xml
-com.raytheon.uf.common.dataplugin.warning/component-deploy.xml
-gov.noaa.nws.ncep.common.dataplugin.ffg/component-deploy.xml
-com.raytheon.uf.edex.plugin.bufrhdw/component-deploy.xml
-com.raytheon.uf.common.dataplugin.bufrascat/component-deploy.xml
-com.raytheon.uf.common.dataplugin.shef/component-deploy.xml
-com.raytheon.edex.plugin.ccfp/component-deploy.xml
-gov.noaa.nws.ncep.edex.plugin.bufrssha/component-deploy.xml
-gov.noaa.nws.ncep.edex.plugin.ffg/component-deploy.xml
-com.raytheon.uf.common.dataplugin.vil/component-deploy.xml
-com.raytheon.edex.plugin.ldadmanual/component-deploy.xml
-com.raytheon.uf.edex.plugin.bufrquikscat/component-deploy.xml
-gov.noaa.nws.ncep.common.dataplugin.nonconvsigmet/component-deploy.xml
-com.raytheon.uf.edex.plugin.vil/component-deploy.xml
-com.raytheon.uf.common.dataplugin.tcg/component-deploy.xml
-com.raytheon.uf.common.dataplugin.lsr/component-deploy.xml
-com.raytheon.edex.plugin.redbook/component-deploy.xml
-com.raytheon.edex.plugin.obs/component-deploy.xml
-com.raytheon.uf.edex.plugin.nwsauth/component-deploy.xml
-com.raytheon.uf.common.dataplugin.sfcobs/component-deploy.xml
-gov.noaa.nws.ncep.common.dataplugin.h5scd/component-deploy.xml
-com.raytheon.uf.common.dataplugin.tcs/component-deploy.xml
-gov.noaa.nws.ncep.edex.plugin.ncccfp/component-deploy.xml
-com.raytheon.edex.plugin.bufrmos/component-deploy.xml
-com.raytheon.uf.edex.plugin.ffmp/component-deploy.xml
-com.raytheon.uf.edex.plugin.scan.common/component-deploy.xml
-gov.noaa.nws.ncep.edex.plugin.mcidas/component-deploy.xml
-com.raytheon.uf.common.dataplugin/component-deploy.xml
-com.raytheon.uf.edex.plugin.fssobs/component-deploy.xml
-com.raytheon.uf.common.dataplugin.profiler/component-deploy.xml
-com.raytheon.uf.common.dataplugin.text/component-deploy.xml
-gov.noaa.nws.ncep.common.dataplugin.ncgrib/component-deploy.xml
-com.raytheon.uf.common.dataplugin.satellite/component-deploy.xml
-gov.noaa.nws.ncep.common.dataplugin.mcidas/component-deploy.xml
-com.raytheon.uf.edex.plugin.lsr/component-deploy.xml
-gov.noaa.nws.ncep.edex.plugin.uair/component-deploy.xml
-com.raytheon.uf.edex.plugin.cwat/component-deploy.xml
-gov.noaa.nws.ncep.common.dataplugin.idft/component-deploy.xml
-com.raytheon.uf.common.dataplugin.ldadhydro/component-deploy.xml
-com.raytheon.uf.common.dataplugin.grib/component-deploy.xml
-com.raytheon.uf.common.dataplugin.acars/component-deploy.xml
-com.raytheon.uf.edex.plugin.scan/component-deploy.xml
-com.raytheon.uf.edex.plugin.bufrncwf/component-deploy.xml
-com.raytheon.uf.common.dataplugin.pirep/component-deploy.xml
-gov.noaa.nws.ncep.common.dataplugin.convsigmet/component-deploy.xml
-com.raytheon.uf.edex.plugin.fog/component-deploy.xml
-com.raytheon.uf.common.dataplugin.airep/component-deploy.xml
-com.raytheon.edex.plugin.modelsounding/component-deploy.xml
-com.raytheon.edex.plugin.taf/component-deploy.xml
-com.raytheon.uf.edex.plugin.preciprate/component-deploy.xml
-com.raytheon.uf.common.dataplugin.npp.viirs/component-deploy.xml
-gov.noaa.nws.ncep.edex.plugin.nonconvsigmet/component-deploy.xml
-gov.noaa.nws.ncep.edex.plugin.idft/component-deploy.xml
-com.raytheon.uf.edex.plugin.bufrascat/component-deploy.xml
-com.raytheon.uf.common.dataplugin.vaa/component-deploy.xml
-gov.noaa.nws.ncep.common.dataplugin.uair/component-deploy.xml
-gov.noaa.nws.ncep.common.dataplugin.atcf/component-deploy.xml
-com.raytheon.uf.common.dataplugin.svrwx/component-deploy.xml
-gov.noaa.nws.ncep.edex.plugin.pafm/component-deploy.xml
-gov.noaa.nws.ncep.edex.plugin.h5uair/component-deploy.xml
-com.raytheon.uf.common.dataplugin.qc/component-deploy.xml
-gov.noaa.nws.ncep.edex.plugin.ncgrib/component-deploy.xml
-com.raytheon.uf.common.dataplugin.qpf/component-deploy.xml
-com.raytheon.uf.common.dataplugin.bufrhdw/component-deploy.xml
-com.raytheon.edex.plugin.gfe/component-deploy.xml
-com.raytheon.uf.edex.plugin.qc/component-deploy.xml
-com.raytheon.uf.common.dataplugin.fssobs/component-deploy.xml
-com.raytheon.uf.common.dataplugin.bufrncwf/component-deploy.xml
-gov.noaa.nws.ncep.common.dataplugin.ncscat/component-deploy.xml
-com.raytheon.uf.edex.plugin.level/component-deploy.xml
-com.raytheon.uf.edex.plugin.bufrmthdw/component-deploy.xml
-com.raytheon.uf.common.dataplugin.acarssounding/component-deploy.xml
-com.raytheon.edex.plugin.text/component-deploy.xml
-com.raytheon.uf.edex.plugin.tcg/component-deploy.xml
-com.raytheon.uf.edex.plugin.qpf/component-deploy.xml
-gov.noaa.nws.ncep.edex.plugin.mosaic/component-deploy.xml
-gov.noaa.nws.ncep.edex.plugin.bufrsgwhv/component-deploy.xml
-com.raytheon.uf.edex.plugin.bufrsigwx/component-deploy.xml
-com.raytheon.uf.edex.plugin.ldadmesonet/component-deploy.xml
-com.raytheon.uf.common.dataplugin.bufrmthdw/component-deploy.xml
-com.raytheon.uf.edex.plugin.acarssounding/component-deploy.xml
-gov.noaa.nws.ncep.edex.plugin.tcm/component-deploy.xml
-com.raytheon.edex.plugin.satellite/component-deploy.xml
-com.raytheon.edex.plugin.recco/component-deploy.xml
-com.raytheon.uf.edex.plugin.vaa/component-deploy.xml
-gov.noaa.nws.ncep.edex.plugin.h5scd/component-deploy.xml
-gov.noaa.nws.ncep.edex.plugin.scd/component-deploy.xml
-gov.noaa.nws.ncep.edex.plugin.wcp/component-deploy.xml
-com.raytheon.uf.common.dataplugin.poessounding/component-deploy.xml
-gov.noaa.nws.ncep.common.dataplugin.bufrssha/component-deploy.xml
-com.raytheon.uf.edex.plugin.bufrssmi/component-deploy.xml
-gov.noaa.nws.ncep.common.dataplugin.h5uair/component-deploy.xml

edexOsgi/com.raytheon.edex.autobldsrv/res/spring/subscription-request.xml

@@ -10,9 +10,12 @@
 	<camelContext id="subscription-camel" 
 	              xmlns="http://camel.apache.org/schema/spring" 
 	              errorHandlerRef="errorHandler">
+	    <endpoint id="subscriptionHTTP_from"
+	    	uri="jetty:http://0.0.0.0:${http.port}/services/subscribe?disableStreamCache=true" />          
+	    
 		<!-- SubscriptionSrv routes -->
 		<route id="subscriptionHTTP">
-			<from uri="jetty:http://0.0.0.0:9581/services/subscribe?disableStreamCache=true" />
+			<from uri="ref:subscriptionHTTP_from" />
 			<bean ref="subscription" method="processRequest"/>
 			<bean ref="serializationUtil" method="marshalToXml" />
 		</route>

edexOsgi/com.raytheon.edex.plugin.airep/src/com/raytheon/edex/plugin/airep/AirepDecoder.java

@@ -22,6 +22,7 @@ package com.raytheon.edex.plugin.airep;
 import java.util.Calendar;
 import java.util.Map;
 
+import com.raytheon.edex.esb.Headers;
 import com.raytheon.edex.exception.DecoderException;
 import com.raytheon.edex.plugin.AbstractDecoder;
 import com.raytheon.edex.plugin.airep.decoder.AIREPWeather;
@@ -89,7 +90,7 @@ public class AirepDecoder extends AbstractDecoder {
      * @throws DecoderException
      *             Thrown if no data is available.
      */
-    public PluginDataObject[] decode(AirepDecoderInput input)
+    public PluginDataObject[] decode(AirepDecoderInput input, Headers header)
             throws DecoderException {
 
         PluginDataObject[] reports = null;
@@ -100,9 +101,14 @@ public class AirepDecoder extends AbstractDecoder {
         try {
             // traceId = getTraceId(hdrMap);
             logger.debug(traceId + "- AirepDecoder.decode()");
-
+            WMOHeader wmoHeader = input.wmoHeader;
-            report = populateRecord(new AirepParser(input.report,
+            if(wmoHeader != null) {
-                    input.wmoHeader), input.wmoHeader);
+                Calendar refTime = TimeTools.findDataTime(
+                        wmoHeader.getYYGGgg(), header);
+                if(refTime != null) {
+                    report = populateRecord(new AirepParser(input.report, refTime));
+                }
+            }
             if (report != null) {
                 report.setTraceId(traceId);
                 report.setPluginName(PLUGIN_NAME);
@@ -130,7 +136,7 @@ public class AirepDecoder extends AbstractDecoder {
      *            The reccon parser that contains the decoded data.
      * @return The populated record.
      */
-    private AirepRecord populateRecord(AirepParser parser, WMOHeader wmoHeader) {
+    private AirepRecord populateRecord(AirepParser parser) {
 
         AirepRecord record = null;
         AircraftObsLocation location = null;

edexOsgi/com.raytheon.edex.plugin.airep/src/com/raytheon/edex/plugin/airep/decoder/AirepParser.java

@@ -19,14 +19,17 @@
  **/
 package com.raytheon.edex.plugin.airep.decoder;
 
+import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.StringTokenizer;
+import java.util.TimeZone;
 import java.util.regex.Pattern;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
+import com.raytheon.edex.esb.Headers;
 import com.raytheon.uf.edex.decodertools.aircraft.AircraftFlightLevel;
 import com.raytheon.uf.edex.decodertools.aircraft.AircraftLatitude;
 import com.raytheon.uf.edex.decodertools.aircraft.AircraftLongitude;
@@ -121,7 +124,8 @@ public class AirepParser {
 
     private AircraftRemarks rptRemarks = null;
 
-    private WMOHeader wmoHeader;
+    private Calendar refTime;
+    
 
     /**
      * Create the parser for and decode an observation from a String.
@@ -129,21 +133,20 @@ public class AirepParser {
      * @param anObservation
      *            A string containing the observation.
      */
-    public AirepParser(String anObservation, WMOHeader wmoHeader) {
+    public AirepParser(String anObservation, Calendar refTime) {
-        this.wmoHeader = wmoHeader;
+        this.refTime = refTime;
         reportData = anObservation;
         parseElements();
     } // AirepParser()
 
     /**
-     * Create the parser for and decode an observation from a String.
+     * Create the parser for and decode an observation from a byte array.
      * 
      * @param anObservation
      *            A string containing the observation.
      */
-    public AirepParser(byte[] anObservation) {
+    public AirepParser(byte[] anObservation, Calendar refTime) {
-        reportData = new String(anObservation);
+        this(new String(anObservation), refTime);
-        parseElements();
     } // AirepParser()
 
     /**
@@ -263,11 +266,10 @@ public class AirepParser {
         for (int i = 0; i < theElements.size(); i++) {
             Object o = theElements.get(i);
             if (o instanceof String) {
-                String[] latLon = null; // AircraftLatitude.splitLatLon((String)
+                String[] latLon = AircraftLatitude.splitLatLon((String) o);
-                                        // o);
+                if ((latLon != null)&&(latLon.length ==2)) {
-                if (latLon != null) {
-                    theElements.set(i, latLon[1]);
                     theElements.add(i, latLon[0]);
+                    theElements.set(i, latLon[1]);
                     break;
                 }
             }
@@ -329,23 +331,25 @@ public class AirepParser {
             if (o instanceof String) {
                 String s = (String) o;
                 if (TIME.matcher(s).matches()) {
+
                     int hour = Integer.parseInt(s.substring(0, 2));
                     int minute = Integer.parseInt(s.substring(2));
-                    Calendar oTime = TimeTools.getSystemCalendar(
-                            wmoHeader.getYear(), wmoHeader.getMonth(),
-                            wmoHeader.getDay());
 
-                    observationTime = TimeTools.copy(oTime);
+                    if (refTime != null) {
+                        
+                        observationTime = TimeTools.copy(refTime);
                         observationTime.set(Calendar.HOUR_OF_DAY, hour);
                         observationTime.set(Calendar.MINUTE, minute);
                         observationTime.set(Calendar.SECOND, 0);
                         observationTime.set(Calendar.MILLISECOND, 0);
-
+                        // If the observation time ends up greater than
-                    if (observationTime.compareTo(oTime) > 0) {
+                        // the reference time, back up a day.
+                        if (observationTime.compareTo(refTime) > 0) {
                             observationTime.add(Calendar.DAY_OF_MONTH, -1);
                         }
 
                         theElements.set(i, observationTime);
+                    }
                     break;
                 }
             }
@@ -679,4 +683,24 @@ public class AirepParser {
         return (rptRemarks != null) ? rptRemarks.toString() : "";
     } // getRemarks()
 
+    
+    public static void main(String [] args) {
+        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+        sdf.setTimeZone(TimeZone.getTimeZone("ZULU"));
+        
+        Calendar refTime = TimeTools.getBaseCalendar(2011, 12, 14);
+        refTime.set(Calendar.HOUR_OF_DAY, 17);
+        refTime.set(Calendar.MINUTE, 15);
+        refTime.set(Calendar.SECOND, 00);
+        refTime.set(Calendar.MILLISECOND, 0);
+        
+        String data = "ARP UAL121 4400N 05700W 1640 F390 MS00 000/099KT TB MOD SK CLEAR=";
+        AirepParser p = new AirepParser(data,refTime);
+        System.out.println(sdf.format(p.getObservationTime().getTime()));
+        
+        data = "ARP UAL121 4400N 05700W 1840 F390 MS00 000/099KT TB MOD SK CLEAR=";
+        p = new AirepParser(data,refTime);
+        System.out.println(sdf.format(p.getObservationTime().getTime()));
+    }
+    
 } // AirepParser

edexOsgi/com.raytheon.edex.plugin.binlightning/res/spring/binlightning_ep-ingest.xml

@@ -13,12 +13,12 @@
 		<constructor-arg value="jms-dist:queue:Ingest.binlightning?destinationResolver=#qpidDurableResolver" />
 	</bean>
 
-    <bean id="binlightningCamelRegistered" factory-bean="contextManager"
+	<bean id="binlightningCamelRegistered" factory-bean="clusteredCamelContextMgr"
 		factory-method="register" depends-on="persistCamelRegistered">
-		<constructor-arg ref="binlightning-camel"/>
+		<constructor-arg ref="clusteredBinLightningRoutes" />
   	</bean>
 
-	<camelContext id="binlightning-camel"
+	<camelContext id="clusteredBinLightningRoutes"
 		xmlns="http://camel.apache.org/schema/spring"
 		errorHandlerRef="errorHandler"
 		autoStartup="false">

edexOsgi/com.raytheon.edex.plugin.binlightning/src/com/raytheon/edex/plugin/binlightning/BinLightningDecoder.java

@@ -30,7 +30,6 @@ import org.apache.commons.logging.LogFactory;
 import com.raytheon.edex.esb.Headers;
 import com.raytheon.edex.exception.DecoderException;
 import com.raytheon.edex.plugin.AbstractDecoder;
-import com.raytheon.edex.plugin.IBinaryDecoder;
 import com.raytheon.edex.plugin.binlightning.impl.BinLightningFactory;
 import com.raytheon.edex.plugin.binlightning.impl.IBinLightningDecoder;
 import com.raytheon.edex.plugin.binlightning.impl.LightningDataSource;
@@ -92,6 +91,8 @@ public class BinLightningDecoder extends AbstractDecoder {
     // Allow ingest up to 10 minutes into the future.
     private static final long TEN_MINUTES = 10 * 60 * 1000L;
 
+    private SimpleDateFormat SDF;
+
     private Log logger = LogFactory.getLog(getClass());
 
     /**
@@ -107,6 +108,8 @@ public class BinLightningDecoder extends AbstractDecoder {
      * will return false, decode() will return a null.
      */
     public BinLightningDecoder() {
+        SDF = new SimpleDateFormat("yyyyMMddHHmmss");
+        SDF.setTimeZone(TimeZone.getTimeZone("Zulu"));
     }
 
     /**
@@ -125,8 +128,12 @@ public class BinLightningDecoder extends AbstractDecoder {
 
             traceId = (String) headers.get(DecoderTools.INGEST_FILE_NAME);
 
-            WMOHeader header = new WMOHeader(data);
+            WMOHeader wmoHdr = new WMOHeader(data);
-            if (header.isValid()) {
+            if (wmoHdr.isValid()) {
+
+                Calendar baseTime = TimeTools.findDataTime(wmoHdr.getYYGGgg(),
+                        headers);
+
                 byte[] pdata = DecoderTools.stripWMOHeader(data, SFUS_PATTERN);
                 if (pdata == null) {
                     pdata = DecoderTools.stripWMOHeader(data, SFPA_PATTERN);
@@ -174,21 +181,19 @@ public class BinLightningDecoder extends AbstractDecoder {
                         return new PluginDataObject[0];
                     }
 
-                    Calendar c = TimeTools.getSystemCalendar();
+                    Calendar c = TimeTools.copy(baseTime);
                     if (c == null) {
                         throw new DecoderException(traceId
                                 + "-Error decoding times");
                     }
-                    report.setInsertTime(c);
 
                     Calendar cStart = report.getStartTime();
                     if (cStart.getTimeInMillis() > c.getTimeInMillis()
                             + TEN_MINUTES) {
-                        SimpleDateFormat sdf = new SimpleDateFormat(
+                        synchronized (SDF) {
-                                "yyyyMMddHHmmss");
-                        sdf.setTimeZone(TimeZone.getTimeZone("Zulu"));
                             logger.info("Discarding future data for " + traceId
-                                + " at " + sdf.format(cStart.getTime()));
+                                    + " at " + SDF.format(cStart.getTime()));
+                        }
                     } else {
                         Calendar cStop = report.getStopTime();
 
@@ -206,8 +211,7 @@ public class BinLightningDecoder extends AbstractDecoder {
                                 report.constructDataURI();
                                 reports = new PluginDataObject[] { report };
                             } catch (PluginException e) {
-                                throw new DecoderException(
+                                logger.error("Error constructing datauri", e);
-                                        "Error constructing datauri", e);
                             }
                         }
                     }

edexOsgi/com.raytheon.edex.plugin.bufrmos/src/com/raytheon/edex/plugin/bufrmos/common/BufrMosData.java

@@ -86,7 +86,8 @@ public abstract class BufrMosData extends PersistablePluginDataObject implements
 	private String wmoHeader;
 
 	@Embedded
-    private PointDataView pdv = null;
+	@DynamicSerializeElement
+	private PointDataView pointDataView = null;
 
 	@ManyToOne(cascade = { CascadeType.REFRESH })
 	@PrimaryKeyJoinColumn
@@ -237,12 +238,12 @@ public abstract class BufrMosData extends PersistablePluginDataObject implements
 
 	@Override
 	public PointDataView getPointDataView() {
-        return this.pdv;
+		return this.pointDataView;
 	}
 
 	@Override
-    public void setPointDataView(PointDataView pdv) {
+	public void setPointDataView(PointDataView pointDataView) {
-        this.pdv = pdv;
+		this.pointDataView = pointDataView;
 
 	}
 }

edexOsgi/com.raytheon.edex.plugin.ccfp/src/com/raytheon/edex/plugin/ccfp/CcfpDecoder.java

@@ -24,6 +24,7 @@ import java.util.Calendar;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import com.raytheon.edex.esb.Headers;
 import com.raytheon.edex.exception.DecoderException;
 import com.raytheon.edex.plugin.AbstractDecoder;
 import com.raytheon.uf.common.dataplugin.PluginDataObject;
@@ -31,6 +32,7 @@ import com.raytheon.uf.common.dataplugin.PluginException;
 import com.raytheon.uf.common.time.DataTime;
 import com.raytheon.uf.common.time.TimeRange;
 import com.raytheon.uf.edex.decodertools.time.TimeTools;
+import com.raytheon.uf.edex.wmo.message.WMOHeader;
 import com.vividsolutions.jts.io.WKTReader;
 
 /**
@@ -84,6 +86,7 @@ public class CcfpDecoder extends AbstractDecoder {
 
     private static final String SPACE = " ";
 
+    private static final PluginDataObject [] EMPTY_PDO = new PluginDataObject [0];
     /**
      * Constructor
      * 
@@ -92,8 +95,17 @@ public class CcfpDecoder extends AbstractDecoder {
     public CcfpDecoder() throws DecoderException {
     }
 
-    public PluginDataObject[] decode(String msg) throws DecoderException,
+    public PluginDataObject[] decode(String msg, Headers headers) throws PluginException {
-            PluginException {
+
+        PluginDataObject [] data = null;
+        
+        Calendar baseTime = null;
+        WMOHeader wmoHdr = new WMOHeader(msg.getBytes());
+        if (wmoHdr.isValid()) {
+            baseTime = TimeTools.findDataTime(wmoHdr.getYYGGgg(), headers);
+        } else {
+            baseTime = TimeTools.getSystemCalendar();
+        }
         
         CcfpRecord record = new CcfpRecord();
         record.setMessageData(msg);
@@ -124,7 +136,7 @@ public class CcfpDecoder extends AbstractDecoder {
                 } else {
                     record.setCanadaflag(Boolean.FALSE);
                 }
-                record.setInsertTime(TimeTools.getSystemCalendar());
+                record.setInsertTime(baseTime);
             }
             if (record.getProducttype().equals("AREA")) {
                 matcher = AREA_PATTERN.matcher(msg);
@@ -205,15 +217,20 @@ public class CcfpDecoder extends AbstractDecoder {
                 }
             }
         } catch (Exception e) {
-            throw new DecoderException("Unable to decode CCFP", e);
+            record = null;
+            logger.error("Unable to decode CCFP", e);
         }
+        data = EMPTY_PDO;
         if(record != null) {
             record.setPluginName(PLUGIN_NAME);
+            try {
                 record.constructDataURI();
-            return new PluginDataObject[] { record };
+                record.setInsertTime(baseTime);
-        } else {
+                data = new PluginDataObject[] { record };
-            return new PluginDataObject[0];
+            } catch (PluginException e) {
-        }
+                logger.error("Error constructing datauri", e);
-
+            }
+        }
+        return data;
     }
 }

edexOsgi/com.raytheon.edex.plugin.goessounding/src/com/raytheon/edex/plugin/goessounding/GOESSoundingDecoder.java

@@ -136,7 +136,6 @@ public class GOESSoundingDecoder extends AbstractDecoder implements
             WMOHeader wmoHeader = input.getWmoHeader();
 
             if ((wmoHeader != null) && (wmoHeader.isValid())) {
-
                 try {
                     byte[] messageData = input.getDocumentData();
 

edexOsgi/com.raytheon.edex.plugin.goessounding/src/com/raytheon/edex/plugin/goessounding/decoder/GOESSoundingDataAdapter.java

@@ -211,8 +211,7 @@ public class GOESSoundingDataAdapter {
 //                seconds = (dp.getValue() != null) ? ((Double) dp.getValue())
 //                        .intValue() : null;
 
-                Calendar baseTime = TimeTools.getSystemCalendar();
+                Calendar baseTime = TimeTools.getBaseCalendar(year, 1, 1);
-                baseTime.set(Calendar.YEAR, year);
                 baseTime.set(Calendar.DAY_OF_YEAR, day);
                 baseTime.set(Calendar.HOUR_OF_DAY, hour);
                 baseTime.set(Calendar.MINUTE, minute);

edexOsgi/com.raytheon.edex.plugin.grib/src/com/raytheon/edex/plugin/grib/GribDecoder.java

@@ -34,7 +34,6 @@ import ucar.unidata.io.RandomAccessFile;
 
 import com.raytheon.uf.common.dataplugin.PluginDataObject;
 import com.raytheon.uf.common.dataplugin.grib.GribRecord;
-import com.raytheon.uf.common.dataplugin.grib.StatusConstants;
 import com.raytheon.uf.common.dataplugin.grib.exception.GribException;
 import com.raytheon.uf.common.status.IUFStatusHandler;
 import com.raytheon.uf.common.status.UFStatus;
@@ -109,7 +108,7 @@ public class GribDecoder {
                 // manual ingest endpoint
                 if (recordLengths.size() > 1) {
                     raf.seek(0);
-                    splitFile(file.getName(), raf, recordLengths);
+                    splitFile(file.getName(), raf, recordLengths, edition);
                     return new GribRecord[] {};
                 }
             }
@@ -165,7 +164,7 @@ public class GribDecoder {
      * @throws IOException
      */
     private void splitFile(String fileName, RandomAccessFile raf,
-            List<Long> sizes) throws IOException {
+            List<Long> sizes, int edition) throws IOException {
         FileOutputStream out = null;
         byte[] transfer = null;
         for (int i = 0; i < sizes.size(); i++) {
@@ -175,7 +174,7 @@ public class GribDecoder {
 
             try {
                 out = new FileOutputStream(System.getProperty("edex.home")
-                        + "/data/manual/" + fileName + "_record_" + (i + 1));
+                        + "/data/manual/grib/grib" + edition + "LargeSplit/" + fileName + "_record_" + (i + 1));
                 out.write(transfer);
                 out.close();
             } finally {

edexOsgi/com.raytheon.edex.plugin.ldadprofiler/src/com/raytheon/edex/plugin/ldadprofiler/common/ProfilerLdadObs.java

@@ -165,7 +165,8 @@ public class ProfilerLdadObs extends PersistablePluginDataObject implements
 	private String stationName;
 
 	@Embedded
-    private PointDataView pdv;
+	@DynamicSerializeElement
+	private PointDataView pointDataView;
 
 	/**
 	 * @return the base_time
@@ -251,23 +252,23 @@ public class ProfilerLdadObs extends PersistablePluginDataObject implements
 	}
 
 	/**
-     * @return the pdv
+	 * @return the pointDataView
 	 */
 	@Override
 	public PointDataView getPointDataView() {
-        return pdv;
+		return pointDataView;
 	}
 
 	public ProfilerLdadObs() {
 	}
 
 	/**
-     * @param pdv
+	 * @param pointDataView
-     *            the pdv to set
+	 *            the pointDataView to set
 	 */
 	@Override
-    public void setPointDataView(PointDataView pdv) {
+	public void setPointDataView(PointDataView pointDataView) {
-        this.pdv = pdv;
+		this.pointDataView = pointDataView;
 	}
 
 	// ----------------------------------------------------

edexOsgi/com.raytheon.edex.plugin.modelsounding/src/com/raytheon/edex/plugin/modelsounding/common/SoundingSite.java

@@ -103,7 +103,8 @@ public class SoundingSite extends PersistablePluginDataObject implements
 	private SurfaceObsLocation location;
 
 	@Embedded
-    private PointDataView pdv;
+	@DynamicSerializeElement
+	private PointDataView pointDataView;
 
 	// Text of the WMO header
 	@Column
@@ -886,11 +887,11 @@ public class SoundingSite extends PersistablePluginDataObject implements
 
 	@Override
 	public PointDataView getPointDataView() {
-        return this.pdv;
+		return this.pointDataView;
 	}
 
 	@Override
-    public void setPointDataView(PointDataView pdv) {
+	public void setPointDataView(PointDataView pointDataView) {
-        this.pdv = pdv;
+		this.pointDataView = pointDataView;
 	}
 }

edexOsgi/com.raytheon.edex.plugin.obs/src/com/raytheon/edex/plugin/obs/metar/MetarDecoder.java

@@ -42,6 +42,7 @@ import com.raytheon.uf.common.time.DataTime;
 import com.raytheon.uf.edex.decodertools.core.DecoderTools;
 import com.raytheon.uf.edex.decodertools.core.IDecoderConstants;
 import com.raytheon.uf.edex.decodertools.time.TimeTools;
+import com.raytheon.uf.edex.wmo.message.WMOHeader;
 import com.vividsolutions.jts.geom.Coordinate;
 import com.vividsolutions.jts.geom.GeometryFactory;
 import com.vividsolutions.jts.geom.Point;
@@ -210,8 +211,19 @@ public class MetarDecoder extends AbstractDecoder {
             throws DecoderException {
 
         MetarSeparator sep = MetarSeparator.separate(inputData, headers);
+        
         List<PluginDataObject> retVal = new ArrayList<PluginDataObject>();
 
+        Calendar baseTime = TimeTools.getSystemCalendar();
+        WMOHeader wmoHdr = sep.getWMOHeader();
+        if(TimeTools.allowArchive()) {
+            if((wmoHdr != null)&&(wmoHdr.isValid())) {
+                baseTime = TimeTools.findDataTime(wmoHdr.getYYGGgg(), headers);
+            } else {
+                logger.error("ARCHIVE MODE-No WMO Header found in file" + headers.get(WMOHeader.INGEST_FILE_NAME));
+            }
+        }
+        
         while (sep.hasNext()) {
             byte[] messageData = sep.next();
             Pattern thePattern;
@@ -278,17 +290,26 @@ public class MetarDecoder extends AbstractDecoder {
                                         traceId, icao));
                         continue;
                     }
-                    String fileName = null;
+                    
-                    if (headers != null) {
+                    
-                        fileName = (String) headers
+                    Calendar obsTime = null;
-                                .get(DecoderTools.INGEST_FILE_NAME);
+                    Integer da = DecoderTools.getInt(timeGroup, 0, 2);
+                    Integer hr = DecoderTools.getInt(timeGroup, 2, 4);
+                    Integer mi = DecoderTools.getInt(timeGroup, 4, 6);
+                    if ((da != null) && (hr != null) && (mi != null)) {
+                        obsTime = TimeTools.copy(baseTime);
+                        obsTime.set(Calendar.DAY_OF_MONTH, da);
+                        obsTime.set(Calendar.HOUR_OF_DAY, hr);
+                        obsTime.set(Calendar.MINUTE, mi);
                     }                    
-                    Calendar obsTime = TimeTools.findCurrentTime(
-                            matcher.group(3), fileName);
                     if (obsTime != null) {
                         record.setTimeObs(obsTime);
                         record.setDataTime(new DataTime(obsTime));
-                        record.setRefHour(Util.findReferenceHour(timeGroup));
+                        Calendar refHour = TimeTools.copyToNearestHour(obsTime);
+                        if(mi >= 45) {
+                            refHour.add(Calendar.HOUR_OF_DAY, 1);
+                        }
+                        record.setRefHour(refHour);
                         // TODO :
                     } else {
                         // couldn't find observation time so exit.
@@ -301,7 +322,16 @@ public class MetarDecoder extends AbstractDecoder {
                 // into the future
                 Calendar obsTime = record.getTimeObs();
                 if (obsTime != null) {
-                    Calendar currTime = TimeTools.getSystemCalendar();
+                    Calendar currTime = TimeTools.copy(baseTime);
+
+                    // Do this only for archive mode!!! Otherwise valid data will not pass if the WMO header
+                    // date/time is much less than the obstime. For instance
+                    //  WMO Header time = dd1200
+                    //  Observed   time = dd1235
+                    // To solve this will require greater precision in the file timestamp.
+                    if(TimeTools.allowArchive()) {
+                        currTime.add(Calendar.HOUR, 1);
+                    }
                     currTime.add(Calendar.MINUTE, METAR_FUTURE_LIMIT);
 
                     long diff = currTime.getTimeInMillis()

edexOsgi/com.raytheon.edex.plugin.obs/src/com/raytheon/edex/plugin/obs/metar/MetarPointDataTransform.java

@@ -475,9 +475,7 @@ public class MetarPointDataTransform {
         mr.setPkWndSpd(pdv.getNumber(PK_WND_SPD).intValue());
         long t = pdv.getNumber(PK_WND_TIME).longValue();
         if (t >= 0) {
-            Calendar c = TimeTools.getSystemCalendar();
+            mr.setPkWndTime(TimeTools.newCalendar(t));
-            c.setTimeInMillis(t);
-            mr.setPkWndTime(c);
         }
 
         return mr;

edexOsgi/com.raytheon.edex.plugin.radar/META-INF/MANIFEST.MF

@@ -25,5 +25,6 @@ Import-Package: com.raytheon.uf.common.comm,
  com.raytheon.uf.common.menus,
  com.raytheon.uf.common.menus.xml,
  com.raytheon.uf.common.site.ingest,
+ com.raytheon.uf.common.dataplugin.text.db,
  org.apache.commons.logging,
  org.apache.tools.bzip2

edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/RadarDecoder.java

@@ -33,6 +33,7 @@ import com.raytheon.edex.plugin.AbstractDecoder;
 import com.raytheon.edex.plugin.radar.dao.RadarStationDao;
 import com.raytheon.edex.plugin.radar.level2.Level2BaseRadar;
 import com.raytheon.edex.plugin.radar.level3.Level3BaseRadar;
+import com.raytheon.edex.plugin.radar.util.RadarEdexTextProductUtil;
 import com.raytheon.uf.common.dataplugin.PluginDataObject;
 import com.raytheon.uf.common.dataplugin.PluginException;
 import com.raytheon.uf.common.dataplugin.radar.RadarRecord;
@@ -165,8 +166,10 @@ public class RadarDecoder extends AbstractDecoder {
                         AFOSProductId afos = new AFOSProductId("WSR", "ROB",
                                 siteId);
                         // store the product ROB that is barely do-able
-                        RadarTextProductUtil.storeTextProduct(afos, header,
+                        Calendar cal = (TimeTools.allowArchive() ? header
-                                dataString, true, Calendar.getInstance());
+                                .getHeaderDate() : Calendar.getInstance());
+                        RadarEdexTextProductUtil.storeTextProduct(afos, header,
+                                dataString, true, cal);
                         return new PluginDataObject[0];
                     }
                 }
@@ -491,8 +494,10 @@ public class RadarDecoder extends AbstractDecoder {
                 RadarTextProductUtil.createAfosId(75, splits[1].substring(1)));
 
         // store the product to the text database
-        RadarTextProductUtil.storeTextProduct(afos, header, temp, true,
+        Calendar cal = (TimeTools.allowArchive() ? header.getHeaderDate()
-                Calendar.getInstance());
+                : Calendar.getInstance());
+        RadarEdexTextProductUtil
+                .storeTextProduct(afos, header, temp, true, cal);
 
         // send message to alertviz
         EDEXUtil.sendMessageAlertViz(Priority.VERBOSE,

edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/level3/Level3BaseRadar.java

@@ -38,6 +38,7 @@ import org.apache.tools.bzip2.CBZip2InputStream;
 
 import com.raytheon.edex.esb.Headers;
 import com.raytheon.edex.plugin.radar.dao.RadarStationDao;
+import com.raytheon.edex.plugin.radar.util.RadarEdexTextProductUtil;
 import com.raytheon.uf.common.dataplugin.radar.RadarStation;
 import com.raytheon.uf.common.dataplugin.radar.level3.AlertAdaptationParameters;
 import com.raytheon.uf.common.dataplugin.radar.level3.AlertMessage;
@@ -54,6 +55,7 @@ import com.raytheon.uf.common.status.UFStatus;
 import com.raytheon.uf.common.status.UFStatus.Priority;
 import com.raytheon.uf.common.time.DataTime;
 import com.raytheon.uf.edex.database.DataAccessLayerException;
+import com.raytheon.uf.edex.decodertools.time.TimeTools;
 import com.raytheon.uf.edex.wmo.message.AFOSProductId;
 import com.raytheon.uf.edex.wmo.message.WMOHeader;
 
@@ -822,9 +824,10 @@ public class Level3BaseRadar {
                 AFOSProductId afos = new AFOSProductId(afosId);
                 if (afos.isFilled()) {
                     try {
-                        RadarTextProductUtil.storeTextProduct(afos, header,
+                        Calendar cal = (TimeTools.allowArchive() ? theMsgTimestamp
-                                tabularBlock.getString(), true,
+                                : Calendar.getInstance());
-                                Calendar.getInstance());
+                        RadarEdexTextProductUtil.storeTextProduct(afos, header,
+                                tabularBlock.getString(), true, cal);
                     } catch (Exception e) {
                         theHandler.handle(Priority.ERROR,
                                 "Could not store text product", e);
@@ -913,9 +916,10 @@ public class Level3BaseRadar {
             AFOSProductId afos = new AFOSProductId(afosId);
             if (afos.isFilled()) {
                 try {
-                    RadarTextProductUtil.storeTextProduct(afos, header,
+                    Calendar cal = (TimeTools.allowArchive() ? theMsgTimestamp
-                            tabularBlock.getString(), true,
+                            : Calendar.getInstance());
-                            Calendar.getInstance());
+                    RadarEdexTextProductUtil.storeTextProduct(afos, header,
+                            tabularBlock.getString(), true, cal);
                 } catch (Exception e) {
                     theHandler.handle(Priority.ERROR,
                             "Could not store text product", e);

edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/util/RadarEdexTextProductUtil.java

@@ -0,0 +1,112 @@
+/**
+ * This software was developed and / or modified by Raytheon Company,
+ * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+ * 
+ * U.S. EXPORT CONTROLLED TECHNICAL DATA
+ * This software product contains export-restricted data whose
+ * export/transfer/disclosure is restricted by U.S. law. Dissemination
+ * to non-U.S. persons whether in the United States or abroad requires
+ * an export license or other authorization.
+ * 
+ * Contractor Name:        Raytheon Company
+ * Contractor Address:     6825 Pine Street, Suite 340
+ *                         Mail Stop B8
+ *                         Omaha, NE 68106
+ *                         402.291.0100
+ * 
+ * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+ * further licensing information.
+ **/
+package com.raytheon.edex.plugin.radar.util;
+
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+
+import com.raytheon.uf.common.dataplugin.text.db.OperationalStdTextProduct;
+import com.raytheon.uf.common.dataplugin.text.db.PracticeStdTextProduct;
+import com.raytheon.uf.common.dataplugin.text.db.StdTextProduct;
+import com.raytheon.uf.common.status.IUFStatusHandler;
+import com.raytheon.uf.common.status.UFStatus;
+import com.raytheon.uf.common.status.UFStatus.Priority;
+import com.raytheon.uf.edex.core.EDEXUtil;
+import com.raytheon.uf.edex.wmo.message.AFOSProductId;
+import com.raytheon.uf.edex.wmo.message.WMOHeader;
+
+/**
+ * TODO Add Description
+ * 
+ * <pre>
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * May 18, 2012            dgilling     Initial creation
+ * 
+ * </pre>
+ * 
+ * @author dgilling
+ * @version 1.0
+ */
+
+public class RadarEdexTextProductUtil {
+
+    private static final transient IUFStatusHandler theHandler = UFStatus
+            .getHandler(RadarEdexTextProductUtil.class);
+
+    private static final String TEXT_ENDPOINT = "textDirectDecodedIngestRoute";
+
+    /**
+     * A private constructor so that Java does not attempt to create one for us.
+     * As this class should not be instantiated, do not attempt to ever call
+     * this constructor; it will simply throw an AssertionError.
+     * 
+     */
+    private RadarEdexTextProductUtil() {
+        throw new AssertionError();
+    }
+
+    public static void storeTextProduct(AFOSProductId afosId, WMOHeader wmoId,
+            String text, boolean operationalMode, Calendar cal) {
+        SimpleDateFormat sdf = new SimpleDateFormat("MMM dd yyyy HH:mm:ss");
+        String temp = "";
+        if (wmoId != null && wmoId.isValid()) {
+            temp = sdf.format(wmoId.getHeaderDate().getTime());
+        } else {
+            temp = sdf.format(cal.getTime());
+        }
+        if (!text.contains("Message Date:")) {
+            text = "Message Date: " + temp + "\n\n" + text;
+        }
+        text = text.replaceAll("Page \\d+", "");
+        boolean isValid = wmoId.isValid();
+        try {
+            if (afosId != null && text != null && !afosId.toString().isEmpty()
+                    && !text.isEmpty()) {
+                StdTextProduct textProd = (operationalMode == true ? new OperationalStdTextProduct()
+                        : new PracticeStdTextProduct());
+                if (!isValid) {
+                    textProd.setWmoid(" ");
+                    textProd.setSite(" ");
+                    textProd.setBbbid(" ");
+                    textProd.setHdrtime(" ");
+                } else {
+                    textProd.setWmoid(wmoId.getTtaaii());
+                    textProd.setSite(wmoId.getCccc());
+                    textProd.setHdrtime(wmoId.getYYGGgg());
+                    textProd.setBbbid(wmoId.getBBBIndicator());
+                }
+                textProd.setRefTime(cal.getTimeInMillis());
+                textProd.setCccid(afosId.getCcc());
+                textProd.setNnnid(afosId.getNnn());
+                textProd.setXxxid(afosId.getXxx());
+                textProd.setProduct(text);
+                EDEXUtil.getMessageProducer()
+                        .sendAsync(TEXT_ENDPOINT, textProd);
+            }
+        } catch (Exception e) {
+            theHandler.handle(Priority.ERROR,
+                    "Unable to store product to text database", e);
+        }
+    }
+}

edexOsgi/com.raytheon.edex.plugin.redbook/src/com/raytheon/edex/plugin/redbook/RedbookDecoder.java

@@ -19,19 +19,18 @@
  **/
 package com.raytheon.edex.plugin.redbook;
 
-
 import java.util.ArrayList;
 import java.util.Date;
 
 import com.raytheon.edex.esb.Headers;
 import com.raytheon.edex.exception.DecoderException;
 import com.raytheon.edex.plugin.AbstractDecoder;
-import com.raytheon.uf.edex.database.plugin.PluginFactory;
 import com.raytheon.edex.plugin.redbook.common.RedbookRecord;
 import com.raytheon.edex.plugin.redbook.dao.RedbookDao;
 import com.raytheon.edex.plugin.redbook.decoder.RedbookParser;
 import com.raytheon.uf.common.dataplugin.PluginDataObject;
 import com.raytheon.uf.common.dataplugin.PluginException;
+import com.raytheon.uf.edex.database.plugin.PluginFactory;
 import com.raytheon.uf.edex.wmo.message.WMOHeader;
 
 /**
@@ -48,6 +47,8 @@ import com.raytheon.uf.edex.wmo.message.WMOHeader;
  * 20080529           1131 jkorman     Added new Separator constructor.
  * 11/11/08           1684 chammack    Refactored to camel
  * 20090327           2019 jkorman     Added code to check for non-redbook data.
+ * 20120524           #647 dgilling    Update persistence time in 
+ *                                     createdBackDatedVersionIfNeeded.
  * </pre>
  * 
  * @author jkorman
@@ -157,12 +158,15 @@ public class RedbookDecoder extends AbstractDecoder {
 
         return reports;
     }
-        private PluginDataObject[] createMergedRecordList(RedbookRecord report) throws PluginException {
+
+    private PluginDataObject[] createMergedRecordList(RedbookRecord report)
+            throws PluginException {
         ArrayList<PluginDataObject> listResult = new ArrayList<PluginDataObject>();
 
         RedbookRecord newRecord = report;
         while (newRecord != null) {
-             // Note that this newRecord may be modified by createdBackDatedVersionIfNeeded.
+            // Note that this newRecord may be modified by
+            // createdBackDatedVersionIfNeeded.
             listResult.add(newRecord);
 
             newRecord = createdBackDatedVersionIfNeeded(newRecord);
@@ -175,34 +179,42 @@ public class RedbookDecoder extends AbstractDecoder {
         RedbookRecord existingRecord;
 
         try {
-             dao = (RedbookDao) PluginFactory.getInstance().getPluginDao(PLUGIN_NAME);
+            dao = (RedbookDao) PluginFactory.getInstance().getPluginDao(
-             existingRecord = (RedbookRecord) dao.getMetadata(record.getDataURI());
+                    PLUGIN_NAME);
+            existingRecord = (RedbookRecord) dao.getMetadata(record
+                    .getDataURI());
         } catch (PluginException e) {
-             logger.error(traceId + "Could not create back-dated copy of " + 
+            logger.error(traceId + "Could not create back-dated copy of "
-                     record.getDataURI(), e);
+                    + record.getDataURI(), e);
             return null;
         }
 
         if (existingRecord != null) {
             try {
-                 existingRecord = dao.getFullRecord((RedbookRecord) existingRecord);
+                existingRecord = dao.getFullRecord(existingRecord);
             } catch (Exception e) {
-                 logger.error(traceId + "Could not retrieve existing " + 
+                logger.error(
-                         record.getDataURI(), e);
+                        traceId + "Could not retrieve existing "
+                                + record.getDataURI(), e);
                 return null;
             }
             RedbookRecord backDatedRecord;
             try {
                 backDatedRecord = existingRecord.createBackdatedVersion();
+                // this must be updated so that the insert time is updated
+                // and the Wes2Bridge archiver properly finds these backdated
+                // records
+                backDatedRecord.setPersistenceTime(new Date());
                 backDatedRecord.setPluginName(PLUGIN_NAME);
                 backDatedRecord.constructDataURI();
             } catch (PluginException e) {
-                 logger.error(traceId + "Could not create back-dated copy of " + 
+                logger.error(traceId + "Could not create back-dated copy of "
-                         record.getDataURI(), e);
+                        + record.getDataURI(), e);
                 return null;
             }
             record.setOverwriteAllowed(true);
-             dao.delete(existingRecord); // replace op does not update metadata ?!
+            dao.delete(existingRecord); // replace op does not update metadata
+                                        // ?!
             logger.info("Storing new version of " + record.getDataURI());
 
             return backDatedRecord;

edexOsgi/com.raytheon.edex.plugin.redbook/src/com/raytheon/edex/plugin/redbook/common/RedbookRecord.java

@@ -21,7 +21,6 @@ package com.raytheon.edex.plugin.redbook.common;
 
 import java.util.Calendar;
 import java.util.Date;
-import java.util.TimeZone;
 
 import javax.persistence.Column;
 import javax.persistence.Entity;
@@ -37,7 +36,7 @@ import com.raytheon.uf.common.dataplugin.IDecoderGettable;
 import com.raytheon.uf.common.dataplugin.PluginException;
 import com.raytheon.uf.common.dataplugin.annotations.DataURI;
 import com.raytheon.uf.common.dataplugin.persist.IPersistable;
-import com.raytheon.uf.common.dataplugin.persist.PersistablePluginDataObject;
+import com.raytheon.uf.common.dataplugin.persist.ServerSpecificPersistablePluginDataObject;
 import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
 import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
 import com.raytheon.uf.common.time.DataTime;
@@ -67,8 +66,8 @@ import com.raytheon.uf.common.time.DataTime;
 @XmlRootElement
 @XmlAccessorType(XmlAccessType.NONE)
 @DynamicSerialize
-public class RedbookRecord extends PersistablePluginDataObject implements
+public class RedbookRecord extends ServerSpecificPersistablePluginDataObject
-        IPersistable, Cloneable {
+        implements IPersistable, Cloneable {
 
     private static final long serialVersionUID = 1L;
 
@@ -322,30 +321,6 @@ public class RedbookRecord extends PersistablePluginDataObject implements
         return null;
     }
 
-    /**
-     * 
-     */
-    @Override
-    public Date getPersistenceTime() {
-        Calendar c = getInsertTime();
-        if (c == null)
-            return null;
-
-        return c.getTime();
-    }
-
-    /**
-     * Set the time to be used for the persistence time for this object.
-     * 
-     * @param persistTime
-     *            The persistence time to be used.
-     */
-    public void setPersistenceTime(Date persistTime) {
-        Calendar c = Calendar.getInstance(TimeZone.getTimeZone("GMT"));
-        c.setTime(persistTime);
-        setInsertTime(c);
-    }
-
     /**
      * @return a mostly shallow copy of the RedbookRecord with the reference
      *         time set back one minute. Clears the id and dataURI fields of the
@@ -362,10 +337,11 @@ public class RedbookRecord extends PersistablePluginDataObject implements
         other.dataURI = null;
 
         Date newRefTime = new Date(dataTime.getRefTime().getTime() - 60 * 1000);
-        if (dataTime.getUtilityFlags().contains(DataTime.FLAG.FCST_USED))
+        if (dataTime.getUtilityFlags().contains(DataTime.FLAG.FCST_USED)) {
             other.dataTime = new DataTime(newRefTime, dataTime.getFcstTime());
-        else
+        } else {
             other.dataTime = new DataTime(newRefTime);
+        }
 
         return other;
     }

edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/ShefDecoder.java

@@ -19,7 +19,6 @@
  **/
 package com.raytheon.edex.plugin.shef;
 
-import java.util.Calendar;
 import java.util.Date;
 
 import org.apache.commons.logging.Log;
@@ -34,7 +33,6 @@ import com.raytheon.edex.plugin.shef.database.PurgeText;
 import com.raytheon.uf.common.dataplugin.PluginDataObject;
 import com.raytheon.uf.common.ohd.AppsDefaults;
 import com.raytheon.uf.edex.decodertools.core.DecoderTools;
-import com.raytheon.uf.edex.wmo.message.WMOHeader;
 
 /**
  * Decoder implementation for SHEF data
@@ -91,6 +89,8 @@ public class ShefDecoder {
      * @return
      */
     public PluginDataObject[] decode(byte[] data, Headers headers) {
+        boolean archiveMode = AppsDefaults.getInstance().getBoolean("ALLOW_ARCHIVE_DATA",false);
+        
         String traceId = null;
 
         if (headers != null) {
@@ -108,8 +108,15 @@ public class ShefDecoder {
             separator = null;
         }
         if (separator != null) {
+            
             long startTime = System.currentTimeMillis();
-            Date postDate = getPostTime(startTime);
+
+            Date postDate = null;
+            if(archiveMode) {
+                postDate = getPostTime(separator.getWmoHeader().getHeaderDate().getTimeInMillis());
+            } else {
+                postDate = getPostTime(startTime);
+            }
 
             PostShef postShef = new PostShef(postDate);
             if(separator.hasNext()) {
@@ -181,10 +188,6 @@ public class ShefDecoder {
     private void doDecode(ShefSeparator separator, String traceId, PostShef postShef) {
         
         long startTime = System.currentTimeMillis();
-        long endTime;
-        
-        // Force time to nearest second.
-        long t = startTime - (startTime % 1000);
 
         AppsDefaults appDefaults = AppsDefaults.getInstance();
         boolean logSHEFOut = appDefaults.getBoolean("shef_out", false);
@@ -226,8 +229,7 @@ public class ShefDecoder {
             }
         } // while()
         if(dataProcessed) {
-            endTime = System.currentTimeMillis();
+            postShef.logStats(traceId, System.currentTimeMillis() - startTime);
-            postShef.logStats(traceId, endTime - startTime);
         }
     }
     
@@ -236,14 +238,15 @@ public class ShefDecoder {
      * @param startTime
      * @return
      */
-    private Date getPostTime(long startTime) {
+    private static Date getPostTime(long startTime) {
         // Force time to nearest second.
-        long t = startTime - (startTime % 1000);
+        return new Date(startTime - (startTime % 1000));
-        return new Date(t);
     }
     
     
-    
+    /*
+     * 
+     */
     public static final void main(String [] args) {
         
         long t = System.currentTimeMillis();

edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/ShefSeparator.java

@@ -152,6 +152,7 @@ public class ShefSeparator extends AbstractRecordSeparator {
                     traceId = wmoHeader.getWmoHeader();
                     log.info("TraceId set to WMOHeader = " + traceId);
                 }
+                // TODO: DR 14 - Shef time changes.
                 Calendar c = wmoHeader.getHeaderDate();
                 if (c != null) {
                     productTime = c.getTime();

edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/dao/ShefDao.java

@@ -27,15 +27,13 @@ import com.raytheon.uf.common.ohd.AppsDefaults;
 
 public class ShefDao extends DefaultPluginDao {
 
-
+    /**
-    private AppsDefaults appsDefaults;
+     * 
-
+     * @param pluginName
-
+     * @throws PluginException
-
+     */
-
     public ShefDao(String pluginName) throws PluginException {
         super(pluginName);
-        appsDefaults = AppsDefaults.getInstance();
     }
 
     @Override
@@ -50,7 +48,7 @@ public class ShefDao extends DefaultPluginDao {
          * Purge the file system
          */
         logger.info("Purging Hydro file system...");
-        File metarInputFile = new File(appsDefaults
+        File metarInputFile = new File(AppsDefaults.getInstance()
                 .getToken("whfs_local_data_dir")
                 + "/metar_input");
         if (!metarInputFile.exists()) {

edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/database/PostShef.java

@@ -25,7 +25,6 @@ import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Date;
-import java.util.GregorianCalendar;
 import java.util.HashMap;
 import java.util.List;
 import java.util.TimeZone;
@@ -37,6 +36,12 @@ import org.apache.commons.logging.LogFactory;
 
 import com.raytheon.edex.plugin.shef.data.ShefData;
 import com.raytheon.edex.plugin.shef.data.ShefRecord;
+import com.raytheon.edex.plugin.shef.util.BitUtils;
+import com.raytheon.edex.plugin.shef.util.SHEFDate;
+import com.raytheon.edex.plugin.shef.util.ShefStats;
+import com.raytheon.edex.plugin.shef.util.ShefUtil;
+import com.raytheon.edex.plugin.shef.util.StoreDisposition;
+import com.raytheon.uf.common.dataplugin.persist.PersistableDataObject;
 import com.raytheon.uf.common.dataplugin.shef.tables.Alertalarmval;
 import com.raytheon.uf.common.dataplugin.shef.tables.AlertalarmvalId;
 import com.raytheon.uf.common.dataplugin.shef.tables.Arealfcst;
@@ -61,24 +66,17 @@ import com.raytheon.uf.common.dataplugin.shef.tables.Stnclass;
 import com.raytheon.uf.common.dataplugin.shef.tables.Unkstn;
 import com.raytheon.uf.common.dataplugin.shef.tables.Unkstnvalue;
 import com.raytheon.uf.common.dataplugin.shef.tables.UnkstnvalueId;
-
+import com.raytheon.uf.common.dataplugin.shef.util.ParameterCode;
-import com.raytheon.edex.plugin.shef.util.BitUtils;
 import com.raytheon.uf.common.dataplugin.shef.util.ParameterCode.DataType;
 import com.raytheon.uf.common.dataplugin.shef.util.ParameterCode.Duration;
 import com.raytheon.uf.common.dataplugin.shef.util.ParameterCode.Extremum;
 import com.raytheon.uf.common.dataplugin.shef.util.ParameterCode.PhysicalElement;
 import com.raytheon.uf.common.dataplugin.shef.util.ParameterCode.PhysicalElementCategory;
 import com.raytheon.uf.common.dataplugin.shef.util.ParameterCode.TypeSource;
-import com.raytheon.uf.common.dataplugin.shef.util.ParameterCode;
 import com.raytheon.uf.common.dataplugin.shef.util.SHEFTimezone;
-import com.raytheon.edex.plugin.shef.util.SHEFDate;
-import com.raytheon.edex.plugin.shef.util.ShefStats;
-import com.raytheon.edex.plugin.shef.util.ShefUtil;
-import com.raytheon.edex.plugin.shef.util.StoreDisposition;
-import com.raytheon.uf.common.dataplugin.persist.PersistableDataObject;
 import com.raytheon.uf.common.dataplugin.shef.util.ShefConstants;
-import com.raytheon.uf.common.dataplugin.shef.util.ShefQC;
 import com.raytheon.uf.common.dataplugin.shef.util.ShefConstants.IngestSwitch;
+import com.raytheon.uf.common.dataplugin.shef.util.ShefQC;
 import com.raytheon.uf.common.ohd.AppsDefaults;
 import com.raytheon.uf.edex.database.dao.CoreDao;
 import com.raytheon.uf.edex.database.dao.DaoConfig;
@@ -130,6 +128,11 @@ public class PostShef {
         QC_DEFAULT, QC_GROSSRANGE_FAILED, QC_REASONRANGE_FAILED, QC_ROC_FAILED, QC_ROC_PASSED, QC_OUTLIER_FAILED, QC_OUTLIER_PASSED, QC_SCC_FAILED, QC_SCC_PASSED, QC_MSC_FAILED, QC_MSC_PASSED, QC_EXTERN_FAILED, QC_EXTERN_QUEST, QC_MANUAL_PASSED, QC_MANUAL_QUEST, QC_MANUAL_FAILED, QC_MANUAL_NEW, QC_PASSED, QC_QUESTIONABLE, QC_FAILED, QC_NOT_PASSED, QC_NOT_FAILED
     };
 
+    private static final SimpleDateFormat DB_TIMESTAMP = new SimpleDateFormat(ShefConstants.POSTGRES_DATE_FORMAT.toPattern());
+    static {
+        DB_TIMESTAMP.setTimeZone(TimeZone.getTimeZone(ShefConstants.GMT));
+    }
+    
     private static final Pattern Q_CODES = Pattern.compile("Q[^BEF]");
     
     private static final String POST_START_MSG = "Posting process started for LID [%s] PEDTSEP [%s] value [%s]";
@@ -203,6 +206,8 @@ public class PostShef {
 
     private boolean perfLog = false;
 
+    private boolean archiveMode = false;
+    
     private HashMap<String, Location> idLocations = new HashMap<String, Location>();
 
     /**
@@ -256,6 +261,8 @@ public class PostShef {
         dataLog = appDefaults.getBoolean(ShefConstants.SHEF_DATA_LOG, false);
         // TODO need to implement this token and the performance logging
         perfLog = appDefaults.getBoolean(ShefConstants.SHEF_PERFLOG, false);
+        
+        archiveMode = appDefaults.getBoolean("ALLOW_ARCHIVE_DATA",false);
     }
 
     /**
@@ -648,7 +655,7 @@ public class PostShef {
                 start = System.currentTimeMillis();
                 // Identifier has been set from the awipsHeader.
 
-                postProductLink(locId, identifier, obsTime, postDate);
+                postProductLink(locId, identifier, obsTime);
                 // postProductLink(locId, shefRecord.getIdentifier(), obsTime);
                 stats.addElapsedTimeIngest(System.currentTimeMillis() - start);
                 
@@ -1108,9 +1115,13 @@ public class PostShef {
         String pe = null;
         String ts = null;
 
-        String query = "select lid,pe,ts " + "from " + tableName + " "
+//      String query = "select lid,pe,ts " + "from " + tableName + " "
-                + "where validtime > CURRENT_TIMESTAMP and "
+//      + "where validtime > CURRENT_TIMESTAMP and "
-                + "probability < 0.0";
+//      + "probability < 0.0";
+        
+        String query = String
+                .format("select lid,pe,ts from %s where validtime > '%s' and probability < 0.0",
+                        tableName, toTimeStamp(postDate));
 
         try {
             dao = new CoreDao(DaoConfig.forDatabase(ShefConstants.IHFS));
@@ -2549,8 +2560,7 @@ public class PostShef {
      * @param obsTime
      *            - The observation time
      */
-    private void postProductLink(String locId, String productId, Date obsTime,
+    private void postProductLink(String locId, String productId, Date obsTime) {
-            Date postDate) {
         if (log.isDebugEnabled()) {
             log.debug("PostShef.postProductLink() called...");
         }
@@ -3004,8 +3014,6 @@ public class PostShef {
                 basisTime = new Date();
             }
 
-            // TODO fix the duplicate code below:
-
             if (dataValue == "") {
                 dataValue = "-9999";
             }
@@ -3247,6 +3255,29 @@ public class PostShef {
         return dataObj;
     }
 
+    
+    /**
+     * 
+     * @param c
+     * @return
+     */
+    private static String toTimeStamp(Date d) {
+        String timeStamp = null;
+        if(d != null) {
+            timeStamp = DB_TIMESTAMP.format(d);
+        }
+        return timeStamp;
+    }
+
+    /**
+     * 
+     * @param c
+     * @return
+     */
+    private static String toTimeStamp(Calendar c) {
+        return toTimeStamp(c.getTime());
+    }
+    
     public static final void main(String[] args) {
 
         Calendar postDate = TimeTools.getBaseCalendar(2011, 1, 12);

edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/database/PostTables.java

@@ -28,7 +28,8 @@ import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.hibernate.Session;
+import org.hibernate.connection.ConnectionProvider;
+import org.hibernate.engine.SessionFactoryImplementor;
 
 import com.raytheon.edex.plugin.shef.data.ShefData;
 import com.raytheon.edex.plugin.shef.data.ShefRecord;
@@ -1037,15 +1038,16 @@ public class PostTables {
         CoreDao dao = null;
         Connection conn = null;
         CallableStatement cs = null;
-        Session ses = null;
         int status = -1;
         if (dataValue == "") {
             dataValue = ShefConstants.SHEF_MISSING;
         }
         try {
             dao = new CoreDao(DaoConfig.forDatabase(ShefConstants.IHFS));
-            ses = dao.getSessionFactory().openSession();
+            SessionFactoryImplementor impl = (SessionFactoryImplementor) dao.getSessionFactory();
-            conn = ses.connection();
+            ConnectionProvider cp = impl.getConnectionProvider();
+            conn = cp.getConnection();
+
             cs = conn.prepareCall("{call " + functionName
                     + "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)}");
             cs.setString(1, locId);
@@ -1129,14 +1131,8 @@ public class PostTables {
             } catch (Exception e) {
                 // Intentionally empty
             }
-            try {
-                ses.close();
-            } catch (Exception e) {
-                // Intentionally empty
-            }
             cs = null;
             conn = null;
-            ses = null;
         }
         return status;
     }
@@ -1160,7 +1156,6 @@ public class PostTables {
         }
         CoreDao dao = null;
         Connection conn = null;
-        Session ses = null;
         CallableStatement cs = null;
         int status = -1;
         if (dataValue == "") {
@@ -1168,8 +1163,10 @@ public class PostTables {
         }
         try {
             dao = new CoreDao(DaoConfig.forDatabase(ShefConstants.IHFS));
-            ses = dao.getSessionFactory().openSession();
+            SessionFactoryImplementor impl = (SessionFactoryImplementor) dao.getSessionFactory();
-            conn = ses.connection();
+            ConnectionProvider cp = impl.getConnectionProvider();
+            conn = cp.getConnection();
+
             cs = conn.prepareCall("{call " + functionName
                     + "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)}");
             cs.setString(1, locId);
@@ -1259,11 +1256,6 @@ public class PostTables {
             } catch (Exception e) {
                 // Intentionally empty
             }
-            try {
-                ses.close();
-            } catch (Exception e) {
-                // Intentionally empty
-            }
             cs = null;
             conn = null;
         }
@@ -1287,7 +1279,6 @@ public class PostTables {
         long start = System.currentTimeMillis();
         CoreDao dao = null;
         Connection conn = null;
-        Session ses = null;
         CallableStatement cs = null;
         java.sql.Timestamp timeStamp = null;
         int status = -1;
@@ -1297,8 +1288,10 @@ public class PostTables {
 
         try {
             dao = new CoreDao(DaoConfig.forDatabase(ShefConstants.IHFS));
-            ses = dao.getSessionFactory().openSession();
+            SessionFactoryImplementor impl = (SessionFactoryImplementor) dao.getSessionFactory();
-            conn = ses.connection();
+            ConnectionProvider cp = impl.getConnectionProvider();
+            conn = cp.getConnection();
+            
             cs = conn.prepareCall("{call " + functionName
                     + "(?, ?, ?, ?, ?, cast(? as real), ?, ?, ?, ?,"
                     + " ?, ?, ?, ?, ?, ?, ?)}");
@@ -1421,11 +1414,6 @@ public class PostTables {
             } catch (Exception e) {
                 // Intentionally empty
             }
-            try {
-                ses.close();
-            } catch (Exception e) {
-                // Intentionally empty
-            }
             cs = null;
             conn = null;
         }
@@ -1490,7 +1478,6 @@ public class PostTables {
 
         CoreDao dao = null;
         Connection conn = null;
-        Session ses = null;
         PreparedStatement ps = null;
         java.sql.Timestamp timeStamp = null;
         java.sql.Timestamp timeStamp2 = null;
@@ -1508,8 +1495,12 @@ public class PostTables {
 
         try {
             dao = new CoreDao(DaoConfig.forDatabase(ShefConstants.IHFS));
-            ses = dao.getSessionFactory().openSession();
+
-            conn = ses.connection();
+            SessionFactoryImplementor impl = (SessionFactoryImplementor) dao.getSessionFactory();
+            ConnectionProvider cp = impl.getConnectionProvider();
+
+            conn = cp.getConnection();
+            
             if (updateFlag) {
                 ps = conn.prepareCall(updateQuery);
             } else {
@@ -1618,11 +1609,6 @@ public class PostTables {
             } catch (Exception e) {
                 // Intentionally empty
             }
-            try {
-                ses.close();
-            } catch (Exception e) {
-                // Intentionally empty
-            }
             ps = null;
             conn = null;
         }

edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/ohdlib/GagePP.java

@@ -89,8 +89,6 @@ public class GagePP {
 
     private static Log logger = LogFactory.getLog(GagePP.class);
 
-    private SimpleDateFormat minSDF = new SimpleDateFormat("mm");
-
     private GagePPWrite gagePPWrite = null;
 
     private char pOffsetCode;
@@ -100,10 +98,6 @@ public class GagePP {
     private short p6HourSlot;
     
     public GagePP() {
-        // used_value_count = 0;
-        // ignored_value_count = 0;
-        // // total_value_count = 0;
-        // group_count = 0;
     }
 
     /**
@@ -135,7 +129,6 @@ public class GagePP {
         // double elapsed_time;
 
         int prev_dur = 0;
-        // String prev_key = null;
         int use_value;
         short precip_val;
         short hour = 1;
@@ -152,10 +145,7 @@ public class GagePP {
         
         logger.info("Processing records at " + btime);
         
-        
+        Calendar dt = TimeTools.newCalendar(rec.getObsTime().getTime());
-
-        Calendar dt = Calendar.getInstance(SHEFTimezone.GMT_TIMEZONE);
-        dt.setTime(rec.getObsTime());
 
         HourlyppId id = new HourlyppId(rec.getLocationId(), rec.getTypeSource()
                 .getCode(), btime);
@@ -545,8 +535,8 @@ public class GagePP {
     private void gage_pp_1hour_slot(final GagePPOptions pOptions,
             final String pPE, PrecipRecord rec) {
         Calendar dt = Calendar.getInstance(SHEFTimezone.GMT_TIMEZONE);
-        Date datetime = rec.getObsTime();
+        dt.setTime(rec.getObsTime());
-        dt.setTime(datetime);
+
         int hour = dt.get(Calendar.HOUR_OF_DAY);
 
         int minute = dt.get(Calendar.MINUTE);
@@ -656,34 +646,30 @@ public class GagePP {
             final Calendar ansi_obstime_year_sec, String ansi_date_year_day,
             short [] p6HourSlot, char[] p6HourOffsetCode) {
 
-        float ppq_window;
+        float ppq_window = pOptions.getIntppq();
+        ppq_window *= SECONDS_PER_HOUR;
+        
         int bottom_6_hour_period;
         int diff1;
         int diff2;
-        int hour;
+        int hour = ansi_obstime_year_sec.get(Calendar.HOUR_OF_DAY);
-        int minute;
+        int minute = ansi_obstime_year_sec.get(Calendar.MINUTE);
         int num_periods;
         int remainder;
         int top_6_hour_period;
-        Calendar num_seconds_since_00z = Calendar
+//        Calendar num_seconds_since_00z = Calendar
-                .getInstance(SHEFTimezone.GMT_TIMEZONE);
+//                .getInstance(SHEFTimezone.GMT_TIMEZONE);
 
-        ppq_window = pOptions.getIntppq();
+        int num_seconds_since_00z = (hour * SECONDS_PER_HOUR)
-        ppq_window *= SECONDS_PER_HOUR;
+                + (minute * SECONDS_PER_MINUTE);
 
-
+        bottom_6_hour_period = num_seconds_since_00z / SECONDS_IN_6HOUR_PERIOD;
-        hour = ansi_obstime_year_sec.get(Calendar.HOUR_OF_DAY);
-        minute = ansi_obstime_year_sec.get(Calendar.MINUTE);
-
-        num_seconds_since_00z.setTimeInMillis((hour * SECONDS_PER_HOUR)
-                + (minute * SECONDS_PER_MINUTE));
-
-        bottom_6_hour_period = (int) (num_seconds_since_00z.getTimeInMillis() / SECONDS_IN_6HOUR_PERIOD);
         top_6_hour_period = bottom_6_hour_period + 1;
 
-        diff1 = (int) (num_seconds_since_00z.getTimeInMillis() - (bottom_6_hour_period * SECONDS_IN_6HOUR_PERIOD));
+        diff1 = num_seconds_since_00z
+                - (bottom_6_hour_period * SECONDS_IN_6HOUR_PERIOD);
         diff2 = (top_6_hour_period * SECONDS_IN_6HOUR_PERIOD)
-                - (int) num_seconds_since_00z.getTimeInMillis();
+                - num_seconds_since_00z;
 
         if (diff1 < diff2) {
             /*

edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/ohdlib/GagePPWrite.java

@@ -28,6 +28,8 @@ import java.util.List;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
+import com.raytheon.edex.plugin.shef.ohdlib.GagePPOptions.upd_action;
+import com.raytheon.uf.common.dataplugin.persist.PersistableDataObject;
 import com.raytheon.uf.common.dataplugin.shef.tables.Dailypp;
 import com.raytheon.uf.common.dataplugin.shef.tables.DailyppId;
 import com.raytheon.uf.common.dataplugin.shef.tables.Hourlypc;
@@ -35,11 +37,8 @@ import com.raytheon.uf.common.dataplugin.shef.tables.HourlypcId;
 import com.raytheon.uf.common.dataplugin.shef.tables.Hourlypp;
 import com.raytheon.uf.common.dataplugin.shef.tables.HourlyppId;
 import com.raytheon.uf.common.dataplugin.shef.tables.IHourlyTS;
-import com.raytheon.edex.plugin.shef.ohdlib.GagePPOptions.upd_action;
-import com.raytheon.uf.common.dataplugin.persist.PersistableDataObject;
 import com.raytheon.uf.common.dataplugin.shef.util.SHEFTimezone;
 import com.raytheon.uf.common.dataplugin.shef.util.ShefConstants;
-import com.raytheon.uf.common.dataplugin.shef.util.ShefQC;
 import com.raytheon.uf.edex.database.dao.CoreDao;
 import com.raytheon.uf.edex.database.dao.DaoConfig;
 import com.raytheon.uf.edex.decodertools.time.TimeTools;
@@ -63,10 +62,6 @@ import com.raytheon.uf.edex.decodertools.time.TimeTools;
 public final class GagePPWrite {
     private static Log logger = LogFactory.getLog(GagePPWrite.class);
     
-    private static final float MISSING_PRECIP = -9999f;
-
-    private Date datetime;
-
     private char hourly_qc[] = new char[PrecipUtils.NUM_HOURLY_SLOTS];
 
     private char minute_offset[] = new char[PrecipUtils.NUM_HOURLY_SLOTS];
@@ -81,8 +76,6 @@ public final class GagePPWrite {
      * Empty constructor.
      */
     public GagePPWrite() {
-        
-        
         // sdf = new SimpleDateFormat(ShefConstants.POSTGRES_DATE_STRING);
         sdf = new SimpleDateFormat("yyyy-MM-dd");
         sdf.setTimeZone(SHEFTimezone.GMT_TIMEZONE);
@@ -106,9 +99,7 @@ public final class GagePPWrite {
             char manual_qc_code) {
 
 
-        datetime = dtime;
+        Calendar dt = TimeTools.newCalendar(dtime.getTime());
-        Calendar dt = TimeTools.getSystemCalendar();
-        dt.setTime(datetime);
 
         int hour_slot = dt.get(Calendar.HOUR_OF_DAY);
         Arrays.fill(hourly_qc, '-');

edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/ohdlib/PrecipUtils.java

@@ -45,8 +45,6 @@ public abstract class PrecipUtils {
 
     private static final char ZERO_OFFSET = '0';
     
-    private static final AppsDefaults APPS_DEFAULTS = AppsDefaults.getInstance();
-
     public static final int NUM_HOURLY_SLOTS = 24;
 
     public static final int NUM_6HOURLY_SLOTS = 4;
@@ -71,6 +69,7 @@ public abstract class PrecipUtils {
      * @param options
      */
     public static void get_precip_window(GagePPOptions options) {
+        AppsDefaults APPS_DEFAULTS = AppsDefaults.getInstance();
         options.setIntpc(APPS_DEFAULTS.getInt("intpc",10));
         options.setIntlppp(APPS_DEFAULTS.getInt("intlppp",10));
         options.setIntuppp(APPS_DEFAULTS.getInt("intuppp",10));
@@ -109,7 +108,7 @@ public abstract class PrecipUtils {
      * @return
      */
     public static float get_6hour_precip_window() {
-        return APPS_DEFAULTS.getFloat("intppq", 2.0f);
+        return AppsDefaults.getInstance().getFloat("intppq", 2.0f);
     }
     
     /**
@@ -771,7 +770,7 @@ public abstract class PrecipUtils {
              */
 
             /* Get the search window around local 7 AM. */
-            int local_7am_window = APPS_DEFAULTS.getInt("ppp_ppd_local_7am_window", 3);
+            int local_7am_window = AppsDefaults.getInstance().getInt("ppp_ppd_local_7am_window", 3);
 
             Calendar c = new GregorianCalendar(SHEFTimezone.GMT_TIMEZONE);
             c.setTime(time12z);

edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/AbstractShefTransformer.java

@@ -35,6 +35,7 @@ import com.raytheon.uf.common.dataplugin.PluginDataObject;
 import com.raytheon.uf.common.ohd.AppsDefaults;
 import com.raytheon.uf.edex.decodertools.core.DecoderTools;
 import com.raytheon.uf.edex.decodertools.time.TimeTools;
+import com.raytheon.uf.edex.wmo.message.WMOHeader;
 
 /**
  * Base class for observation data to SHEF conversions.
@@ -142,7 +143,6 @@ public abstract class AbstractShefTransformer<T extends PluginDataObject>
      * @param objects
      * @return
      */
-    @SuppressWarnings("unchecked")
     public static Iterator<?> iterate(PluginDataObject[] objects) {
         Iterator<PluginDataObject> it = null;
         if (objects != null) {
@@ -203,11 +203,15 @@ public abstract class AbstractShefTransformer<T extends PluginDataObject>
      * @return
      */
     protected StringBuilder makeWMOHeader(StringBuilder buffer,
-            String stationId, Headers headers) {
+            String stationId, Headers headers, WMOHeader hdr) {
 
-        Calendar c = TimeTools.getSystemCalendar((String) headers
+        Calendar c = null;
-                .get(DecoderTools.INGEST_FILE_NAME));
         
+        if((hdr != null)&&(headers != null)) {
+            c = TimeTools.findDataTime(hdr.getYYGGgg(), headers);
+        } else {
+            c = TimeTools.getSystemCalendar();
+        }
         buffer.append(String.format(WMO_HEADER_FMT, stationId, c));
         
         return buffer;

edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/MetarToShefTransformer.java

@@ -116,7 +116,7 @@ public class MetarToShefTransformer extends
                     String stnId = rec.getStationId();
                     place = 2;
                     StringBuilder sb = makeWMOHeader(openWMOMessage(0, 200),
-                            stnId, headers);
+                            stnId, headers, hdr);
                     place = 3;
 
                     startMessageLine(sb);
@@ -415,5 +415,13 @@ public class MetarToShefTransformer extends
         
         System.out.println(newobs);
      
+        Calendar c = TimeTools.getBaseCalendar(2011, 11, 30);
+        c.set(Calendar.HOUR_OF_DAY, 16);
+        c.set(Calendar.MINUTE, 21);
+        
+        
+        System.out.println(String.format(WMO_HEADER_FMT, "KOMA", c));
+       
+        
     }
 }

edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/SMToShefTransformer.java

@@ -30,6 +30,7 @@ import com.raytheon.uf.common.dataplugin.sfcobs.ObsCommon;
 import com.raytheon.uf.edex.decodertools.core.DecoderTools;
 import com.raytheon.uf.edex.decodertools.core.IDecoderConstants;
 import com.raytheon.uf.edex.decodertools.time.TimeTools;
+import com.raytheon.uf.edex.wmo.message.WMOHeader;
 
 /**
  * Transforms a decoded synoptic observation into a series of SHEF encoded data
@@ -84,8 +85,10 @@ public class SMToShefTransformer extends AbstractShefTransformer<ObsCommon> {
                 // Currently returns false, so nothing is encoded at this time.
                 if (encodeThisStation(rec)) {
 
+                    WMOHeader hdr = new WMOHeader(rec.getObsText().getBytes());
+
                     StringBuilder sb = makeWMOHeader(openWMOMessage(0, 200),
-                            "KWOH", headers);
+                            "KWOH", headers, hdr);
 
                     startMessageLine(sb)
                             .append(":SHEF derived data created by SMToShefTransformer");

edexOsgi/com.raytheon.edex.plugin.text/META-INF/MANIFEST.MF

@@ -19,4 +19,5 @@ Export-Package: com.raytheon.edex.plugin.text,
 Bundle-RequiredExecutionEnvironment: JavaSE-1.6
 Import-Package: com.raytheon.uf.common.dataplugin.text,
  com.raytheon.uf.common.dataplugin.text.db,
- com.raytheon.uf.common.dataplugin.text.request
+ com.raytheon.uf.common.dataplugin.text.request,
+ com.raytheon.uf.edex.maintenance.archive

edexOsgi/com.raytheon.edex.plugin.text/res/spring/text-ingest.xml

@@ -41,6 +41,14 @@
 		<property name="maxPoolSize" value="2" />
 	</bean>
 	
+	<!-- Special handler for text plugin archives that bases filenames off 
+	     creation time-->
+    <bean id="textArchiveNamer" class="com.raytheon.edex.plugin.text.maintenance.archiver.TextArchiveFileNameFormatter" />
+	<bean factory-bean="databaseArchiver" factory-method="registerPluginArchiveFormatter" depends-on="databaseArchiver">
+		<constructor-arg value="text" />
+		<constructor-arg ref="textArchiveNamer" />
+	</bean>
+
 
 	<camelContext id="text-camel"
 		xmlns="http://camel.apache.org/schema/spring"

edexOsgi/com.raytheon.edex.plugin.text/src/com/raytheon/edex/plugin/text/TextDecoder.java

@@ -144,7 +144,7 @@ public class TextDecoder extends AbstractDecoder {
                     pdo.setProductId(textProduct.getCccid()
                             + textProduct.getNnnid() + textProduct.getXxxid());
                     DataTime dt = new DataTime(
-                            TimeTools.newCalendar(textProduct.getCreatetime()));
+                            TimeTools.newCalendar(textProduct.getRefTime()));
                     pdo.setDataTime(dt);
                 } else {
                     // throw new Exception("product already exists");

edexOsgi/com.raytheon.edex.plugin.text/src/com/raytheon/edex/plugin/text/dao/TextDao.java

@@ -20,11 +20,18 @@
 package com.raytheon.edex.plugin.text.dao;
 
 import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
 
 import com.raytheon.edex.db.dao.DefaultPluginDao;
+import com.raytheon.edex.textdb.dao.StdTextProductDao;
 import com.raytheon.edex.textdb.dbapi.impl.TextDB;
 import com.raytheon.uf.common.dataplugin.PluginException;
+import com.raytheon.uf.common.dataplugin.persist.PersistableDataObject;
+import com.raytheon.uf.common.dataquery.db.QueryParam.QueryOperand;
+import com.raytheon.uf.edex.database.DataAccessLayerException;
 import com.raytheon.uf.edex.database.purge.PurgeLogger;
+import com.raytheon.uf.edex.database.query.DatabaseQuery;
 
 /**
  * DAO for text products
@@ -57,6 +64,7 @@ public class TextDao extends DefaultPluginDao {
 		// no op
 	}
 
+    @Override
 	public void purgeExpiredData() throws PluginException {
 		int deletedRecords = 0;
 
@@ -69,4 +77,41 @@ public class TextDao extends DefaultPluginDao {
 		PurgeLogger.logInfo("Purged " + deletedRecords + " items total.",
 				"text");
     }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public List<PersistableDataObject> getRecordsToArchive(
+            Calendar insertStartTime, Calendar insertEndTime)
+            throws DataAccessLayerException {
+        StdTextProductDao dao = new StdTextProductDao(true);
+        DatabaseQuery dbQuery = new DatabaseQuery(dao.getDaoClass());
+        dbQuery.addQueryParam("insertTime", insertStartTime,
+                QueryOperand.GREATERTHANEQUALS);
+        dbQuery.addQueryParam("insertTime", insertEndTime,
+                QueryOperand.LESSTHAN);
+        dbQuery.addOrder("insertTime", true);
+
+        return (List<PersistableDataObject>) dao.queryByCriteria(dbQuery);
+    }
+
+    @Override
+    public Date getMinInsertTime(String productKey)
+            throws DataAccessLayerException {
+        StdTextProductDao dao = new StdTextProductDao(true);
+        DatabaseQuery query = new DatabaseQuery(dao.getDaoClass());
+        List<String[]> keys = this.getProductKeyParameters(productKey);
+        for (String[] key : keys) {
+            query.addQueryParam(key[0], key[1]);
+        }
+        query.addReturnedField("insertTime");
+        query.addOrder("insertTime", true);
+        query.setMaxResults(1);
+        @SuppressWarnings("unchecked")
+        List<Calendar> result = (List<Calendar>) dao.queryByCriteria(query);
+        if (result.isEmpty()) {
+            return null;
+        } else {
+            return result.get(0).getTime();
+        }
+	}
 }

edexOsgi/com.raytheon.edex.plugin.text/src/com/raytheon/edex/plugin/text/handler/StdTextProductRequestHandler.java

@@ -95,7 +95,7 @@ public class StdTextProductRequestHandler implements
         if (createtime == null) {
             createtime = new Date().getTime();
         }
-        text.setCreatetime(createtime);
+        text.setRefTime(createtime);
         text.setProduct(product);
         boolean success = dao.writeProduct(text);
 

edexOsgi/com.raytheon.edex.plugin.text/src/com/raytheon/edex/plugin/text/maintenance/archiver/TextArchiveFileNameFormatter.java

@@ -0,0 +1,126 @@
+/**
+ * This software was developed and / or modified by Raytheon Company,
+ * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+ * 
+ * U.S. EXPORT CONTROLLED TECHNICAL DATA
+ * This software product contains export-restricted data whose
+ * export/transfer/disclosure is restricted by U.S. law. Dissemination
+ * to non-U.S. persons whether in the United States or abroad requires
+ * an export license or other authorization.
+ * 
+ * Contractor Name:        Raytheon Company
+ * Contractor Address:     6825 Pine Street, Suite 340
+ *                         Mail Stop B8
+ *                         Omaha, NE 68106
+ *                         402.291.0100
+ * 
+ * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+ * further licensing information.
+ **/
+package com.raytheon.edex.plugin.text.maintenance.archiver;
+
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.raytheon.uf.common.dataplugin.persist.DefaultPathProvider;
+import com.raytheon.uf.common.dataplugin.persist.PersistableDataObject;
+import com.raytheon.uf.common.dataplugin.text.db.StdTextProduct;
+import com.raytheon.uf.common.status.IUFStatusHandler;
+import com.raytheon.uf.common.status.UFStatus;
+import com.raytheon.uf.edex.database.DataAccessLayerException;
+import com.raytheon.uf.edex.database.plugin.PluginDao;
+import com.raytheon.uf.edex.maintenance.archive.IPluginArchiveFileNameFormatter;
+
+/**
+ * TODO Add Description
+ * 
+ * <pre>
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Apr 20, 2012            dgilling     Initial creation
+ * 
+ * </pre>
+ * 
+ * @author dgilling
+ * @version 1.0
+ */
+
+public class TextArchiveFileNameFormatter implements
+        IPluginArchiveFileNameFormatter {
+
+    private static final transient IUFStatusHandler statusHandler = UFStatus
+            .getHandler(TextArchiveFileNameFormatter.class);
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see
+     * com.raytheon.uf.edex.maintenance.archive.IPluginArchiveFileNameFormatter
+     * #getPdosByFile(java.lang.String,
+     * com.raytheon.uf.edex.database.plugin.PluginDao, java.util.Map,
+     * java.util.Calendar, java.util.Calendar)
+     */
+    @Override
+    public Map<String, List<PersistableDataObject>> getPdosByFile(
+            String pluginName, PluginDao dao,
+            Map<String, List<PersistableDataObject>> pdoMap,
+            Calendar startTime, Calendar endTime)
+            throws DataAccessLayerException {
+        List<PersistableDataObject> pdos = dao.getRecordsToArchive(startTime,
+                endTime);
+
+        Set<String> newFileEntries = new HashSet<String>();
+        if (pdos != null && !pdos.isEmpty()) {
+            if (pdos.get(0) instanceof StdTextProduct) {
+                for (PersistableDataObject pdo : pdos) {
+                    StdTextProduct casted = (StdTextProduct) pdo;
+
+                    // no refTime to use, so we use creation time
+                    Date time = new Date(casted.getRefTime());
+                    String timeString = null;
+                    synchronized (DefaultPathProvider.fileNameFormat) {
+                        timeString = DefaultPathProvider.fileNameFormat
+                                .format(time);
+                    }
+                    String path = pluginName + timeString;
+
+                    newFileEntries.add(path);
+                    List<PersistableDataObject> list = pdoMap.get(path);
+                    if (list == null) {
+                        list = new ArrayList<PersistableDataObject>(pdos.size());
+                        pdoMap.put(path, list);
+                    }
+                    list.add(pdo);
+                }
+            } else {
+                statusHandler.error("Invalid PersistableDataObject class "
+                        + pdos.get(0).getClass()
+                        + "sent to TextArchiveFileNameFormatter to archive");
+            }
+        }
+
+        Iterator<String> iter = pdoMap.keySet().iterator();
+        Map<String, List<PersistableDataObject>> pdosToSave = new HashMap<String, List<PersistableDataObject>>(
+                pdoMap.size() - newFileEntries.size());
+
+        while (iter.hasNext()) {
+            String key = iter.next();
+            if (!newFileEntries.contains(key)) {
+                pdosToSave.put(key, pdoMap.get(key));
+                iter.remove();
+            }
+        }
+
+        return pdosToSave;
+    }
+}

edexOsgi/com.raytheon.edex.productsrv/res/spring/uengine-request.xml

@@ -22,10 +22,10 @@
     <property name="connectors">
       <list>
         <bean id="Connector" class="org.eclipse.jetty.server.nio.SelectChannelConnector">
-          <property name="port" value="8080"/>
+          <property name="port" value="${web.port}"/>
           <property name="maxIdleTime" value="30000"/>
           <property name="acceptors" value="2"/>
-          <property name="confidentialPort" value="8443"/>
+          <property name="confidentialPort" value="${confidential.port}"/>
         </bean>
       </list>
     </property>
@@ -61,15 +61,19 @@
   </bean>
   
     <camelContext id="uengine-camel" xmlns="http://camel.apache.org/schema/spring" errorHandlerRef="errorHandler">                                           
+        <endpoint id="uEngineHttpThrift_from"
+        	uri="jetty:http://0.0.0.0:${http.port}/services/pyproductthrift?disableStreamCache=true&amp;chunked=false&amp;httpBindingRef=#binaryHttpBinding" />
+        <endpoint id="uEngineHttpJaxb_from"
+        	uri="jetty:http://0.0.0.0:${http.port}/services/pyproductjaxb" />
         
     	<!-- UEngine routes -->
         <route id="uEngineHttpThrift">
-            <from uri="jetty:http://0.0.0.0:9581/services/pyproductthrift?disableStreamCache=true&amp;chunked=false&amp;httpBindingRef=#binaryHttpBinding" />
+            <from uri="ref:uEngineHttpThrift_from" />
             <bean ref="uEngine" method="executePython" />
             <bean ref="serializationUtil" method="transformToThrift" />
         </route>
 		<route id="uEngineHttpJaxb">		
-	    	<from uri="jetty:http://0.0.0.0:9581/services/pyproductjaxb" />
+	    	<from uri="ref:uEngineHttpJaxb_from" />
             <bean ref="uEngine" method="executePython" />
             <bean ref="serializationUtil" method="marshalToXml" />
 		</route>  

edexOsgi/com.raytheon.edex.textdb/src/com/raytheon/edex/textdb/dao/StdTextProductDao.java

@@ -106,7 +106,9 @@ public class StdTextProductDao extends CoreDao {
 
     private static final String CCC_ID = "cccid";
 
-    private static final String CREATETIME = "createtime";
+    private static final String REFTIME = "refTime";
+
+    private static final String INSERTTIME = "insertTime";
 
     private static final String ProdHDRTIME = "prodId.hdrtime";
 
@@ -128,7 +130,7 @@ public class StdTextProductDao extends CoreDao {
 
     private static final String PRACTICE_TABLE = "practicestdtextproducts";
 
-    private static final String TM_QUERY_FMT = "select createtime from table_name where cccid='%s' and nnnid='%s' and xxxid='%s';";
+    private static final String TM_QUERY_FMT = "select refTime from table_name where cccid='%s' and nnnid='%s' and xxxid='%s';";
 
     private static final String AFOS_QUERY_STMT = "from StdTextProduct prod where "
             + ProdCCC_ID
@@ -143,7 +145,7 @@ public class StdTextProductDao extends CoreDao {
             + " = :"
             + XXX_ID
             + " order by "
-            + CREATETIME + " desc";
+            + REFTIME + " desc" + ", " + INSERTTIME + " desc";
 
     private Log logger = LogFactory.getLog(getClass());
 
@@ -183,7 +185,7 @@ public class StdTextProductDao extends CoreDao {
         prodId.setXxxid(xxx);
         try {
             Query query = this.getSession().createQuery(
-                    "SELECT createtime from "
+                    "SELECT refTime from "
                             + textProduct.getClass().getSimpleName()
                             + " where prodId = :prodid");
             query.setParameter("prodid", prodId);
@@ -391,11 +393,12 @@ public class StdTextProductDao extends CoreDao {
             Criteria criteria = session
                     .createCriteria(getStdTextProductInstance().getClass());
             criteria.add(Restrictions.allEq(tmp));
-            criteria.add(Restrictions.gt(CREATETIME, new Long(searchTime)));
+            criteria.add(Restrictions.gt(REFTIME, new Long(searchTime)));
             criteria.addOrder(Order.asc(ProdCCC_ID));
             criteria.addOrder(Order.asc(ProdNNN_ID));
             criteria.addOrder(Order.asc(ProdXXX_ID));
-            criteria.addOrder(Order.desc(CREATETIME));
+            criteria.addOrder(Order.desc(REFTIME));
+            criteria.addOrder(Order.desc(INSERTTIME));
             criteria.addOrder(Order.desc(ProdHDRTIME));
 
             Iterator<?> iter = criteria.list().iterator();
@@ -467,7 +470,7 @@ public class StdTextProductDao extends CoreDao {
 
             Criteria criteria = sess.createCriteria(getStdTextProductInstance()
                     .getClass());
-            criteria.setProjection(Projections.max(CREATETIME));
+            criteria.setProjection(Projections.max(REFTIME));
             criteria.add(Restrictions.allEq(tmp));
 
             List<?> list = criteria.list();
@@ -602,8 +605,8 @@ public class StdTextProductDao extends CoreDao {
     }
 
     /**
-     * Simple purge routine. Deletes all data that has a createtime older than
+     * Simple purge routine. Deletes all data that has a refTime older than that
-     * that specified.
+     * specified.
      */
     public int versionPurge() {
         return versionPurge(null);
@@ -650,20 +653,21 @@ public class StdTextProductDao extends CoreDao {
                 String nnnid = null;
                 String xxxid = null;
 
-                String cTimeQueryString = null;
+                String refTimeQueryString = null;
                 {
-                    StringBuilder cTimeQueryBuilder = new StringBuilder(200);
+                    StringBuilder refTimeQueryBuilder = new StringBuilder(200);
-                    cTimeQueryBuilder.append("SELECT createtime FROM ");
+                    refTimeQueryBuilder.append("SELECT refTime FROM ");
-                    cTimeQueryBuilder.append(getStdTextProductInstance()
+                    refTimeQueryBuilder.append(getStdTextProductInstance()
                             .getClass().getSimpleName());
-                    cTimeQueryBuilder.append(" WHERE ");
+                    refTimeQueryBuilder.append(" WHERE ");
-                    cTimeQueryBuilder.append(ProdCCC_ID).append(" = :cccid")
+                    refTimeQueryBuilder.append(ProdCCC_ID).append(" = :cccid")
                             .append(" AND ");
-                    cTimeQueryBuilder.append(ProdNNN_ID).append(" = :nnnid")
+                    refTimeQueryBuilder.append(ProdNNN_ID).append(" = :nnnid")
                             .append(" AND ");
-                    cTimeQueryBuilder.append(ProdXXX_ID).append(" = :xxxid");
+                    refTimeQueryBuilder.append(ProdXXX_ID).append(" = :xxxid");
-                    cTimeQueryBuilder.append(" ORDER BY createtime DESC");
+                    refTimeQueryBuilder.append(" ORDER BY refTime DESC");
-                    cTimeQueryString = cTimeQueryBuilder.toString();
+                    refTimeQueryBuilder.append(", insertTime DESC");
+                    refTimeQueryString = refTimeQueryBuilder.toString();
                 }
 
                 String delQueryString = null;
@@ -679,7 +683,7 @@ public class StdTextProductDao extends CoreDao {
                             .append(" AND ");
                     delQueryBuilder.append(ProdXXX_ID).append(" = :xxxid")
                             .append(" AND ");
-                    delQueryBuilder.append("createtime < :createtime");
+                    delQueryBuilder.append("refTime < :refTime");
                     delQueryString = delQueryBuilder.toString();
                 }
 
@@ -693,29 +697,29 @@ public class StdTextProductDao extends CoreDao {
 
                     try {
                         tx = session.beginTransaction();
-                        Query cTimeQuery = session
+                        Query refTimeQuery = session
-                                .createQuery(cTimeQueryString);
+                                .createQuery(refTimeQueryString);
-                        cTimeQuery.setString("cccid", cccid);
+                        refTimeQuery.setString("cccid", cccid);
-                        cTimeQuery.setString("nnnid", nnnid);
+                        refTimeQuery.setString("nnnid", nnnid);
-                        cTimeQuery.setString("xxxid", xxxid);
+                        refTimeQuery.setString("xxxid", xxxid);
-                        cTimeQuery.setMaxResults(prodInfo.getVersionstokeep());
+                        refTimeQuery
-                        List<?> createTimes = cTimeQuery.list();
+                                .setMaxResults(prodInfo.getVersionstokeep());
-                        if (createTimes.size() >= prodInfo.getVersionstokeep()) {
+                        List<?> refTimes = refTimeQuery.list();
-                            long createTime = ((Number) createTimes
+                        if (refTimes.size() >= prodInfo.getVersionstokeep()) {
-                                    .get(prodInfo.getVersionstokeep() - 1))
+                            long refTime = ((Number) refTimes.get(prodInfo
-                                    .longValue();
+                                    .getVersionstokeep() - 1)).longValue();
                             Query delQuery = session
                                     .createQuery(delQueryString);
                             delQuery.setString("cccid", cccid);
                             delQuery.setString("nnnid", nnnid);
                             delQuery.setString("xxxid", xxxid);
-                            delQuery.setLong("createtime", createTime);
+                            delQuery.setLong("refTime", refTime);
 
                             if (PurgeLogger.isDebugEnabled()) {
                                 PurgeLogger.logDebug("Purging records for ["
                                         + cccid + nnnid + xxxid
-                                        + "] before createTime [" + createTime
+                                        + "] before refTime [" + refTime + "]",
-                                        + "]", PLUGIN_NAME);
+                                        PLUGIN_NAME);
                             }
 
                             int rowsDeleted = delQuery.executeUpdate();
@@ -876,7 +880,7 @@ public class StdTextProductDao extends CoreDao {
             ProjectionList projectionList = Projections.projectionList();
             projectionList.add(Projections.distinct(Projections
                     .property(ProdWMO_ID)));
-            projectionList.add(Projections.max(CREATETIME));
+            projectionList.add(Projections.max(REFTIME));
             projectionList.add(Projections.groupProperty(ProdWMO_ID));
             projectionList.add(Projections.groupProperty(ProdSITE));
             projectionList.add(Projections.groupProperty(ProdCCC_ID));
@@ -894,23 +898,17 @@ public class StdTextProductDao extends CoreDao {
             List<StdTextProduct> tmpProducts = new ArrayList<StdTextProduct>();
             for (int i = 0; i < products.size(); i++) {
                 StdTextProduct stdTextProduct = getStdTextProductInstance();
-                stdTextProduct
+                stdTextProduct.setWmoid((String) ((Object[]) (products
-                        .setWmoid((String) ((Object[]) ((Object[]) products
                         .toArray())[i])[0]);
-                stdTextProduct
+                stdTextProduct.setRefTime((Long) ((Object[]) (products
-                        .setCreatetime((Long) ((Object[]) ((Object[]) products
                         .toArray())[i])[1]);
                 stdTextProduct
-                        .setSite((String) ((Object[]) ((Object[]) products
+                        .setSite((String) ((Object[]) (products.toArray())[i])[3]);
-                                .toArray())[i])[3]);
+                stdTextProduct.setCccid((String) ((Object[]) (products
-                stdTextProduct
-                        .setCccid((String) ((Object[]) ((Object[]) products
                         .toArray())[i])[4]);
-                stdTextProduct
+                stdTextProduct.setNnnid((String) ((Object[]) (products
-                        .setNnnid((String) ((Object[]) ((Object[]) products
                         .toArray())[i])[5]);
-                stdTextProduct
+                stdTextProduct.setXxxid((String) ((Object[]) (products
-                        .setXxxid((String) ((Object[]) ((Object[]) products
                         .toArray())[i])[6]);
                 tmpProducts.add(stdTextProduct);
             }
@@ -949,7 +947,7 @@ public class StdTextProductDao extends CoreDao {
                 projectionList.add(Projections.distinct(Projections
                         .property(BBB_ID)));
                 projectionList.add(Projections.property(ProdWMO_ID));
-                projectionList.add(Projections.property(CREATETIME));
+                projectionList.add(Projections.property(REFTIME));
                 projectionList.add(Projections.property(ProdSITE));
                 projectionList.add(Projections.property(ProdCCC_ID));
                 projectionList.add(Projections.property(ProdNNN_ID));
@@ -961,10 +959,10 @@ public class StdTextProductDao extends CoreDao {
                         .createCriteria(getStdTextProductInstance().getClass());
                 if (readAllVersions && startTimeMillis != null) {
                     criteria.add(Restrictions.and(Restrictions.allEq(map),
-                            Restrictions.ge(CREATETIME, startTimeMillis)));
+                            Restrictions.ge(REFTIME, startTimeMillis)));
                 } else {
                     criteria.add(Restrictions.and(Restrictions.allEq(map),
-                            Restrictions.eq(CREATETIME, p.getCreatetime())));
+                            Restrictions.eq(REFTIME, p.getRefTime())));
                 }
 
                 criteria.setProjection(projectionList);
@@ -977,32 +975,23 @@ public class StdTextProductDao extends CoreDao {
                 List<StdTextProduct> tmpProducts = new ArrayList<StdTextProduct>();
                 for (int i = 0; i < list.size(); i++) {
                     StdTextProduct stdTextProduct = getStdTextProductInstance();
-                    stdTextProduct
+                    stdTextProduct.setBbbid((String) ((Object[]) (list
-                            .setBbbid((String) ((Object[]) ((Object[]) list
                             .toArray())[i])[0]);
-                    stdTextProduct
+                    stdTextProduct.setWmoid((String) ((Object[]) (list
-                            .setWmoid((String) ((Object[]) ((Object[]) list
                             .toArray())[i])[1]);
-                    stdTextProduct
+                    stdTextProduct.setRefTime((Long) ((Object[]) (list
-                            .setCreatetime((Long) ((Object[]) ((Object[]) list
                             .toArray())[i])[2]);
                     stdTextProduct
-                            .setSite((String) ((Object[]) ((Object[]) list
+                            .setSite((String) ((Object[]) (list.toArray())[i])[3]);
-                                    .toArray())[i])[3]);
+                    stdTextProduct.setCccid((String) ((Object[]) (list
-                    stdTextProduct
-                            .setCccid((String) ((Object[]) ((Object[]) list
                             .toArray())[i])[4]);
-                    stdTextProduct
+                    stdTextProduct.setNnnid((String) ((Object[]) (list
-                            .setNnnid((String) ((Object[]) ((Object[]) list
                             .toArray())[i])[5]);
-                    stdTextProduct
+                    stdTextProduct.setXxxid((String) ((Object[]) (list
-                            .setXxxid((String) ((Object[]) ((Object[]) list
                             .toArray())[i])[6]);
-                    stdTextProduct
+                    stdTextProduct.setHdrtime((String) ((Object[]) (list
-                            .setHdrtime((String) ((Object[]) ((Object[]) list
                             .toArray())[i])[7]);
-                    stdTextProduct
+                    stdTextProduct.setProduct((String) ((Object[]) (list
-                            .setProduct((String) ((Object[]) ((Object[]) list
                             .toArray())[i])[8]);
                     tmpProducts.add(stdTextProduct);
                 }
@@ -1053,11 +1042,11 @@ public class StdTextProductDao extends CoreDao {
         int version = 0;
         final String query1 = "SELECT DISTINCT site FROM stdTextProducts "
                 + "WHERE wmoId = ? ORDER BY site ASC";
-        final String query2 = "SELECT MAX(createTime) FROM stdTextProducts "
+        final String query2 = "SELECT MAX(refTime) FROM stdTextProducts "
                 + "WHERE site = ? and wmoId = ?";
         final String query3 = "SELECT cccId, nnnId, xxxId, hdrTime, bbbId, version "
                 + "FROM stdTextProducts "
-                + "WHERE site = ? AND wmoId = ? AND createTime = ? "
+                + "WHERE site = ? AND wmoId = ? AND refTime = ? "
                 + "ORDER BY cccId ASC, nnnId ASC, xxxId ASC, hdrTime DESC, bbbId DESC";
         List<String> retVal = new ArrayList<String>();
 
@@ -1077,10 +1066,10 @@ public class StdTextProductDao extends CoreDao {
                 rs2 = ps2.executeQuery();
 
                 if (rs2.next()) {
-                    int createTime = rs2.getInt("createTime");
+                    int refTime = rs2.getInt("refTime");
                     ps3.setString(1, site);
                     ps3.setString(2, wmoId);
-                    ps3.setInt(3, createTime);
+                    ps3.setInt(3, refTime);
                     rs3 = ps3.executeQuery();
 
                     while (rs3.next()) {
@@ -1158,7 +1147,7 @@ public class StdTextProductDao extends CoreDao {
      */
     public List<String> read_wh(String site, long startTimeMillis) {
         return read_wh(site,
-                (int) (Math.round(startTimeMillis / (float) MILLIS_PER_SECOND)));
+                (Math.round(startTimeMillis / (float) MILLIS_PER_SECOND)));
     }
 
     /*
@@ -1181,8 +1170,8 @@ public class StdTextProductDao extends CoreDao {
         ResultSet rs1 = null;
         String product = null;
         final String query = "SELECT product "
-                + "FROM stdTextProducts WHERE wmoId = ? AND createTime >= ? "
+                + "FROM stdTextProducts WHERE wmoId = ? AND refTime >= ? "
-                + "ORDER BY createTime DESC";
+                + "ORDER BY refTime DESC, insertTime DESC";
         List<String> retVal = new ArrayList<String>();
 
         try {
@@ -1244,11 +1233,11 @@ public class StdTextProductDao extends CoreDao {
         int version = 0;
         final String query1 = "SELECT DISTINCT wmoId "
                 + "FROM stdTextProducts WHERE site = ? " + "ORDER BY wmoId ASC";
-        final String query2 = "SELECT MAX(createTime) "
+        final String query2 = "SELECT MAX(refTime) "
                 + "FROM stdTextProducts WHERE site = ? and wmoId = ?";
         final String query3 = "SELECT cccId, nnnId, xxxId, hdrTime, bbbId, version "
                 + "FROM stdTextProducts "
-                + "WHERE site = ? AND wmoId = ? AND createTime = ? "
+                + "WHERE site = ? AND wmoId = ? AND refTime = ? "
                 + "ORDER BY cccId ASC, nnnId ASC, xxxId ASC, hdrTime DESC, bbbId DESC";
         List<String> retVal = new ArrayList<String>();
 
@@ -1268,11 +1257,11 @@ public class StdTextProductDao extends CoreDao {
                 rs2 = ps2.executeQuery();
 
                 if (rs2.next()) {
-                    int createTime = rs2.getInt("createTime");
+                    int refTime = rs2.getInt("refTime");
                     ps3 = c.prepareStatement(query3);
                     ps3.setString(1, site);
                     ps3.setString(2, wmoId);
-                    ps3.setInt(3, createTime);
+                    ps3.setInt(3, refTime);
                     rs3 = ps3.executeQuery();
 
                     while (rs3.next()) {
@@ -1349,7 +1338,7 @@ public class StdTextProductDao extends CoreDao {
      */
     public List<String> read_sh(String site, long startTimeMillis) {
         return read_sh(site,
-                (int) (Math.round(startTimeMillis / (float) MILLIS_PER_SECOND)));
+                (Math.round(startTimeMillis / (float) MILLIS_PER_SECOND)));
     }
 
     /*
@@ -1372,8 +1361,8 @@ public class StdTextProductDao extends CoreDao {
         ResultSet rs1 = null;
         String product = null;
         final String query = "SELECT product "
-                + "FROM stdTextProducts WHERE site = ? AND createTime >= ? "
+                + "FROM stdTextProducts WHERE site = ? AND refTime >= ? "
-                + "ORDER BY createTime DESC";
+                + "ORDER BY refTime DESC, insertTime DESC";
         List<String> retVal = new ArrayList<String>();
 
         try {
@@ -1441,7 +1430,7 @@ public class StdTextProductDao extends CoreDao {
             final String query2 = "SELECT cccId, hdrTime, bbbId, version "
                     + "FROM stdTextProducts "
                     + "WHERE site = ? AND wmoId = ? AND nnnId = ? AND xxxId = ? "
-                    + "ORDER BY cccId ASC, createTime DESC";
+                    + "ORDER BY cccId ASC, refTime DESC, insertTime DESC";
 
             try {
                 session = getSession();
@@ -1519,10 +1508,9 @@ public class StdTextProductDao extends CoreDao {
      * 
      * PDL: Read the version that matches the input Return
      * 
-     * Note: The MAX function was used just in case createTimes are same. Since
+     * Note: The MAX function was used just in case refTimes are same. Since the
-     * the product createTime is the UNIX time which has the unit of
+     * product refTime is the UNIX time which has the unit of microseconds, it
-     * microseconds, it is assumed there should not have two products with the
+     * is assumed there should not have two products with the same refTime.
-     * same createTime.
      * ====================================================================
      */
     public int readLatestIntr(String wmoId, String site) {
@@ -1533,12 +1521,12 @@ public class StdTextProductDao extends CoreDao {
         ResultSet rs1 = null;
         ResultSet rs2 = null;
         int version = -1;
-        final String query1 = "SELECT MAX(createTime) "
+        final String query1 = "SELECT MAX(refTime) "
                 + "FROM stdTextProducts "
                 + "WHERE wmoId = ? AND site = ? AND cccId = 'NOA' AND nnnId = 'FOS' AND xxxId = 'PIL'";
         final String query2 = "SELECT MAX(version) "
                 + "FROM stdTextProducts "
-                + "WHERE wmoId = ? AND site = ? AND cccId = 'NOA' AND nnnId = 'FOS' AND xxxId = 'PIL' AND createTime = ?";
+                + "WHERE wmoId = ? AND site = ? AND cccId = 'NOA' AND nnnId = 'FOS' AND xxxId = 'PIL' AND refTime = ?";
 
         try {
             session = getSession();
@@ -1549,11 +1537,11 @@ public class StdTextProductDao extends CoreDao {
             rs1 = ps1.executeQuery();
 
             if (rs1.next()) {
-                int createTime = rs1.getInt("createTime");
+                int refTime = rs1.getInt("refTime");
                 ps2 = c.prepareStatement(query2);
                 ps2.setString(1, site);
                 ps2.setString(2, wmoId);
-                ps2.setInt(3, createTime);
+                ps2.setInt(3, refTime);
                 rs2 = ps2.executeQuery();
 
                 if (rs2.next()) {
@@ -1586,10 +1574,9 @@ public class StdTextProductDao extends CoreDao {
      * 
      * PDL: Read the version that matches the input Return
      * 
-     * Note: The MAX function was used just in case createTimes are same. Since
+     * Note: The MAX function was used just in case refTimes are same. Since the
-     * the product createTime is the UNIX time which has the unit of
+     * product refTime is the UNIX time which has the unit of microseconds, it
-     * microseconds, it is assumed there should not have two products with the
+     * is assumed there should not have two products with the same createTim
-     * same createTim
      * ====================================================================
      */
     public int readLatest(String wmoId, String site, String cccId,
@@ -1601,12 +1588,12 @@ public class StdTextProductDao extends CoreDao {
         ResultSet rs1 = null;
         ResultSet rs2 = null;
         int version = -1;
-        final String query1 = "SELECT MAX(createTime) "
+        final String query1 = "SELECT MAX(refTime) "
                 + "FROM stdTextProducts "
                 + "WHERE wmoId = ? AND site = ? AND cccId = ? AND nnnId = ? AND xxxId = ? AND hdrTime = ?";
         final String query2 = "SELECT MAX(version) "
                 + "FROM stdTextProducts "
-                + "WHERE wmoId = ? AND site = ? AND cccId = ? AND nnnId = ? AND xxxId = ? AND hdrTime = ? AND createTime = ?";
+                + "WHERE wmoId = ? AND site = ? AND cccId = ? AND nnnId = ? AND xxxId = ? AND hdrTime = ? AND refTime = ?";
 
         try {
             session = getSession();
@@ -1621,7 +1608,7 @@ public class StdTextProductDao extends CoreDao {
             rs1 = ps1.executeQuery();
 
             if (rs1.next()) {
-                int createTime = rs1.getInt("createTime");
+                int refTime = rs1.getInt("refTime");
                 ps2 = c.prepareStatement(query2);
                 ps2.setString(1, wmoId);
                 ps2.setString(2, site);
@@ -1629,7 +1616,7 @@ public class StdTextProductDao extends CoreDao {
                 ps2.setString(4, nnnId);
                 ps2.setString(5, xxxId);
                 ps2.setString(6, hdrTime);
-                ps2.setInt(7, createTime);
+                ps2.setInt(7, refTime);
                 rs2 = ps2.executeQuery();
 
                 if (rs2.next()) {
@@ -1696,10 +1683,10 @@ public class StdTextProductDao extends CoreDao {
         final String abbrIdQuery = "SELECT DISTINCT nnnId, xxxId "
                 + "FROM stdTextProducts "
                 + "WHERE wmoId = ? AND site = ? ORDER BY nnnId ASC, xxxId ASC";
-        final String hdrQuery = "SELECT DISTINCT cccId, hdrTime, bbbId, version, createTime "
+        final String hdrQuery = "SELECT DISTINCT cccId, hdrTime, bbbId, version, refTime "
                 + "FROM stdTextProducts "
                 + "WHERE wmoId = ? AND site = ? AND nnnId = ? AND xxxId = ? "
-                + "ORDER by cccId ASC, createTime DESC";
+                + "ORDER by cccId ASC, refTime DESC, insertTime DESC";
 
         try {
             session = getSession();
@@ -1767,10 +1754,12 @@ public class StdTextProductDao extends CoreDao {
                         bbbId = rs2.getString("bbbId");
                         version = rs2.getInt("version");
 
-                        if (bbbId == null || bbbId.length() > 0)
+                        if (bbbId == null || bbbId.length() > 0) {
                             bbbId = "-";
-                        if (cccId == null)
+                        }
+                        if (cccId == null) {
                             cccId = "";
+                        }
 
                         retVal.add(generateHeader(wmoId, site, hdrTime, bbbId,
                                 cccId, nnnId, xxxId));
@@ -1821,7 +1810,7 @@ public class StdTextProductDao extends CoreDao {
      */
     public List<String> read_wsh(String wmoId, String site, long startTimeMillis) {
         return read_wsh(wmoId, site,
-                (int) (Math.round(startTimeMillis / (float) MILLIS_PER_SECOND)));
+                (Math.round(startTimeMillis / (float) MILLIS_PER_SECOND)));
     }
 
     /*
@@ -1844,8 +1833,8 @@ public class StdTextProductDao extends CoreDao {
         ResultSet rs1 = null;
         String product = null;
         final String query = "SELECT product FROM stdTextProducts "
-                + "WHERE wmoId = ? site = ? AND createTime >= ? "
+                + "WHERE wmoId = ? site = ? AND refTime >= ? "
-                + "ORDER BY createTime DESC";
+                + "ORDER BY refTime DESC, insertTime DESC";
         List<String> retVal = new ArrayList<String>();
 
         try {
@@ -1903,7 +1892,7 @@ public class StdTextProductDao extends CoreDao {
         final String hdrQuery = "SELECT cccId, nnnId, xxxId, hdrTime, bbbId, version "
                 + "FROM stdTextProducts "
                 + "WHERE wmoId = ? AND site = ? "
-                + "ORDER BY cccId ASC, nnnId ASC, xxxId ASC, createTime DESC";
+                + "ORDER BY cccId ASC, nnnId ASC, xxxId ASC, refTime DESC, insertTime DESC";
         List<String> retVal = new ArrayList<String>();
 
         try {
@@ -1922,14 +1911,18 @@ public class StdTextProductDao extends CoreDao {
                 bbbId = rs1.getString("bbbId");
                 version = rs1.getInt("version");
 
-                if (bbbId == null || bbbId.length() > 0)
+                if (bbbId == null || bbbId.length() > 0) {
                     bbbId = "-";
-                if (cccId == null)
+                }
+                if (cccId == null) {
                     cccId = "";
-                if (nnnId == null || nnnId.length() > 0)
+                }
+                if (nnnId == null || nnnId.length() > 0) {
                     nnnId = "-";
-                if (xxxId == null)
+                }
+                if (xxxId == null) {
                     xxxId = "";
+                }
 
                 retVal.add(generateHeader(wmoId, site, hdrTime, bbbId, cccId,
                         nnnId, xxxId));
@@ -1990,7 +1983,7 @@ public class StdTextProductDao extends CoreDao {
             final String hdrQuery = "SELECT cccId, hdrTime, bbbId, version "
                     + "FROM stdTextProducts "
                     + "WHERE wmoId = ? AND site = ? AND nnnId = ? AND xxxId = ? "
-                    + "ORDER BY cccId ASC, createTime DESC";
+                    + "ORDER BY cccId ASC, refTime DESC, insertTime DESC";
 
             try {
                 session = getSession();
@@ -2008,10 +2001,12 @@ public class StdTextProductDao extends CoreDao {
                     bbbId = rs1.getString("bbbId");
                     version = rs1.getInt("version");
 
-                    if (bbbId == null || bbbId.length() > 0)
+                    if (bbbId == null || bbbId.length() > 0) {
                         bbbId = "-";
-                    if (cccId == null)
+                    }
+                    if (cccId == null) {
                         cccId = "";
+                    }
 
                     retVal.add(generateHeader(wmoId, site, hdrTime, bbbId,
                             cccId, nnnId, xxxId));
@@ -2076,12 +2071,12 @@ public class StdTextProductDao extends CoreDao {
             final String hdrQuery = "SELECT cccId, hdrTime, bbbId, version "
                     + "FROM stdTextProducts "
                     + "WHERE wmoId = ? AND site = ? AND nnnId = ? AND xxxId = ? "
-                    + "ORDER BY cccId ASC, createTime DESC";
+                    + "ORDER BY cccId ASC, refTime DESC, insertTime DESC";
 
             final String noXxxQuery = "SELECT cccId, xxxId hdrTime, bbbId, version "
                     + "FROM stdTextProducts "
                     + "WHERE wmoId = ? AND site = ? AND nnnId = ? "
-                    + "ORDER BY cccId ASC, xxxId ASC, createTime DESC";
+                    + "ORDER BY cccId ASC, xxxId ASC, refTime DESC, insertTime DESC";
 
             if (abbrId.length() > 3) {
                 xxxId = abbrId.substring(3);
@@ -2115,10 +2110,12 @@ public class StdTextProductDao extends CoreDao {
                         xxxId = rs1.getString("xxxId");
                     }
 
-                    if (bbbId == null || bbbId.length() > 0)
+                    if (bbbId == null || bbbId.length() > 0) {
                         bbbId = "-";
-                    if (cccId == null)
+                    }
+                    if (cccId == null) {
                         cccId = "";
+                    }
 
                     retVal.add(generateHeader(wmoId, site, hdrTime, bbbId,
                             cccId, nnnId, xxxId));
@@ -2184,7 +2181,7 @@ public class StdTextProductDao extends CoreDao {
             final String hdrQuery = "SELECT cccId, bbbId, version "
                     + "FROM stdTextProducts "
                     + "WHERE wmoId = ? AND site = ? AND nnnId = ? AND xxxId = ? AND hdrTime = ?"
-                    + "ORDER BY cccId ASC, createTime DESC, version DESC";
+                    + "ORDER BY cccId ASC, refTime DESC, insertTime DESC, version DESC";
 
             try {
                 session = getSession();
@@ -2204,15 +2201,17 @@ public class StdTextProductDao extends CoreDao {
                     bbbId = rs1.getString("bbbId");
                     version = rs1.getInt("version");
 
-                    if (bbbId == null || bbbId.length() > 0)
+                    if (bbbId == null || bbbId.length() > 0) {
                         bbbId = "-";
-                    if (cccId == null)
+                    }
+                    if (cccId == null) {
                         cccId = "";
+                    }
 
                     String hdr = generateHeader(wmoId, site, hdrTime, bbbId,
                             cccId, nnnId, xxxId);
 
-                    // order by ensures MAX(createTime) and MAX(version) is
+                    // order by ensures MAX(refTime) and MAX(version) is
                     // loaded first
                     if (retVal.size() > 0) {
                         if (!retVal.get(retVal.size() - 1).equals(hdr)) {
@@ -2279,7 +2278,7 @@ public class StdTextProductDao extends CoreDao {
             final String hdrQuery = "SELECT cccId, version "
                     + "FROM stdTextProducts "
                     + "WHERE wmoId = ? AND site = ? AND nnnId = ? AND xxxId = ? AND hdrTime = ? AND bbbId = ? "
-                    + "ORDER BY cccId ASC, createTime DESC, version DESC";
+                    + "ORDER BY cccId ASC, refTime DESC, insertTime DESC, version DESC";
 
             try {
                 session = getSession();
@@ -2299,13 +2298,14 @@ public class StdTextProductDao extends CoreDao {
                     cccId = rs1.getString("cccId");
                     version = rs1.getInt("version");
 
-                    if (cccId == null)
+                    if (cccId == null) {
                         cccId = "";
+                    }
 
                     String hdr = generateHeader(wmoId, site, hdrTime, bbbId,
                             cccId, nnnId, xxxId);
 
-                    // order by ensures MAX(createTime) and MAX(version) is
+                    // order by ensures MAX(refTime) and MAX(version) is
                     // loaded first
                     if (retVal.size() > 0) {
                         if (!retVal.get(retVal.size() - 1).equals(hdr)) {
@@ -2390,7 +2390,7 @@ public class StdTextProductDao extends CoreDao {
                 final String hdrQuery = "SELECT cccId, nnnId, xxxId, bbbId, version "
                         + "FROM stdTextProducts "
                         + "WHERE wmoId = ? AND site = ? AND hdrTime = ? AND cccId IS NOT NULL AND nnnId IS NOT NULL AND xxxId IS NOT NULL "
-                        + "ORDER BY cccId ASC, nnnId ASC, xxxId ASC, bbbId DESC, createTime DESC, version DESC";
+                        + "ORDER BY cccId ASC, nnnId ASC, xxxId ASC, bbbId DESC, refTime DESC, insertTime DESC, version DESC";
                 ps1 = c.prepareStatement(hdrQuery);
                 break;
             }
@@ -2398,7 +2398,7 @@ public class StdTextProductDao extends CoreDao {
                 final String hdrQuery = "SELECT version "
                         + "FROM stdTextProducts "
                         + "WHERE wmoId = ? AND site = ? AND hdrTime = ? AND cccId = 'NOA' AND nnnId = 'FOS' AND xxxId = 'PIL' "
-                        + "ORDER BY createTime DESC, version DESC";
+                        + "ORDER BY refTime DESC, insertTime DESC, version DESC";
                 ps1 = c.prepareStatement(hdrQuery);
                 break;
             }
@@ -2406,7 +2406,7 @@ public class StdTextProductDao extends CoreDao {
                 final String hdrQuery = "SELECT version "
                         + "FROM stdTextProducts "
                         + "WHERE wmoId = ? AND site = ? AND hdrTime = ? AND cccId = 'NOA' AND nnnId = 'FOS' AND xxxId = 'PIL' "
-                        + "ORDER BY createTime DESC, version DESC";
+                        + "ORDER BY refTime DESC, insertTime DESC, version DESC";
                 ps1 = c.prepareStatement(hdrQuery);
                 break;
             }
@@ -2440,7 +2440,7 @@ public class StdTextProductDao extends CoreDao {
                 String hdr = generateHeader(wmoId, site, hdrTime, bbbId, cccId,
                         nnnId, xxxId);
 
-                // order by ensures MAX(createTime) and MAX(version) is
+                // order by ensures MAX(refTime) and MAX(version) is
                 // loaded first
                 if (retVal.size() > 0) {
                     if (!retVal.get(retVal.size() - 1).equals(hdr)) {
@@ -2514,7 +2514,7 @@ public class StdTextProductDao extends CoreDao {
                 final String hdrQuery = "SELECT cccId, nnnId, xxxId, version "
                         + "FROM stdTextProducts "
                         + "WHERE wmoId = ? AND site = ? AND hdrTime = ? AND bbbId = ? AND cccId IS NOT NULL AND nnnId IS NOT NULL AND xxxId IS NOT NULL "
-                        + "ORDER BY cccId ASC, nnnId ASC, xxxId ASC, createTime DESC";
+                        + "ORDER BY cccId ASC, nnnId ASC, xxxId ASC, refTime DESC, insertTime DESC";
                 ps1 = c.prepareStatement(hdrQuery);
                 break;
             }
@@ -2522,7 +2522,7 @@ public class StdTextProductDao extends CoreDao {
                 final String hdrQuery = "SELECT version "
                         + "FROM stdTextProducts "
                         + "WHERE wmoId = ? AND site = ? AND hdrTime = ? AND bbbId = ? AND cccId = 'NOA' AND nnnId = 'FOS' AND xxxId = 'PIL' "
-                        + "ORDER BY createTime DESC";
+                        + "ORDER BY refTime DESC, insertTime DESC";
                 ps1 = c.prepareStatement(hdrQuery);
                 break;
             }
@@ -2530,7 +2530,7 @@ public class StdTextProductDao extends CoreDao {
                 final String hdrQuery = "SELECT version "
                         + "FROM stdTextProducts "
                         + "WHERE wmoId = ? AND site = ? AND hdrTime = ? AND bbbId = ? AND cccId = 'NOA' AND nnnId = 'FOS' AND xxxId = 'PIL' "
-                        + "ORDER BY createTime DESC";
+                        + "ORDER BY refTime DESC, insertTime DESC";
                 ps1 = c.prepareStatement(hdrQuery);
                 break;
             }
@@ -2560,7 +2560,7 @@ public class StdTextProductDao extends CoreDao {
                 String hdr = generateHeader(wmoId, site, hdrTime, bbbId, cccId,
                         nnnId, xxxId);
 
-                // order by ensures MAX(createTime) and MAX(version) is
+                // order by ensures MAX(refTime) and MAX(version) is
                 // loaded first
                 if (retVal.size() > 0) {
                     if (!retVal.get(retVal.size() - 1).equals(hdr)) {
@@ -2769,7 +2769,7 @@ public class StdTextProductDao extends CoreDao {
     public static final void main(String[] args) {
         long time = System.currentTimeMillis() / 1000L;
 
-        final String TM_QUERY_FMT = "delete from table_name where createtime < %d;";
+        final String TM_QUERY_FMT = "delete from table_name where refTime < %d;";
         Matcher m = Pattern.compile("table_name").matcher(TM_QUERY_FMT);
         String tempQuery = m.replaceAll(TM_QUERY_FMT);
 

edexOsgi/com.raytheon.edex.textdb/src/com/raytheon/edex/textdb/dbapi/impl/TextDB.java

@@ -50,6 +50,7 @@ import com.raytheon.uf.common.message.Header;
 import com.raytheon.uf.common.serialization.SerializationUtil;
 import com.raytheon.uf.common.site.SiteMap;
 import com.raytheon.uf.edex.core.props.PropertiesFactory;
+import com.raytheon.uf.edex.decodertools.time.TimeTools;
 import com.raytheon.uf.edex.wmo.message.AFOSProductId;
 import com.raytheon.uf.edex.wmo.message.WMOHeader;
 
@@ -808,9 +809,11 @@ public class TextDB {
         StdTextProductDao dao = new StdTextProductDao(operationalMode);
         boolean success = false;
         try {
-            if (textProduct.getCreatetime() == null) {
+            if (textProduct.getRefTime() == null) {
-                textProduct.setCreatetime(System.currentTimeMillis());
+                textProduct.setRefTime(System.currentTimeMillis());
             }
+            textProduct.setInsertTime(Calendar.getInstance(TimeZone
+                    .getTimeZone("GMT")));
             success = dao.write(textProduct);
         } catch (Exception e) {
             logger.error(e);
@@ -866,7 +869,13 @@ public class TextDB {
         }
 
         product.append(reportData);
-        long writeTime = System.currentTimeMillis();
+
+        Long writeTime = new Long(System.currentTimeMillis());
+        if (TimeTools.allowArchive()) {
+            Calendar c = header.getHeaderDate();
+            writeTime = new Long(c.getTimeInMillis());
+
+        }
 
         StdTextProduct textProduct = (operationalMode ? new OperationalStdTextProduct()
                 : new PracticeStdTextProduct());
@@ -877,7 +886,7 @@ public class TextDB {
         textProduct.setNnnid(prodId.getNnn());
         textProduct.setHdrtime(hdrTime);
         textProduct.setBbbid(bbbIndicator);
-        textProduct.setCreatetime(writeTime);
+        textProduct.setRefTime(writeTime);
         textProduct.setProduct(product.toString());
         boolean success = writeProduct(textProduct);
         if (success) {
@@ -958,8 +967,9 @@ public class TextDB {
             boolean operationalMode, Headers headers) {
         // Look for a WMO heading on the first line
         String[] pieces = reportData.split("\r*\n", 2);
-        if (pieces.length > 1)
+        if (pieces.length > 1) {
             pieces[0] += "\n"; // WMOHeader expects this
+        }
         WMOHeader header = new WMOHeader(pieces[0].getBytes(), headers);
 
         // Need to construct an AFOSProductId from the productId

edexOsgi/com.raytheon.uf.common.dataplugin.airep/src/com/raytheon/uf/common/dataplugin/airep/AirepRecord.java

@@ -19,6 +19,10 @@
  **/
 package com.raytheon.uf.common.dataplugin.airep;
 
+import java.math.RoundingMode;
+import java.text.DateFormat;
+import java.text.DecimalFormat;
+import java.text.SimpleDateFormat;
 import java.util.Calendar;
 import java.util.Collection;
 import java.util.HashMap;
@@ -62,6 +66,8 @@ import com.vividsolutions.jts.geom.Geometry;
  * ------------ ---------- ----------- --------------------------
  * 20080103            384 jkorman     Initial Coding.
  * 20080107            720 jkorman     remove default assignments from attributes.
+ * 20120405            435 dgilling    Prevent NullPointerExceptions in
+ *                                     buildMessageData().
  * </pre>
  * 
  * @author jkorman
@@ -87,11 +93,13 @@ public class AirepRecord extends PluginDataObject implements ISpatialEnabled,
 
     public static final Unit<Angle> LOCATION_UNIT = NonSI.DEGREE_ANGLE;
 
-    private static UnitConverter ftToHft = NonSI.FOOT.getConverterTo(SI.HECTO(NonSI.FOOT));
+    private static UnitConverter ftToHft = NonSI.FOOT.getConverterTo(SI
+            .HECTO(NonSI.FOOT));
 
     private static final HashMap<String, String> PARM_MAP = new HashMap<String, String>();
 
-    //private static final HashMap<Integer, String> WX_MAP = new HashMap<Integer, String>();
+    // private static final HashMap<Integer, String> WX_MAP = new
+    // HashMap<Integer, String>();
 
     static {
         PARM_MAP.put("T", SFC_TEMP);
@@ -569,70 +577,81 @@ public class AirepRecord extends PluginDataObject implements ISpatialEnabled,
     }
 
     private String buildMessageData() {
-        String s = "";
+        boolean validLocation = (location != null);
 
-        String lat = String.valueOf(location.getLatitude());
+        StringBuilder messageData = new StringBuilder("ARP ");
-        String lon = String.valueOf(location.getLongitude());
+        if (validLocation && getStationId() != null) {
-        String latDir = location.getLatitude() > 0 ? "N" : "S";
+            messageData.append(getStationId());
-        String lonDir = location.getLongitude() > 0 ? "E" : "W";
-        String hour = "";
-        String minute = "";
-        if(timeObs != null){
-            hour = String.valueOf(timeObs.get(Calendar.HOUR_OF_DAY));
-            minute = String.valueOf(timeObs.get(Calendar.MINUTE));
         }
-        String flightLevel = String.valueOf((int)ftToHft.convert(location.getFlightLevel()));
+        messageData.append(' ');
-        String wind = windDirection != null? 
+
-                String.valueOf(windDirection.intValue()) + "/" 
+        if ((validLocation) && (!Double.isNaN(getLatitude()))
-                + String.valueOf(windSpeed.intValue()) + "KT": "";
+                && (!Double.isNaN(getLongitude()))) {
-        //String wx = flightWeather != null? WX_MAP.get(flightWeather) : "";        
+            messageData.append(formatLatLon(getLatitude(), true));
-        lat = formatLatLon(lat);
+            messageData.append(' ');
-        lon = formatLatLon(lon);
+            messageData.append(formatLatLon(getLongitude(), false));
-        String temperature = "";
+            messageData.append(' ');
+        }
+
+        if (timeObs != null) {
+            DateFormat df = new SimpleDateFormat("HHmm");
+            messageData.append(df.format(timeObs.getTime()));
+        }
+        messageData.append(" F");
+
+        if (validLocation && getFlightLevel() != null) {
+            int flightLevel = (int) ftToHft.convert(getFlightLevel());
+            messageData.append(flightLevel);
+        }
+        messageData.append(' ');
+
         if (temp != null) {
             if (temp > 0) {
-                temperature = "P" + temp.intValue();
+                messageData.append('P');
             } else {
-                temperature = "M" + String.valueOf(temp.intValue()).substring(1);
+                messageData.append('M');
             }
+            messageData.append(Math.abs(temp.intValue()));
         }
-        if (hour.length() < 2) {
+        messageData.append(' ');
-            hour = "0" + hour;
+
+        if ((windDirection != null) && (windSpeed != null)) {
+            messageData.append(windDirection.intValue());
+            messageData.append('/');
+            messageData.append(windSpeed.intValue());
+            messageData.append("KT");
         }
-        if (minute.length() < 2) {
+        messageData.append("TB");
-            minute = "0" + minute;
+
-        }
+        return messageData.toString();
-        s = "ARP " + location.getStationId() + " " 
-                + lat + latDir + " " + lon + lonDir + " " 
-                +  hour + minute + " F" + flightLevel 
-                + " " + temperature + " " + wind 
-                + "TB";
-        return s;
     }
 
-    private String formatLatLon(String str){
+    private String formatLatLon(double value, boolean isLatitude) {
-        str = str.startsWith("-")? str.substring(1) : str;
+        char dir;
-        
+        if (isLatitude) {
-        int decimalIndex = str.indexOf(".");
+            if (value > 0) {
-        
+                dir = 'N';
-        if(decimalIndex != -1){
+            } else {
-            String temp = str.substring(decimalIndex + 1);
+                dir = 'S';
-            if(temp.length() > 3){
+            }
-                temp = temp.substring(0, 3);
+        } else {
-            } else if (temp.length() != 3) {
+            if (value > 0) {
-                while(temp.length() != 3){
+                dir = 'E';
-                    temp += "0";
+            } else {
+                dir = 'W';
             }
         }
-            str = str.substring(0,decimalIndex) + temp;
-        }
 
-        return str;
+        DecimalFormat df = new DecimalFormat("###.000");
+        df.setRoundingMode(RoundingMode.DOWN);
+
+        return df.format(Math.abs(value)) + dir;
     }
 
     /**
-     * Returns the hashCode for this object. This implementation
+     * Returns the hashCode for this object. This implementation returns the
-     * returns the hashCode of the generated dataURI.
+     * hashCode of the generated dataURI.
+     * 
      * @see java.lang.Object#hashCode()
      */
     @Override
@@ -645,26 +664,31 @@ public class AirepRecord extends PluginDataObject implements ISpatialEnabled,
     }
 
     /**
-     * Checks if this record is equal to another by checking the
+     * Checks if this record is equal to another by checking the generated
-     * generated dataURI.
+     * dataURI.
+     * 
      * @param obj
      * @see java.lang.Object#equals(java.lang.Object)
      */
     @Override
     public boolean equals(Object obj) {
-        if (this == obj)
+        if (this == obj) {
             return true;
-        if (obj == null)
+        }
+        if (obj == null) {
             return false;
-        if (getClass() != obj.getClass())
+        }
+        if (getClass() != obj.getClass()) {
             return false;
+        }
         AirepRecord other = (AirepRecord) obj;
         if (getDataURI() == null) {
             if (other.getDataURI() != null) {
                 return false;
             }
-        } else if (!getDataURI().equals(other.getDataURI()))
+        } else if (!getDataURI().equals(other.getDataURI())) {
             return false;
+        }
         return true;
     }
 

edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/src/com/raytheon/uf/common/dataplugin/binlightning/BinLightningRecord.java

@@ -20,8 +20,6 @@
 package com.raytheon.uf.common.dataplugin.binlightning;
 
 import java.util.Calendar;
-import java.util.Date;
-import java.util.TimeZone;
 
 import javax.persistence.Column;
 import javax.persistence.Entity;
@@ -37,7 +35,7 @@ import com.raytheon.uf.common.dataplugin.IDecoderGettable;
 import com.raytheon.uf.common.dataplugin.annotations.DataURI;
 import com.raytheon.uf.common.dataplugin.binlightning.impl.LightningStrikePoint;
 import com.raytheon.uf.common.dataplugin.persist.IPersistable;
-import com.raytheon.uf.common.dataplugin.persist.PersistablePluginDataObject;
+import com.raytheon.uf.common.dataplugin.persist.ServerSpecificPersistablePluginDataObject;
 import com.raytheon.uf.common.datastorage.IDataStore;
 import com.raytheon.uf.common.datastorage.StorageException;
 import com.raytheon.uf.common.datastorage.records.ByteDataRecord;
@@ -80,8 +78,8 @@ import com.raytheon.uf.edex.decodertools.time.TimeTools;
 @XmlRootElement
 @DynamicSerialize
 @XmlAccessorType(XmlAccessType.NONE)
-public class BinLightningRecord extends PersistablePluginDataObject implements
+public class BinLightningRecord extends
-        IPersistable {
+        ServerSpecificPersistablePluginDataObject implements IPersistable {
 
     /** Serializable id * */
     private static final long serialVersionUID = 1L;
@@ -187,33 +185,6 @@ public class BinLightningRecord extends PersistablePluginDataObject implements
     // super(message);
     // }
 
-    /**
-     * Get the time to use for persisting this data.
-     * 
-     * @return The persistence time for this data. Returns null if either start
-     *         or stop time are not valid.
-     */
-    @Override
-    public Date getPersistenceTime() {
-        Calendar c = getInsertTime();
-        if (c == null)
-            return null;
-
-        return c.getTime();
-    }
-
-    /**
-     * Set the time to be used for the persistence time for this object.
-     * 
-     * @param persistTime
-     *            The persistence time to be used.
-     */
-    public void setPersistenceTime(Date persistTime) {
-        Calendar c = Calendar.getInstance(TimeZone.getTimeZone("GMT"));
-        c.setTime(persistTime);
-        setInsertTime(c);
-    }
-
     /**
      * Track the current persistence time for the data set.
      */
@@ -239,8 +210,8 @@ public class BinLightningRecord extends PersistablePluginDataObject implements
         if (insertIndex < obsTimes.length) {
             long t1 = startTimeMillis;
 
-            Calendar c = TimeTools.getBaseCalendar(strike.getYear(), strike
+            Calendar c = TimeTools.getBaseCalendar(strike.getYear(),
-                    .getMonth(), strike.getDay());
+                    strike.getMonth(), strike.getDay());
 
             c.set(Calendar.HOUR_OF_DAY, strike.getHour());
             c.set(Calendar.MINUTE, strike.getMinute());

edexOsgi/com.raytheon.uf.common.dataplugin.bufrascat/src/com/raytheon/uf/common/dataplugin/bufrascat/AScatObs.java

@@ -73,7 +73,8 @@ public class AScatObs extends PersistablePluginDataObject implements
 	private static final long serialVersionUID = 1L;
 
 	@Embedded
-    private PointDataView pdv;
+	@DynamicSerializeElement
+	private PointDataView pointDataView;
 
 	@DataURI(position = 1)
 	@XmlAttribute
@@ -361,14 +362,14 @@ public class AScatObs extends PersistablePluginDataObject implements
      */
 	@Override
 	public PointDataView getPointDataView() {
-        return pdv;
+		return pointDataView;
 	}
 
 	/**
      * 
      */
 	@Override
-    public void setPointDataView(PointDataView pdv) {
+	public void setPointDataView(PointDataView pointDataView) {
-        this.pdv = pdv;
+		this.pointDataView = pointDataView;
 	}
 }

edexOsgi/com.raytheon.uf.common.dataplugin.bufrhdw/src/com/raytheon/uf/common/dataplugin/bufrhdw/BufrHDWObs.java

@@ -89,7 +89,8 @@ public class BufrHDWObs extends PersistablePluginDataObject implements
 	private SurfaceObsLocation location;
 
 	@Embedded
-    private PointDataView pdv;
+	@DynamicSerializeElement
+	private PointDataView pointDataView;
 
 	// Text of the WMO header
 	@Column(length = 32)
@@ -289,21 +290,6 @@ public class BufrHDWObs extends PersistablePluginDataObject implements
 		this.wmoHeader = wmoHeader;
 	}
 
-    /**
-     * @return the pdv
-     */
-    public PointDataView getPdv() {
-        return pdv;
-    }
-
-    /**
-     * @param pdv
-     *            the pdv to set
-     */
-    public void setPdv(PointDataView pdv) {
-        this.pdv = pdv;
-    }
-
 	/**
 	 * @return the windDir
 	 */
@@ -619,15 +605,15 @@ public class BufrHDWObs extends PersistablePluginDataObject implements
      */
 	@Override
 	public PointDataView getPointDataView() {
-        return pdv;
+		return pointDataView;
 	}
 
 	/**
      * 
      */
 	@Override
-    public void setPointDataView(PointDataView pdv) {
+	public void setPointDataView(PointDataView pointDataView) {
-        this.pdv = pdv;
+		this.pointDataView = pointDataView;
 	}
 
 	/**

edexOsgi/com.raytheon.uf.common.dataplugin.bufrmthdw/src/com/raytheon/uf/common/dataplugin/bufrmthdw/BufrMTHDWObs.java

@@ -89,7 +89,8 @@ public class BufrMTHDWObs extends PersistablePluginDataObject implements
 	private SurfaceObsLocation location;
 
 	@Embedded
-    private PointDataView pdv;
+	@DynamicSerializeElement
+	private PointDataView pointDataView;
 
 	// Text of the WMO header
 	@Column(length = 32)
@@ -289,21 +290,6 @@ public class BufrMTHDWObs extends PersistablePluginDataObject implements
 		this.wmoHeader = wmoHeader;
 	}
 
-    /**
-     * @return the pdv
-     */
-    public PointDataView getPdv() {
-        return pdv;
-    }
-
-    /**
-     * @param pdv
-     *            the pdv to set
-     */
-    public void setPdv(PointDataView pdv) {
-        this.pdv = pdv;
-    }
-
 	/**
 	 * @return the windDir
 	 */
@@ -619,15 +605,15 @@ public class BufrMTHDWObs extends PersistablePluginDataObject implements
          */
 	@Override
 	public PointDataView getPointDataView() {
-        return pdv;
+		return pointDataView;
 	}
 
 	/**
      * 
      */
 	@Override
-    public void setPointDataView(PointDataView pdv) {
+	public void setPointDataView(PointDataView pointDataView) {
-        this.pdv = pdv;
+		this.pointDataView = pointDataView;
 	}
 
 	/**

edexOsgi/com.raytheon.uf.common.dataplugin.bufrncwf/src/com/raytheon/uf/common/dataplugin/ncwf/BUFRncwf.java

@@ -70,7 +70,8 @@ public class BUFRncwf extends PersistablePluginDataObject implements
 	private static final long serialVersionUID = 1L;
 
 	@Embedded
-    private PointDataView pdv;
+	@DynamicSerializeElement
+	private PointDataView pointDataView;
 
 	@Embedded
 	@DataURI(position = 1, embedded = true)
@@ -247,29 +248,14 @@ public class BUFRncwf extends PersistablePluginDataObject implements
      */
 	@Override
 	public PointDataView getPointDataView() {
-        return pdv;
+		return pointDataView;
-    }
-
-    /**
-     * @return the pdv
-     */
-    public PointDataView getPdv() {
-        return pdv;
-    }
-
-    /**
-     * @param pdv
-     *            the pdv to set
-     */
-    public void setPdv(PointDataView pdv) {
-        this.pdv = pdv;
 	}
 
 	/**
      * 
      */
 	@Override
-    public void setPointDataView(PointDataView pdv) {
+	public void setPointDataView(PointDataView pointDataView) {
-        this.pdv = pdv;
+		this.pointDataView = pointDataView;
 	}
 }

edexOsgi/com.raytheon.uf.common.dataplugin.bufrquikscat/src/com/raytheon/uf/common/dataplugin/bufrquikscat/QUIKScatObs.java

@@ -73,7 +73,8 @@ public class QUIKScatObs extends PersistablePluginDataObject implements
 	private static final long serialVersionUID = 1L;
 
 	@Embedded
-    private PointDataView pdv;
+	@DynamicSerializeElement
+	private PointDataView pointDataView;
 
 	@DataURI(position = 1)
 	@XmlAttribute
@@ -361,14 +362,14 @@ public class QUIKScatObs extends PersistablePluginDataObject implements
      */
 	@Override
 	public PointDataView getPointDataView() {
-        return pdv;
+		return pointDataView;
 	}
 
 	/**
      * 
      */
 	@Override
-    public void setPointDataView(PointDataView pdv) {
+	public void setPointDataView(PointDataView pointDataView) {
-        this.pdv = pdv;
+		this.pointDataView = pointDataView;
 	}
 }

edexOsgi/com.raytheon.uf.common.dataplugin.bufrsigwx/src/com/raytheon/uf/common/dataplugin/bufrsigwx/SigWxData.java

@@ -99,7 +99,8 @@ public class SigWxData extends PersistablePluginDataObject implements
 	private Integer topHeight;
 
 	@Embedded
-    private PointDataView pdv;
+	@DynamicSerializeElement
+	private PointDataView pointDataView;
 
 	// Text of the WMO header
 	@Column(length = 32)
@@ -236,21 +237,6 @@ public class SigWxData extends PersistablePluginDataObject implements
 		this.key = key;
 	}
 
-    /**
-     * @return the pdv
-     */
-    public PointDataView getPdv() {
-        return pdv;
-    }
-
-    /**
-     * @param pdv
-     *            the pdv to set
-     */
-    public void setPdv(PointDataView pdv) {
-        this.pdv = pdv;
-    }
-
 	@Override
 	public IDecoderGettable getDecoderGettable() {
 		return null;
@@ -281,15 +267,15 @@ public class SigWxData extends PersistablePluginDataObject implements
      */
 	@Override
 	public PointDataView getPointDataView() {
-        return pdv;
+		return pointDataView;
 	}
 
 	/**
      * 
      */
 	@Override
-    public void setPointDataView(PointDataView pdv) {
+	public void setPointDataView(PointDataView pointDataView) {
-        this.pdv = pdv;
+		this.pointDataView = pointDataView;
 	}
 
 	/**