Issue #2693 Data set information infrastructure for availability offset calculations

Change-Id: I0f8d90a753ce37d37c682ff17613eb9a44554a3b

Former-commit-id: b64df5806a [formerly 129163bf87 [formerly 48fad391deb95616556628424b1a1799c2929f89]]
Former-commit-id: 129163bf87
Former-commit-id: ef8eb701da

edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/util/DataSetInformationXmlWriter.java

@@ -0,0 +1,54 @@
+package com.raytheon.uf.common.datadelivery.retrieval.util;
+
+/**
+ * This software was developed and / or modified by Raytheon Company,
+ * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+ * 
+ * U.S. EXPORT CONTROLLED TECHNICAL DATA
+ * This software product contains export-restricted data whose
+ * export/transfer/disclosure is restricted by U.S. law. Dissemination
+ * to non-U.S. persons whether in the United States or abroad requires
+ * an export license or other authorization.
+ * 
+ * Contractor Name:        Raytheon Company
+ * Contractor Address:     6825 Pine Street, Suite 340
+ *                         Mail Stop B8
+ *                         Omaha, NE 68106
+ *                         402.291.0100
+ * 
+ * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+ * further licensing information.
+ **/
+
+import com.raytheon.uf.common.datadelivery.retrieval.xml.DataSetInformationLookup;
+
+/**
+ * Writes out DataSet Information data to xml.
+ * 
+ * <pre>
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Jan 14, 2014            dhladky     Initial creation
+ * 
+ * </pre>
+ * 
+ * @author dhladky
+ * @version 1.0
+ */
+
+public interface DataSetInformationXmlWriter {
+
+    /**
+     * Writes the DataSetInformation lookup to XML
+     * 
+     * @param aol
+     * @param modelName
+     * @throws Exception
+     */
+    void writeDataSetInformationXml(DataSetInformationLookup dsi)
+            throws Exception;
+
+}

edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/util/LookupManager.java

@@ -26,10 +26,13 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 
 import javax.xml.bind.JAXBException;
 
 import com.raytheon.uf.common.datadelivery.registry.Parameter;
+import com.raytheon.uf.common.datadelivery.retrieval.xml.DataSetInformation;
+import com.raytheon.uf.common.datadelivery.retrieval.xml.DataSetInformationLookup;
 import com.raytheon.uf.common.datadelivery.retrieval.xml.LevelLookup;
 import com.raytheon.uf.common.datadelivery.retrieval.xml.ParameterConfig;
 import com.raytheon.uf.common.datadelivery.retrieval.xml.ParameterLookup;
@@ -60,6 +63,7 @@ import com.raytheon.uf.common.util.ServiceLoaderUtil;
  * Jan 18, 2013   1513     dhladky     Level lookup refit.
  * Mar 21, 2013   1794     djohnson    ServiceLoaderUtil now requires the requesting class.
  * Nov 07, 2013   2361     njensen     Use JAXBManager for XML
+ * Jam 14, 2014            dhladky     AvailabilityOffset calculations
  * 
  * </pre>
  * 
@@ -72,7 +76,7 @@ public class LookupManager {
      * Implementation of the xml writers that writes to localization files.
      */
     private class LocalizationXmlWriter implements LevelXmlWriter,
-            ParameterXmlWriter {
+            ParameterXmlWriter, DataSetInformationXmlWriter {
         /**
          * {@inheritDoc}
          */
@@ -107,6 +111,23 @@ public class LookupManager {
 
             getJaxbManager().marshalToXmlFile(pl, file.getAbsolutePath());
         }
+
+
+        @Override
+        public void writeDataSetInformationXml(DataSetInformationLookup dsi)
+                throws Exception {
+            
+            IPathManager pm = PathManagerFactory.getPathManager();
+            LocalizationContext lc = pm.getContext(
+                    LocalizationType.COMMON_STATIC, LocalizationLevel.SITE);
+            String fileName = getDataSetInformationFileName();
+            LocalizationFile lf = pm.getLocalizationFile(lc, fileName);
+            File file = lf.getFile();
+            
+            getJaxbManager().marshalToXmlFile(dsi, file.getAbsolutePath());
+            
+        }
+
     }
 
     private static final IUFStatusHandler statusHandler = UFStatus
@@ -120,6 +141,8 @@ public class LookupManager {
             + File.separatorChar + "lookups" + File.separatorChar;
 
     private static final String CONFIG_FILE_PARAM = "ParameterLookup.xml";
+    
+    private static final String CONFIG_FILE_DATASETINFO = "DataSetInformationLookup.xml";
 
     private static final String CONFIG_FILE_LEVEL = "LevelLookup.xml";
 
@@ -135,9 +158,11 @@ public class LookupManager {
         return instance;
     }
 
-    private final Map<String, ParameterLookup> parameters = new HashMap<String, ParameterLookup>();
+    private final Map<String, ParameterLookup> parameters = new HashMap<String, ParameterLookup>(1);
 
-    private final Map<String, LevelLookup> levels = new HashMap<String, LevelLookup>();
+    private final Map<String, LevelLookup> levels = new HashMap<String, LevelLookup>(1);
+    
+    private final Map<String, DataSetInformation> dataSetInformations = new HashMap<String, DataSetInformation>(1);
 
     private UnitLookup unitLookup = null;
 
@@ -148,13 +173,17 @@ public class LookupManager {
     private final ParameterXmlWriter parameterXmlWriter = ServiceLoaderUtil
             .load(LookupManager.class, ParameterXmlWriter.class,
                     new LocalizationXmlWriter());
+    
+    private final DataSetInformationXmlWriter dataSetInformationXmlWriter = ServiceLoaderUtil
+            .load(LookupManager.class, DataSetInformationXmlWriter.class,
+                    new LocalizationXmlWriter());
 
     private static JAXBManager jaxb;
 
-    private static JAXBManager getJaxbManager()
+    private static JAXBManager getJaxbManager() throws JAXBException {
-            throws JAXBException {
         if (jaxb == null) {
             jaxb = new JAXBManager(LevelLookup.class, ParameterLookup.class,
+                    DataSetInformationLookup.class, DataSetInformation.class,
                     UnitLookup.class);
         }
 
@@ -295,6 +324,168 @@ public class LookupManager {
 
         return paramLoookup;
     }
+    
+    /**
+     * Gets the Data Set Information
+     * 
+     * @param modelName
+     * @return
+     */
+    public DataSetInformation getDataSetInformation(String modelName) {
+
+        DataSetInformation dsi = null;
+
+        if (dataSetInformations.isEmpty()) {
+            
+            DataSetInformationLookup dataSetInformationLookup = getDataSetInformationLookupFromFile();
+            
+            if (dataSetInformationLookup == null) {
+                dataSetInformationLookup = new DataSetInformationLookup();
+            }
+            
+            for (DataSetInformation dataSetinfo: dataSetInformationLookup.getDataSetInformations()){
+                dataSetInformations.put(dataSetinfo.getModelName(), dataSetinfo);
+            }
+        } 
+        
+        if (dataSetInformations.containsKey(modelName)) {
+            dsi = dataSetInformations.get(modelName);
+        }
+      
+        return dsi;
+    }
+
+    private DataSetInformationLookup getDataSetInformationLookupFromFile() {
+
+        DataSetInformationLookup dataSetInformationLookup = null;
+        LocalizationFile file = null;
+        String fileName = getDataSetInformationFileName();
+
+        try {
+            file = getLocalizationFile(fileName);
+        } catch (Exception e) {
+            statusHandler
+                    .error(" Failed to read Data Set Information Lookup: " + fileName, e);
+        }
+
+        if (file != null) {
+            try {
+                dataSetInformationLookup = readDataSetInformationXml(file.getFile());
+            } catch (Exception e) {
+                statusHandler.handle(
+                        Priority.PROBLEM,
+                        "Failed to Read Data Set Information Lookup from file: "
+                                + file.getName(), e);
+            }
+        }
+
+        return dataSetInformationLookup;
+    }
+    
+    /**
+     * Availability Offset file name
+     * 
+     * @param modelName
+     * @return
+     */
+    private String getDataSetInformationFileName() {
+        return CONFIG_FILE_ROOT + CONFIG_FILE_DATASETINFO;
+    }
+    
+    /**
+     * Read Data Set Information lookup
+     * 
+     * @param file
+     * @return
+     * @throws Exception
+     */
+    private DataSetInformationLookup readDataSetInformationXml(File file) throws Exception {
+
+        DataSetInformationLookup dsi = null;
+
+        if (file != null && file.exists()) {
+            dsi = getJaxbManager().unmarshalFromXmlFile(
+                    DataSetInformationLookup.class, file);
+        }
+
+        return dsi;
+    }
+    
+    /**
+     * Modify or create a data set information lookup
+     * 
+     * @param modelName
+     */
+    public void modifyDataSetInformationLookup(DataSetInformation dsi) {
+
+        try {
+            DataSetInformationLookup dataSetInformationLookup = null;
+            // update map and write out file
+            synchronized (dataSetInformations) {
+
+                dataSetInformations.put(dsi.getModelName(), dsi);
+                dataSetInformationLookup = getDataSetInformationLookupFromFile();
+                
+                if (dataSetInformationLookup == null) {
+                    dataSetInformationLookup = new DataSetInformationLookup();
+                }
+
+                if (dataSetInformationLookup.getDataSetInformations().contains(
+                        dsi)) {
+                    // no changes
+                    return;
+                } else {
+                    // clear the hash and rebuild it with what is set in current
+                    dataSetInformationLookup.getDataSetInformations().clear();
+
+                    for (Entry<String, DataSetInformation> entry : dataSetInformations
+                            .entrySet()) {
+                        dataSetInformationLookup.getDataSetInformations().add(
+                                entry.getValue());
+                    }
+                }
+            }
+
+            if (dataSetInformationLookup != null) {
+                dataSetInformationXmlWriter
+                        .writeDataSetInformationXml(dataSetInformationLookup);
+
+                statusHandler
+                        .info("Updated/Created Data Set Information lookup! "
+                                + dsi.getModelName());
+            }
+
+        } catch (Exception e) {
+            statusHandler.error(
+                    "Couldn't create/update Data Set Information lookup! ", e);
+        }
+    }
+
+    /**
+     * Does a availability offset lookup exist?
+     * 
+     * @param modelName
+     * @return
+     */
+    public boolean dataSetInformationLookupExists() {
+
+        LocalizationFile file = null;
+        String fileName = getDataSetInformationFileName();
+        try {
+            file = getLocalizationFile(fileName);
+        } catch (Exception fnfe) {
+            statusHandler.error(
+                    "Failed to lookup Data Set Information localization file: "
+                            + fileName, fnfe);
+        }
+        if (file != null) {
+            return file.exists();
+        }
+
+        return false;
+    }
+
+
 
     /**
      * param file name

edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/xml/DataSetInformation.java

@@ -0,0 +1,118 @@
+package com.raytheon.uf.common.datadelivery.retrieval.xml;
+
+/**
+ * This software was developed and / or modified by Raytheon Company,
+ * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+ * 
+ * U.S. EXPORT CONTROLLED TECHNICAL DATA
+ * This software product contains export-restricted data whose
+ * export/transfer/disclosure is restricted by U.S. law. Dissemination
+ * to non-U.S. persons whether in the United States or abroad requires
+ * an export license or other authorization.
+ * 
+ * Contractor Name:        Raytheon Company
+ * Contractor Address:     6825 Pine Street, Suite 340
+ *                         Mail Stop B8
+ *                         Omaha, NE 68106
+ *                         402.291.0100
+ * 
+ * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+ * further licensing information.
+ **/
+
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
+import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
+
+/**
+ * Data Set Info XML Object.
+ * 
+ * <pre>
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Jan 14, 2014            dhladky      Initial creation.
+ * 
+ * </pre>
+ * 
+ * @author dhladky
+ * @version 1.0
+ */
+
+@XmlRootElement(name = "DataSetInformation")
+@XmlAccessorType(XmlAccessType.NONE)
+@DynamicSerialize
+public class DataSetInformation {
+
+    @XmlElement(name = "modelName", type = String.class)
+    @DynamicSerializeElement
+    protected String modelName;
+    
+    @XmlElement(name = "multiplier", type = Double.class)
+    @DynamicSerializeElement
+    protected Double multiplier;
+    
+    @XmlElement(name = "modelRunIncrement", type = Integer.class)
+    @DynamicSerializeElement
+    protected Integer modelRunIncrement;
+    
+    @XmlElement(name = "defaultOffset", type = Integer.class)
+    @DynamicSerializeElement
+    protected Integer defaultOffset;
+        
+    public DataSetInformation() {
+        
+    }
+    
+    public DataSetInformation(String modelName, Double multiplier, int modelRunIncrement, int defaultOffset) {
+  
+        this.modelName = modelName;
+        this.multiplier = multiplier;
+        this.modelRunIncrement = modelRunIncrement;
+        this.defaultOffset = defaultOffset;
+    }
+
+    public String getModelName() {
+        return modelName;
+    }
+
+    public void setModelName(String modelName) {
+        this.modelName = modelName;
+    }
+
+    public Double getMultiplier() {
+        return multiplier;
+    }
+
+    public void setMultiplier(Double multiplier) {
+        this.multiplier = multiplier;
+    }
+
+    public Integer getModelRunIncrement() {
+        return modelRunIncrement;
+    }
+
+    public void setModelRunIncrement(Integer modelRunIncrement) {
+        this.modelRunIncrement = modelRunIncrement;
+    }
+
+    public Integer getDefaultOffset() {
+        return defaultOffset;
+    }
+
+    public void setDefaultOffset(Integer defaultOffset) {
+        this.defaultOffset = defaultOffset;
+    }
+    
+    public int getRange() {
+        return (int) (getMultiplier() * getModelRunIncrement());
+    }
+
+}

edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/xml/DataSetInformationLookup.java

@@ -0,0 +1,74 @@
+package com.raytheon.uf.common.datadelivery.retrieval.xml;
+
+/**
+ * This software was developed and / or modified by Raytheon Company,
+ * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+ * 
+ * U.S. EXPORT CONTROLLED TECHNICAL DATA
+ * This software product contains export-restricted data whose
+ * export/transfer/disclosure is restricted by U.S. law. Dissemination
+ * to non-U.S. persons whether in the United States or abroad requires
+ * an export license or other authorization.
+ * 
+ * Contractor Name:        Raytheon Company
+ * Contractor Address:     6825 Pine Street, Suite 340
+ *                         Mail Stop B8
+ *                         Omaha, NE 68106
+ *                         402.291.0100
+ * 
+ * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+ * further licensing information.
+ **/
+
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlElements;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
+import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
+
+/**
+ * Data Set Lookup XML Object.
+ * 
+ * <pre>
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Jan 14, 2014            dhladky      Initial creation.
+ * 
+ * </pre>
+ * 
+ * @author dhladky
+ * @version 1.0
+ */
+
+@XmlRootElement(name = "DataSetInformationLookup")
+@XmlAccessorType(XmlAccessType.NONE)
+@DynamicSerialize
+public class DataSetInformationLookup {
+    
+    public DataSetInformationLookup() {
+        dataSetInformations = new ArrayList<DataSetInformation>(1);
+    }
+    
+    @XmlElements({ @XmlElement(name = "dataSetInformation", type = DataSetInformation.class) })
+    @DynamicSerializeElement
+    private List<DataSetInformation> dataSetInformations;
+
+    public List<DataSetInformation> getDataSetInformations() {
+        return dataSetInformations;
+    }
+
+    public void setDataSetInformations(List<DataSetInformation> dataSetInformations) {
+        this.dataSetInformations = dataSetInformations;
+    }
+
+}

edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/OPENDAPServiceConfig.xml

@@ -86,5 +86,8 @@
     <constant name="UNIT_PATTERN" value=".*(\[(.*?)\])\s*"/>
     <constant name="TWOM" value="2m"/>
     <constant name="TENM" value="10m"/>
+    <constant name="DEFAULT_MULTIPLIER" value="1.5"/>
+    <constant name="DEFAULT_RUN_INCREMENT" value="6"/>
+    <constant name="DEFAULT_OFFSET" value="270"/>
 
 </serviceConfig>

edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPMetaDataParser.java

@@ -47,6 +47,7 @@ import com.raytheon.uf.common.datadelivery.registry.Provider.ServiceType;
 import com.raytheon.uf.common.datadelivery.registry.Time;
 import com.raytheon.uf.common.datadelivery.retrieval.util.HarvesterServiceManager;
 import com.raytheon.uf.common.datadelivery.retrieval.util.LookupManager;
+import com.raytheon.uf.common.datadelivery.retrieval.xml.DataSetInformation;
 import com.raytheon.uf.common.datadelivery.retrieval.xml.ParameterConfig;
 import com.raytheon.uf.common.datadelivery.retrieval.xml.ParameterLookup;
 import com.raytheon.uf.common.gridcoverage.Corner;
@@ -95,6 +96,7 @@ import dods.dap.DAS;
  * Sept 25, 2013 1797      dhladky      separated time from gridded time
  * Oct 10, 2013 1797       bgonzale     Refactored registry Time objects.
  * Dec 18, 2013 2636       mpduff       Calculate a data availability delay for the dataset and dataset meta data.
+ * Jan 14, 2014            dhladky      Data set info used for availability delay calculations.
  * 
  * </pre>
  * 
@@ -261,21 +263,7 @@ class OpenDAPMetaDataParser extends MetaDataParser {
                 time.setStepUnit(Time.findStepUnit(step.get(1))
                         .getDurationUnit());
                 gdsmd.setTime(time);
-
+                
-                // Calculate dataset availability delay
-                long startMillis = time.getStart().getTime();
-                long now = TimeUtil.newGmtCalendar().getTimeInMillis();
-                long offset = (now - startMillis) / TimeUtil.MILLIS_PER_MINUTE;
-                dataSet.setAvailabilityOffset((int) offset);
-                gdsmd.setAvailabilityOffset((int) offset);
-
-                if (statusHandler.isPriorityEnabled(Priority.DEBUG)) {
-                    statusHandler.debug("Dataset Name: "
-                            + dataSet.getDataSetName());
-                    statusHandler.debug("StartTime:    " + time.getStart());
-                    statusHandler.debug("Offset:       "
-                            + dataSet.getAvailabilityOffset());
-                }
             } catch (Exception le) {
                 logParsingException(timecon, "Time", collectionName, url);
             }
@@ -756,12 +744,54 @@ class OpenDAPMetaDataParser extends MetaDataParser {
                 throw new IllegalStateException(
                         "The time cannot be null for a DataSet object!");
             }
+            
+            // Calculate dataset availability delay
+            DataSetInformation dsi = LookupManager.getInstance().getDataSetInformation(gdsmd.getDataSetName());
+            long offset = 0l;
+            long startMillis = gdsmd.getTime().getStart().getTime();
+            long endMillis = TimeUtil.newGmtCalendar().getTimeInMillis();
+            
+            /**
+             * This is here for if no one has configured this particular model
+             * They are gross defaults and will not guarantee this model working
+             */
+            if (dsi == null) {
+                Double multi = Double.parseDouble(serviceConfig
+                        .getConstantValue("DEFAULT_MULTIPLIER"));
+                Integer runIncrement = Integer.parseInt(serviceConfig
+                        .getConstantValue("DEFAULT_RUN_INCREMENT"));
+                Integer defaultOffest = Integer.parseInt(serviceConfig
+                        .getConstantValue("DEFAULT_OFFSET"));
+                dsi = new DataSetInformation(gdsmd.getDataSetName(), multi,
+                        runIncrement, defaultOffest);
+                // writes out a place holder DataSetInformation object in the file
+                LookupManager.getInstance().modifyDataSetInformationLookup(dsi);
+            } 
+            
+            offset = (endMillis - startMillis) / TimeUtil.MILLIS_PER_MINUTE;
+            // this is the actually ranging check
+            if (dsi.getRange() < offset) {
+                offset = dsi.getDefaultOffset();
+            }
 
+            dataSet.setAvailabilityOffset((int) offset);
+            gdsmd.setAvailabilityOffset((int) offset);
+
+            if (statusHandler.isPriorityEnabled(Priority.DEBUG)) {
+                statusHandler.debug("Dataset Name: "
+                        + dataSet.getDataSetName());
+                statusHandler.debug("StartTime:    " + gdsmd.getTime());
+                statusHandler.debug("Offset:       "
+                        + dataSet.getAvailabilityOffset());
+            }
+            
             List<DataSetMetaData<?>> toStore = metaDatas.get(dataSet);
+            
             if (toStore == null) {
                 toStore = new ArrayList<DataSetMetaData<?>>();
                 metaDatas.put(dataSet, toStore);
             }
+            
             toStore.add(gdsmd);
         }