xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #2693 Data set information infrastructure for availability offset calculations

Browse source 
Change-Id: I0f8d90a753ce37d37c682ff17613eb9a44554a3b

Former-commit-id: b64df5806a [formerly 129163bf87 [formerly 48fad391deb95616556628424b1a1799c2929f89]]
Former-commit-id: 129163bf87
Former-commit-id: ef8eb701da
This commit is contained in:
Dave Hladky 2014-01-15 13:29:18 -06:00
parent 821fc0f0b9
commit f475d7a4f5
7 changed files with 1925 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1435
edexOsgi/com.raytheon.uf.common.datadelivery.harvester/utility/common_static/base/datadelivery/lookups/DataSetInformationLookup.xml Normal file
View file

File diff suppressed because it is too large Load diff

54
edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/util/DataSetInformationXmlWriter.java Normal file
View file

@ -0,0 +1,54 @@
package com.raytheon.uf.common.datadelivery.retrieval.util;
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
import com.raytheon.uf.common.datadelivery.retrieval.xml.DataSetInformationLookup;
/**
* Writes out DataSet Information data to xml.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 14, 2014 dhladky Initial creation
*
* </pre>
*
* @author dhladky
* @version 1.0
*/
public interface DataSetInformationXmlWriter {
/**
* Writes the DataSetInformation lookup to XML
*
* @param aol
* @param modelName
* @throws Exception
*/
void writeDataSetInformationXml(DataSetInformationLookup dsi)
throws Exception;
}

201
edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/util/LookupManager.java
View file

@ -26,10 +26,13 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import javax.xml.bind.JAXBException;
import com.raytheon.uf.common.datadelivery.registry.Parameter;
import com.raytheon.uf.common.datadelivery.retrieval.xml.DataSetInformation;
import com.raytheon.uf.common.datadelivery.retrieval.xml.DataSetInformationLookup;
import com.raytheon.uf.common.datadelivery.retrieval.xml.LevelLookup;
import com.raytheon.uf.common.datadelivery.retrieval.xml.ParameterConfig;
import com.raytheon.uf.common.datadelivery.retrieval.xml.ParameterLookup;
@ -60,6 +63,7 @@ import com.raytheon.uf.common.util.ServiceLoaderUtil;
* Jan 18, 2013 1513 dhladky Level lookup refit.
* Mar 21, 2013 1794 djohnson ServiceLoaderUtil now requires the requesting class.
* Nov 07, 2013 2361 njensen Use JAXBManager for XML
* Jam 14, 2014 dhladky AvailabilityOffset calculations
*
* </pre>
*
@ -72,7 +76,7 @@ public class LookupManager {
* Implementation of the xml writers that writes to localization files.
*/
private class LocalizationXmlWriter implements LevelXmlWriter,
ParameterXmlWriter {
ParameterXmlWriter, DataSetInformationXmlWriter {
/**
* {@inheritDoc}
*/
@ -107,6 +111,23 @@ public class LookupManager {
getJaxbManager().marshalToXmlFile(pl, file.getAbsolutePath());
}
@Override
public void writeDataSetInformationXml(DataSetInformationLookup dsi)
throws Exception {
IPathManager pm = PathManagerFactory.getPathManager();
LocalizationContext lc = pm.getContext(
LocalizationType.COMMON_STATIC, LocalizationLevel.SITE);
String fileName = getDataSetInformationFileName();
LocalizationFile lf = pm.getLocalizationFile(lc, fileName);
File file = lf.getFile();
getJaxbManager().marshalToXmlFile(dsi, file.getAbsolutePath());
}
}
private static final IUFStatusHandler statusHandler = UFStatus
@ -120,6 +141,8 @@ public class LookupManager {
+ File.separatorChar + "lookups" + File.separatorChar;
private static final String CONFIG_FILE_PARAM = "ParameterLookup.xml";
private static final String CONFIG_FILE_DATASETINFO = "DataSetInformationLookup.xml";
private static final String CONFIG_FILE_LEVEL = "LevelLookup.xml";
@ -135,9 +158,11 @@ public class LookupManager {
return instance;
}
private final Map<String, ParameterLookup> parameters = new HashMap<String, ParameterLookup>();
private final Map<String, ParameterLookup> parameters = new HashMap<String, ParameterLookup>(1);
private final Map<String, LevelLookup> levels = new HashMap<String, LevelLookup>();
private final Map<String, LevelLookup> levels = new HashMap<String, LevelLookup>(1);
private final Map<String, DataSetInformation> dataSetInformations = new HashMap<String, DataSetInformation>(1);
private UnitLookup unitLookup = null;
@ -148,13 +173,17 @@ public class LookupManager {
private final ParameterXmlWriter parameterXmlWriter = ServiceLoaderUtil
.load(LookupManager.class, ParameterXmlWriter.class,
new LocalizationXmlWriter());
private final DataSetInformationXmlWriter dataSetInformationXmlWriter = ServiceLoaderUtil
.load(LookupManager.class, DataSetInformationXmlWriter.class,
new LocalizationXmlWriter());
private static JAXBManager jaxb;
private static JAXBManager getJaxbManager()
throws JAXBException {
private static JAXBManager getJaxbManager() throws JAXBException {
if (jaxb == null) {
jaxb = new JAXBManager(LevelLookup.class, ParameterLookup.class,
DataSetInformationLookup.class, DataSetInformation.class,
UnitLookup.class);
}
@ -295,6 +324,168 @@ public class LookupManager {
return paramLoookup;
}
/**
* Gets the Data Set Information
*
* @param modelName
* @return
*/
public DataSetInformation getDataSetInformation(String modelName) {
DataSetInformation dsi = null;
if (dataSetInformations.isEmpty()) {
DataSetInformationLookup dataSetInformationLookup = getDataSetInformationLookupFromFile();
if (dataSetInformationLookup == null) {
dataSetInformationLookup = new DataSetInformationLookup();
}
for (DataSetInformation dataSetinfo: dataSetInformationLookup.getDataSetInformations()){
dataSetInformations.put(dataSetinfo.getModelName(), dataSetinfo);
}
}
if (dataSetInformations.containsKey(modelName)) {
dsi = dataSetInformations.get(modelName);
}
return dsi;
}
private DataSetInformationLookup getDataSetInformationLookupFromFile() {
DataSetInformationLookup dataSetInformationLookup = null;
LocalizationFile file = null;
String fileName = getDataSetInformationFileName();
try {
file = getLocalizationFile(fileName);
} catch (Exception e) {
statusHandler
.error(" Failed to read Data Set Information Lookup: " + fileName, e);
}
if (file != null) {
try {
dataSetInformationLookup = readDataSetInformationXml(file.getFile());
} catch (Exception e) {
statusHandler.handle(
Priority.PROBLEM,
"Failed to Read Data Set Information Lookup from file: "
+ file.getName(), e);
}
}
return dataSetInformationLookup;
}
/**
* Availability Offset file name
*
* @param modelName
* @return
*/
private String getDataSetInformationFileName() {
return CONFIG_FILE_ROOT + CONFIG_FILE_DATASETINFO;
}
/**
* Read Data Set Information lookup
*
* @param file
* @return
* @throws Exception
*/
private DataSetInformationLookup readDataSetInformationXml(File file) throws Exception {
DataSetInformationLookup dsi = null;
if (file != null && file.exists()) {
dsi = getJaxbManager().unmarshalFromXmlFile(
DataSetInformationLookup.class, file);
}
return dsi;
}
/**
* Modify or create a data set information lookup
*
* @param modelName
*/
public void modifyDataSetInformationLookup(DataSetInformation dsi) {
try {
DataSetInformationLookup dataSetInformationLookup = null;
// update map and write out file
synchronized (dataSetInformations) {
dataSetInformations.put(dsi.getModelName(), dsi);
dataSetInformationLookup = getDataSetInformationLookupFromFile();
if (dataSetInformationLookup == null) {
dataSetInformationLookup = new DataSetInformationLookup();
}
if (dataSetInformationLookup.getDataSetInformations().contains(
dsi)) {
// no changes
return;
} else {
// clear the hash and rebuild it with what is set in current
dataSetInformationLookup.getDataSetInformations().clear();
for (Entry<String, DataSetInformation> entry : dataSetInformations
.entrySet()) {
dataSetInformationLookup.getDataSetInformations().add(
entry.getValue());
}
}
}
if (dataSetInformationLookup != null) {
dataSetInformationXmlWriter
.writeDataSetInformationXml(dataSetInformationLookup);
statusHandler
.info("Updated/Created Data Set Information lookup! "
+ dsi.getModelName());
}
} catch (Exception e) {
statusHandler.error(
"Couldn't create/update Data Set Information lookup! ", e);
}
}
/**
* Does a availability offset lookup exist?
*
* @param modelName
* @return
*/
public boolean dataSetInformationLookupExists() {
LocalizationFile file = null;
String fileName = getDataSetInformationFileName();
try {
file = getLocalizationFile(fileName);
} catch (Exception fnfe) {
statusHandler.error(
"Failed to lookup Data Set Information localization file: "
+ fileName, fnfe);
}
if (file != null) {
return file.exists();
}
return false;
}
/**
* param file name

118
edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/xml/DataSetInformation.java Normal file
View file

@ -0,0 +1,118 @@
package com.raytheon.uf.common.datadelivery.retrieval.xml;
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
/**
* Data Set Info XML Object.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 14, 2014 dhladky Initial creation.
*
* </pre>
*
* @author dhladky
* @version 1.0
*/
@XmlRootElement(name = "DataSetInformation")
@XmlAccessorType(XmlAccessType.NONE)
@DynamicSerialize
public class DataSetInformation {
@XmlElement(name = "modelName", type = String.class)
@DynamicSerializeElement
protected String modelName;
@XmlElement(name = "multiplier", type = Double.class)
@DynamicSerializeElement
protected Double multiplier;
@XmlElement(name = "modelRunIncrement", type = Integer.class)
@DynamicSerializeElement
protected Integer modelRunIncrement;
@XmlElement(name = "defaultOffset", type = Integer.class)
@DynamicSerializeElement
protected Integer defaultOffset;
public DataSetInformation() {
}
public DataSetInformation(String modelName, Double multiplier, int modelRunIncrement, int defaultOffset) {
this.modelName = modelName;
this.multiplier = multiplier;
this.modelRunIncrement = modelRunIncrement;
this.defaultOffset = defaultOffset;
}
public String getModelName() {
return modelName;
}
public void setModelName(String modelName) {
this.modelName = modelName;
}
public Double getMultiplier() {
return multiplier;
}
public void setMultiplier(Double multiplier) {
this.multiplier = multiplier;
}
public Integer getModelRunIncrement() {
return modelRunIncrement;
}
public void setModelRunIncrement(Integer modelRunIncrement) {
this.modelRunIncrement = modelRunIncrement;
}
public Integer getDefaultOffset() {
return defaultOffset;
}
public void setDefaultOffset(Integer defaultOffset) {
this.defaultOffset = defaultOffset;
}
public int getRange() {
return (int) (getMultiplier() * getModelRunIncrement());
}
}

74
edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/xml/DataSetInformationLookup.java Normal file
View file

@ -0,0 +1,74 @@
package com.raytheon.uf.common.datadelivery.retrieval.xml;
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
import java.util.ArrayList;
import java.util.List;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlElements;
import javax.xml.bind.annotation.XmlRootElement;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
/**
* Data Set Lookup XML Object.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 14, 2014 dhladky Initial creation.
*
* </pre>
*
* @author dhladky
* @version 1.0
*/
@XmlRootElement(name = "DataSetInformationLookup")
@XmlAccessorType(XmlAccessType.NONE)
@DynamicSerialize
public class DataSetInformationLookup {
public DataSetInformationLookup() {
dataSetInformations = new ArrayList<DataSetInformation>(1);
}
@XmlElements({ @XmlElement(name = "dataSetInformation", type = DataSetInformation.class) })
@DynamicSerializeElement
private List<DataSetInformation> dataSetInformations;
public List<DataSetInformation> getDataSetInformations() {
return dataSetInformations;
}
public void setDataSetInformations(List<DataSetInformation> dataSetInformations) {
this.dataSetInformations = dataSetInformations;
}
}

3
edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/OPENDAPServiceConfig.xml
View file

@ -86,5 +86,8 @@
<constant name="UNIT_PATTERN" value=".*(\[(.*?)\])\s*"/>
<constant name="TWOM" value="2m"/>
<constant name="TENM" value="10m"/>
<constant name="DEFAULT_MULTIPLIER" value="1.5"/>
<constant name="DEFAULT_RUN_INCREMENT" value="6"/>
<constant name="DEFAULT_OFFSET" value="270"/>
</serviceConfig>

60
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPMetaDataParser.java
View file

@ -47,6 +47,7 @@ import com.raytheon.uf.common.datadelivery.registry.Provider.ServiceType;
import com.raytheon.uf.common.datadelivery.registry.Time;
import com.raytheon.uf.common.datadelivery.retrieval.util.HarvesterServiceManager;
import com.raytheon.uf.common.datadelivery.retrieval.util.LookupManager;
import com.raytheon.uf.common.datadelivery.retrieval.xml.DataSetInformation;
import com.raytheon.uf.common.datadelivery.retrieval.xml.ParameterConfig;
import com.raytheon.uf.common.datadelivery.retrieval.xml.ParameterLookup;
import com.raytheon.uf.common.gridcoverage.Corner;
@ -95,6 +96,7 @@ import dods.dap.DAS;
* Sept 25, 2013 1797 dhladky separated time from gridded time
* Oct 10, 2013 1797 bgonzale Refactored registry Time objects.
* Dec 18, 2013 2636 mpduff Calculate a data availability delay for the dataset and dataset meta data.
* Jan 14, 2014 dhladky Data set info used for availability delay calculations.
*
* </pre>
*
@ -261,21 +263,7 @@ class OpenDAPMetaDataParser extends MetaDataParser {
time.setStepUnit(Time.findStepUnit(step.get(1))
.getDurationUnit());
gdsmd.setTime(time);
// Calculate dataset availability delay
long startMillis = time.getStart().getTime();
long now = TimeUtil.newGmtCalendar().getTimeInMillis();
long offset = (now - startMillis) / TimeUtil.MILLIS_PER_MINUTE;
dataSet.setAvailabilityOffset((int) offset);
gdsmd.setAvailabilityOffset((int) offset);
if (statusHandler.isPriorityEnabled(Priority.DEBUG)) {
statusHandler.debug("Dataset Name: "
+ dataSet.getDataSetName());
statusHandler.debug("StartTime: " + time.getStart());
statusHandler.debug("Offset: "
+ dataSet.getAvailabilityOffset());
}
} catch (Exception le) {
logParsingException(timecon, "Time", collectionName, url);
}
@ -756,12 +744,54 @@ class OpenDAPMetaDataParser extends MetaDataParser {
throw new IllegalStateException(
"The time cannot be null for a DataSet object!");
}
// Calculate dataset availability delay
DataSetInformation dsi = LookupManager.getInstance().getDataSetInformation(gdsmd.getDataSetName());
long offset = 0l;
long startMillis = gdsmd.getTime().getStart().getTime();
long endMillis = TimeUtil.newGmtCalendar().getTimeInMillis();
/**
* This is here for if no one has configured this particular model
* They are gross defaults and will not guarantee this model working
*/
if (dsi == null) {
Double multi = Double.parseDouble(serviceConfig
.getConstantValue("DEFAULT_MULTIPLIER"));
Integer runIncrement = Integer.parseInt(serviceConfig
.getConstantValue("DEFAULT_RUN_INCREMENT"));
Integer defaultOffest = Integer.parseInt(serviceConfig
.getConstantValue("DEFAULT_OFFSET"));
dsi = new DataSetInformation(gdsmd.getDataSetName(), multi,
runIncrement, defaultOffest);
// writes out a place holder DataSetInformation object in the file
LookupManager.getInstance().modifyDataSetInformationLookup(dsi);
}
offset = (endMillis - startMillis) / TimeUtil.MILLIS_PER_MINUTE;
// this is the actually ranging check
if (dsi.getRange() < offset) {
offset = dsi.getDefaultOffset();
}
dataSet.setAvailabilityOffset((int) offset);
gdsmd.setAvailabilityOffset((int) offset);
if (statusHandler.isPriorityEnabled(Priority.DEBUG)) {
statusHandler.debug("Dataset Name: "
+ dataSet.getDataSetName());
statusHandler.debug("StartTime: " + gdsmd.getTime());
statusHandler.debug("Offset: "
+ dataSet.getAvailabilityOffset());
}
List<DataSetMetaData<?>> toStore = metaDatas.get(dataSet);
if (toStore == null) {
toStore = new ArrayList<DataSetMetaData<?>>();
metaDatas.put(dataSet, toStore);
}
toStore.add(gdsmd);
}