Issue #1552 - Implement FFMP Data Access

Peer review comments
Change-Id: Ibf11c96248972a86a6a0d779ba482cd133125b55

Former-commit-id: 8bddd467b1 [formerly 544c90e8a0b5aba5470e60beac18d86d972c05cd]
Former-commit-id: 81649d346a
This commit is contained in:
Mike Duff 2013-02-20 11:31:59 -06:00
parent fd95a99596
commit b1a549a99f
15 changed files with 470 additions and 13 deletions

View file

@ -24,6 +24,7 @@ Require-Bundle: org.eclipse.ui,
com.raytheon.uf.common.stats;bundle-version="1.0.0",
com.raytheon.uf.viz.stats;bundle-version="1.0.0",
com.raytheon.uf.common.event;bundle-version="1.0.0",
com.raytheon.uf.common.dataaccess;bundle-version="1.0.0",
org.apache.log4j;bundle-version="1.0.0"
Bundle-RequiredExecutionEnvironment: JavaSE-1.6
Bundle-ActivationPolicy: lazy

View file

@ -42,6 +42,7 @@ import com.raytheon.uf.viz.core.rsc.LoadProperties;
* Jan 31, 2013 bkowal Initial creation
* Feb 14, 2013 1614 bsteffen Refactor data access framework to use
* single request.
* Feb 19, 2013 1552 mpduff Handle empty legend text.
*
* </pre>
*
@ -54,7 +55,7 @@ public abstract class AbstractDataAccessResource<T extends AbstractDataAccessRes
protected static final String _SPACE_ = " ";
private String genericLegendText;
private String genericLegendText = StringUtils.EMPTY;
/**
* Constructor
@ -77,6 +78,9 @@ public abstract class AbstractDataAccessResource<T extends AbstractDataAccessRes
}
}
/**
* {@inheritDoc}
*/
@Override
protected void initInternal(IGraphicsTarget target) throws VizException {
DataTime[] timesToLoad = descriptor.getFramesInfo().getTimeMap()
@ -92,6 +96,9 @@ public abstract class AbstractDataAccessResource<T extends AbstractDataAccessRes
}
}
/**
* {@inheritDoc}
*/
@Override
public void remove(DataTime dataTime) {
;// for now never remove anything from dataTimes since there are no
@ -132,7 +139,10 @@ public abstract class AbstractDataAccessResource<T extends AbstractDataAccessRes
* the request-type specific legend text
*/
private final String buildLegendText(String genericLegendText) {
StringBuilder stringBuilder = new StringBuilder(genericLegendText);
StringBuilder stringBuilder = new StringBuilder();
if (genericLegendText != null) {
stringBuilder.append(genericLegendText);
}
stringBuilder.append(this.padWithSeparator(this
.buildLegendTextInternal()));
return stringBuilder.toString();

View file

@ -54,7 +54,6 @@ import com.raytheon.viz.dataaccess.rsc.AbstractDataAccessResourceData;
@XmlAccessorType(XmlAccessType.NONE)
public class GenericGeometryResourceData extends
AbstractDataAccessResourceData<IGeometryData> {
/**
* Constructor
*/

View file

@ -0,0 +1 @@
com.raytheon.uf.common.dataaccess.impl.DefaultDataRequest

View file

@ -25,6 +25,7 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import com.raytheon.uf.common.dataaccess.IDataFactory;
import com.raytheon.uf.common.dataaccess.IDataRequest;
import com.raytheon.uf.common.dataaccess.exception.InvalidIdentifiersException;
@ -42,6 +43,7 @@ import com.raytheon.uf.common.dataaccess.exception.InvalidIdentifiersException;
* Nov 13, 2012 njensen Initial creation
* Feb 14, 2013 1614 bsteffen Refactor data access framework to use
* single request.
* Feb 19, 2012 1552 mpduff Implement IDataFactory.
*
* </pre>
*
@ -49,7 +51,7 @@ import com.raytheon.uf.common.dataaccess.exception.InvalidIdentifiersException;
* @version 1.0
*/
public abstract class AbstractDataFactory {
public abstract class AbstractDataFactory implements IDataFactory {
/**
* Returns the identifiers that must be set on a request for the request to
@ -57,7 +59,7 @@ public abstract class AbstractDataFactory {
*
* @return the required identifiers
*/
public String[] getRequiredIdentifiers(){
public String[] getRequiredIdentifiers() {
return null;
}
@ -97,8 +99,8 @@ public abstract class AbstractDataFactory {
}
if (!missing.isEmpty() || !invalid.isEmpty()) {
throw new InvalidIdentifiersException(request.getDatatype(), missing,
invalid);
throw new InvalidIdentifiersException(request.getDatatype(),
missing, invalid);
}
}

View file

@ -7,7 +7,8 @@ Bundle-Vendor: RAYTHEON
Eclipse-RegisterBuddy: com.raytheon.edex.common, com.raytheon.uf.common.serialization, com.raytheon.uf.viz.core
Bundle-RequiredExecutionEnvironment: JavaSE-1.6
Export-Package: com.raytheon.uf.common.dataplugin.ffmp,
com.raytheon.uf.common.dataplugin.ffmp.dao
com.raytheon.uf.common.dataplugin.ffmp.dao,
com.raytheon.uf.common.dataplugin.ffmp.dataaccess
Require-Bundle: javax.persistence;bundle-version="1.0.0",
com.raytheon.edex.common;bundle-version="1.11.1",
com.raytheon.uf.common.monitor;bundle-version="1.11.5",
@ -19,4 +20,8 @@ Require-Bundle: javax.persistence;bundle-version="1.0.0",
com.raytheon.uf.common.dataplugin.shef;bundle-version="1.12.1174",
com.raytheon.uf.common.cache;bundle-version="1.12.1174",
com.raytheon.uf.common.serialization.comm;bundle-version="1.12.1174",
com.raytheon.uf.common.dataplugin.grid;bundle-version="1.0.0"
com.raytheon.uf.common.dataplugin.grid;bundle-version="1.0.0",
com.raytheon.uf.common.dataaccess;bundle-version="1.0.0",
com.google.guava;bundle-version="1.0.0",
javax.measure;bundle-version="1.0.0",
com.raytheon.uf.common.message;bundle-version="1.12.1174"

View file

@ -1,2 +1 @@
com.raytheon.uf.common.dataplugin.ffmp.FFMPRecord
com.raytheon.uf.common.dataplugin.ffmp.FFMPRecord

View file

@ -1,4 +1,5 @@
source.. = src/
output.. = bin/
bin.includes = META-INF/,\
.
.,\
res/

View file

@ -0,0 +1,12 @@
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:amq="http://activemq.apache.org/schema/core" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd">
<bean id="ffmpDataFactory"
class="com.raytheon.uf.common.dataplugin.ffmp.dataaccess.FFMPGeometryFactory" />
<bean factory-bean="dataAccessRegistry" factory-method="register" id="ffmpDataAccessRegister">
<constructor-arg value="ffmp" />
<constructor-arg ref="ffmpDataFactory" />
</bean>
</beans>

View file

@ -0,0 +1,313 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.dataplugin.ffmp.dataaccess;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.measure.unit.Unit;
import com.raytheon.uf.common.dataaccess.IDataRequest;
import com.raytheon.uf.common.dataaccess.exception.DataRetrievalException;
import com.raytheon.uf.common.dataaccess.exception.UnsupportedOutputTypeException;
import com.raytheon.uf.common.dataaccess.geom.IGeometryData;
import com.raytheon.uf.common.dataaccess.grid.IGridData;
import com.raytheon.uf.common.dataaccess.impl.AbstractDataPluginFactory;
import com.raytheon.uf.common.dataaccess.impl.DefaultGeometryData;
import com.raytheon.uf.common.dataaccess.util.DatabaseQueryUtil;
import com.raytheon.uf.common.dataaccess.util.DatabaseQueryUtil.QUERY_MODE;
import com.raytheon.uf.common.dataaccess.util.PDOUtil;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPBasin;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPBasinData;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPRecord;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPTemplates;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPTemplates.MODE;
import com.raytheon.uf.common.dataplugin.ffmp.HucLevelGeometriesFactory;
import com.raytheon.uf.common.dataquery.requests.RequestConstraint;
import com.raytheon.uf.common.dataquery.requests.RequestConstraint.ConstraintType;
import com.raytheon.uf.common.dataquery.responses.DbQueryResponse;
import com.raytheon.uf.common.datastorage.IDataStore;
import com.raytheon.uf.common.monitor.config.FFMPRunConfigurationManager;
import com.raytheon.uf.common.monitor.config.FFMPSourceConfigurationManager;
import com.raytheon.uf.common.monitor.xml.DomainXML;
import com.raytheon.uf.common.monitor.xml.SourceXML;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.vividsolutions.jts.geom.Geometry;
/**
* A data factory for retrieving FFMP data.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 24, 2013 1552 mpduff Initial creation
*
* </pre>
*
* @author mpduff
* @version 1.0
*/
public class FFMPGeometryFactory extends AbstractDataPluginFactory {
private static final IUFStatusHandler statusHandler = UFStatus
.getHandler(FFMPGeometryFactory.class);
/** Site key constant */
public static final String SITE_KEY = "siteKey";
/** Data key constant */
public static final String DATA_KEY = "dataKey";
/** wfo constant */
public static final String WFO = "wfo";
/** plugin constant */
public static final String PLUGIN_NAME = "ffmp";
/** huc constant */
public static final String HUC = "huc";
/** source name constant */
public static final String SOURCE_NAME = "sourceName";
/** FFMP Templates object */
private FFMPTemplates templates;
/**
* Constructor.
*/
public FFMPGeometryFactory() {
}
/**
* Not Supported.
*/
@Override
protected IGridData[] getGridData(IDataRequest request,
DbQueryResponse dbQueryResponse) {
// Not currently handled
throw new UnsupportedOutputTypeException(request.getDatatype(),
PLUGIN_NAME);
}
/**
* {@inheritDoc}
*/
@Override
protected IGeometryData[] getGeometryData(IDataRequest request,
DbQueryResponse dbQueryResponse) {
List<Map<String, Object>> results = dbQueryResponse.getResults();
Map<Long, DefaultGeometryData> cache = new HashMap<Long, DefaultGeometryData>();
for (Map<String, Object> map : results) {
for (Map.Entry<String, Object> es : map.entrySet()) {
FFMPRecord rec = (FFMPRecord) es.getValue();
try {
IDataStore dataStore = PDOUtil.getDataStore(rec);
rec.retrieveMapFromDataStore(dataStore, rec.getDataURI(),
templates,
(String) request.getIdentifiers().get(HUC), rec
.getDataTime().getRefTime(), rec
.getSourceName());
} catch (Exception e) {
throw new DataRetrievalException(
"Failed to retrieve the IDataRecord for PluginDataObject: "
+ rec.toString(), e);
}
try {
cache = makeGeometryData(rec, request, cache);
} catch (Exception e) {
statusHandler.handle(Priority.PROBLEM,
e.getLocalizedMessage(), e);
}
}
}
return cache.values().toArray(
new DefaultGeometryData[cache.values().size()]);
}
/**
* {@inheritDoc}
*/
@Override
protected Map<String, RequestConstraint> buildConstraintsFromRequest(
IDataRequest request) {
Map<String, RequestConstraint> map = new HashMap<String, RequestConstraint>();
Map<String, Object> identifiers = request.getIdentifiers();
String siteKey = (String) identifiers.get(SITE_KEY);
for (Map.Entry<String, Object> entry : request.getIdentifiers()
.entrySet()) {
String key = entry.getKey();
String value = (String) entry.getValue();
if (!key.equals(HUC)) {
RequestConstraint rc = new RequestConstraint(value);
map.put(key, rc);
}
}
RequestConstraint parameterConstraint = new RequestConstraint();
parameterConstraint.setConstraintValueList(request.getParameters());
parameterConstraint.setConstraintType(ConstraintType.IN);
map.put(SOURCE_NAME, parameterConstraint);
String domain = (String) request.getIdentifiers().get(WFO);
DomainXML domainXml = FFMPRunConfigurationManager.getInstance()
.getDomain(domain);
templates = FFMPTemplates.getInstance(domainXml, siteKey, MODE.EDEX);
return map;
}
/**
* Create the IGeometryData objects.
*
* @param rec
* The FFMPRecord
* @param cache
* @param huc
* The HUC level
* @param siteKey
* The siteKey
* @param cwa
* The CWA
* @param dataKey
* The dataKey
* @throws Exception
*/
private Map<Long, DefaultGeometryData> makeGeometryData(FFMPRecord rec,
IDataRequest request, Map<Long, DefaultGeometryData> cache)
throws Exception {
String huc = (String) request.getIdentifiers().get(HUC);
String dataKey = (String) request.getIdentifiers().get(DATA_KEY);
String siteKey = (String) request.getIdentifiers().get(SITE_KEY);
String cwa = (String) request.getIdentifiers().get(WFO);
FFMPBasinData basinData = rec.getBasinData(huc);
HashMap<Long, FFMPBasin> basinDataMap = basinData.getBasins();
HucLevelGeometriesFactory geomFactory = HucLevelGeometriesFactory
.getInstance();
Map<Long, Geometry> geomMap = geomFactory.getGeometries(templates,
dataKey, cwa, huc);
FFMPSourceConfigurationManager srcConfigMan = FFMPSourceConfigurationManager
.getInstance();
SourceXML sourceXml = srcConfigMan.getSource(rec.getSourceName());
DefaultGeometryData data = null;
String[] locationNames = request.getLocationNames();
List<Long> pfafList = null;
if (locationNames != null) {
pfafList = convertLocations(locationNames);
}
for (Long pfaf : geomMap.keySet()) {
if (pfafList == null || pfafList.contains(pfaf)) {
if (cache.containsKey(pfaf)) {
data = cache.get(pfaf);
} else {
data = new DefaultGeometryData();
Map<String, Object> attrs = new HashMap<String, Object>();
attrs.put(DATA_KEY, dataKey);
attrs.put(SITE_KEY, siteKey);
attrs.put(WFO, cwa);
attrs.put(HUC, huc);
data.setAttributes(attrs);
data.setLocationName(String.valueOf(pfaf));
data.setGeometry(geomMap.get(pfaf));
cache.put(pfaf, data);
}
FFMPBasin basin = basinDataMap.get(pfaf);
if (basin == null) {
continue;
}
Float value = basin.getValue(rec.getDataTime().getRefTime());
String parameter = rec.getSourceName();
String unitStr = sourceXml.getUnit();
Unit<?> unit = null;
if (unitStr.equals(SourceXML.UNIT_TXT)) {
unit = Unit.valueOf("in");
}
if (unit != null) {
data.addData(parameter, value, unit);
} else {
data.addData(parameter, value);
}
}
}
return cache;
}
/**
* Convert list of PFAF strings to list of PFAF longs
*
* @param locationNames
* @return
*/
private List<Long> convertLocations(String[] locationNames) {
List<Long> pfafList = new ArrayList<Long>();
for (String s : locationNames) {
try {
pfafList.add(Long.parseLong(s));
} catch (NumberFormatException e) {
statusHandler.handle(Priority.PROBLEM,
"Error parsing pfaf id: " + s, e);
}
}
return pfafList;
}
/**
* {@inheritDoc}
*/
@Override
public String[] getAvailableLocationNames(IDataRequest request) {
List<String> pfafList = new ArrayList<String>();
String domain = (String) request.getIdentifiers().get("wfo");
String sql = "select pfaf_id from mapdata.ffmp_basins where cwa = '"
+ domain + "';";
List<Object[]> results = DatabaseQueryUtil.executeDatabaseQuery(
QUERY_MODE.MODE_SQLQUERY, sql, "metadata", "ffmp");
for (Object[] oa : results) {
pfafList.add((String) oa[0]);
}
return pfafList.toArray(new String[pfafList.size()]);
}
}

View file

@ -74,8 +74,15 @@
<classpathentry combineaccessrules="false" exported="true" kind="src" path="/org.dom4j"/>
<classpathentry combineaccessrules="false" exported="true" kind="src" path="/com.raytheon.uf.edex.plugin.level"/>
<classpathentry combineaccessrules="false" exported="true" kind="src" path="/com.raytheon.uf.common.dataplugin.level"/>
<classpathentry kind="src" path="/com.raytheon.uf.common.dataplugin.ffmp"/>
<classpathentry combineaccessrules="false" kind="src" path="/com.raytheon.uf.common.dataaccess"/>
<classpathentry kind="src" path="/com.raytheon.uf.common.monitor"/>
<classpathentry combineaccessrules="false" exported="true" kind="src" path="/com.raytheon.uf.common.dataplugin.grid"/>
<classpathentry combineaccessrules="false" exported="true" kind="src" path="/com.raytheon.uf.edex.log"/>
<classpathentry kind="src" path="/com.raytheon.viz.core"/>
<classpathentry combineaccessrules="false" kind="src" path="/com.raytheon.uf.edex.decodertools"/>
<classpathentry kind="lib" path="/opt/uframe-eclipse/plugins/org.eclipse.swt.gtk.linux.x86_3.6.1.v3655c.jar"/>
<classpathentry kind="lib" path="/opt/uframe-eclipse/plugins/org.eclipse.swt.gtk.linux.x86_64.source_3.6.1.v3655c.jar"/>
<classpathentry kind="lib" path="/opt/uframe-eclipse/plugins/org.eclipse.swt.gtk.linux.x86_64_3.6.1.v3655c.jar"/>
<classpathentry kind="output" path="bin"/>
</classpath>

View file

@ -24,6 +24,8 @@ import java.util.List;
import org.junit.Ignore;
import com.raytheon.uf.common.monitor.xml.FFMPTemplateXML;
/**
* Implementation of {@link IJaxbableClassesLocator} that returns a static list
* of classes.
@ -86,7 +88,10 @@ public class TestJaxbableClassesLocator implements IJaxbableClassesLocator {
com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.BandwidthMap.class,
com.raytheon.uf.common.datadelivery.retrieval.xml.ServiceConfig.class,
com.raytheon.uf.common.datadelivery.retrieval.xml.UnitLookup.class,
com.raytheon.uf.common.datadelivery.retrieval.xml.LevelLookup.class };
com.raytheon.uf.common.datadelivery.retrieval.xml.LevelLookup.class,
com.raytheon.uf.common.monitor.xml.FFMPSourceConfigXML.class,
com.raytheon.uf.common.monitor.xml.FFMPRunConfigXML.class,
FFMPTemplateXML.class };
JAXB_CLASSES = Arrays.asList(array);
}

View file

@ -0,0 +1,56 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<FFMPRunConfig xmlns:ns2="group">
<runner>
<!-- configure uri filter sources -->
<sourceIngestConfig uriSubLocation="3" name="DHR">
<dataKey>kdvn</dataKey>
<dataKey>tmsp</dataKey>
<dataKey>karx</dataKey>
<dataKey>kdmx</dataKey>
</sourceIngestConfig>
<sourceIngestConfig uriSubLocation="3" name="DPR">
<dataKey>kdvn</dataKey>
<dataKey>tmsp</dataKey>
<dataKey>karx</dataKey>
<dataKey>kdmx</dataKey>
</sourceIngestConfig>
<sourceIngestConfig uriSubLocation="3" name="FFG0124hr">
<dataKey>KMSR</dataKey>
</sourceIngestConfig>
<sourceIngestConfig uriSubLocation="3" name="FFG0324hr">
<dataKey>KMSR</dataKey>
</sourceIngestConfig>
<sourceIngestConfig uriSubLocation="3" name="FFG0624hr">
<dataKey>KMSR</dataKey>
</sourceIngestConfig>
<sourceIngestConfig uriSubLocation="3" name="QPFSCAN">
<dataKey>kdvn</dataKey>
<dataKey>tmsp</dataKey>
<dataKey>karx</dataKey>
<dataKey>kdmx</dataKey>
</sourceIngestConfig>
<!-- overrides of source config -->
<sourceOverride name="DHR">
<dataKey dataKey="kdvn">
<override param="expirationMinutes" value="20"/>
</dataKey>
<dataKey dataKey="kdmx">
<override param="expirationMinutes" value="25"/>
</dataKey>
</sourceOverride>
<!-- setup product definitions -->
<product key="kdvn" name="DHR"/>
<product key="kdvn" name="DPR"/>
<product key="kdmx" name="DHR"/>
<product key="kdmx" name="DPR"/>
<product key="karx" name="DHR"/>
<product key="karx" name="DPR"/>
<product key="tmsp" name="DHR"/>
<!-- domains to be run -->
<domain cwa="DVN" primary="true"/>
<domain cwa="DMX" primary="false"/>
<domain cwa="ARX" primary="false"/>
<domain cwa="LOT" primary="false"/>
<domain cwa="MPX" primary="false"/>
</runner>
</FFMPRunConfig>

View file

@ -0,0 +1,4 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<FFMPTemplate extents="20000.0" virtual="true" numberOfHuc="6" hucDepth="4" xmlns:ns2="group">
<excludedVGBs/>
</FFMPTemplate>

View file

@ -0,0 +1,42 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<FFMPRunConfig xmlns:ns2="group">
<runner>
<!-- Source Ingest Config's, used by URI Filters -->
<sourceIngestConfig name="FFG0124hr" uriSubLocation="3">
<dataKey>KKRF</dataKey>
<dataKey>KMSR</dataKey>
</sourceIngestConfig>
<sourceIngestConfig name="FFG0324hr" uriSubLocation="3">
<dataKey>KKRF</dataKey>
<dataKey>KMSR</dataKey>
</sourceIngestConfig>
<sourceIngestConfig name="FFG0624hr" uriSubLocation="3">
<dataKey>KKRF</dataKey>
<dataKey>KMSR</dataKey>
</sourceIngestConfig>
<sourceIngestConfig name="QPFSCAN" uriSubLocation="3">
<dataKey>koax</dataKey>
<dataKey>kdmx</dataKey>
<dataKey>kuex</dataKey>
</sourceIngestConfig>
<sourceIngestConfig name="DHR" uriSubLocation="3">
<dataKey>koax</dataKey>
<dataKey>kdmx</dataKey>
<dataKey>kuex</dataKey>
</sourceIngestConfig>
<sourceIngestConfig name="DPR" uriSubLocation="3">
<dataKey>kdmx</dataKey>
</sourceIngestConfig>
<!-- Product definitions to be run against -->
<product name="DHR" key="koax"/>
<product name="DHR" key="kuex"/>
<product name="DHR" key="kdmx"/>
<product name="DPR" key="kdmx"/>
<product name="DHRMOSAIC" key="hpe"/>
<product name="BDHRMOSAIC" key="bhpe"/>
<!-- domains to be used -->
<domain primary="true" cwa="OAX"/>
<domain primary="false" cwa="GID"/>
<domain primary="false" cwa="DMX"/>
</runner>
</FFMPRunConfig>