xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge "Omaha #4756 First step to adding ARI FFG grids for FFMP processing. Change-Id: I5a7cef83a617319ac3d5e9dd1eafa12353a99f43" into omaha_16.2.1

Browse source 
Former-commit-id: 57b83f8424836c732a32d6e020db213e73e92089
This commit is contained in:
Richard Peter 2015-09-10 14:50:55 -05:00 • committed by Gerrit Code Review
commit b7c05d7663
6 changed files with 187 additions and 98 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
edexOsgi/com.raytheon.edex.plugin.grib/utility/edex_static/base/grib/tables/-1/-1/4.2.0.16.table
View file

@ -1,3 +1,8 @@
# Product Discipline 0 - Meteorological products, Parameter Category 16: Forecast Radar Imagery
# 0-191 Reserved
255:255:Missing
0:0:Equivalent radar reflectivity factor for rain:m m6 m-3:REFZR
1:1:Equivalent radar reflectivity factor for snow:m m6 m-3:REFZI
2:2:Equivalent radar reflectivity factor for parameterized convection:m m6 m-3:REFZC
3:3:Echo Top:m:RETOP
4:4:Reflectivity:dB:REFD
5:5:Composite reflectivity:dB:REFC

2
edexOsgi/com.raytheon.edex.plugin.grib/utility/edex_static/base/grib/tables/7/0/4.2.0.16.table
View file

@ -8,4 +8,4 @@
196:196:Maximum/Composite radar reflectivity:dB:REFC
197:197:Echo Top:m:RETOP
198:198:Hourly Maximum of Simulated Reflectivity at 1 km AGL:dB:MAXREF
255:255:Missing
255:255:Missing

18
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPUtils.java
View file

@ -57,6 +57,7 @@ import com.raytheon.uf.common.gridcoverage.GridCoverage;
import com.raytheon.uf.common.hydro.spatial.HRAPCoordinates;
import com.raytheon.uf.common.hydro.spatial.HRAPSubGrid;
import com.raytheon.uf.common.message.response.ResponseMessageGeneric;
import com.raytheon.uf.common.monitor.config.FFMPSourceConfigurationManager.GUIDANCE_TYPE;
import com.raytheon.uf.common.monitor.scan.ScanUtils;
import com.raytheon.uf.common.mpe.util.XmrgFile;
import com.raytheon.uf.common.serialization.comm.RequestRouter;
@ -963,19 +964,26 @@ public class FFMPUtils {
}
/**
* Gets the datauri for this partiular FFG
* Gets the datauri for this particular FFG
*
* @param id
* @return
*/
public static String getFFGDataURI(String rfc, String parameter,
public static String getFFGDataURI(GUIDANCE_TYPE type, String datasetid, String parameter,
String plugin) {
DbQueryRequest request = new DbQueryRequest();
request.setEntityClass(GridRecord.class.getName());
request.addConstraint(GridConstants.PARAMETER_ABBREVIATION,
new RequestConstraint(parameter));
request.addConstraint(GridConstants.DATASET_ID, new RequestConstraint(
"FFG-" + rfc.substring(1)));
if (type == GUIDANCE_TYPE.RFC) {
request.addConstraint(GridConstants.DATASET_ID, new RequestConstraint(
"FFG-" + datasetid.substring(1)));
} else {
request.addConstraint(GridConstants.DATASET_ID, new RequestConstraint(
datasetid));
}
request.setOrderByField("dataTime.refTime", OrderMode.DESC);
try {
DbQueryResponse response = (DbQueryResponse) RequestRouter
@ -1468,7 +1476,7 @@ public class FFMPUtils {
* @param dataPath
* @return
*/
private static String[] parseGridDataPath(String dataPath) {
public static String[] parseGridDataPath(String dataPath) {
// parse the path given as the URI match in the source config
return dataPath.split(DataURI.SEPARATOR);
}

54
edexOsgi/com.raytheon.uf.edex.cpgsrv/src/com/raytheon/uf/edex/cpgsrv/CompositeProductGenerator.java
View file

@ -38,6 +38,7 @@ import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.time.DataTime;
import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.edex.core.EDEXUtil;
import com.raytheon.uf.edex.core.EdexException;
import com.raytheon.uf.edex.core.IContextStateProcessor;
@ -64,6 +65,7 @@ import com.raytheon.uf.edex.database.plugin.PluginDao;
* Feb 15, 2013 1638 mschenke Moved DataURINotificationMessage to uf.common.dataplugin
* Apr 17, 2014 2726 rjpeter Updated to send alerts directly to notification route.
* Aug 26, 2014 3503 bclement moved initialization to context state processor pre-start method
* Sep.09, 2015 4756 dhladky Check for possible null filters indicating bad configurations.
* </pre>
*
* @author dhladky
@ -112,6 +114,12 @@ public abstract class CompositeProductGenerator implements
protected String routeId = null;
/** Allowed interval between filter config errors **/
protected static final long ERROR_COUNT_INTERVAL_TIME = TimeUtil.MILLIS_PER_MINUTE * 2;
/** Keeps track of last config error message time */
protected long last_error_time = 0l;
public CompositeProductGenerator(String name, String compositeProductType) {
this(name, compositeProductType, null);
}
@ -189,25 +197,35 @@ public abstract class CompositeProductGenerator implements
URIFilter[] filters = getFilters();
if (filters != null) {
for (URIFilter filter : filters) {
synchronized (filter) {
if (filter.isMatched(messages)) {
try {
EDEXUtil.getMessageProducer()
.sendAsync(
routeId,
SerializationUtil
.transformToThrift(filter
.createGenerateMessage()));
} catch (Exception e) {
logger.error(
getGeneratorName()
+ ": filter: "
+ filter.getName()
+ ": failed to route filter to generator",
e);
}
// Safety, badly mis-configured filters can show up as null
if (filter != null) {
synchronized (filter) {
if (filter.isMatched(messages)) {
try {
EDEXUtil.getMessageProducer()
.sendAsync(
routeId,
SerializationUtil
.transformToThrift(filter
.createGenerateMessage()));
} catch (Exception e) {
logger.error(
getGeneratorName()
+ ": filter: "
+ filter.getName()
+ ": failed to route filter to generator",
e);
}
filter.reset();
filter.reset();
}
}
} else {
long time = System.currentTimeMillis();
if (((time - last_error_time) > ERROR_COUNT_INTERVAL_TIME) || last_error_time == 0l) {
last_error_time = time;
logger.error(getGeneratorName()
+ ": Filter for this generator is null, check configuration!");
}
}
}

89
edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/FFMPGenerator.java
View file

@ -69,6 +69,7 @@ import com.raytheon.uf.common.localization.exception.LocalizationOpFailedExcepti
import com.raytheon.uf.common.monitor.config.FFMPRunConfigurationManager;
import com.raytheon.uf.common.monitor.config.FFMPSourceConfigurationManager;
import com.raytheon.uf.common.monitor.config.FFMPSourceConfigurationManager.DATA_TYPE;
import com.raytheon.uf.common.monitor.config.FFMPSourceConfigurationManager.GUIDANCE_TYPE;
import com.raytheon.uf.common.monitor.config.FFMPSourceConfigurationManager.SOURCE_TYPE;
import com.raytheon.uf.common.monitor.config.FFMPTemplateConfigurationManager;
import com.raytheon.uf.common.monitor.events.MonitorConfigEvent;
@ -139,6 +140,7 @@ import com.raytheon.uf.edex.plugin.ffmp.common.FFTIRatioDiff;
* Jul 10, 2014 2914 garmendariz Remove EnvProperties
* Aug 26, 2014 3503 bclement removed constructDataURI() call
* Aug 08, 2015 4722 dhladky Generalized the processing of FFMP data types.
* Sep 09, 2015 4756 dhladky Further generalization of FFG processing.
* </pre>
*
* @author dhladky
@ -1050,7 +1052,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
}
/**
* Do pull strategy on FFG data
* Do pull strategy on FFG data, currently works with
* Gridded FFG sources only. (There are only gridded sources so far)
*
* @param filter
* @return
@ -1059,48 +1062,78 @@ public class FFMPGenerator extends CompositeProductGenerator implements
ArrayList<String> uris = new ArrayList<String>();
// Check RFC types
for (String rfc : filter.getRFC()) {
// get a hash of the sources and their grib ids
Set<String> sources = FFMPUtils.getFFGParameters(rfc);
if (sources != null) {
if (sources.size() > 0) {
for (String source : sources) {
if (sources != null && sources.size() > 0) {
for (String source : sources) {
SourceXML sourceXml = getSourceConfig().getSource(
source);
SourceXML sourceXml = getSourceConfig().getSource(source);
if (sourceXml != null) {
if (sourceXml != null) {
String plugin = getSourceConfig().getSource(source)
.getPlugin();
uris.add(FFMPUtils.getFFGDataURI(rfc, source,
plugin));
}
String plugin = getSourceConfig().getSource(source)
.getPlugin();
uris.add(FFMPUtils.getFFGDataURI(GUIDANCE_TYPE.RFC, rfc, source, plugin));
}
}
}
}
// Check for ARCHIVE types
ArrayList<String> guidSources = getSourceConfig().getGuidances();
if (guidSources != null && guidSources.size() > 0) {
for (String guidSource : guidSources) {
SourceXML sourceXml = getSourceConfig().getSource(guidSource);
if (sourceXml != null
&& sourceXml.getGuidanceType().equals(
GUIDANCE_TYPE.ARCHIVE.getGuidanceType())) {
String plugin = getSourceConfig().getSource(guidSource)
.getPlugin();
String[] uriComps = FFMPUtils.parseGridDataPath(sourceXml
.getDataPath());
/*
* datasetid is UriComp[3], parameter abbreviation is
* UriComp[7]
*/
uris.add(FFMPUtils.getFFGDataURI(GUIDANCE_TYPE.ARCHIVE,
uriComps[3], uriComps[7], plugin));
}
}
}
// treat it like a regular uri in the filter.
if (uris.size() > 0) {
for (String dataUri : uris) {
// add your pattern checks to the key
for (Pattern pattern : filter.getMatchURIs().keySet()) {
statusHandler.handle(Priority.DEBUG,
"Pattern: " + pattern.toString() + " Key: "
+ dataUri);
try {
if (pattern.matcher(dataUri).find()) {
// matches one of them, which one?
String matchKey = filter.getPatternName(pattern);
// put the sourceName:dataPath key into the sources
// array list
filter.getSources().put(matchKey, dataUri);
/*
* Safety, eliminates chance of unattached source config
* uri's coming in that have no pattern attached to them.
*/
if (dataUri != null && pattern != null) {
statusHandler.handle(Priority.INFO, "Pattern: "
+ pattern.toString() + " Key: " + dataUri);
try {
if (pattern.matcher(dataUri).find()) {
// matches one of them, which one?
String matchKey = filter
.getPatternName(pattern);
/*
* put the sourceName:dataPath key into the
* sources array list.
*/
filter.getSources().put(matchKey, dataUri);
}
} catch (Exception e) {
statusHandler.handle(
Priority.ERROR,
"Unable to locate new FFG file. "
+ dataUri, e);
}
} catch (Exception e) {
statusHandler.handle(
Priority.ERROR,
"Unable to locate new FFG file. "
+ pattern.toString(), e);
}
}
}
@ -1528,7 +1561,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
// write it, allowing, and in fact encouraging replacing
// the last one
dataStore.addDataRecord(rec, sp);
dataStore.store(StoreOp.OVERWRITE);
dataStore.store(StoreOp.REPLACE);
} catch (Exception e) {
statusHandler.handle(

113
edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/common/FFMPConfig.java
View file

@ -59,6 +59,7 @@ import com.vividsolutions.jts.geom.GeometryFactory;
* 06/30/2009 2521 dhladky Initial Creation.
* Apr 24, 2014 2060 njensen Removed unnecessary catch
* Aug 08, 2015 4722 dhladky Simplified source map additions, config.
* Sep.09, 2015 4756 dhladky Further simplified configuration.
*
* </pre>
*
@ -337,17 +338,17 @@ public class FFMPConfig {
sourceKey = keys[1];
} else {
checkSourceName = dataKey;
}
}
if (checkSourceName.equals(sourceName)) {
String dataUri = sourceMap.get(dataKey);
Object dataObject = null;
if (source.getDataType().equals(
FFMPSourceConfigurationManager.DATA_TYPE.XMRG
.getDataType())) {
dataObject = getXMRGFile(dataUri);
dataObject = getXMRGFile(dataUri);
} else if (source.getDataType().equals(
FFMPSourceConfigurationManager.DATA_TYPE.PDO
.getDataType())) {
@ -361,47 +362,8 @@ public class FFMPConfig {
.getDataType())) {
dataObject = getGrib(dataUri);
}
if (dataObject != null) {
// process as a VGB too
ProductXML product = sourceConfig.getProduct(source
.getSourceName());
if (product != null) {
// This indicates a primary source and QPE source
for (ProductRunXML productRun : ffmpgen
.getRunConfig().getRunner(getCWA())
.getProducts()) {
if (productRun.getProductName().equals(
product.getPrimarySource())) {
sourceKey = productRun.getProductKey();
}
}
HashMap<String, Object> virtSourceHash = new HashMap<String, Object>();
virtSourceHash.put(sourceKey, dataObject);
sources.put(product.getVirtual(),
virtSourceHash);
} else {
// NON-Primary sources and non-RFCFFG
if (!source.isRfc()) {
String primarySource = ffmpgen
.getSourceConfig()
.getPrimarySource(source);
for (ProductRunXML productRun : ffmpgen
.getRunConfig().getRunner(getCWA())
.getProducts()) {
if (productRun.getProductName().equals(
primarySource)) {
sourceKey = productRun
.getProductKey();
}
}
}
}
sourceHash.put(sourceKey, dataObject);
}
sourceHash = processSource(sourceHash, dataObject, source, sourceKey);
}
}
@ -417,6 +379,7 @@ public class FFMPConfig {
e);
}
}
/**
* Grab the XMRG file for use
@ -469,5 +432,67 @@ public class FFMPConfig {
public HashMap<String, Object> getSourceData(String sourceName) {
return sources.get(sourceName);
}
/**
* Process the sources from the URIfilter and ready them for processing.
* @param sourceHash
* @param dataObject
* @param source
* @param sourceKey
* @return sourceHash
*/
private HashMap<String, Object> processSource(HashMap<String, Object> sourceHash, Object dataObject, SourceXML source, String sourceKey) {
if (dataObject != null) {
// Is this a primary source?
ProductXML product = ffmpgen.getSourceConfig().getProduct(
source.getSourceName());
// Check for a primary source
if (product != null) {
/*
* Some primary sources derive the sourceKey from their Run
* Config product name.
*/
if (source.getDataType().equals(
FFMPSourceConfigurationManager.DATA_TYPE.XMRG
.getDataType())
|| sourceKey == null) {
for (ProductRunXML productRun : ffmpgen.getRunConfig()
.getRunner(getCWA()).getProducts()) {
if (productRun.getProductName().equals(
product.getPrimarySource())) {
sourceKey = productRun.getProductKey();
break;
}
}
}
// If primary, create a virtual too.
HashMap<String, Object> virtSourceHash = new HashMap<String, Object>();
virtSourceHash.put(sourceKey, dataObject);
sources.put(product.getVirtual(), virtSourceHash);
} else {
// NON Primary sources, find the primary.
String primarySource = ffmpgen
.getSourceConfig()
.getPrimarySource(source);
// Find the sourceKey to run against.
for (ProductRunXML productRun : ffmpgen
.getRunConfig().getRunner(getCWA())
.getProducts()) {
if (productRun.getProductName().equals(
primarySource)) {
sourceKey = productRun
.getProductKey();
break;
}
}
}
// Add to hash of sources to be processed.
sourceHash.put(sourceKey, dataObject);
}
return sourceHash;
}
}