Issue #1961 Add ability to disable grib tables.
Former-commit-id: 1370f32e8a2e2efd4b238b0a9b0e3621ad13043d
This commit is contained in:
Ben Steffensmeier
parent
48ecfd2124
commit
63d20bc2e1
13 changed files with 483 additions and 208 deletions
60
deltaScripts/13.4.1/upgradeGribLocalization.sh
Normal file
60
deltaScripts/13.4.1/upgradeGribLocalization.sh
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
#!/bin/bash
|
||||
# This script will update any gribParamInfo files to use the gridParamInfo tag
|
||||
# and move gribModels files from common_static to edex_static and remove
|
||||
# deprecated fields.
|
||||
#
|
||||
# This update is optional with 13.4.1 but it must be performed before grib can
|
||||
# move to the futue.
|
||||
#
|
||||
# This update is only for edex servers which host the cave localization files
|
||||
|
||||
echo "INFO: Moving all parameterInfo files to grid."
|
||||
|
||||
IFS=$'\n'
|
||||
gribFiles=`find /awips2/edex/data/utility/edex_static/*/*/grib/parameterInfo/ -iname '*.xml'`
|
||||
|
||||
for f in $gribFiles; do
|
||||
newf=${f//grib/grid}
|
||||
if [ -e "$newf" ]; then
|
||||
echo Cannot upgrade $f because $newf already exists
|
||||
else
|
||||
mkdir -p `dirname $newf`
|
||||
mv $f $newf
|
||||
fi
|
||||
done
|
||||
|
||||
echo "INFO: Upgrading all parameterInfo files."
|
||||
|
||||
gridFiles=`find /awips2/edex/data/utility/edex_static/*/*/grid/parameterInfo/ -iname '*.xml'`
|
||||
|
||||
for f in $gridFiles; do
|
||||
sed -n 's/gribParamInfo/gridParamInfo/g;p;' -i $f
|
||||
done
|
||||
|
||||
echo "INFO: Moving all gribModels to edex_static."
|
||||
|
||||
commonFiles=`find /awips2/edex/data/utility/common_static/*/*/grib/models/ -iname '*.xml'`
|
||||
|
||||
for f in $commonFiles; do
|
||||
newf=${f//common_static/edex_static}
|
||||
if [ -e "$newf" ]; then
|
||||
echo Cannot upgrade $f because $newf already exists
|
||||
else
|
||||
mkdir -p `dirname $newf`
|
||||
mv $f $newf
|
||||
fi
|
||||
done
|
||||
|
||||
echo "INFO: Cleaning all gribModel files."
|
||||
|
||||
edexFiles=`find /awips2/edex/data/utility/edex_static/*/*/grib/models/ -iname '*.xml'`
|
||||
|
||||
for f in $edexFiles; do
|
||||
sed '/^\s*<title>.*<\/title>\s*$/d' -i $f
|
||||
sed '/^\s*<alias>.*<\/alias>\s*$/d' -i $f
|
||||
sed '/^\s*<dt>.*<\/dt>\s*$/d' -i $f
|
||||
sed '/^\s*<paramInfo>.*<\/paramInfo>\s*$/d' -i $f
|
||||
done
|
||||
|
||||
echo "INFO: The update finished successfully."
|
||||
exit 0
|
||||
|
|
@ -9,6 +9,15 @@ export IH_DB_NAME=hd_ob83oax
|
|||
|
||||
### end of localization variables ###
|
||||
|
||||
### flag to control grib deprecation ###
|
||||
if [ -z "$gribMode" ]; then
|
||||
# uncomment only one of the following two lines
|
||||
export gribMode=deprecated
|
||||
#export gribMode=future
|
||||
fi
|
||||
|
||||
## end of grib deprecation flag ###
|
||||
|
||||
# setup environment for HPE
|
||||
export DATA_ARCHIVE_ROOT=/tmp/sbn
|
||||
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ import com.raytheon.uf.common.util.mapping.MultipleMappingException;
|
|||
* extension.
|
||||
* Mar 20, 2013 #1774 randerso Added getModelInfo,
|
||||
* added Dflt if no levels specified
|
||||
* Apr 30, 2013 1961 bsteffen Add ability to disable grib tables.
|
||||
*
|
||||
* </pre>
|
||||
*
|
||||
|
|
@ -73,6 +74,12 @@ public class GridParamInfoLookup {
|
|||
/** The singleton instance */
|
||||
private static GridParamInfoLookup instance;
|
||||
|
||||
/**
|
||||
* Temporary boolean to enable or disable loading deprecated grib
|
||||
* definitions
|
||||
*/
|
||||
private static boolean loadGribDefs = false;
|
||||
|
||||
/** Parameter information map */
|
||||
private Map<String, GridParamInfo> modelParamMap;
|
||||
|
||||
|
|
@ -88,6 +95,17 @@ public class GridParamInfoLookup {
|
|||
return instance;
|
||||
}
|
||||
|
||||
public static synchronized boolean enableLoadGribDefs() {
|
||||
GridParamInfoLookup.loadGribDefs = true;
|
||||
if(instance != null){
|
||||
System.err.println("setLoadGribDefs was called too late.");
|
||||
// this will trigger a complete reload. In testing it is never
|
||||
// called too late, this is paranoia code.
|
||||
instance = null;
|
||||
}
|
||||
return GridParamInfoLookup.loadGribDefs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new GribParamInfoLookup instance
|
||||
*/
|
||||
|
|
@ -172,9 +190,16 @@ public class GridParamInfoLookup {
|
|||
private void init() {
|
||||
Unmarshaller um = null;
|
||||
try {
|
||||
JAXBContext context = JAXBContext.newInstance(ParameterInfo.class,
|
||||
GridParamInfo.class, GribParamInfo.class);
|
||||
um = context.createUnmarshaller();
|
||||
if (loadGribDefs) {
|
||||
JAXBContext context = JAXBContext.newInstance(
|
||||
ParameterInfo.class, GridParamInfo.class,
|
||||
GribParamInfo.class);
|
||||
um = context.createUnmarshaller();
|
||||
} else {
|
||||
JAXBContext context = JAXBContext.newInstance(
|
||||
ParameterInfo.class, GridParamInfo.class);
|
||||
um = context.createUnmarshaller();
|
||||
}
|
||||
} catch (JAXBException e) {
|
||||
statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(), e);
|
||||
return;
|
||||
|
|
@ -194,33 +219,41 @@ public class GridParamInfoLookup {
|
|||
if (!modelParamMap.containsKey(key)) {
|
||||
modelParamMap.put(key, paramInfo);
|
||||
}
|
||||
if (paramInfo instanceof GribParamInfo) {
|
||||
statusHandler.info("Loaded deprecated gribParamInfo for "
|
||||
+ key);
|
||||
}
|
||||
} catch (JAXBException e) {
|
||||
statusHandler.handle(Priority.PROBLEM,
|
||||
"Error unmarshalling grid parameter information", e);
|
||||
}
|
||||
}
|
||||
if (loadGribDefs) {
|
||||
// Deprecated grib SITE level files.
|
||||
files = pm.listFiles(pm.getContext(LocalizationType.EDEX_STATIC,
|
||||
LocalizationLevel.SITE), "grib" + IPathManager.SEPARATOR
|
||||
+ "parameterInfo", new String[] { ".xml" }, true, true);
|
||||
for (LocalizationFile file : files) {
|
||||
statusHandler.info("Loading deprecated paramInfo file: "
|
||||
+ file.getFile());
|
||||
String name = file.getFile().getName().replace(".xml", "");
|
||||
// Do not override grid files.
|
||||
if (modelParamMap.get(name) != null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Deprecated grib SITE level files.
|
||||
files = pm.listFiles(pm.getContext(LocalizationType.EDEX_STATIC,
|
||||
LocalizationLevel.SITE), "grib" + IPathManager.SEPARATOR
|
||||
+ "parameterInfo", new String[] { ".xml" }, true, true);
|
||||
for (LocalizationFile file : files) {
|
||||
String name = file.getFile().getName().replace(".xml", "");
|
||||
// Do not override grid files.
|
||||
if (modelParamMap.get(name) != null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
GridParamInfo paramInfo = (GridParamInfo) um.unmarshal(file
|
||||
.getFile());
|
||||
modelParamMap.put(name, paramInfo);
|
||||
} catch (JAXBException e) {
|
||||
statusHandler.handle(Priority.PROBLEM,
|
||||
"Error unmarshalling grid parameter information", e);
|
||||
try {
|
||||
GridParamInfo paramInfo = (GridParamInfo) um.unmarshal(file
|
||||
.getFile());
|
||||
modelParamMap.put(name, paramInfo);
|
||||
} catch (JAXBException e) {
|
||||
statusHandler
|
||||
.handle(Priority.PROBLEM,
|
||||
"Error unmarshalling grid parameter information",
|
||||
e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (GridParamInfo gridParamInfo : modelParamMap.values()) {
|
||||
for (String parmName : gridParamInfo.getParmNames()) {
|
||||
ParameterInfo parameterInfo = gridParamInfo
|
||||
|
|
|
|||
|
|
@ -0,0 +1,43 @@
|
|||
<beans xmlns="http://www.springframework.org/schema/beans"
|
||||
xmlns:amq="http://activemq.apache.org/schema/core" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
|
||||
http://activemq.apache.org/schema/core http://activemq.apache.org/schema/core/activemq-core.xsd
|
||||
http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd">
|
||||
<bean id="gribPathProvider" class="com.raytheon.uf.common.dataplugin.grib.GribPathProvider" />
|
||||
|
||||
<bean id="gribProperties" class="com.raytheon.uf.common.dataplugin.PluginProperties">
|
||||
<property name="pluginName" value="grib" />
|
||||
<property name="pluginFQN" value="com.raytheon.uf.common.dataplugin.grib" />
|
||||
<property name="dao" value="com.raytheon.edex.plugin.grib.dao.GribDao" />
|
||||
<property name="record" value="com.raytheon.uf.common.dataplugin.grib.GribRecord" />
|
||||
<property name="dependencyFQNs">
|
||||
<list>
|
||||
<value>com.raytheon.uf.common.dataplugin.level</value>
|
||||
<value>com.raytheon.uf.common.parameter</value>
|
||||
<value>com.raytheon.uf.common.gridcoverage</value>
|
||||
</list>
|
||||
</property>
|
||||
<property name="pathProvider">
|
||||
<ref bean="gribPathProvider"/>
|
||||
</property>
|
||||
<property name="compression" value="LZF"/>
|
||||
</bean>
|
||||
|
||||
<bean id="gribRegistered" factory-bean="pluginRegistry" factory-method="register"
|
||||
depends-on="levelRegistered,parameterRegistered,gridcoverageRegistered">
|
||||
<constructor-arg value="grib"/>
|
||||
<constructor-arg ref="gribProperties"/>
|
||||
</bean>
|
||||
|
||||
<bean id="gfeLoadGribParamInfo"
|
||||
class="org.springframework.beans.factory.config.MethodInvokingFactoryBean">
|
||||
<property name="targetClass">
|
||||
<value>com.raytheon.edex.plugin.gfe.paraminfo.GridParamInfoLookup</value>
|
||||
</property>
|
||||
<property name="targetMethod">
|
||||
<value>enableLoadGribDefs</value>
|
||||
</property>
|
||||
</bean>
|
||||
|
||||
|
||||
</beans>
|
||||
|
|
@ -0,0 +1,129 @@
|
|||
<beans xmlns="http://www.springframework.org/schema/beans"
|
||||
xmlns:amq="http://activemq.apache.org/schema/core" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
|
||||
http://activemq.apache.org/schema/core http://activemq.apache.org/schema/core/activemq-core.xsd
|
||||
http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd">
|
||||
|
||||
<bean id="gribDecoder" class="com.raytheon.edex.plugin.grib.GribDecoder" />
|
||||
|
||||
<bean id="ingest-grib" class="org.apache.camel.component.jms.JmsComponent">
|
||||
<constructor-arg ref="jmsIngestGribConfig" />
|
||||
<property name="taskExecutor" ref="gribThreadPool" />
|
||||
</bean>
|
||||
|
||||
<bean id="jmsIngestGribConfig" class="org.apache.camel.component.jms.JmsConfiguration"
|
||||
factory-bean="jmsConfig" factory-method="copy">
|
||||
</bean>
|
||||
|
||||
<bean id="gribDecodeThreads" class="java.lang.Integer">
|
||||
<constructor-arg value="${grib-decode.count.threads}" />
|
||||
</bean>
|
||||
|
||||
<bean id="gribThreadPool"
|
||||
class="com.raytheon.uf.edex.esb.camel.spring.JmsThreadPoolTaskExecutor">
|
||||
<property name="corePoolSize" ref="gribDecodeThreads" />
|
||||
<property name="maxPoolSize" ref="gribDecodeThreads" />
|
||||
</bean>
|
||||
|
||||
<bean id="largeFileChecker" class="com.raytheon.edex.plugin.grib.GribLargeFileChecker" />
|
||||
|
||||
<bean id="gribSplitter" class="com.raytheon.edex.plugin.grib.GribSplitter">
|
||||
<constructor-arg value="${edex.home}/data/tmp/"/>
|
||||
</bean>
|
||||
|
||||
<bean id="useLatestAggregationStrategy" class="org.apache.camel.processor.aggregate.UseLatestAggregationStrategy" />
|
||||
|
||||
<bean id="largeFileLockRelease" class="com.raytheon.edex.plugin.grib.GribLockRelease" />
|
||||
|
||||
<bean id="gribDecodeCamelRegistered" factory-bean="contextManager"
|
||||
factory-method="register" depends-on="persistCamelRegistered">
|
||||
<constructor-arg ref="grib-decode"/>
|
||||
</bean>
|
||||
|
||||
<bean id="gribPostProcessor"
|
||||
class="com.raytheon.edex.plugin.grib.decoderpostprocessors.GribPostProcessor"
|
||||
factory-method="getInstance" />
|
||||
|
||||
<bean id="gridToGribConverter"
|
||||
class="com.raytheon.edex.plugin.grib.GridToGribConverter" factory-method="getInstance" />
|
||||
|
||||
<bean id="gribTableLookup" class="com.raytheon.edex.util.grib.GribTableLookup"
|
||||
factory-method="getInstance" depends-on="gridRegistered" />
|
||||
|
||||
<bean id="gribModelLookup"
|
||||
class="com.raytheon.edex.plugin.grib.util.GribModelLookup"
|
||||
factory-method="getInstance" init-method="initCommonStaticModels"/>
|
||||
|
||||
<bean id="gribSpatialCache"
|
||||
class="com.raytheon.edex.plugin.grib.spatial.GribSpatialCache"
|
||||
factory-method="getInstance" depends-on="gridcoveragelookup"/>
|
||||
|
||||
<camelContext id="grib-decode" xmlns="http://camel.apache.org/schema/spring"
|
||||
errorHandlerRef="errorHandler"
|
||||
autoStartup="false">
|
||||
|
||||
<endpoint id="gribFileEndpoint" uri="file:${edex.home}/data/sbn/grib?noop=true&idempotent=false" />
|
||||
|
||||
<route id="gribFileConsumerRoute">
|
||||
<from ref="gribFileEndpoint" />
|
||||
<bean ref="fileToString" />
|
||||
<setHeader headerName="pluginName">
|
||||
<constant>grid</constant>
|
||||
</setHeader>
|
||||
<to uri="ingest-grib:queue:Ingest.Grib" />
|
||||
</route>
|
||||
|
||||
<!-- Begin Grib Decode Route -->
|
||||
<route id="gribIngestRoute">
|
||||
<from uri="ingest-grib:queue:Ingest.Grib?concurrentConsumers=#gribDecodeThreads&destinationResolver=#qpidDurableResolver" />
|
||||
<setHeader headerName="pluginName">
|
||||
<constant>grid</constant>
|
||||
</setHeader>
|
||||
<!-- strategyRef is needed because of camel bug https://issues.apache.org/activemq/browse/CAMEL-3333,
|
||||
without the strategy it uses the original message in the multicast and it loses the largeFileLock header -->
|
||||
<split strategyRef="useLatestAggregationStrategy">
|
||||
<method bean="gribSplitter" method="split" />
|
||||
<choice>
|
||||
<when>
|
||||
<simple>${header.CamelSplitSize} == 1</simple>
|
||||
<doTry>
|
||||
<pipeline>
|
||||
<bean ref="stringToFile" />
|
||||
<bean ref="largeFileChecker" />
|
||||
<bean ref="gribDecoder" />
|
||||
<bean ref="gribSplitter" method="clean" />
|
||||
<!-- send for processing -->
|
||||
<bean ref="gribPostProcessor" method="process" />
|
||||
<multicast parallelProcessing="false">
|
||||
<!-- send to persistence -->
|
||||
<to uri="directvm:persistIndexAlert" />
|
||||
<!-- send to transform -->
|
||||
<to uri="directvm:gridToGrib"/>
|
||||
</multicast>
|
||||
</pipeline>
|
||||
<doCatch>
|
||||
<exception>java.lang.Throwable</exception>
|
||||
<to uri="log:grib?level=ERROR&showBody=false&showCaughtException=true&showStackTrace=true"/>
|
||||
</doCatch>
|
||||
<doFinally>
|
||||
<bean ref="largeFileLockRelease" />
|
||||
</doFinally>
|
||||
</doTry>
|
||||
</when>
|
||||
<otherwise>
|
||||
<to uri="ingest-grib:queue:Ingest.Grib" />
|
||||
</otherwise>
|
||||
</choice>
|
||||
</split>
|
||||
</route>
|
||||
|
||||
<route id="gridToGrib">
|
||||
<from uri="directvm:gridToGrib"/>
|
||||
<bean ref="gridToGribConverter" method="convert"/>
|
||||
<bean ref="index" method="index"/>
|
||||
<bean ref="toDataURI" method="toDataURI"/>
|
||||
<to uri="vm:stageNotification"/>
|
||||
</route>
|
||||
|
||||
</camelContext>
|
||||
</beans>
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
<beans xmlns="http://www.springframework.org/schema/beans"
|
||||
xmlns:amq="http://activemq.apache.org/schema/core" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
|
||||
http://activemq.apache.org/schema/core http://activemq.apache.org/schema/core/activemq-core.xsd
|
||||
http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd">
|
||||
|
||||
<bean id="gridDataQueryHandler" class="com.raytheon.edex.plugin.grib.handler.GridDataHandler" />
|
||||
|
||||
<bean factory-bean="handlerRegistry" factory-method="register">
|
||||
<constructor-arg value="com.raytheon.uf.common.dataplugin.grib.request.GridDataRequestMessage"/>
|
||||
<constructor-arg ref="gridDataQueryHandler"/>
|
||||
</bean>
|
||||
|
||||
<bean id="deleteAllModelDataHandler" class="com.raytheon.edex.plugin.grib.handler.DeleteAllModelDataHandler" />
|
||||
<bean factory-bean="handlerRegistry" factory-method="register">
|
||||
<constructor-arg value="com.raytheon.uf.common.dataplugin.grib.request.DeleteAllModelDataRequest"/>
|
||||
<constructor-arg ref="deleteAllModelDataHandler"/>
|
||||
</bean>
|
||||
</beans>
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
<beans xmlns="http://www.springframework.org/schema/beans"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd">
|
||||
<!-- In the future this file doesn't exist. -->
|
||||
|
||||
</beans>
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
<beans xmlns="http://www.springframework.org/schema/beans"
|
||||
xmlns:amq="http://activemq.apache.org/schema/core" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
|
||||
http://activemq.apache.org/schema/core http://activemq.apache.org/schema/core/activemq-core.xsd
|
||||
http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd">
|
||||
|
||||
<bean id="gribDecoder" class="com.raytheon.edex.plugin.grib.GribDecoder" />
|
||||
|
||||
<bean id="ingest-grib" class="org.apache.camel.component.jms.JmsComponent">
|
||||
<constructor-arg ref="jmsIngestGribConfig" />
|
||||
<property name="taskExecutor" ref="gribThreadPool" />
|
||||
</bean>
|
||||
|
||||
<bean id="jmsIngestGribConfig" class="org.apache.camel.component.jms.JmsConfiguration"
|
||||
factory-bean="jmsConfig" factory-method="copy">
|
||||
</bean>
|
||||
|
||||
<bean id="gribDecodeThreads" class="java.lang.Integer">
|
||||
<constructor-arg value="${grib-decode.count.threads}" />
|
||||
</bean>
|
||||
|
||||
<bean id="gribThreadPool"
|
||||
class="com.raytheon.uf.edex.esb.camel.spring.JmsThreadPoolTaskExecutor">
|
||||
<property name="corePoolSize" ref="gribDecodeThreads" />
|
||||
<property name="maxPoolSize" ref="gribDecodeThreads" />
|
||||
</bean>
|
||||
|
||||
<bean id="largeFileChecker" class="com.raytheon.edex.plugin.grib.GribLargeFileChecker" />
|
||||
|
||||
<bean id="gribSplitter" class="com.raytheon.edex.plugin.grib.GribSplitter">
|
||||
<constructor-arg value="${edex.home}/data/tmp/"/>
|
||||
</bean>
|
||||
|
||||
<bean id="useLatestAggregationStrategy" class="org.apache.camel.processor.aggregate.UseLatestAggregationStrategy" />
|
||||
|
||||
<bean id="largeFileLockRelease" class="com.raytheon.edex.plugin.grib.GribLockRelease" />
|
||||
|
||||
<bean id="gribDecodeCamelRegistered" factory-bean="contextManager"
|
||||
factory-method="register" depends-on="persistCamelRegistered">
|
||||
<constructor-arg ref="grib-decode"/>
|
||||
</bean>
|
||||
|
||||
<bean id="gribPostProcessor"
|
||||
class="com.raytheon.edex.plugin.grib.decoderpostprocessors.GribPostProcessor"
|
||||
factory-method="getInstance" />
|
||||
|
||||
<bean id="gribTableLookup" class="com.raytheon.edex.util.grib.GribTableLookup"
|
||||
factory-method="getInstance" depends-on="gridRegistered" />
|
||||
|
||||
<bean id="gribModelLookup"
|
||||
class="com.raytheon.edex.plugin.grib.util.GribModelLookup"
|
||||
factory-method="getInstance" />
|
||||
|
||||
<bean id="gribSpatialCache"
|
||||
class="com.raytheon.edex.plugin.grib.spatial.GribSpatialCache"
|
||||
factory-method="getInstance" depends-on="gridcoveragelookup"/>
|
||||
|
||||
<camelContext id="grib-decode" xmlns="http://camel.apache.org/schema/spring"
|
||||
errorHandlerRef="errorHandler"
|
||||
autoStartup="false">
|
||||
|
||||
<endpoint id="gribFileEndpoint" uri="file:${edex.home}/data/sbn/grib?noop=true&idempotent=false" />
|
||||
|
||||
<route id="gribFileConsumerRoute">
|
||||
<from ref="gribFileEndpoint" />
|
||||
<bean ref="fileToString" />
|
||||
<setHeader headerName="pluginName">
|
||||
<constant>grid</constant>
|
||||
</setHeader>
|
||||
<to uri="ingest-grib:queue:Ingest.Grib" />
|
||||
</route>
|
||||
|
||||
<!-- Begin Grib Decode Route -->
|
||||
<route id="gribIngestRoute">
|
||||
<from uri="ingest-grib:queue:Ingest.Grib?concurrentConsumers=#gribDecodeThreads&destinationResolver=#qpidDurableResolver" />
|
||||
<setHeader headerName="pluginName">
|
||||
<constant>grid</constant>
|
||||
</setHeader>
|
||||
<!-- strategyRef is needed because of camel bug https://issues.apache.org/activemq/browse/CAMEL-3333,
|
||||
without the strategy it uses the original message in the multicast and it loses the largeFileLock header -->
|
||||
<split strategyRef="useLatestAggregationStrategy">
|
||||
<method bean="gribSplitter" method="split" />
|
||||
<choice>
|
||||
<when>
|
||||
<simple>${header.CamelSplitSize} == 1</simple>
|
||||
<doTry>
|
||||
<pipeline>
|
||||
<bean ref="stringToFile" />
|
||||
<bean ref="largeFileChecker" />
|
||||
<bean ref="gribDecoder" />
|
||||
<bean ref="gribSplitter" method="clean" />
|
||||
<!-- send for processing -->
|
||||
<bean ref="gribPostProcessor" method="process" />
|
||||
<to uri="directvm:persistIndexAlert" />
|
||||
</pipeline>
|
||||
<doCatch>
|
||||
<exception>java.lang.Throwable</exception>
|
||||
<to uri="log:grib?level=ERROR&showBody=false&showCaughtException=true&showStackTrace=true"/>
|
||||
</doCatch>
|
||||
<doFinally>
|
||||
<bean ref="largeFileLockRelease" />
|
||||
</doFinally>
|
||||
</doTry>
|
||||
</when>
|
||||
<otherwise>
|
||||
<to uri="ingest-grib:queue:Ingest.Grib" />
|
||||
</otherwise>
|
||||
</choice>
|
||||
</split>
|
||||
</route>
|
||||
|
||||
</camelContext>
|
||||
</beans>
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
<beans xmlns="http://www.springframework.org/schema/beans"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd">
|
||||
<!-- In the future this file doesn't exist. -->
|
||||
|
||||
</beans>
|
||||
|
|
@ -1,32 +1,4 @@
|
|||
<beans xmlns="http://www.springframework.org/schema/beans"
|
||||
xmlns:amq="http://activemq.apache.org/schema/core" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
|
||||
http://activemq.apache.org/schema/core http://activemq.apache.org/schema/core/activemq-core.xsd
|
||||
http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd">
|
||||
<bean id="gribPathProvider" class="com.raytheon.uf.common.dataplugin.grib.GribPathProvider" />
|
||||
|
||||
<bean id="gribProperties" class="com.raytheon.uf.common.dataplugin.PluginProperties">
|
||||
<property name="pluginName" value="grib" />
|
||||
<property name="pluginFQN" value="com.raytheon.uf.common.dataplugin.grib" />
|
||||
<property name="dao" value="com.raytheon.edex.plugin.grib.dao.GribDao" />
|
||||
<property name="record" value="com.raytheon.uf.common.dataplugin.grib.GribRecord" />
|
||||
<property name="dependencyFQNs">
|
||||
<list>
|
||||
<value>com.raytheon.uf.common.dataplugin.level</value>
|
||||
<value>com.raytheon.uf.common.parameter</value>
|
||||
<value>com.raytheon.uf.common.gridcoverage</value>
|
||||
</list>
|
||||
</property>
|
||||
<property name="pathProvider">
|
||||
<ref bean="gribPathProvider"/>
|
||||
</property>
|
||||
<property name="compression" value="LZF"/>
|
||||
</bean>
|
||||
|
||||
<bean id="gribRegistered" factory-bean="pluginRegistry" factory-method="register"
|
||||
depends-on="levelRegistered,parameterRegistered,gridcoverageRegistered">
|
||||
<constructor-arg value="grib"/>
|
||||
<constructor-arg ref="gribProperties"/>
|
||||
</bean>
|
||||
|
||||
<beans xmlns="http://www.springframework.org/schema/beans" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd">
|
||||
<import resource="../spring.${gribMode}/grib-common.xml" />
|
||||
</beans>
|
||||
|
|
@ -1,129 +1,4 @@
|
|||
<beans xmlns="http://www.springframework.org/schema/beans"
|
||||
xmlns:amq="http://activemq.apache.org/schema/core" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
|
||||
http://activemq.apache.org/schema/core http://activemq.apache.org/schema/core/activemq-core.xsd
|
||||
http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd">
|
||||
|
||||
<bean id="gribDecoder" class="com.raytheon.edex.plugin.grib.GribDecoder" />
|
||||
|
||||
<bean id="ingest-grib" class="org.apache.camel.component.jms.JmsComponent">
|
||||
<constructor-arg ref="jmsIngestGribConfig" />
|
||||
<property name="taskExecutor" ref="gribThreadPool" />
|
||||
</bean>
|
||||
|
||||
<bean id="jmsIngestGribConfig" class="org.apache.camel.component.jms.JmsConfiguration"
|
||||
factory-bean="jmsConfig" factory-method="copy">
|
||||
</bean>
|
||||
|
||||
<bean id="gribDecodeThreads" class="java.lang.Integer">
|
||||
<constructor-arg value="${grib-decode.count.threads}" />
|
||||
</bean>
|
||||
|
||||
<bean id="gribThreadPool"
|
||||
class="com.raytheon.uf.edex.esb.camel.spring.JmsThreadPoolTaskExecutor">
|
||||
<property name="corePoolSize" ref="gribDecodeThreads" />
|
||||
<property name="maxPoolSize" ref="gribDecodeThreads" />
|
||||
</bean>
|
||||
|
||||
<bean id="largeFileChecker" class="com.raytheon.edex.plugin.grib.GribLargeFileChecker" />
|
||||
|
||||
<bean id="gribSplitter" class="com.raytheon.edex.plugin.grib.GribSplitter">
|
||||
<constructor-arg value="${edex.home}/data/tmp/"/>
|
||||
</bean>
|
||||
|
||||
<bean id="useLatestAggregationStrategy" class="org.apache.camel.processor.aggregate.UseLatestAggregationStrategy" />
|
||||
|
||||
<bean id="largeFileLockRelease" class="com.raytheon.edex.plugin.grib.GribLockRelease" />
|
||||
|
||||
<bean id="gribDecodeCamelRegistered" factory-bean="contextManager"
|
||||
factory-method="register" depends-on="persistCamelRegistered">
|
||||
<constructor-arg ref="grib-decode"/>
|
||||
</bean>
|
||||
|
||||
<bean id="gribPostProcessor"
|
||||
class="com.raytheon.edex.plugin.grib.decoderpostprocessors.GribPostProcessor"
|
||||
factory-method="getInstance" />
|
||||
|
||||
<bean id="gridToGribConverter"
|
||||
class="com.raytheon.edex.plugin.grib.GridToGribConverter" factory-method="getInstance" />
|
||||
|
||||
<bean id="gribTableLookup" class="com.raytheon.edex.util.grib.GribTableLookup"
|
||||
factory-method="getInstance" depends-on="gridRegistered" />
|
||||
|
||||
<bean id="gribModelLookup"
|
||||
class="com.raytheon.edex.plugin.grib.util.GribModelLookup"
|
||||
factory-method="getInstance" />
|
||||
|
||||
<bean id="gribSpatialCache"
|
||||
class="com.raytheon.edex.plugin.grib.spatial.GribSpatialCache"
|
||||
factory-method="getInstance" depends-on="gridcoveragelookup"/>
|
||||
|
||||
<camelContext id="grib-decode" xmlns="http://camel.apache.org/schema/spring"
|
||||
errorHandlerRef="errorHandler"
|
||||
autoStartup="false">
|
||||
|
||||
<endpoint id="gribFileEndpoint" uri="file:${edex.home}/data/sbn/grib?noop=true&idempotent=false" />
|
||||
|
||||
<route id="gribFileConsumerRoute">
|
||||
<from ref="gribFileEndpoint" />
|
||||
<bean ref="fileToString" />
|
||||
<setHeader headerName="pluginName">
|
||||
<constant>grid</constant>
|
||||
</setHeader>
|
||||
<to uri="ingest-grib:queue:Ingest.Grib" />
|
||||
</route>
|
||||
|
||||
<!-- Begin Grib Decode Route -->
|
||||
<route id="gribIngestRoute">
|
||||
<from uri="ingest-grib:queue:Ingest.Grib?concurrentConsumers=#gribDecodeThreads&destinationResolver=#qpidDurableResolver" />
|
||||
<setHeader headerName="pluginName">
|
||||
<constant>grid</constant>
|
||||
</setHeader>
|
||||
<!-- strategyRef is needed because of camel bug https://issues.apache.org/activemq/browse/CAMEL-3333,
|
||||
without the strategy it uses the original message in the multicast and it loses the largeFileLock header -->
|
||||
<split strategyRef="useLatestAggregationStrategy">
|
||||
<method bean="gribSplitter" method="split" />
|
||||
<choice>
|
||||
<when>
|
||||
<simple>${header.CamelSplitSize} == 1</simple>
|
||||
<doTry>
|
||||
<pipeline>
|
||||
<bean ref="stringToFile" />
|
||||
<bean ref="largeFileChecker" />
|
||||
<bean ref="gribDecoder" />
|
||||
<bean ref="gribSplitter" method="clean" />
|
||||
<!-- send for processing -->
|
||||
<bean ref="gribPostProcessor" method="process" />
|
||||
<multicast parallelProcessing="false">
|
||||
<!-- send to persistence -->
|
||||
<to uri="directvm:persistIndexAlert" />
|
||||
<!-- send to transform -->
|
||||
<to uri="directvm:gridToGrib"/>
|
||||
</multicast>
|
||||
</pipeline>
|
||||
<doCatch>
|
||||
<exception>java.lang.Throwable</exception>
|
||||
<to uri="log:grib?level=ERROR&showBody=false&showCaughtException=true&showStackTrace=true"/>
|
||||
</doCatch>
|
||||
<doFinally>
|
||||
<bean ref="largeFileLockRelease" />
|
||||
</doFinally>
|
||||
</doTry>
|
||||
</when>
|
||||
<otherwise>
|
||||
<to uri="ingest-grib:queue:Ingest.Grib" />
|
||||
</otherwise>
|
||||
</choice>
|
||||
</split>
|
||||
</route>
|
||||
|
||||
<route id="gridToGrib">
|
||||
<from uri="directvm:gridToGrib"/>
|
||||
<bean ref="gridToGribConverter" method="convert"/>
|
||||
<bean ref="index" method="index"/>
|
||||
<bean ref="toDataURI" method="toDataURI"/>
|
||||
<to uri="vm:stageNotification"/>
|
||||
</route>
|
||||
|
||||
</camelContext>
|
||||
</beans>
|
||||
<beans xmlns="http://www.springframework.org/schema/beans" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd">
|
||||
<import resource="../spring.${gribMode}/grib-decode.xml" />
|
||||
</beans>
|
||||
|
|
@ -1,19 +1,4 @@
|
|||
<beans xmlns="http://www.springframework.org/schema/beans"
|
||||
xmlns:amq="http://activemq.apache.org/schema/core" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
|
||||
http://activemq.apache.org/schema/core http://activemq.apache.org/schema/core/activemq-core.xsd
|
||||
http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd">
|
||||
|
||||
<bean id="gridDataQueryHandler" class="com.raytheon.edex.plugin.grib.handler.GridDataHandler" />
|
||||
|
||||
<bean factory-bean="handlerRegistry" factory-method="register">
|
||||
<constructor-arg value="com.raytheon.uf.common.dataplugin.grib.request.GridDataRequestMessage"/>
|
||||
<constructor-arg ref="gridDataQueryHandler"/>
|
||||
</bean>
|
||||
|
||||
<bean id="deleteAllModelDataHandler" class="com.raytheon.edex.plugin.grib.handler.DeleteAllModelDataHandler" />
|
||||
<bean factory-bean="handlerRegistry" factory-method="register">
|
||||
<constructor-arg value="com.raytheon.uf.common.dataplugin.grib.request.DeleteAllModelDataRequest"/>
|
||||
<constructor-arg ref="deleteAllModelDataHandler"/>
|
||||
</bean>
|
||||
</beans>
|
||||
<beans xmlns="http://www.springframework.org/schema/beans" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd">
|
||||
<import resource="../spring.${gribMode}/grib-request.xml" />
|
||||
</beans>
|
||||
|
|
@ -48,6 +48,24 @@ import com.raytheon.uf.common.util.mapping.Alias;
|
|||
import com.raytheon.uf.common.util.mapping.AliasList;
|
||||
import com.raytheon.uf.common.util.mapping.MultipleMappingException;
|
||||
|
||||
/**
|
||||
*
|
||||
* Lookup a GridModel based off the information in the grib file.
|
||||
*
|
||||
* <pre>
|
||||
*
|
||||
* SOFTWARE HISTORY
|
||||
*
|
||||
* Date Ticket# Engineer Description
|
||||
* ------------ ---------- ----------- --------------------------
|
||||
* Apr 30, 2013 bsteffen Initial javadoc
|
||||
* Apr 30, 2013 1961 bsteffen Add ability to disable grib tables.
|
||||
*
|
||||
* </pre>
|
||||
*
|
||||
* @author unknown
|
||||
* @version 1.0
|
||||
*/
|
||||
public class GribModelLookup {
|
||||
private static final transient IUFStatusHandler statusHandler = UFStatus
|
||||
.getHandler(GribModelLookup.class);
|
||||
|
|
@ -199,8 +217,12 @@ public class GribModelLookup {
|
|||
}
|
||||
}
|
||||
|
||||
modelSet.addModels(initCommonStaticModels());
|
||||
addModels(modelSet);
|
||||
long endTime = System.currentTimeMillis();
|
||||
logger.info("Grib models initialized: " + (endTime - startTime) + "ms");
|
||||
}
|
||||
|
||||
private void addModels(GridModelSet modelSet) {
|
||||
for (GridModel model : modelSet.getModels()) {
|
||||
modelsByName.put(model.getName(), model);
|
||||
for (int process : model.getProcess()) {
|
||||
|
|
@ -218,8 +240,6 @@ public class GribModelLookup {
|
|||
}
|
||||
}
|
||||
}
|
||||
long endTime = System.currentTimeMillis();
|
||||
logger.info("Grib models initialized: " + (endTime - startTime) + "ms");
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -234,8 +254,8 @@ public class GribModelLookup {
|
|||
* @return
|
||||
* @throws GribException
|
||||
*/
|
||||
private List<GridModel> initCommonStaticModels() {
|
||||
List<GridModel> modelSet = new ArrayList<GridModel>();
|
||||
public Boolean initCommonStaticModels() {
|
||||
GridModelSet modelSet = new GridModelSet();
|
||||
LocalizationContext commonStaticSite = PathManagerFactory
|
||||
.getPathManager().getContext(
|
||||
LocalizationContext.LocalizationType.COMMON_STATIC,
|
||||
|
|
@ -250,9 +270,11 @@ public class GribModelLookup {
|
|||
|
||||
for (LocalizationFile modelFile : legacyFiles) {
|
||||
try {
|
||||
logger.info("Loading deprecated gribModel file: "
|
||||
+ modelFile.getFile());
|
||||
GridModelSet fileSet = JAXB.unmarshal(modelFile.getFile(),
|
||||
GridModelSet.class);
|
||||
modelSet.addAll(fileSet.getModels());
|
||||
modelSet.addModels(fileSet.getModels());
|
||||
ArrayList<DatasetInfo> infoList = new ArrayList<DatasetInfo>(
|
||||
fileSet.getModels().size());
|
||||
for (GridModel model : fileSet.getModels()) {
|
||||
|
|
@ -315,7 +337,10 @@ public class GribModelLookup {
|
|||
}
|
||||
}
|
||||
}
|
||||
return modelSet;
|
||||
if (modelSet.getModels() != null) {
|
||||
addModels(modelSet);
|
||||
}
|
||||
return modelSet.getModels() != null && !modelSet.getModels().isEmpty();
|
||||
}
|
||||
|
||||
private String toKey(Integer center, Integer subcenter, String grid,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue