xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

git checkout origin/master_18.1.1 edexOsgi/com.raytheon.edex.plugin.shef. closes #238

Browse source
This commit is contained in:
mjames-upc 2018-07-13 14:51:54 -06:00
parent dca7d4a875
commit bfa6f10d6e
17 changed files with 1672 additions and 1854 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
edexOsgi/com.raytheon.edex.plugin.shef/.classpath
View file

@ -3,5 +3,8 @@
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
<classpathentry kind="src" path="src"/>
<classpathentry kind="src" path="test"/>
<classpathentry combineaccessrules="false" kind="src" path="/org.mockito"/>
<classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/>
<classpathentry kind="output" path="bin"/>
</classpath>

23
edexOsgi/com.raytheon.edex.plugin.shef/res/spring/shef-common.xml
View file

@ -1,16 +1,17 @@
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd">
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd">
<!-- pluginFQN etc not set so as to not load hibernate objects into metadata -->
<bean id="shefProperties" class="com.raytheon.uf.common.dataplugin.PluginProperties">
<property name="pluginName" value="shef" />
<property name="dao" value="com.raytheon.edex.plugin.shef.dao.ShefDao" />
</bean>
<bean factory-bean="pluginRegistry" factory-method="register">
<constructor-arg value="shef"/>
<constructor-arg ref="shefProperties"/>
</bean>
<bean id="shefProperties" class="com.raytheon.uf.common.dataplugin.PluginProperties">
<property name="pluginName" value="shef" />
<property name="dao" value="com.raytheon.edex.plugin.shef.dao.ShefDao" />
</bean>
<bean factory-bean="pluginRegistry" factory-method="register">
<constructor-arg value="shef"/>
<constructor-arg ref="shefProperties"/>
</bean>
</beans>

228
edexOsgi/com.raytheon.edex.plugin.shef/res/spring/shef-ingest.xml
View file

@ -1,46 +1,46 @@
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd">
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://camel.apache.org/schema/spring
http://camel.apache.org/schema/spring/camel-spring.xsd">
<bean class="com.raytheon.uf.common.logback.appender.ThreadBasedAppender" factory-method="registerThreadPattern">
<constructor-arg value="shef" />
<constructor-arg value="Ingest.Shef.*" />
</bean>
<bean id="shefDecoder" class="com.raytheon.edex.plugin.shef.ShefDecoder">
<constructor-arg value="shef" />
</bean>
<bean id="shefDecoder" class="com.raytheon.edex.plugin.shef.ShefDecoder">
<constructor-arg value="shef" />
</bean>
<bean id="shefSeparator" class="com.raytheon.edex.plugin.shef.ShefSeparator" />
<bean id="shefSeparator" class="com.raytheon.edex.plugin.shef.ShefSeparator" />
<bean id="metarToShef" class="com.raytheon.edex.transform.shef.MetarToShefTransformer">
<constructor-arg type="java.lang.String"
value=" -a -b -p1 -p6 -p24 -round -w -strip " />
</bean>
<bean id="metarToShef" class="com.raytheon.edex.transform.shef.MetarToShefTransformer">
<constructor-arg type="java.lang.String"
value=" -a -b -p1 -p6 -p24 -round -w -strip " />
</bean>
<bean id="metarToShefFilter"
class="com.raytheon.edex.transform.shef.MetarToShefFilter">
<constructor-arg type="java.lang.String" value="metarToShefFilter.xml" />
<constructor-arg type="java.lang.String" value="SITE" />
</bean>
<bean id="metarToShefFilter"
class="com.raytheon.edex.transform.shef.MetarToShefFilter">
<constructor-arg type="java.lang.String" value="metarToShefFilter.xml" />
</bean>
<bean id="synopticToShef" class="com.raytheon.edex.transform.shef.SMToShefTransformer">
<constructor-arg type="java.lang.String"
value=" -a -v -b -p1 -p6 -p24 -round -w -strip " />
</bean>
<bean id="synopticToShef" class="com.raytheon.edex.transform.shef.SMToShefTransformer">
<constructor-arg type="java.lang.String"
value=" -a -v -b -p1 -p6 -p24 -round -w -strip " />
</bean>
<bean id="synopticToShefFilter"
class="com.raytheon.edex.transform.shef.SynopticToShefFilter">
<constructor-arg type="java.lang.String" value="synopticToShefFilter.xml" />
<constructor-arg type="java.lang.String" value="SITE" />
</bean>
<bean id="synopticToShefFilter"
class="com.raytheon.edex.transform.shef.SynopticToShefFilter">
<constructor-arg type="java.lang.String" value="synopticToShefFilter.xml" />
</bean>
<bean id="shefDistRegistry" factory-bean="distributionSrv"
factory-method="register">
<constructor-arg value="shef" />
<constructor-arg
value="jms-durable:queue:Ingest.Shef"/>
</bean>
<bean id="shefDistRegistry" factory-bean="distributionSrv"
factory-method="register">
<constructor-arg value="shef" />
<constructor-arg
value="jms-durable:queue:Ingest.Shef"/>
</bean>
<bean id="shefHandleoupDistRegistry" factory-bean="handleoupDistributionSrv"
factory-method="register">
@ -61,7 +61,7 @@
errorHandlerRef="errorHandler">
<endpoint id="shefManualFileEndpoint"
uri="file:${edex.home}/data/share/hydroapps/shefdecode/input?delete=true&amp;maxMessagesPerPoll=1000&amp;delay=15000&amp;exclusiveReadLockStrategy=#shefFileChangedStrategy" />
<route id="shefManualFileScan">
<from ref="shefManualFileEndpoint" />
<bean ref="manualProc" method="copyFileToArchive" />
@ -73,93 +73,93 @@
<!-- End add for manual input -->
<camelContext id="shef-camel" xmlns="http://camel.apache.org/schema/spring"
errorHandlerRef="errorHandler">
<camelContext id="shef-camel" xmlns="http://camel.apache.org/schema/spring"
errorHandlerRef="errorHandler">
<!-- Begin shef routes -->
<route id="shefIngestRoute">
<from
uri="jms-durable:queue:Ingest.Shef"/>
<setHeader headerName="pluginName">
<constant>shef</constant>
</setHeader>
<pipeline>
<bean ref="stringToFile" />
<to uri="direct-vm:shefProcessing" />
</pipeline>
</route>
<route id="shefStagedRoute">
<from
uri="jms-durable:queue:Ingest.ShefStaged"/>
<setHeader headerName="pluginName">
<constant>shef</constant>
</setHeader>
<to uri="direct-vm:shefProcessing" />
</route>
<!-- Begin shef routes -->
<route id="shefIngestRoute">
<from
uri="jms-durable:queue:Ingest.Shef"/>
<setHeader headerName="pluginName">
<constant>shef</constant>
</setHeader>
<pipeline>
<bean ref="stringToFile" />
<to uri="direct-vm:shefProcessing" />
</pipeline>
</route>
<route id="shefStagedRoute">
<from
uri="jms-durable:queue:Ingest.ShefStaged"/>
<setHeader headerName="pluginName">
<constant>shef</constant>
</setHeader>
<to uri="direct-vm:shefProcessing" />
</route>
<!-- direct-vm will be run under original thread, should this be moved to
a queue?? but will cause message to be serialized to the jms queue... -->
<route id="synopticToShefRoute">
<from uri="direct-vm:synopticToShef" />
<bean ref="synopticToShefFilter" method="filter" />
<pipeline>
<split streaming="true">
<method bean="synopticToShef" method="iterate" />
<bean ref="synopticToShef" method="transform" />
<to uri="jms-durable:queue:Ingest.ShefStaged" />
</split>
</pipeline>
</route>
<!-- direct-vm will be run under original thread, should this be moved to
a queue?? but will cause message to be serialized to the jms queue... -->
<route id="synopticToShefRoute">
<from uri="direct-vm:synopticToShef" />
<bean ref="synopticToShefFilter" method="filter" />
<pipeline>
<split streaming="true">
<method bean="synopticToShef" method="iterate" />
<bean ref="synopticToShef" method="transform" />
<to uri="jms-durable:queue:Ingest.ShefStaged" />
</split>
</pipeline>
</route>
<!-- direct-vm will be run under original thread, should this be moved to
a queue?? but will cause message to be serialized to the jms queue... -->
<route id="metarToShefRoute">
<from uri="direct-vm:metarToShef" />
<bean ref="metarToShefFilter" method="filter" />
<pipeline>
<split streaming="true">
<method bean="metarToShef" method="iterate" />
<bean ref="metarToShef" method="transformMetar" />
<to
uri="jms-durable:queue:Ingest.ShefStaged"/>
</split>
</pipeline>
</route>
<!-- direct-vm will be run under original thread, should this be moved to
a queue?? but will cause message to be serialized to the jms queue... -->
<route id="metarToShefRoute">
<from uri="direct-vm:metarToShef" />
<bean ref="metarToShefFilter" method="filter" />
<pipeline>
<split streaming="true">
<method bean="metarToShef" method="iterate" />
<bean ref="metarToShef" method="transformMetar" />
<to
uri="jms-durable:queue:Ingest.ShefStaged"/>
</split>
</pipeline>
</route>
<route id="shefProcessingRoute">
<from uri="direct-vm:shefProcessing" />
<doTry>
<pipeline>
<bean ref="shefDecoder" method="decode" />
<bean ref="processUtil" method="log"/>
</pipeline>
<doCatch>
<exception>java.lang.Throwable</exception>
<to
uri="log:shef?level=ERROR" />
</doCatch>
</doTry>
</route>
<route id="shefProcessingRoute">
<from uri="direct-vm:shefProcessing" />
<doTry>
<pipeline>
<bean ref="shefDecoder" method="decode" />
<bean ref="processUtil" method="log"/>
</pipeline>
<doCatch>
<exception>java.lang.Throwable</exception>
<to
uri="log:shef?level=ERROR" />
</doCatch>
</doTry>
</route>
<route id="shefManualIngestRoute">
<from
uri="jms-durable:queue:Ingest.ShefManual"/>
<setHeader headerName="pluginName">
<constant>shef</constant>
</setHeader>
<doTry>
<pipeline>
<route id="shefManualIngestRoute">
<from
uri="jms-durable:queue:Ingest.ShefManual"/>
<setHeader headerName="pluginName">
<constant>shef</constant>
</setHeader>
<doTry>
<pipeline>
<bean ref="stringToFile" />
<bean ref="shefDecoder" method="decode" />
<bean ref="processUtil" method="log"/>
</pipeline>
<doCatch>
<exception>java.lang.Throwable</exception>
<to
uri="log:shef?level=ERROR" />
</doCatch>
</doTry>
</route>
<bean ref="shefDecoder" method="decode" />
<bean ref="processUtil" method="log"/>
</pipeline>
<doCatch>
<exception>java.lang.Throwable</exception>
<to
uri="log:shef?level=ERROR" />
</doCatch>
</doTry>
</route>
</camelContext>
</camelContext>
</beans>

799
edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/SHEFParser.java
View file

File diff suppressed because it is too large Load diff

60
edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/ShefDecoder.java
View file

@ -26,6 +26,7 @@ import com.raytheon.edex.plugin.shef.ShefSeparator.ShefDecoderInput;
import com.raytheon.edex.plugin.shef.data.ShefRecord;
import com.raytheon.edex.plugin.shef.database.PostShef;
import com.raytheon.edex.plugin.shef.database.PurgeText;
import com.raytheon.edex.plugin.shef.util.ShefParm;
import com.raytheon.uf.common.dataplugin.PluginDataObject;
import com.raytheon.uf.common.ohd.AppsDefaults;
import com.raytheon.uf.common.status.IUFStatusHandler;
@ -39,8 +40,8 @@ import com.raytheon.uf.edex.decodertools.core.DecoderTools;
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 03/19/2008 387 M. Duff Initial creation.
* 06/02/2008 1166 M. Duff Added checks for null data objects.
* 10/16/2008 1548 jelkins Integrated ParameterCode Types
@ -55,6 +56,7 @@ import com.raytheon.uf.edex.decodertools.core.DecoderTools;
* 12/--/2009 jkorman Major refactor - split into ShefDecoder/SHEFParser
* 03/07/2013 15071 W. Kwock Skip empty data files.
* 04/28/2014 3088 mpduff Use UFStatus logging, various cleanup.
* 01/10/2018 5049 mduff Pass ShefParm to the ShefParser.
* </pre>
*/
public class ShefDecoder {
@ -66,6 +68,8 @@ public class ShefDecoder {
// here.
private PluginDataObject[] records = new PluginDataObject[0];
private ShefParm shefParm;
/**
* Constructor
*/
@ -79,6 +83,8 @@ public class ShefDecoder {
* @param name
*/
public ShefDecoder(String name) {
shefParm = new ShefParm();
shefParm.populate();
}
/**
@ -91,8 +97,8 @@ public class ShefDecoder {
* @return PluginDataObject[] of decoded data
*/
public PluginDataObject[] decode(byte[] data, Headers headers) {
boolean archiveMode = AppsDefaults.getInstance().getBoolean(
"ALLOW_ARCHIVE_DATA", false);
boolean archiveMode = AppsDefaults.getInstance()
.getBoolean("ALLOW_ARCHIVE_DATA", false);
String traceId = null;
@ -123,7 +129,7 @@ public class ShefDecoder {
postDate = getPostTime(startTime);
}
PostShef postShef = new PostShef(postDate);
PostShef postShef = new PostShef(postDate, shefParm);
if (separator.hasNext()) {
PurgeText pText = new PurgeText(postDate);
pText.storeTextProduct(separator);
@ -138,12 +144,6 @@ public class ShefDecoder {
return records;
}
/**
*
* @param data
* @param headers
* @return
*/
public PluginDataObject[] decodeNoWMOHeader(byte[] data, Headers headers) {
String traceId = null;
@ -169,7 +169,7 @@ public class ShefDecoder {
PostShef postShef = null;
try {
postShef = new PostShef(postDate);
postShef = new PostShef(postDate, shefParm);
} catch (Exception e) {
logger.error("Could not create PostShef", e);
}
@ -199,7 +199,8 @@ public class ShefDecoder {
while (separator.hasNext()) {
ShefDecoderInput sdi = separator.next();
try {
SHEFParser parser = new SHEFParser(sdi);
SHEFParser parser = new SHEFParser(sdi, appDefaults,
shefParm);
ShefRecord shefRecord = parser.decode();
if (shefRecord != null) {
if (shefRecord.getDataValues() != null) {
@ -209,8 +210,9 @@ public class ShefDecoder {
}
postShef.post(shefRecord);
} catch (Throwable tt) {
logger.error(traceId
+ "- Could not post record.", tt);
logger.error(
traceId + "- Could not post record.",
tt);
}
} else {
logger.info(traceId + "- No data records in file.");
@ -219,39 +221,21 @@ public class ShefDecoder {
logger.info(traceId + "- No records in file.");
}
} catch (Exception ee) {
logger.error(traceId + "- Could not parse SHEF report.", ee);
logger.error(traceId + "- Could not parse SHEF report.",
ee);
}
} // while()
if (dataProcessed) {
postShef.logStats(traceId, System.currentTimeMillis()
- startTime);
postShef.logStats(traceId,
System.currentTimeMillis() - startTime);
}
} finally {
postShef.close();
}
}
/**
*
* @param startTime
* @return
*/
private static Date getPostTime(long startTime) {
private Date getPostTime(long startTime) {
// Force time to nearest second.
return new Date(startTime - (startTime % 1000));
}
/*
*
*/
public static final void main(String[] args) {
long t = System.currentTimeMillis();
Date postDateA = new Date(t);
t = t - (t % 1000);
Date postDateB = new Date(t);
System.out.println(postDateA.getTime());
System.out.println(postDateB.getTime());
}
}

87
edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/data/ShefData.java
View file

@ -20,8 +20,6 @@
package com.raytheon.edex.plugin.shef.data;
import java.util.Date;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.raytheon.edex.plugin.shef.util.SHEFDate;
import com.raytheon.edex.plugin.shef.util.ShefParm;
@ -34,7 +32,8 @@ import com.raytheon.uf.common.dataplugin.shef.util.ParameterCode.Probability;
import com.raytheon.uf.common.dataplugin.shef.util.ParameterCode.TypeSource;
import com.raytheon.uf.common.dataplugin.shef.util.SHEFTimezone;
import com.raytheon.uf.common.dataplugin.shef.util.ShefConstants;
import com.raytheon.uf.common.serialization.ISerializableObject;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
/**
* SHEF Data parent object
@ -43,16 +42,21 @@ import com.raytheon.uf.common.serialization.ISerializableObject;
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Date Ticket# Engineer Description
* ------------ --------- ----------- --------------------------
* 03/19/08 387 M. Duff Initial creation.
* 10/16/2008 1548 jelkins Integrated ParameterCode Types
* 04/29/2014 3088 mpduff cleanup.
* 06/26/2014 3321 mpduff Added ingestfilter primary key getter.
* 01/10/2018 5049 mduff ShefParm is now provided to this class.
*
* </pre>
*/
public class ShefData implements ISerializableObject {
public class ShefData {
private final IUFStatusHandler logger = UFStatus.getHandler(ShefData.class);
private ShefParm shefParm;
private String stringValue = null;
@ -109,10 +113,12 @@ public class ShefData implements ISerializableObject {
private boolean revisedRecord = false;
/**
* Empty constructor
* Constructor.
*
* @param ShefParm
*/
public ShefData() {
public ShefData(ShefParm shefParm) {
this.shefParm = shefParm;
}
/**
@ -128,6 +134,11 @@ public class ShefData implements ISerializableObject {
*/
public void setStringValue(String stringValue) {
this.stringValue = stringValue;
if (stringValue == null) {
value = null;
return;
}
try {
boolean neg = false;
int negPos = stringValue.indexOf('-');
@ -140,8 +151,7 @@ public class ShefData implements ISerializableObject {
value *= -1.0;
}
} catch (NumberFormatException nfe) {
value = null;
} catch (NullPointerException npe) {
logger.warn("Invalid value: " + stringValue, nfe);
value = null;
}
}
@ -279,15 +289,13 @@ public class ShefData implements ISerializableObject {
// check to see if this is a valid typesource
String key = paramType + paramSource;
Integer n = ShefParm.getTypeSourceCode(key);
Integer n = shefParm.getTypeSourceCode(key);
if ((n != null) && (n == 1)) {
TypeSource ts = TypeSource.getEnum(key);
dataTypeCode = paramType;
dataSourceCode = paramSource;
setTypeSource(ts);
} else {
}
Duration duration = Duration.INSTANTENOUS;
@ -524,10 +532,7 @@ public class ShefData implements ISerializableObject {
* the creationDateObj to set
*/
public void setCreationDateObj(Date creationDate) {
SHEFDate d = new SHEFDate(creationDate, SHEFTimezone.GMT_TIMEZONE);
if (d != null) {
createTime = d;
}
createTime = new SHEFDate(creationDate, SHEFTimezone.GMT_TIMEZONE);
}
/**
@ -586,10 +591,7 @@ public class ShefData implements ISerializableObject {
* the observationTimeObj to set
*/
public void setObservationTimeObj(Date observationTime) {
SHEFDate d = new SHEFDate(observationTime, SHEFTimezone.GMT_TIMEZONE);
if (d != null) {
obsTime = d;
}
obsTime = new SHEFDate(observationTime, SHEFTimezone.GMT_TIMEZONE);
}
public void setObsTime(SHEFDate date) {
@ -669,7 +671,7 @@ public class ShefData implements ISerializableObject {
if ("S".equals(unitsCode)) {
if (physicalElement != null) {
String key = physicalElement.getCode();
Double cf = ShefParm.getPhysicalElementConversionFactor(key);
Double cf = shefParm.getPhysicalElementConversionFactor(key);
Double n = doConversion(physicalElement, unitsCode, value);
if (n == null) {
if (cf != null) {
@ -775,8 +777,8 @@ public class ShefData implements ISerializableObject {
receiver.append((revisedRecord) ? " 1" : " 0");
receiver.append(" ");
// Data source
receiver.append(String.format("%-8s", (dataSource != null) ? dataSource
: " "));
receiver.append(
String.format("%-8s", (dataSource != null) ? dataSource : " "));
receiver.append(" ");
// Time series indicator
receiver.append(String.format("%3d", timeSeriesId));
@ -868,39 +870,4 @@ public class ShefData implements ISerializableObject {
}
return dValue;
}
/**
*
* @param args
*/
public static final void main(String[] args) {
// ShefData d = new ShefData();
//
// d.setParameterCodeString("AD","Z");
//
// System.out.println(d);
//
// double dv = 0.04;
//
// System.out.println(String.format("[%.3f]",dv));
//
double adjustedValue = 10;
double divisor = 1;
double base = 0;
double multiplier = 1000;
double adder = 0;
double n = (adjustedValue / divisor + base) * multiplier + adder;
System.out.println(n);
Pattern Q_CODES = Pattern.compile("Q[^BEF]");
Matcher m = Q_CODES.matcher("QI");
if (m.matches()) {
System.out.println("found");
}
}
}

666
edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/database/PostShef.java
View file

File diff suppressed because it is too large Load diff

483
edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/database/PostTables.java
View file

@ -19,6 +19,7 @@
**/
package com.raytheon.edex.plugin.shef.database;
import java.io.Serializable;
import java.sql.CallableStatement;
import java.sql.Connection;
import java.sql.PreparedStatement;
@ -39,6 +40,10 @@ import com.raytheon.edex.plugin.shef.ohdlib.GagePP;
import com.raytheon.edex.plugin.shef.ohdlib.GagePPOptions;
import com.raytheon.edex.plugin.shef.ohdlib.GagePPOptions.shef_dup;
import com.raytheon.edex.plugin.shef.ohdlib.PrecipUtils;
import com.raytheon.edex.plugin.shef.util.FcstPEPDOLookupUtil;
import com.raytheon.edex.plugin.shef.util.ObsPEPDOLookupUtil;
import com.raytheon.edex.plugin.shef.util.ObsPrecipPDOLookupUtil;
import com.raytheon.edex.plugin.shef.util.PDOLookupReturn;
import com.raytheon.edex.plugin.shef.util.PrecipitationUtils;
import com.raytheon.edex.plugin.shef.util.ShefStats;
import com.raytheon.uf.common.dataplugin.persist.PersistableDataObject;
@ -47,6 +52,9 @@ import com.raytheon.uf.common.dataplugin.shef.tables.Arealfcst;
import com.raytheon.uf.common.dataplugin.shef.tables.Arealobs;
import com.raytheon.uf.common.dataplugin.shef.tables.Commentvalue;
import com.raytheon.uf.common.dataplugin.shef.tables.Contingencyvalue;
import com.raytheon.uf.common.dataplugin.shef.tables.ICheckValue;
import com.raytheon.uf.common.dataplugin.shef.tables.Latestobsvalue;
import com.raytheon.uf.common.dataplugin.shef.tables.LatestobsvalueId;
import com.raytheon.uf.common.dataplugin.shef.tables.Pairedvalue;
import com.raytheon.uf.common.dataplugin.shef.tables.PairedvalueId;
import com.raytheon.uf.common.dataplugin.shef.tables.Procvalue;
@ -89,10 +97,11 @@ import com.raytheon.uf.edex.database.dao.DaoConfig;
* 09/03/2014 mpduff postRiverStatus() writes directly, not via batch
* 10/16/2014 3454 bphillip Upgrading to Hibernate 4
* Aug 05, 2015 4486 rjpeter Changed Timestamp to Date.
* Dec 18, 2017 6554 bkowal Added checkForAndHandleDuplicate to avoid running SQL procedures that are guaranteed
* to fail because the configuration does not allow record updates.
* </pre>
*
* @author mduff
* @version 1.0
*/
public class PostTables {
@ -106,13 +115,8 @@ public class PostTables {
private static final String RIVER_STATUS_INSERT_STATEMENT = "INSERT INTO riverstatus values(?,?,?,?,?,?,?,?,?)";
private static final String RIVER_STATUS_UPDATE_STATEMENT = "UPDATE riverstatus SET lid = ? , "
+ "pe = ? , "
+ "dur = ? , "
+ "ts = ? , "
+ "extremum = ? ,"
+ "probability = ? , "
+ "validtime = ? , "
+ "basistime = ? , "
+ "pe = ? , " + "dur = ? , " + "ts = ? , " + "extremum = ? ,"
+ "probability = ? , " + "validtime = ? , " + "basistime = ? , "
+ "value = ? " + "WHERE lid= ? AND pe= ? AND ts= ?";
private static GagePPOptions gagePPOptions;
@ -190,20 +194,39 @@ public class PostTables {
basisTime.setTime(0);
}
String procName = "latestobs";
if (dataValue.equals("")) {
dataValue = "-9999.0";
}
final String procName = "latestobs";
/* now call the PostgreSQL function */
start = System.currentTimeMillis();
execFunction(procName, record, shefData, locId, dataValue, qualifier,
qualityCode, productId, productTime, postTime, duplicateOption,
stats);
end = System.currentTimeMillis();
end = System.currentTimeMillis();
stats.addElapsedTimeIngest(end - start);
stats.incrementLatestObs();
LatestobsvalueId id = new LatestobsvalueId();
id.setLid(locId);
id.setPe(shefData.getPhysicalElement().getCode());
id.setDur(shefData.getDurationValue());
id.setTs(shefData.getTypeSource().getCode());
id.setExtremum(shefData.getExtremum().getCode());
id.setObstime(shefData.getObservationTimeObj());
if (dataValue.isEmpty()) {
dataValue = ShefConstants.SHEF_MISSING;
}
id.setValue(Double.parseDouble(dataValue));
id.setRevision((record.isRevisedRecord() ? (short) 1 : (short) 0));
id.setShefQualCode(qualifier);
id.setQualityCode((int) qualityCode);
id.setProductId(productId);
id.setProducttime(productTime);
id.setPostingtime(postTime);
if (checkForAndHandleDuplicate(shefData, locId, dataValue,
duplicateOption, Latestobsvalue.class, id, false)) {
stats.incrementIgnored();
} else {
execFunction(procName, record, shefData, locId, dataValue,
qualifier, qualityCode, productId, productTime, postTime,
duplicateOption, stats);
end = System.currentTimeMillis();
stats.addElapsedTimeIngest(end - start);
stats.incrementLatestObs();
}
}
/**
@ -231,8 +254,8 @@ public class PostTables {
short dur = Short.parseShort(shefData.getDuration().getValue() + "");
String ts = shefData.getTypeSource().getCode();
String extremum = shefData.getExtremum().getCode();
float probability = Float.parseFloat(shefData.getProbability()
.getValue() + "");
float probability = Float
.parseFloat(shefData.getProbability().getValue() + "");
Date validTime = shefData.getObservationTimeObj();
// CONCERN : Is this the correct behavior when the basisTime is missing?
@ -299,8 +322,8 @@ public class PostTables {
id.setExtremum(shefData.getExtremum().getCode());
id.setLid(locId);
id.setPe(shefData.getPhysicalElement().getCode());
id.setProbability(Float.parseFloat(shefData.getProbability().getValue()
+ ""));
id.setProbability(
Float.parseFloat(shefData.getProbability().getValue() + ""));
pairedValue.setPostingtime(postTime);
pairedValue.setProductId(productId);
@ -321,8 +344,8 @@ public class PostTables {
// lid, pe, dur, ts, extremum, probability, validtime, basistime,
// ref_value
sql = "select value from pairedvalue where lid = '" + locId
+ "' and pe = '" + pe + "' and dur = " + dur
+ " and ts = '" + ts + "' and extremum = '" + extremum
+ "' and pe = '" + pe + "' and dur = " + dur + " and ts = '"
+ ts + "' and extremum = '" + extremum
+ "' and probability = " + probability
+ " and validtime = '"
+ ShefConstants.POSTGRES_DATE_FORMAT.format(validTime)
@ -356,8 +379,8 @@ public class PostTables {
/* data was properly added to table */
stats.incrementRejected();
} else {
if (AppsDefaults.getInstance().getBoolean(
ShefConstants.DUP_MESSAGE, false)) {
if (AppsDefaults.getInstance()
.getBoolean(ShefConstants.DUP_MESSAGE, false)) {
log.info("Ignoring duplicate PairedValue for " + locId
+ ", " + productId + ", "
+ shefData.getObservationTime());
@ -397,19 +420,31 @@ public class PostTables {
Date validTime, Date postTime, DataType type) {
String procName = null;
Class<?> daoClass = null;
Serializable id = null;
if (DataType.READING.equals(type)) {
/* see if the data is a precip report of some kind */
int precipIndex = PrecipitationUtils.getPrecipitationIndex(shefData
.getPhysicalElement());
int precipIndex = PrecipitationUtils
.getPrecipitationIndex(shefData.getPhysicalElement());
/* post all non-precip observed data */
if (precipIndex == ShefConstants.NOT_PRECIP) {
procName = "obs_pe";
PDOLookupReturn pdoLookupReturn = ObsPEPDOLookupUtil.lookupPDO(
shefData.getPhysicalElement().getCode(), locId,
shefData);
daoClass = pdoLookupReturn.getDaoClass();
id = pdoLookupReturn.getId();
/* now call the PostgreSQL function */
} else {
procName = "obs_precip";
PDOLookupReturn pdoLookupReturn = ObsPrecipPDOLookupUtil
.lookupPDO(shefData.getPhysicalElement().getCode(),
locId, shefData);
daoClass = pdoLookupReturn.getDaoClass();
id = pdoLookupReturn.getId();
/*
* if gpp is enabled, and the switch for this record dictates,
@ -417,12 +452,11 @@ public class PostTables {
* a file that will be sent to the gpp server after the product
* is fully processed. if PP, only consider hourly data.
*/
boolean gage_pp_enable = AppsDefaults.getInstance().getBoolean(
"gage_pp_enable", false);
boolean gage_pp_enable = AppsDefaults.getInstance()
.getBoolean("gage_pp_enable", false);
if (gage_pp_enable
&& (ingestSwitch == ShefConstants.IngestSwitch.POST_PE_AND_HOURLY)) {
PrecipRecord precip = new PrecipRecord(shefData);
precip.setPostingTime(postTime);
precip.setQualCode(qualityCode);
@ -433,9 +467,10 @@ public class PostTables {
if ((PhysicalElement.PRECIPITATION_INCREMENT.equals(pe))
&& ((shefData.getDuration() == Duration._1_DAY)
|| (shefData.getDuration() == Duration._1_PERIOD) || (shefData
.getDuration() == Duration._6_HOUR))) {
|| (shefData
.getDuration() == Duration._1_PERIOD)
|| (shefData
.getDuration() == Duration._6_HOUR))) {
PrecipitationUtils.writePrecipGpp(shefData, record,
qualityCode, productId, productTime, postTime,
locId, qualifier, dataValue);
@ -444,15 +479,17 @@ public class PostTables {
}
if ((PhysicalElement.PRECIPITATION_ACCUMULATOR.equals(pe))
|| ((PhysicalElement.PRECIPITATION_INCREMENT
.equals(pe)) && ((shefData.getDuration() == Duration._60_MINUTES) || (shefData
.getDuration() == Duration._1_HOUR)))) {
if (dataValue.equals("")) {
.equals(pe))
&& ((shefData
.getDuration() == Duration._60_MINUTES)
|| (shefData
.getDuration() == Duration._1_HOUR)))) {
if (dataValue.isEmpty()) {
dataValue = ShefConstants.SHEF_MISSING;
}
if (PrecipUtils.checkPrecipWindow(
shefData.getObsTime(), pe, gagePPOptions)) {
if (PrecipUtils.checkPrecipWindow(shefData.getObsTime(),
pe, gagePPOptions)) {
PrecipitationUtils.writePrecipGpp(shefData, record,
qualityCode, productId, productTime,
postTime, locId, qualifier, dataValue);
@ -462,63 +499,81 @@ public class PostTables {
}
}
}
} else if (DataType.FORECAST.equals(type)) {
procName = "fcst_pe";
PDOLookupReturn pdoLookupReturn = FcstPEPDOLookupUtil.lookupPDO(
shefData.getPhysicalElement().getCode(), locId, shefData,
validTime);
daoClass = pdoLookupReturn.getDaoClass();
id = pdoLookupReturn.getId();
}
long start = System.currentTimeMillis();
int status = -1;
if (DataType.FORECAST.equals(type)) {
status = execFcstFunc(procName, record, shefData, locId, dataValue,
qualifier, qualityCode, productId, productTime, postTime,
duplicateOption, ingestSwitch, stats, validTime);
if (checkForAndHandleDuplicate(shefData, locId, dataValue,
duplicateOption, daoClass, id, true)) {
stats.incrementIgnored();
} else {
/* now call the PostgreSQL function */
status = execFunction(procName, record, shefData, locId, dataValue,
qualifier, qualityCode, productId, productTime, postTime,
duplicateOption, ingestSwitch, stats);
}
if (DataType.FORECAST.equals(type)) {
status = execFcstFunc(procName, record, shefData, locId,
dataValue, qualifier, qualityCode, productId,
productTime, postTime, duplicateOption, ingestSwitch,
stats, validTime);
} else {
/* now call the PostgreSQL function */
status = execFunction(procName, record, shefData, locId,
dataValue, qualifier, qualityCode, productId,
productTime, postTime, duplicateOption, ingestSwitch,
stats);
}
long end = System.currentTimeMillis();
long end = System.currentTimeMillis();
if (status < 0) {
log.error(record.getTraceId() + " - PostgresSQL error " + status
+ " executing " + procName + " function for " + locId
+ ", " + shefData.getObservationTimeObj().toString() + ", "
+ productTime.toString() + ", " + productId + ", "
+ postTime.toString());
stats.incrementErrorMessages();
} else {
if ((DataType.READING.equals(type))
|| (DataType.PROCESSED.equals(type))) {
stats.incrementObsPe();
if (shefData.getPhysicalElement().getCategory() == PhysicalElementCategory.HEIGHT) {
stats.incrementObsHeight();
stats.setElapsedTimeHeightIngest(end - start);
if (status < 0) {
log.error(record.getTraceId() + " - PostgresSQL error " + status
+ " executing " + procName + " function for " + locId
+ ", " + shefData.getObservationTimeObj().toString()
+ ", " + productTime.toString() + ", " + productId
+ ", " + postTime.toString());
stats.incrementErrorMessages();
} else {
if ((DataType.READING.equals(type))
|| (DataType.PROCESSED.equals(type))) {
stats.incrementObsPe();
if (shefData.getPhysicalElement()
.getCategory() == PhysicalElementCategory.HEIGHT) {
stats.incrementObsHeight();
stats.setElapsedTimeHeightIngest(end - start);
} else if ((shefData.getPhysicalElement() != PhysicalElement.PRESSURE_ATMOSPHERIC)
&& (shefData.getPhysicalElement() != PhysicalElement.PRESSURE_ATMOSPHERIC_3HR)
&& (shefData.getPhysicalElement() != PhysicalElement.PRESSURE_CHARACTERISTIC)
&& (shefData.getPhysicalElement() != PhysicalElement.PRESSURE_SEA_LEVEL)) {
stats.incrementObsPrecip();
stats.addElapsedTimePrecipIngest(end - start);
} else {
stats.addElapsedTimeOtherIngest(end - start);
}
} else if (DataType.FORECAST.equals(type)) {
stats.incrementForecastPe();
stats.addElapsedTimeForecastIngest(end - start);
} else if ((shefData
.getPhysicalElement() != PhysicalElement.PRESSURE_ATMOSPHERIC)
&& (shefData
.getPhysicalElement() != PhysicalElement.PRESSURE_ATMOSPHERIC_3HR)
&& (shefData
.getPhysicalElement() != PhysicalElement.PRESSURE_CHARACTERISTIC)
&& (shefData
.getPhysicalElement() != PhysicalElement.PRESSURE_SEA_LEVEL)) {
stats.incrementObsPrecip();
stats.addElapsedTimePrecipIngest(end - start);
} else {
stats.addElapsedTimeOtherIngest(end - start);
}
} else if (DataType.FORECAST.equals(type)) {
stats.incrementForecastPe();
stats.addElapsedTimeForecastIngest(end - start);
/*
* track which lid/pe combinations have been processed for
* forecast river data.
*/
if ((shefData.getPhysicalElement().getCategory() == PhysicalElementCategory.HEIGHT)
|| (shefData.getPhysicalElement().getCategory() == PhysicalElementCategory.DISCHARGE)) {
loadForecastInfo(locId, shefData.getPhysicalElement(),
stats);
/*
* track which lid/pe combinations have been processed for
* forecast river data.
*/
if ((shefData.getPhysicalElement()
.getCategory() == PhysicalElementCategory.HEIGHT)
|| (shefData.getPhysicalElement()
.getCategory() == PhysicalElementCategory.DISCHARGE)) {
loadForecastInfo(locId, shefData.getPhysicalElement(),
stats);
}
}
}
}
@ -591,10 +646,10 @@ public class PostTables {
probability = value.getId().getProbability();
dataValue = value.getValue();
revision = value.getRevision();
validTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId()
.getValidtime());
basisTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId()
.getBasistime());
validTime = ShefConstants.POSTGRES_DATE_FORMAT
.format(value.getId().getValidtime());
basisTime = ShefConstants.POSTGRES_DATE_FORMAT
.format(value.getId().getBasistime());
appendStr = "probability = " + probability + " and "
+ "validtime = '" + validTime + "' and " + "basistime = '"
@ -609,10 +664,10 @@ public class PostTables {
probability = value.getId().getProbability();
dataValue = value.getValue();
revision = value.getRevision();
validTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId()
.getValidtime());
basisTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId()
.getBasistime());
validTime = ShefConstants.POSTGRES_DATE_FORMAT
.format(value.getId().getValidtime());
basisTime = ShefConstants.POSTGRES_DATE_FORMAT
.format(value.getId().getBasistime());
appendStr = "probability = " + probability + " and "
+ "validtime = '" + validTime + "' and " + "basistime = '"
@ -626,8 +681,8 @@ public class PostTables {
extremum = value.getId().getExtremum();
dataValue = value.getValue();
revision = value.getRevision();
validTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId()
.getObstime());
validTime = ShefConstants.POSTGRES_DATE_FORMAT
.format(value.getId().getObstime());
appendStr = "obstime = '" + validTime + "'";
} else if (dataObj instanceof Rejecteddata) {
@ -641,10 +696,10 @@ public class PostTables {
probability = value.getId().getProbability();
dataValue = value.getValue();
revision = value.getRevision();
validTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId()
.getValidtime());
basisTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId()
.getBasistime());
validTime = ShefConstants.POSTGRES_DATE_FORMAT
.format(value.getId().getValidtime());
basisTime = ShefConstants.POSTGRES_DATE_FORMAT
.format(value.getId().getBasistime());
String postingTime = ShefConstants.POSTGRES_DATE_FORMAT
.format(value.getId().getPostingtime());
@ -662,10 +717,10 @@ public class PostTables {
probability = value.getId().getProbability();
dataValue = value.getValue();
revision = value.getRevision();
validTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId()
.getValidtime());
basisTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId()
.getBasistime());
validTime = ShefConstants.POSTGRES_DATE_FORMAT
.format(value.getId().getValidtime());
basisTime = ShefConstants.POSTGRES_DATE_FORMAT
.format(value.getId().getBasistime());
appendStr = "probability = " + probability + " and "
+ "validtime = '" + validTime + "' and " + "basistime = '"
@ -681,8 +736,8 @@ public class PostTables {
extremum = value.getId().getExtremum();
dataValue = value.getValue();
revision = value.getRevision();
validTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId()
.getObstime());
validTime = ShefConstants.POSTGRES_DATE_FORMAT
.format(value.getId().getObstime());
appendStr = "obstime = '" + validTime + "'";
} else if (dataObj instanceof Arealfcst) {
@ -695,10 +750,10 @@ public class PostTables {
dataValue = value.getValue();
revision = value.getRevision();
probability = value.getId().getProbability();
validTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId()
.getValidtime());
basisTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId()
.getBasistime());
validTime = ShefConstants.POSTGRES_DATE_FORMAT
.format(value.getId().getValidtime());
basisTime = ShefConstants.POSTGRES_DATE_FORMAT
.format(value.getId().getBasistime());
appendStr = "probability = " + probability + " and "
+ "validtime = '" + validTime + "' and " + "basistime = '"
@ -712,8 +767,8 @@ public class PostTables {
extremum = value.getId().getExtremum();
dataValue = value.getId().getValue();
revision = value.getId().getRevision();
validTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId()
.getObstime());
validTime = ShefConstants.POSTGRES_DATE_FORMAT
.format(value.getId().getObstime());
appendStr = "obstime = '" + validTime + "'";
@ -792,8 +847,8 @@ public class PostTables {
/*
* don't perform the overwrite since conditions were not met
*/
if (AppsDefaults.getInstance().getBoolean(
ShefConstants.DUP_MESSAGE, false)) {
if (AppsDefaults.getInstance()
.getBoolean(ShefConstants.DUP_MESSAGE, false)) {
log.info("Ignoring duplicate " + tableName + " for "
+ locId + ", " + validTime);
}
@ -806,8 +861,10 @@ public class PostTables {
} else {
log.error("Query = [" + sql + "]");
}
log.error(dataObj.getTraceId() + " - PostgresSQL error updating "
+ tableName + " for " + locId + ", " + validTime, e);
log.error(
dataObj.getTraceId() + " - PostgresSQL error updating "
+ tableName + " for " + locId + ", " + validTime,
e);
stats.incrementErrorMessages();
}
}
@ -820,8 +877,8 @@ public class PostTables {
public void postUnknownStation(Unkstn unkstn, ShefStats stats) {
/* Build the sql query string */
StringBuilder sql = new StringBuilder();
sql.append("select lid from unkstn where lid = '" + unkstn.getLid()
+ "'");
sql.append(
"select lid from unkstn where lid = '" + unkstn.getLid() + "'");
try {
Object[] result = dao.executeSQLQuery(sql.toString());
@ -836,12 +893,11 @@ public class PostTables {
}
} catch (Exception e) {
log.error("Query = [" + sql.toString() + "]");
log.error(
unkstn.getTraceId()
+ " - PostgresSQL error updating UnkStn for "
+ unkstn.getLid() + ", "
+ unkstn.getProducttime().toString() + ", "
+ unkstn.getPostingtime().toString(), e);
log.error(unkstn.getTraceId()
+ " - PostgresSQL error updating UnkStn for "
+ unkstn.getLid() + ", "
+ unkstn.getProducttime().toString() + ", "
+ unkstn.getPostingtime().toString(), e);
stats.incrementErrorMessages();
}
}
@ -938,13 +994,12 @@ public class PostTables {
}
private int execFunction(String functionName, ShefRecord record,
ShefData shefData, String locId, String dataValue,
String qualifier, long qualityCode, String productId,
Date productTime, Date postTime, String duplicateOption,
ShefStats stats) {
ShefData shefData, String locId, String dataValue, String qualifier,
long qualityCode, String productId, Date productTime, Date postTime,
String duplicateOption, ShefStats stats) {
CallableStatement cs = null;
int status = -1;
if (dataValue.equals("")) {
if (dataValue.isEmpty()) {
dataValue = ShefConstants.SHEF_MISSING;
}
try {
@ -960,8 +1015,8 @@ public class PostTables {
cs.setInt(3, shefData.getDurationValue());
cs.setString(4, shefData.getTypeSource().getCode());
cs.setString(5, shefData.getExtremum().getCode());
cs.setTimestamp(6, new Timestamp(shefData.getObservationTimeObj()
.getTime()));
cs.setTimestamp(6,
new Timestamp(shefData.getObservationTimeObj().getTime()));
cs.setDouble(7, Double.parseDouble(dataValue));
cs.setString(8, qualifier);
cs.setInt(9, (int) qualityCode);
@ -986,17 +1041,16 @@ public class PostTables {
if (status == 0) {
conn.commit();
} else {
throw new Exception("PostgresSQL error executing function "
+ functionName);
throw new Exception(
"PostgresSQL error executing function " + functionName);
}
} catch (Exception e) {
log.error("Error updating/committing PE insert for PE "
+ shefData.getPhysicalElement());
log.error("Record Data: " + record);
log.error(
record.getTraceId()
+ " - PostgresSQL error executing function "
+ functionName, e);
log.error(record.getTraceId()
+ " - PostgresSQL error executing function " + functionName,
e);
}
return status;
}
@ -1009,13 +1063,13 @@ public class PostTables {
* @return - status of action, 1 is good, 0 is bad
*/
private int execFunction(String functionName, ShefRecord record,
ShefData shefData, String locId, String dataValue,
String qualifier, long qualityCode, String productId,
Date productTime, Date postTime, String duplicateOption,
ShefConstants.IngestSwitch ingestSwitch, ShefStats stats) {
ShefData shefData, String locId, String dataValue, String qualifier,
long qualityCode, String productId, Date productTime, Date postTime,
String duplicateOption, ShefConstants.IngestSwitch ingestSwitch,
ShefStats stats) {
CallableStatement cs = null;
int status = -1;
if (dataValue.equals("")) {
if (dataValue.isEmpty()) {
dataValue = ShefConstants.SHEF_MISSING;
}
try {
@ -1031,8 +1085,8 @@ public class PostTables {
cs.setInt(3, shefData.getDurationValue());
cs.setString(4, shefData.getTypeSource().getCode());
cs.setString(5, shefData.getExtremum().getCode());
cs.setTimestamp(6, new java.sql.Timestamp(shefData
.getObservationTimeObj().getTime()));
cs.setTimestamp(6, new java.sql.Timestamp(
shefData.getObservationTimeObj().getTime()));
cs.setDouble(7, Double.parseDouble(dataValue));
cs.setString(8, qualifier);
cs.setInt(9, (int) qualityCode);
@ -1058,16 +1112,16 @@ public class PostTables {
if (status == 0) {
conn.commit();
} else {
throw new Exception("PostgresSQL error executing function "
+ functionName);
throw new Exception(
"PostgresSQL error executing function " + functionName);
}
} catch (Exception e) {
log.error("Record Data: " + record);
log.error(record.getTraceId()
+ " - PostgresSQL error executing function " + functionName);
log.error(
"Error updating/committing PE insert for PE "
+ shefData.getPhysicalElement(), e);
+ " - PostgresSQL error executing function "
+ functionName);
log.error("Error updating/committing PE insert for PE "
+ shefData.getPhysicalElement(), e);
}
return status;
}
@ -1080,19 +1134,19 @@ public class PostTables {
* @return - status of action, 1 is good, 0 is bad
*/
private int execFcstFunc(String functionName, ShefRecord record,
ShefData shefData, String locId, String dataValue,
String qualifier, long qualityCode, String productId,
Date productTime, Date postTime, String duplicateOption,
ShefConstants.IngestSwitch ingestSwitch, ShefStats stats,
Date validTime) {
ShefData shefData, String locId, String dataValue, String qualifier,
long qualityCode, String productId, Date productTime, Date postTime,
String duplicateOption, ShefConstants.IngestSwitch ingestSwitch,
ShefStats stats, Date validTime) {
CallableStatement cs = null;
java.sql.Timestamp timeStamp = null;
int status = -1;
if (dataValue.equals("")) {
if (dataValue.isEmpty()) {
dataValue = ShefConstants.SHEF_MISSING;
}
int doOverwrite = determineUpdateAction(duplicateOption,
shefData.isRevisedRecord());
try {
conn = getConnection();
cs = statementMap.get(functionName);
@ -1140,11 +1194,6 @@ public class PostTables {
timeStamp = new java.sql.Timestamp(postTime.getTime());
cs.setTimestamp(15, timeStamp);
int doOverwrite = 0;
doOverwrite = determineUpdateAction(duplicateOption,
shefData.isRevisedRecord());
cs.setInt(16, doOverwrite);
cs.registerOutParameter(17, java.sql.Types.INTEGER);
@ -1155,10 +1204,10 @@ public class PostTables {
} catch (Exception e) {
log.error("Record Data: " + record);
log.error(record.getTraceId()
+ " - PostgresSQL error executing function " + functionName);
log.error(
"Error updating/committing PE insert for PE "
+ shefData.getPhysicalElement(), e);
+ " - PostgresSQL error executing function "
+ functionName);
log.error("Error updating/committing PE insert for PE "
+ shefData.getPhysicalElement(), e);
stats.incrementErrorMessages();
}
@ -1173,16 +1222,20 @@ public class PostTables {
StringBuilder message = new StringBuilder("shef_duplicate : ");
if ("ALWAYS_OVERWRITE".equals(token)) {
gagePPOptions.setShef_duplicate(shef_dup.ALWAYS_OVERWRITE);
message.append("ALWAYS_OVERWRITE (always process duplicate reports)");
message.append(
"ALWAYS_OVERWRITE (always process duplicate reports)");
} else if ("USE_REVCODE".equals(token)) {
gagePPOptions.setShef_duplicate(shef_dup.USE_REVCODE);
message.append("USE_REVCODE (only process duplicate reports if they are revisions");
message.append(
"USE_REVCODE (only process duplicate reports if they are revisions");
} else if ("IF_DIFFERENT_AND_REVCODE".equals(token)) {
gagePPOptions.setShef_duplicate(shef_dup.IF_DIFFERENT_AND_REVCODE);
message.append("IF_DIFFERENT_OR_REVCODE (only process duplicate reports if they are revisions or their values are different");
message.append(
"IF_DIFFERENT_OR_REVCODE (only process duplicate reports if they are revisions or their values are different");
} else {
gagePPOptions.setShef_duplicate(shef_dup.IF_DIFFERENT);
message.append("IF_DIFFERENT (only process duplicate reports if their values are different");
message.append(
"IF_DIFFERENT (only process duplicate reports if their values are different");
}
log.info(message.toString());
@ -1242,8 +1295,8 @@ public class PostTables {
.floatValue();
ps.setFloat(6, probability);
timeStamp = new java.sql.Timestamp(shefDataValue
.getObservationTimeObj().getTime());
timeStamp = new java.sql.Timestamp(
shefDataValue.getObservationTimeObj().getTime());
ps.setTimestamp(7, timeStamp);
Date basisDate = shefDataValue.getCreationDateObj();
@ -1269,7 +1322,8 @@ public class PostTables {
} catch (Exception e) {
if (updateFlag) {
log.error(String.format(
"Error updating into RiverStatus with [%s]", record), e);
"Error updating into RiverStatus with [%s]", record),
e);
} else {
log.error(String.format(
"Error inserting into RiverStatus with [%s]", record),
@ -1290,8 +1344,8 @@ public class PostTables {
return status;
}
private PreparedStatement getRiverStatusPreparedStatement(boolean updateFlag)
throws SQLException {
private PreparedStatement getRiverStatusPreparedStatement(
boolean updateFlag) throws SQLException {
if (updateFlag) {
PreparedStatement riverStatusUpdateStatement = conn
.prepareCall(RIVER_STATUS_UPDATE_STATEMENT);
@ -1351,4 +1405,81 @@ public class PostTables {
}
}
}
}
private boolean checkForAndHandleDuplicate(final ShefData shefData,
final String locId, final String dataValue,
final String duplicateOption, final Class<?> daoClass,
final Serializable id, final boolean alwaysUpdateIfMissing) {
if (daoClass == null || id == null) {
/*
* The Stored Procedure will not recognize the associated shef data.
*/
return false;
}
final CoreDao lookupDao = new CoreDao(
DaoConfig.forClass("ihfs", daoClass));
PersistableDataObject<?> pdo = lookupDao.queryById(id);
if (pdo == null) {
/*
* No duplicate records exist.
*/
return false;
}
/*
* Are updates allowed?
*/
final int doOverwrite = determineUpdateAction(duplicateOption,
shefData.isRevisedRecord());
if (doOverwrite == ShefConstants.UPDATE_ACTION) {
/*
* Record will always be updated no matter what.
*/
return false;
}
if (pdo instanceof ICheckValue) {
/*
* Two possibilities here. (1) the value will always be updated if
* it is currently "missing" regardless of how the update action has
* been defined in Apps Defaults (2) the record will be updated if
* the new value is different per the update action specified in
* Apps Defaults.
*/
ICheckValue checkValue = (ICheckValue) pdo;
final String compareValue = checkValue.getCompareValue();
if (compareValue == null) {
/*
* This should never happen because the current hydro ihfs has
* the concept of a Missing value filler.
*/
throw new IllegalStateException(
"Encountered unexpected null data value for "
+ daoClass.getSimpleName()
+ " with identifier: " + id.toString() + ".");
}
if ((alwaysUpdateIfMissing
&& compareValue.startsWith(ShefConstants.SHEF_MISSING))
|| (doOverwrite == ShefConstants.IF_DIFFERENT_UPDATE_ACTION
&& !compareValue.equals(dataValue))) {
return false;
} else {
/*
* The value is a duplicate and there is no path to update it.
* Ensure that no SQL procedures are executed beyond this point
* because they are guaranteed to fail.
*/
if (AppsDefaults.getInstance()
.getBoolean(ShefConstants.DUP_MESSAGE, false)) {
log.info("Ignoring duplicate " + daoClass.getSimpleName()
+ " for: " + id.toString() + ".");
}
return true;
}
}
/*
* Nothing to indicate that they record may not be a candidate for
* updates.
*/
return false;
}
}

688
edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/util/ParserToken.java
View file

File diff suppressed because it is too large Load diff

123
edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/util/ShefParm.java
View file

@ -45,51 +45,49 @@ import com.raytheon.uf.common.localization.PathManagerFactory;
* ------------ ---------- ----------- --------------------------
* Apr 18, 2008 387 M. Duff Initial Version.
* Dec 16, 2015 5166 kbisanz Update logging to use SLF4J
* Jan 10, 2018 5049 mduff Made class not so static.
*
* </pre>
*
* @author mduff
* @version 1.0
*/
public class ShefParm {
static {
log = LoggerFactory
.getLogger(com.raytheon.edex.plugin.shef.util.ShefParm.class);
physicalElementConversion = new HashMap<String, Double>();
durationCodeValues = new HashMap<String, Short>();
typeSourceCodes = new HashMap<String, Integer>();
extremumCodes = new HashMap<String, Integer>();
probabilityCodeValues = new HashMap<String, Float>();
sendCodesDuration = new HashMap<String, String>();
dataQualifierCodes = new HashMap<String, Integer>();
MAX_ERRORS = 0;
populate();
}
private static Map<String, Double> physicalElementConversion;
private static Map<String, Short> durationCodeValues;
private static Map<String, Integer> typeSourceCodes;
private static Map<String, Integer> extremumCodes;
private static Map<String, Float> probabilityCodeValues;
private static Map<String, String> sendCodesDuration;
private static Map<String, Integer> dataQualifierCodes;
private static int MAX_ERRORS;
private static final Logger log;
private static final String PLUGIN_NAME = "shef";
private static final String PROPFILE_NAME = "SHEFPARM";
private static int fileSection = 0;
private Map<String, Double> physicalElementConversion;
private Map<String, Short> durationCodeValues;
private Map<String, Integer> typeSourceCodes;
private Map<String, Integer> extremumCodes;
private Map<String, Float> probabilityCodeValues;
private Map<String, String> sendCodesDuration;
private Map<String, Integer> dataQualifierCodes;
private int MAX_ERRORS;
private static final Logger log = LoggerFactory
.getLogger(com.raytheon.edex.plugin.shef.util.ShefParm.class);;
private int fileSection = 0;
public ShefParm() {
durationCodeValues = new HashMap<>();
typeSourceCodes = new HashMap<>();
extremumCodes = new HashMap<>();
probabilityCodeValues = new HashMap<>();
sendCodesDuration = new HashMap<>();
dataQualifierCodes = new HashMap<>();
physicalElementConversion = new HashMap<>();
MAX_ERRORS = 0;
}
/**
* Get a Physical Element conversion factor
@ -98,7 +96,7 @@ public class ShefParm {
* - Physical Element
* @return - the conversion factor
*/
public static Double getPhysicalElementConversionFactor(String key) {
public Double getPhysicalElementConversionFactor(String key) {
return physicalElementConversion.get(key);
}
@ -109,7 +107,7 @@ public class ShefParm {
* - Duration Code
* @return - the Duration Code value
*/
public static Short getDurationCodeValue(String key) {
public Short getDurationCodeValue(String key) {
return durationCodeValues.get(key);
}
@ -120,7 +118,7 @@ public class ShefParm {
* - Type Source
* @return - 1 if valid code, null if invalid
*/
public static Integer getTypeSourceCode(String key) {
public Integer getTypeSourceCode(String key) {
return typeSourceCodes.get(key);
}
@ -131,7 +129,7 @@ public class ShefParm {
* - Extremum Code
* @return - 1 if valid code, null if invalid
*/
public static Integer getExtremumCode(String key) {
public Integer getExtremumCode(String key) {
return extremumCodes.get(key);
}
@ -142,7 +140,7 @@ public class ShefParm {
* - Probability Code
* @return - Probability Code's value
*/
public static Float getProbabilityCodeValue(String key) {
public Float getProbabilityCodeValue(String key) {
return probabilityCodeValues.get(key);
}
@ -153,7 +151,7 @@ public class ShefParm {
* - Code
* @return - Default Values
*/
public static String getSendCodeDurationDefaults(String key) {
public String getSendCodeDurationDefaults(String key) {
return sendCodesDuration.get(key);
}
@ -164,7 +162,7 @@ public class ShefParm {
* - Extremum Code
* @return - 1 if valid code, null if invalid
*/
public static Integer getDataQualifierCodes(String key) {
public Integer getDataQualifierCodes(String key) {
return dataQualifierCodes.get(key);
}
@ -173,14 +171,14 @@ public class ShefParm {
*
* @return - the maximum number of errors
*/
public static Integer getMaxErrors() {
public Integer getMaxErrors() {
return MAX_ERRORS;
}
/**
* Populate the values from the file.
*/
private static void populate() {
public void populate() {
PathManager pathMgr = (PathManager) PathManagerFactory.getPathManager();
@ -189,43 +187,18 @@ public class ShefParm {
File baseDir = pathMgr.getFile(ctx, PLUGIN_NAME);
File srcFile = new File(baseDir, PROPFILE_NAME);
BufferedReader in = null;
try {
in = new BufferedReader(new FileReader(srcFile));
try (BufferedReader in = new BufferedReader(new FileReader(srcFile))) {
String line = null;
while ((line = in.readLine()) != null) {
processLine(line);
}
in.close();
} catch (IOException ioe) {
ioe.printStackTrace();
log.error("Error loading " + PROPFILE_NAME);
log.error("Error loading " + PROPFILE_NAME, ioe);
}
}
private static String expandPE(String pe) {
// 0123456
// ADZZZZZ
StringBuilder peCode = new StringBuilder("--IRZZ");
if ((pe != null) && (pe.length() >= 2)) {
for (int i = 0; i < pe.length() && (i < peCode.length()); i++) {
peCode.setCharAt(i, pe.charAt(i));
}
}
char z4 = peCode.charAt(3);
char z5 = peCode.charAt(5);
if ('Z' == z4) {
if ('Z' == z5) {
peCode.setCharAt(3, 'R');
} else {
// FIXME: This is an error
}
}
return peCode.toString();
}
private static void processLine(String line) {
private void processLine(String line) {
String[] pair = null;
if (line.startsWith("$")) {
return;
@ -237,7 +210,7 @@ public class ShefParm {
}
return;
} else {
if (line.equals("SHEFPARM")) {
if ("SHEFPARM".equals(line)) {
return;
}
switch (fileSection) {
@ -276,10 +249,4 @@ public class ShefParm {
}
}
}
public static final void main(String[] args) {
String s = expandPE("ADZZZZZ");
System.out.println(s);
}
}

113
edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/util/ShefUtil.java
View file

@ -37,13 +37,14 @@ import com.raytheon.uf.common.wmo.WMOHeader;
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 03/19/08 387 M. Duff Initial creation.
* 12/12/08 1786 J. Sanchez Handled date parsing exceptions.
* 01/12/08 1846 J. Sanchez Update parseCreationDate to use default year.
* May 14, 2014 2536 bclement moved WMO Header to common, removed unused log
* Aug 05, 2015 4486 rjpeter Changed Timestamp to Date.
* Jan 10, 2018 5049 mduff ShefParm is passed in when needed.
* </pre>
*/
public final class ShefUtil {
@ -78,7 +79,7 @@ public final class ShefUtil {
Calendar recordTime = Calendar.getInstance();
recordTime.setTime(recordDate);
if (time.indexOf("/") > -1) {
if (time.indexOf('/') > -1) {
/* Split the string apart and set the individual values */
parts = time.split("/");
for (String part : parts) {
@ -105,7 +106,7 @@ public final class ShefUtil {
* Put the pieces together to get a full time stamp Start with the
* century and work to the right
*/
StringBuffer sb = new StringBuffer();
StringBuilder sb = new StringBuilder();
if ((dt != null) && (dy != null)) {
sb.append(dt + dy);
@ -113,7 +114,7 @@ public final class ShefUtil {
dy = String.valueOf(recordTime.get(Calendar.YEAR)).substring(0,
2);
sb.append(dt + dy);
} else if (dy == null) {
} else {
int year = recordTime.get(Calendar.YEAR);
sb.append(year);
}
@ -174,7 +175,7 @@ public final class ShefUtil {
timeString = sb.toString();
} else {
StringBuffer sb = new StringBuffer();
StringBuilder sb = new StringBuilder();
String year = String.valueOf(recordTime.get(Calendar.YEAR));
String month = String.valueOf(recordTime.get(Calendar.MONTH) + 1);
String day = String.valueOf(recordTime.get(Calendar.DAY_OF_MONTH));
@ -410,14 +411,14 @@ public final class ShefUtil {
int year = 0;
cal.setTimeInMillis(System.currentTimeMillis());
year = cal.get(Calendar.YEAR); // retrieves current year
year = cal.get(Calendar.YEAR);
currentDate = cal.getTime();
cal.setTime(creationDate);
cal.set(Calendar.YEAR, year);
if (cal.after(currentDate)) {
--year; // uses the previous year
--year;
}
return String.valueOf(year);
@ -446,7 +447,7 @@ public final class ShefUtil {
positive = false;
}
String relativeAmount = null;
if (relativeValue.equals("DRE")) {
if ("DRE".equals(relativeValue)) {
relativeAmount = relativeValue;
} else {
relativeAmount = relativeValue.substring(4);
@ -457,41 +458,41 @@ public final class ShefUtil {
if (positive) {
obsTimeCal.add(Calendar.SECOND, new Integer(relativeAmount));
} else {
obsTimeCal.add(Calendar.SECOND, (new Integer(relativeAmount))
* -1);
obsTimeCal.add(Calendar.SECOND,
(new Integer(relativeAmount)) * -1);
}
break;
case 'N':
if (positive) {
obsTimeCal.add(Calendar.MINUTE, new Integer(relativeAmount));
} else {
obsTimeCal.add(Calendar.MINUTE, (new Integer(relativeAmount))
* -1);
obsTimeCal.add(Calendar.MINUTE,
(new Integer(relativeAmount)) * -1);
}
break;
case 'H':
if (positive) {
obsTimeCal.add(Calendar.HOUR, new Integer(relativeAmount));
} else {
obsTimeCal.add(Calendar.HOUR, (new Integer(relativeAmount))
* -1);
obsTimeCal.add(Calendar.HOUR,
(new Integer(relativeAmount)) * -1);
}
break;
case 'D':
if (positive) {
obsTimeCal.add(Calendar.DAY_OF_MONTH, new Integer(
relativeAmount));
obsTimeCal.add(Calendar.DAY_OF_MONTH,
new Integer(relativeAmount));
} else {
obsTimeCal.add(Calendar.DAY_OF_MONTH, (new Integer(
relativeAmount)) * -1);
obsTimeCal.add(Calendar.DAY_OF_MONTH,
(new Integer(relativeAmount)) * -1);
}
break;
case 'M':
if (positive) {
obsTimeCal.add(Calendar.MONTH, new Integer(relativeAmount));
} else {
obsTimeCal.add(Calendar.MONTH, (new Integer(relativeAmount))
* -1);
obsTimeCal.add(Calendar.MONTH,
(new Integer(relativeAmount)) * -1);
}
break;
case 'E':
@ -503,8 +504,8 @@ public final class ShefUtil {
if (positive) {
obsTimeCal.add(Calendar.YEAR, new Integer(relativeAmount));
} else {
obsTimeCal.add(Calendar.YEAR, (new Integer(relativeAmount))
* -1);
obsTimeCal.add(Calendar.YEAR,
(new Integer(relativeAmount)) * -1);
}
break;
}
@ -596,7 +597,7 @@ public final class ShefUtil {
century--;
}
returnVal = (century * 100) + pYear;
} else { // recordDate length must be 8
} else {
date = ShefConstants.YYYYMMDD_FORMAT.parse(dateString);
calendar.setTime(date);
returnVal = calendar.get(Calendar.YEAR);
@ -612,19 +613,17 @@ public final class ShefUtil {
* @return - the identifier
*/
public static synchronized String createIdentifier(WMOHeader wmoHeader) {
String retVal = null;
StringBuffer sb = new StringBuffer();
sb.append(wmoHeader.getWmoHeader() + "::::");
sb.append(wmoHeader.getA1() + "::::");
sb.append(wmoHeader.getA2() + "::::");
sb.append(wmoHeader.getIi() + "::::");
sb.append(wmoHeader.getT1() + "::::");
sb.append(wmoHeader.getT2() + "::::");
sb.append(wmoHeader.getBBBIndicator() + "::::");
sb.append(wmoHeader.getCccc() + "::::");
StringBuilder sb = new StringBuilder();
sb.append(wmoHeader.getWmoHeader()).append("::::");
sb.append(wmoHeader.getA1()).append("::::");
sb.append(wmoHeader.getA2()).append("::::");
sb.append(wmoHeader.getIi()).append("::::");
sb.append(wmoHeader.getT1()).append("::::");
sb.append(wmoHeader.getT2()).append("::::");
sb.append(wmoHeader.getBBBIndicator()).append("::::");
sb.append(wmoHeader.getCccc()).append("::::");
sb.append(wmoHeader.getYYGGgg());
retVal = sb.toString();
return retVal;
return sb.toString();
}
/**
@ -656,7 +655,7 @@ public final class ShefUtil {
return ((lower < value) && (value < upper));
}
public static final int validatePEDTSEP(String pedtsep) {
public static final int validatePEDTSEP(String pedtsep, ShefParm shefParm) {
// Set to non-error code for switch statement flowdown.
int err = -1;
@ -671,47 +670,59 @@ public final class ShefUtil {
switch (pedtsep.length()) {
case 7: {
if (ShefParm.getProbabilityCodeValue(pedtsep.substring(6, 7)) == null) {
// "Non-existent value for given probability parameter code",
if (shefParm.getProbabilityCodeValue(
pedtsep.substring(6, 7)) == null) {
// "Non-existent value for given probability parameter
// code",
err = SHEFErrorCodes.LOG_063;
} else {
err = 0;
}
break;
}
case 6: {
if (err <= 0) {
if (ShefParm.getExtremumCode(pedtsep.substring(5, 6)) == null) {
// "Non-existent value for given extremum parameter code"
if (shefParm
.getExtremumCode(pedtsep.substring(5, 6)) == null) {
// "Non-existent value for given extremum parameter
// code"
err = SHEFErrorCodes.LOG_061;
} else {
err = 0;
}
}
break;
}
case 5: // Type Source
case 5:
if (err <= 0) {
if (ShefParm.getTypeSourceCode(pedtsep.substring(3, 5)) == null) {
// "Non-existent value for given type and source parameter code"
if (shefParm.getTypeSourceCode(
pedtsep.substring(3, 5)) == null) {
// "Non-existent value for given type and source
// parameter code"
err = SHEFErrorCodes.LOG_034;
} else {
err = 0;
}
}
break;
case 3: {
// Duration
if (err <= 0) {
if (ShefParm.getDurationCodeValue(pedtsep.substring(2, 3)) == null) {
// "Non-existent value for given duration parameter code"
if (shefParm.getDurationCodeValue(
pedtsep.substring(2, 3)) == null) {
// "Non-existent value for given duration parameter
// code"
err = SHEFErrorCodes.LOG_060;
} else {
err = 0;
}
}
break;
}
case 2: {
if (err <= 0) {
if (ShefParm.getPhysicalElementConversionFactor(pedtsep
.substring(0, 2)) == null) {
if (shefParm.getPhysicalElementConversionFactor(
pedtsep.substring(0, 2)) == null) {
// "An expected parameter code is missing"
err = SHEFErrorCodes.LOG_003;
} else {
@ -722,12 +733,14 @@ public final class ShefUtil {
}
case 1: {
// This is an error
// "Parameter code too short or field misinterpreted as parameter code",
// "Parameter code too short or field misinterpreted as
// parameter code",
err = 64;
break;
}
default: {
// "Parameter code too short or field misinterpreted as parameter code",
// "Parameter code too short or field misinterpreted as
// parameter code",
err = SHEFErrorCodes.LOG_064;
}
}

75
edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/AbstractShefFilter.java
View file

@ -1,36 +1,34 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.edex.transform.shef;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.raytheon.uf.common.dataplugin.PluginDataObject;
import com.raytheon.uf.common.localization.ILocalizationFile;
import com.raytheon.uf.common.localization.IPathManager;
import com.raytheon.uf.common.localization.LocalizationContext;
import com.raytheon.uf.common.localization.LocalizationContext.LocalizationLevel;
import com.raytheon.uf.common.localization.LocalizationContext.LocalizationType;
import com.raytheon.uf.common.localization.PathManagerFactory;
import com.raytheon.uf.common.serialization.JAXBManager;
@ -44,28 +42,27 @@ import com.raytheon.uf.edex.decodertools.core.filterimpl.WMOHeaderFilterElement;
/**
* Abstraction of a Shef Filter.
*
*
* <pre>
*
*
* SOFTWARE HISTORY
*
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Oct 28, 2015 4783 bkowal Initial creation
* Dec 16, 2015 5166 kbisanz Update logging to use SLF4J
* Jul 13, 2016 5744 mapeters Retrieve config files from common_static
* instead of edex_static
*
* Nov 02, 2017 6132 mapeters Support localization overrides for filter file
*
* </pre>
*
*
* @author bkowal
*/
public abstract class AbstractShefFilter {
private static final String ERROR_1_FMT = "Could not create {%s} context for file \"%s\"";
private static final String ERROR_2_FMT = "File %s does not exist";
private static final String ERROR_FMT = "File %s does not exist";
protected static final String METAR_CFG = "metar.cfg";
@ -81,36 +78,27 @@ public abstract class AbstractShefFilter {
this.init();
}
protected AbstractShefFilter(String configFile, String localContext,
protected AbstractShefFilter(String configFile,
final Class<? extends AbstractShefFilter> shefFilterClass) {
this.init();
this.filterConfigFile = configFile;
this.readConfig(localContext, shefFilterClass);
this.readConfig(shefFilterClass);
}
private void readConfig(String localContext,
private void readConfig(
final Class<? extends AbstractShefFilter> shefFilterClass) {
try {
/*
* Retrieve and verify the existence of the configuration file.
*/
String configPath = FILTERS_DIR + IPathManager.SEPARATOR
+ filterConfigFile;
IPathManager manager = PathManagerFactory.getPathManager();
if (manager == null) {
throw new Exception("Failed to construct a Path Manager.");
}
LocalizationContext context = manager.getContext(
LocalizationType.COMMON_STATIC,
LocalizationLevel.valueOf(localContext));
if (context == null) {
throw new Exception(String.format(ERROR_1_FMT, localContext,
this.filterConfigFile));
}
Path configPath = Paths.get(
manager.getFile(context, FILTERS_DIR).getAbsolutePath())
.resolve(filterConfigFile);
if (!Files.exists(configPath)) {
throw new Exception(String.format(ERROR_2_FMT,
configPath.toString()));
ILocalizationFile configFile = manager.getStaticLocalizationFile(
LocalizationType.COMMON_STATIC, configPath);
if (configFile == null || !configFile.exists()) {
throw new FileNotFoundException(
String.format(ERROR_FMT, configPath));
}
/*
@ -120,8 +108,10 @@ public abstract class AbstractShefFilter {
shefFilterClass, RadiusFilterElement.class,
RectFilterElement.class, StationIdFilterElement.class,
WMOHeaderFilterElement.class);
byte[] data = Files.readAllBytes(configPath);
Object obj = jaxb.unmarshalFromXml(new String(data));
Object obj;
try (InputStream is = configFile.openInputStream()) {
obj = jaxb.unmarshalFromInputStream(is);
}
if (shefFilterClass.isInstance(obj)
|| obj instanceof PluginDataObjectFilter) {
this.buildRun(obj, this.filterConfigFile);
@ -147,19 +137,18 @@ public abstract class AbstractShefFilter {
* Only allow keep to be set to false. Once false it stays that
* way.
*/
if (AbstractObsFilter.INCLUDE_TYPE.equals(element
.getFilterType())) {
if (AbstractObsFilter.INCLUDE_TYPE
.equals(element.getFilterType())) {
// Did the filter pass?
if (r == null) {
// If we fail an element, exit now.
keep = false;
break;
}
} else if (AbstractObsFilter.EXCLUDE_TYPE.equals(element
.getFilterType())) {
} else if (AbstractObsFilter.EXCLUDE_TYPE
.equals(element.getFilterType())) {
if (r != null) {
// There was a match, so we want to remove this
// item.
// There was a match, so we want to remove this item.
keep = false;
// And there's no reason for further checks.
break;

36
edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/MetarToShefFilter.java
View file

@ -1,19 +1,19 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
@ -41,9 +41,9 @@ import com.raytheon.uf.edex.decodertools.core.filterimpl.PluginDataObjectFilter;
* Use information in metarToShefFilter.xml, MetarToShefFilter filters out the
* metar messages before send the message to MetarToShefTransformer to encode to
* a SHEF message.
*
*
* <pre>
*
*
* SOFTWARE HISTORY
* Date Ticket# Engineer Description
* ---------- ------- -------- --------------------------
@ -52,16 +52,17 @@ import com.raytheon.uf.edex.decodertools.core.filterimpl.PluginDataObjectFilter;
* 08/08/2013 16408 wkwock Use different metar.cfg file and options
* 09/09/2014 3548 mapeters Replaced SerializationUtil usage with JAXBManager.
* 10/28/2015 4783 bkowal Refactored and abstracted into {@link AbstractShefFilter}.
*
* 11/02/2017 6132 mapeters Support localization overrides for filter file
*
* </pre>
*
*
* @author wkwock
* @version 1.0
*/
@XmlRootElement
@XmlAccessorType(XmlAccessType.NONE)
@DynamicSerialize
public class MetarToShefFilter extends AbstractShefFilter {
@XmlElement
@DynamicSerializeElement
protected List<MetarToShefRun> metarToShefRun;
@ -75,8 +76,8 @@ public class MetarToShefFilter extends AbstractShefFilter {
public MetarToShefFilter() {
}
public MetarToShefFilter(String configFile, String localContext) {
super(configFile, localContext, MetarToShefFilter.class);
public MetarToShefFilter(String configFile) {
super(configFile, MetarToShefFilter.class);
for (MetarToShefRun mtsr : metarToShefRun) {
logger.info("Filter name = " + mtsr.getFilterName()
@ -109,12 +110,11 @@ public class MetarToShefFilter extends AbstractShefFilter {
/**
* Apply the list of filters against given input data.
*
*/
@Override
public PluginDataObject[] filter(PluginDataObject[] reports) {
Map<Integer, MetarToShefRun> matchList = new HashMap<>();
List<PluginDataObject> reportList = new ArrayList<PluginDataObject>();
List<PluginDataObject> reportList = new ArrayList<>();
for (PluginDataObject report : reports) {
for (MetarToShefRun mtsr : metarToShefRun) {
PluginDataObject resultRpt = filterARun(report,
@ -126,10 +126,10 @@ public class MetarToShefFilter extends AbstractShefFilter {
}
}
}
if (matchList.isEmpty() == false) {
if (!matchList.isEmpty()) {
MetarToShefTransformer.setMatchList(matchList);
}
return (PluginDataObject[]) reportList.toArray(new PluginDataObject[0]);
return reportList.toArray(new PluginDataObject[0]);
}
@Override
@ -155,18 +155,10 @@ public class MetarToShefFilter extends AbstractShefFilter {
metarToShefRun.add(element);
}
/**
*
* @return
*/
public List<MetarToShefRun> getMetarToShefRun() {
return metarToShefRun;
}
/**
*
* @param elements
*/
public void setMetarToShefRun(List<MetarToShefRun> elements) {
metarToShefRun = elements;
}

49
edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/SynopticToShefFilter.java
View file

@ -1,34 +1,34 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.edex.transform.shef;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import com.raytheon.uf.common.dataplugin.PluginDataObject;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
@ -39,24 +39,25 @@ import com.raytheon.uf.edex.decodertools.core.filterimpl.PluginDataObjectFilter;
/**
* Used to filter synoptic messages before sending them to
* {@link SMToShefTransformer}.
*
*
* <pre>
*
*
* SOFTWARE HISTORY
*
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Oct 28, 2015 4783 bkowal Initial creation
*
* Nov 02, 2017 6132 mapeters Support localization overrides for filter file
*
* </pre>
*
*
* @author bkowal
* @version 1.0
*/
@XmlRootElement
@XmlAccessorType(XmlAccessType.NONE)
@DynamicSerialize
public class SynopticToShefFilter extends AbstractShefFilter {
@XmlElement
@DynamicSerializeElement
private List<SynopticToShefRun> synopticToShefRun;
@ -68,8 +69,8 @@ public class SynopticToShefFilter extends AbstractShefFilter {
this.synopticToShefRun = new ArrayList<>();
}
public SynopticToShefFilter(String configFile, String localContext) {
super(configFile, localContext, SynopticToShefFilter.class);
public SynopticToShefFilter(String configFile) {
super(configFile, SynopticToShefFilter.class);
for (SynopticToShefRun run : synopticToShefRun) {
logger.info("Filter name = " + run.getFilterName()
@ -82,13 +83,6 @@ public class SynopticToShefFilter extends AbstractShefFilter {
this.synopticToShefRun = new ArrayList<>();
}
/*
* (non-Javadoc)
*
* @see
* com.raytheon.edex.transform.shef.AbstractShefFilter#buildRun(java.lang
* .Object, java.lang.String)
*/
@Override
protected void buildRun(Object obj, String configFile) {
if (obj instanceof PluginDataObjectFilter) {
@ -127,7 +121,7 @@ public class SynopticToShefFilter extends AbstractShefFilter {
@Override
public PluginDataObject[] filter(PluginDataObject[] reports) {
Map<Integer, SynopticToShefRun> matchList = new HashMap<>();
List<PluginDataObject> reportList = new ArrayList<PluginDataObject>();
List<PluginDataObject> reportList = new ArrayList<>();
for (PluginDataObject report : reports) {
for (SynopticToShefRun run : this.synopticToShefRun) {
PluginDataObject resultRpt = filterARun(report,
@ -139,10 +133,10 @@ public class SynopticToShefFilter extends AbstractShefFilter {
}
}
}
if (matchList.isEmpty() == false) {
if (!matchList.isEmpty()) {
SMToShefTransformer.setMatchMap(matchList);
}
return (PluginDataObject[]) reportList.toArray(new PluginDataObject[0]);
return reportList.toArray(new PluginDataObject[0]);
}
/**
@ -156,7 +150,8 @@ public class SynopticToShefFilter extends AbstractShefFilter {
* @param synopticToShefRun
* the synopticToShefRun to set
*/
public void setSynopticToShefRun(List<SynopticToShefRun> synopticToShefRun) {
public void setSynopticToShefRun(
List<SynopticToShefRun> synopticToShefRun) {
this.synopticToShefRun = synopticToShefRun;
}
}

12
edexOsgi/com.raytheon.edex.plugin.shef/utility/common_static/base/distribution/shef.xml
View file

@ -2,22 +2,28 @@
<!--
This_software_was_developed_and_/_or_modified_by_Raytheon_Company,
pursuant_to_Contract_DG133W-05-CQ-1067_with_the_US_Government.
U.S._EXPORT_CONTROLLED_TECHNICAL_DATA
This_software_product_contains_export-restricted_data_whose
export/transfer/disclosure_is_restricted_by_U.S._law._Dissemination
to_non-U.S._persons_whether_in_the_United_States_or_abroad_requires
an_export_license_or_other_authorization.
Contractor_Name:________Raytheon_Company
Contractor_Address:_____6825_Pine_Street,_Suite_340
________________________Mail_Stop_B8
________________________Omaha,_NE_68106
________________________402.291.0100
See_the_AWIPS_II_Master_Rights_File_("Master_Rights_File.pdf")_for
further_licensing_information.
-->
<!--
This is an absolute override file, indicating that a higher priority
version of the file will completely replace a lower priority version
of the file.
-->
<requestPatterns xmlns:ns2="group">
<regex>^[AF][BS].... (KOMA|KOAX|KLSE|KARX|KDSM|KDMX|KDVN|KMLI|KEAX|KMCI|KFSD|KGRI|KGID|KLBF|KSTL|KLSX|KMSP|KMPX|KTOP|KZMP|KPQR).*</regex>
<regex>^FGUS.. (KKRF|KMSR|KSTR ).*</regex>

77
edexOsgi/com.raytheon.edex.plugin.shef/utility/common_static/base/hydro/shefGadTokens.xml
View file

@ -1,28 +1,53 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<!--
This_software_was_developed_and_/_or_modified_by_Raytheon_Company,
pursuant_to_Contract_DG133W-05-CQ-1067_with_the_US_Government.
U.S._EXPORT_CONTROLLED_TECHNICAL_DATA
This_software_product_contains_export-restricted_data_whose
export/transfer/disclosure_is_restricted_by_U.S._law._Dissemination
to_non-U.S._persons_whether_in_the_United_States_or_abroad_requires
an_export_license_or_other_authorization.
Contractor_Name:________Raytheon_Company
Contractor_Address:_____6825_Pine_Street,_Suite_340
________________________Mail_Stop_B8
________________________Omaha,_NE_68106
________________________402.291.0100
See_the_AWIPS_II_Master_Rights_File_("Master_Rights_File.pdf")_for
further_licensing_information.
-->
<!--
This is an absolute override file, indicating that a higher priority
version of the file will completely replace a lower priority version
of the file.
-->
<SHEFGadTokens>
<token>db_name</token>
<token>whfs_local_data_dir</token>
<token>dupmess</token>
<token>shef_post_unk</token>
<token>shef_duplicate</token>
<token>shef_alertalarm</token>
<token>locmess</token>
<token>shef_winpast</token>
<token>shef_winfuture</token>
<token>shef_post_link</token>
<token>shef_post_latest</token>
<token>shef_load_maxfcst</token>
<token>shef_post_baddata</token>
<token>basis_hours_filter</token>
<token>elgmess</token>
<token>shef_load_ingest</token>
<token>ingest_mess</token>
<token>shef_procobs</token>
<token>shef_data_log</token>
<token>shef_perflog</token>
<token>shef_storetext</token>
<token>gage_pp_enable</token>
<token>gage_pp_data</token>
<token>gage_pp_log</token>
<token>gage_pp_sleep</token>
</SHEFGadTokens>
<token>db_name</token>
<token>whfs_local_data_dir</token>
<token>dupmess</token>
<token>shef_post_unk</token>
<token>shef_duplicate</token>
<token>shef_alertalarm</token>
<token>locmess</token>
<token>shef_winpast</token>
<token>shef_winfuture</token>
<token>shef_post_link</token>
<token>shef_post_latest</token>
<token>shef_load_maxfcst</token>
<token>shef_post_baddata</token>
<token>basis_hours_filter</token>
<token>elgmess</token>
<token>shef_load_ingest</token>
<token>ingest_mess</token>
<token>shef_procobs</token>
<token>shef_data_log</token>
<token>shef_perflog</token>
<token>shef_storetext</token>
<token>gage_pp_enable</token>
<token>gage_pp_data</token>
<token>gage_pp_log</token>
<token>gage_pp_sleep</token>
</SHEFGadTokens>

4
edexOsgi/com.raytheon.edex.plugin.shef/utility/common_static/base/shef/metar.cfg
View file

@ -1,3 +1,7 @@
# This is an absolute override file,
# indicating that a higher priority version of the file
# will completely replace a lower priority version of the file.
/tmp/queue/metar/in
/tmp/queue/metar/out
/tmp/queue/metar/err