From bfa6f10d6e61981b5426ee4dcb63d51bc4dd8b04 Mon Sep 17 00:00:00 2001 From: mjames-upc Date: Fri, 13 Jul 2018 14:51:54 -0600 Subject: [PATCH] git checkout origin/master_18.1.1 edexOsgi/com.raytheon.edex.plugin.shef. closes #238 --- .../com.raytheon.edex.plugin.shef/.classpath | 3 + .../res/spring/shef-common.xml | 23 +- .../res/spring/shef-ingest.xml | 228 ++--- .../raytheon/edex/plugin/shef/SHEFParser.java | 799 +++++++----------- .../edex/plugin/shef/ShefDecoder.java | 60 +- .../edex/plugin/shef/data/ShefData.java | 87 +- .../edex/plugin/shef/database/PostShef.java | 666 ++++++++------- .../edex/plugin/shef/database/PostTables.java | 483 +++++++---- .../edex/plugin/shef/util/ParserToken.java | 688 +++++++-------- .../edex/plugin/shef/util/ShefParm.java | 123 +-- .../edex/plugin/shef/util/ShefUtil.java | 113 +-- .../transform/shef/AbstractShefFilter.java | 75 +- .../transform/shef/MetarToShefFilter.java | 36 +- .../transform/shef/SynopticToShefFilter.java | 49 +- .../common_static/base/distribution/shef.xml | 12 +- .../base/hydro/shefGadTokens.xml | 77 +- .../utility/common_static/base/shef/metar.cfg | 4 + 17 files changed, 1672 insertions(+), 1854 deletions(-) diff --git a/edexOsgi/com.raytheon.edex.plugin.shef/.classpath b/edexOsgi/com.raytheon.edex.plugin.shef/.classpath index 1fa3e6803d..0617e9b5b4 100644 --- a/edexOsgi/com.raytheon.edex.plugin.shef/.classpath +++ b/edexOsgi/com.raytheon.edex.plugin.shef/.classpath @@ -3,5 +3,8 @@ + + + diff --git a/edexOsgi/com.raytheon.edex.plugin.shef/res/spring/shef-common.xml b/edexOsgi/com.raytheon.edex.plugin.shef/res/spring/shef-common.xml index 59b07ea826..b4e3b2bbca 100644 --- a/edexOsgi/com.raytheon.edex.plugin.shef/res/spring/shef-common.xml +++ b/edexOsgi/com.raytheon.edex.plugin.shef/res/spring/shef-common.xml @@ -1,16 +1,17 @@ + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://www.springframework.org/schema/beans + http://www.springframework.org/schema/beans/spring-beans.xsd"> - - - - - - - - - + + + + + + + + + \ No newline at end of file diff --git a/edexOsgi/com.raytheon.edex.plugin.shef/res/spring/shef-ingest.xml b/edexOsgi/com.raytheon.edex.plugin.shef/res/spring/shef-ingest.xml index b8242f6a6d..24961dd3a9 100644 --- a/edexOsgi/com.raytheon.edex.plugin.shef/res/spring/shef-ingest.xml +++ b/edexOsgi/com.raytheon.edex.plugin.shef/res/spring/shef-ingest.xml @@ -1,46 +1,46 @@ + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://www.springframework.org/schema/beans + http://www.springframework.org/schema/beans/spring-beans.xsd + http://camel.apache.org/schema/spring + http://camel.apache.org/schema/spring/camel-spring.xsd"> - - - + + + - + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - - + + + + @@ -61,7 +61,7 @@ errorHandlerRef="errorHandler"> - + @@ -73,93 +73,93 @@ - + - - - - - shef - - - - - - - - - - shef - - - + + + + + shef + + + + + + + + + + shef + + + - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - java.lang.Throwable - - - - + + + + + + + + + java.lang.Throwable + + + + - - - - shef - - - + + + + shef + + + - - - - - java.lang.Throwable - - - - + + + + + java.lang.Throwable + + + + - + diff --git a/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/SHEFParser.java b/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/SHEFParser.java index dcc6ab734f..c2d4e6794a 100644 --- a/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/SHEFParser.java +++ b/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/SHEFParser.java @@ -24,9 +24,7 @@ import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; -import java.util.StringTokenizer; import java.util.TimeZone; import com.raytheon.edex.plugin.shef.ShefSeparator.ShefDecoderInput; @@ -63,6 +61,8 @@ import com.raytheon.uf.common.wmo.WMOHeader; * Apr 29, 2014 3088 mpduff Changed to use UFStatus logging. * Apr 27, 2015 4377 skorolev Corrected set up the default product id. * Mar 10, 2016 5352 mduff Truncated identifier to 10 characters. + * Jan 10, 2018 5049 mduff Rewrote parsing. + * May 25, 2018 5049 mduff Fixed bug introduced during refactor. * * * @@ -77,7 +77,9 @@ public class SHEFParser { private static final SHEFErrors ERR_LOGGER = SHEFErrors .registerLogger(SHEFParser.class); - private static final HashMap STARTTYPES = new HashMap(); + private static final String NEWLINE = "\n"; + + private static final HashMap STARTTYPES = new HashMap<>(); static { STARTTYPES.put(TokenType.A_REC, 0); STARTTYPES.put(TokenType.A_REC_R, 0); @@ -87,7 +89,7 @@ public class SHEFParser { STARTTYPES.put(TokenType.E_REC_R, 0); } - private static final HashSet VALID_TRACE_PE = new HashSet(); + private static final HashSet VALID_TRACE_PE = new HashSet<>(); static { VALID_TRACE_PE.add(PhysicalElement.PRECIPITATION_ACCUMULATOR); VALID_TRACE_PE.add(PhysicalElement.PRECIPITATION_INCREMENT); @@ -98,15 +100,13 @@ public class SHEFParser { } - private static final HashSet EOD_SENDCODES = new HashSet(); + private static final HashSet EOD_SENDCODES = new HashSet<>(); static { EOD_SENDCODES.add("HY"); EOD_SENDCODES.add("PY"); EOD_SENDCODES.add("QY"); } - private static final String CARRIAGECONTROL = "\r\n"; - private String message; private String traceId; @@ -117,8 +117,6 @@ public class SHEFParser { private Date productDate; - private String recordType; - private List parts; private int partsIndex; @@ -131,7 +129,7 @@ public class SHEFParser { // The observation time is the second positional field in each of // the "A", "B", or "E" data records. This is a mandatory item and - // will be completely evaluationed once the timezone has been set. + // will be completely evaluated once the timezone has been set. // After that time the obsTime and timeZone must not change. private String obsTime = null; @@ -139,9 +137,6 @@ public class SHEFParser { private SHEFDate obsDate = null; - // - private boolean was24Hour = false; - // The timezone is the third positional field in a SHEF record. This // item is optional and if missing shall default to "Z". Do not set // this value until the timezone positional field has been seen or @@ -150,9 +145,6 @@ public class SHEFParser { private TimeZone tz = null; - // - // - // private SHEFDate adjObsDate = null; // The createTime may be evaluated at any time during the data scan. This @@ -179,29 +171,31 @@ public class SHEFParser { private String bRecordDataSource = null; - private AppsDefaults appDefaults = AppsDefaults.getInstance(); - private boolean emitSkippedValues = false; private String reportLead = null; + private ShefParm shefParm; + /** * Constructor * * @param sdi * ShefDecoderInput */ - public SHEFParser(ShefDecoderInput sdi) { + public SHEFParser(ShefDecoderInput sdi, AppsDefaults appDefaults, + ShefParm shefParm) { message = sdi.record; traceId = sdi.traceId; awipsHeader = sdi.awipsHeader; wmoHeader = sdi.wmoHeader; productDate = sdi.productDate; + this.shefParm = shefParm; + message = clearTabs(message); + parts = tokenize(message); - parts = tokenize(clearTabs(message)); - - emitSkippedValues = appDefaults.getBoolean( - ShefConstants.SHEF_EMIT_SKIPPED, false); + emitSkippedValues = appDefaults + .getBoolean(ShefConstants.SHEF_EMIT_SKIPPED, false); partsIndex = 0; } @@ -285,8 +279,8 @@ public class SHEFParser { */ public void incrementAdjObsDate(ParserToken increment) { if ((adjObsDate != null) && (increment != null)) { - SHEFDate date = SHEFDate.increment(adjObsDate, - increment.getToken(), 2); + SHEFDate date = SHEFDate.increment(adjObsDate, increment.getToken(), + 2); if (date != null) { adjObsDate = date; } @@ -403,52 +397,48 @@ public class SHEFParser { * @return The decoded ShefRecord */ public ShefRecord decode() { - boolean revision = false; ShefRecord record = null; // Synchronize to the start of the data. ParserToken t = null; - while (parts.size() > 0) { + while (!parts.isEmpty()) { t = parts.remove(partsIndex); if (STARTTYPES.containsKey(t.getType())) { break; } } - if (parts.size() <= 0) { + if (parts.isEmpty()) { log.error("No data parsed from report.\n" + message); return record; } record = new ShefRecord(); setCurrentQualifier("Z"); switch (t.getType()) { - case A_REC_R: { - revision = true; - } + case A_REC_R: case A_REC: { record.setShefType(ShefRecord.ShefType.A); + boolean revision = t.getType() == TokenType.A_REC_R; record.setRevisedRecord(revision); if ((record = parseARecord(record)) != null) { record.setLocationId(locationId); } break; } - case B_REC_R: { - revision = true; - } + case B_REC_R: case B_REC: { record.setShefType(ShefRecord.ShefType.B); + boolean revision = t.getType() == TokenType.B_REC_R; record.setRevisedRecord(revision); if ((record = parseBRecord(record)) != null) { record.setLocationId(locationId); } break; } - case E_REC_R: { - revision = true; - } + case E_REC_R: case E_REC: { record.setShefType(ShefRecord.ShefType.E); + boolean revision = t.getType() == TokenType.E_REC_R; record.setRevisedRecord(revision); if ((record = parseERecord(record)) != null) { record.setLocationId(locationId); @@ -493,8 +483,8 @@ public class SHEFParser { d.toPostData(); i++; } else { - if (ShefConstants.SHEF_SKIPPED.equals(d - .getStringValue())) { + if (ShefConstants.SHEF_SKIPPED + .equals(d.getStringValue())) { data.remove(i); } else { d.toPostData(); @@ -520,12 +510,14 @@ public class SHEFParser { reportLead = null; if (getPositionalData()) { StringBuilder sb = new StringBuilder(); - PRIMARY: for (int i = 0; i < parts.size();) { - ParserToken t = parts.remove(i); + boolean exit = false; + while (!parts.isEmpty() && !exit) { + ParserToken t = parts.remove(0); sb.append(t.getRawToken()); switch (t.getType()) { case TIMEZONE: { - break PRIMARY; + exit = true; + break; } case OBS_DATE_4: case OBS_DATE_6: @@ -764,10 +756,10 @@ public class SHEFParser { } s = token.getToken(); - int currError = ShefUtil.validatePEDTSEP(s); + int currError = ShefUtil.validatePEDTSEP(s, shefParm); if (currError == 0) { - PhysicalElement pe = PhysicalElement.getEnum(s - .substring(0, 2)); + PhysicalElement pe = PhysicalElement + .getEnum(s.substring(0, 2)); if (!PhysicalElement.UNKNOWN.equals(pe)) { pedtsep = s; } else { @@ -847,7 +839,7 @@ public class SHEFParser { } } // switch if ((pedtsep != null) && (value != null)) { - ShefData data = new ShefData(); + ShefData data = new ShefData(shefParm); data.setParameterCodeString(pedtsep, currentDuration); data.setLocationId(getLocationId()); data.setObservationTime(record.getRecordDate()); @@ -891,7 +883,7 @@ public class SHEFParser { break; } - } // for + } } return record; } @@ -900,7 +892,8 @@ public class SHEFParser { * Invalidate this record if PE in {HY QY PY} and the timezone is zulu. * */ - private boolean validateRecord(List tokens, ShefRecord record) { + private boolean validateRecord(List tokens, + ShefRecord record) { boolean isValid = true; int error = 0; for (ParserToken token : tokens) { @@ -908,13 +901,11 @@ public class SHEFParser { if (token != null) { String pe = token.getSendCode(); if (pe != null) { - if (pe != null) { - if (pe.startsWith("HY") || pe.startsWith("QY") - || pe.startsWith("PY")) { - if ("Z".equals(timeZone)) { - isValid = false; - error = SHEFErrorCodes.LOG_035; - } + if (pe.startsWith("HY") || pe.startsWith("QY") + || pe.startsWith("PY")) { + if ("Z".equals(timeZone)) { + isValid = false; + error = SHEFErrorCodes.LOG_035; } } } @@ -931,11 +922,6 @@ public class SHEFParser { // * B Record specific methods. // ********************************* - /** - * - * @param record - * @return - */ private ShefRecord parseBRecord(ShefRecord record) { reportLead = null; @@ -974,8 +960,8 @@ public class SHEFParser { parts = saveList; fixupDates(parts, tz); - List pattern = new ArrayList(); - List> bdata = new ArrayList>(); + List pattern = new ArrayList<>(); + List> bdata = new ArrayList<>(); int idx = 0; boolean end = false; boolean addToken = false; @@ -991,20 +977,20 @@ public class SHEFParser { pattern.add(t); } } - List bLine = new ArrayList(); + List bLine = new ArrayList<>(); addToken = false; for (; !end && idx < parts.size(); idx++) { ParserToken t = parts.get(idx); if (TokenType.B_END.equals(t.getType())) { - if (bLine.size() > 0) { + if (!bLine.isEmpty()) { bdata.add(bLine); } break; } else if (TokenType.B_DATA.equals(t.getType())) { - if (bLine.size() > 0) { + if (!bLine.isEmpty()) { bdata.add(bLine); } - bLine = new ArrayList(); + bLine = new ArrayList<>(); addToken = true; continue; } @@ -1016,13 +1002,14 @@ public class SHEFParser { if (validatePattern(pattern)) { for (List subList : bdata) { identifyBData(subList); - identifyUnknownToken( - subList.subList(1, subList.size()), true); + identifyUnknownToken(subList.subList(1, subList.size()), + true); // Make a copy of the master date so that each data line // gets a clean copy. SHEFDate localMaster = new SHEFDate(masterDate); try { - interpretData(record, pattern, subList, localMaster); + interpretData(record, pattern, subList, + localMaster); } catch (Exception e) { ERR_LOGGER.error(getClass(), createRecordHeader(record, reportLead) @@ -1030,10 +1017,8 @@ public class SHEFParser { ERR_LOGGER.error(getClass(), createDataLine(subList)); ERR_LOGGER.error(getClass(), "?"); - ERR_LOGGER.error(getClass(), - "Exception " + e.getLocalizedMessage()); - ERR_LOGGER - .error(getClass(), SHEFErrorCodes.LOG_090); + ERR_LOGGER.error(getClass(), SHEFErrorCodes.LOG_090, + e); } } @@ -1131,7 +1116,8 @@ public class SHEFParser { TokenType tt = t.getType(); switch (tt) { case PEDTSEP: { - error = ShefUtil.validatePEDTSEP(t.getRawToken()); + error = ShefUtil.validatePEDTSEP(t.getRawToken(), + shefParm); valid = (error == 0); break; } @@ -1141,7 +1127,8 @@ public class SHEFParser { break; } case QUAL_CODE: { - valid = isValidQualityCode(t.getRawToken().substring(2)); + valid = isValidQualityCode( + t.getRawToken().substring(2)); error = SHEFErrorCodes.LOG_085; break; } @@ -1176,7 +1163,6 @@ public class SHEFParser { * @param subList */ private void identifyBData(List subList) { - TokenType last = TokenType.NIL; TokenType next = TokenType.LOC_ID; for (int i = 0; i < subList.size();) { @@ -1219,7 +1205,8 @@ public class SHEFParser { if (TokenType.SLASH.equals(tt.getType())) { if (TokenType.SLASH.equals(last)) { // only set an empty if the last token was a SLASH - subList.set(i, new ParserToken("", TokenType.EMPTY)); + subList.set(i, + new ParserToken("", TokenType.EMPTY)); i++; } else { subList.remove(i); @@ -1246,7 +1233,7 @@ public class SHEFParser { * */ private void parseLineData() { - List newTokens = new ArrayList(); + List newTokens = new ArrayList<>(); newTokens.add(new ParserToken("", TokenType.B_PATTERN)); TokenType last = TokenType.NIL; for (ParserToken t : parts) { @@ -1283,7 +1270,7 @@ public class SHEFParser { } } } - if (newTokens.size() > 0) { + if (!newTokens.isEmpty()) { parts = newTokens; } } @@ -1449,7 +1436,7 @@ public class SHEFParser { } s = pToken.getToken(); - int currError = ShefUtil.validatePEDTSEP(s); + int currError = ShefUtil.validatePEDTSEP(s, shefParm); PhysicalElement pe = null; if (currError == 0) { pe = PhysicalElement.getEnum(s.substring(0, 2)); @@ -1525,10 +1512,10 @@ public class SHEFParser { // this if (!reSync && (value != null)) { - ShefData data = new ShefData(); - data.setParameterCodeString( - pedtsep, - (currentDurationOverride == null) ? currentDuration + ShefData data = new ShefData(shefParm); + data.setParameterCodeString(pedtsep, + (currentDurationOverride == null) + ? currentDuration : currentDurationOverride); data.setLocationId(lid); data.setDataSource(bRecordDataSource); @@ -1554,22 +1541,23 @@ public class SHEFParser { data.setQualifier(qualifierOverride); } data.setRetainedComment(retainedComment); - data.setRevisedRecord(record - .isRevisedRecord()); - data.fixupDuration((durationValueOverride == null) ? durationValue - : durationValueOverride); + data.setRevisedRecord( + record.isRevisedRecord()); + data.fixupDuration( + (durationValueOverride == null) + ? durationValue + : durationValueOverride); if (trace) { - if (legalTraceValue(data - .getPhysicalElement())) { + if (legalTraceValue( + data.getPhysicalElement())) { record.addDataValue(data); } else { - ERR_LOGGER - .error(getClass(), - createRecordHeader( - record, - reportLead) - + createDataLine(pattern)); + ERR_LOGGER.error(getClass(), + createRecordHeader(record, + reportLead) + + createDataLine( + pattern)); ERR_LOGGER.error(getClass(), createDataLine(bdata)); ERR_LOGGER.error(getClass(), " ?"); @@ -1592,7 +1580,8 @@ public class SHEFParser { trace = false; break; } - bToken = bdata.get(bDataPtr++); + bToken = bdata.get(bDataPtr); + bDataPtr++; exitStatus = tokenError(record, pattern, bdata, bToken); if (exitStatus == 1) { @@ -1733,8 +1722,8 @@ public class SHEFParser { case UNKNOWN: { if (isMissingValue(bToken.getToken())) { value = ShefConstants.SHEF_MISSING; - qualifier = getMissingQualifier(bToken - .getToken()); + qualifier = getMissingQualifier( + bToken.getToken()); } else if (isTraceValue(bToken.getToken())) { value = ShefConstants.SHEF_TRACE; trace = true; @@ -1747,10 +1736,10 @@ public class SHEFParser { // We only want to process a new data item if we've // set the data. if (value != null) { - ShefData data = new ShefData(); - data.setParameterCodeString( - pedtsep, - (currentDurationOverride == null) ? currentDuration + ShefData data = new ShefData(shefParm); + data.setParameterCodeString(pedtsep, + (currentDurationOverride == null) + ? currentDuration : currentDurationOverride); data.setLocationId(lid); data.setDataSource(bRecordDataSource); @@ -1774,29 +1763,30 @@ public class SHEFParser { data.setQualifier(qualifierOverride); } if (retainedComment != null) { - data.setRetainedComment(retainedComment); + data.setRetainedComment( + retainedComment); retainedComment = null; } else { lastData = data; } - data.setRevisedRecord(record - .isRevisedRecord()); + data.setRevisedRecord( + record.isRevisedRecord()); - data.fixupDuration((durationValueOverride == null) ? durationValue - : durationValueOverride); - if (trace - && ShefConstants.SHEF_TRACE - .equals(value)) { - if (legalTraceValue(data - .getPhysicalElement())) { + data.fixupDuration( + (durationValueOverride == null) + ? durationValue + : durationValueOverride); + if (trace && ShefConstants.SHEF_TRACE + .equals(value)) { + if (legalTraceValue( + data.getPhysicalElement())) { record.addDataValue(data); } else { - ERR_LOGGER - .error(getClass(), - createRecordHeader( - record, - reportLead) - + createDataLine(pattern)); + ERR_LOGGER.error(getClass(), + createRecordHeader(record, + reportLead) + + createDataLine( + pattern)); ERR_LOGGER.error(getClass(), createDataLine(bdata)); ERR_LOGGER.error(getClass(), " ?"); @@ -1846,7 +1836,7 @@ public class SHEFParser { } } // for if (value != null) { - ShefData data = new ShefData(); + ShefData data = new ShefData(shefParm); data.setLocationId(lid); data.setDataSource(bRecordDataSource); data.setObservationTime(record.getRecordDate()); @@ -1871,8 +1861,8 @@ public class SHEFParser { currentDurationOverride); data.setRetainedComment(retainedComment); data.setRevisedRecord(record.isRevisedRecord()); - data.fixupDuration((durationValueOverride == null) ? durationValue - : durationValueOverride); + data.fixupDuration((durationValueOverride == null) + ? durationValue : durationValueOverride); if (trace && ShefConstants.SHEF_TRACE.equals(value)) { if (legalTraceValue(data.getPhysicalElement())) { record.addDataValue(data); @@ -1882,8 +1872,8 @@ public class SHEFParser { + createDataLine(pattern)); ERR_LOGGER.error(getClass(), createDataLine(bdata)); ERR_LOGGER.error(getClass(), " ?"); - ERR_LOGGER - .error(getClass(), SHEFErrorCodes.LOG_031); + ERR_LOGGER.error(getClass(), + SHEFErrorCodes.LOG_031); } } else { record.addDataValue(data); @@ -1986,19 +1976,12 @@ public class SHEFParser { return errorCondition; } - /** - * - * @param baseTime - * @param drOuter - * @param drInner - * @param record - * @return - */ private SHEFDate getRelativeDate(SHEFDate baseTime, ParserToken drOuter, ParserToken drInner, ShefRecord record, boolean overRide) { SHEFDate date = null; ParserToken dateRelative = null; - if ((drOuter != null) && (TokenType.DATE_REL.equals(drOuter.getType()))) { + if ((drOuter != null) + && (TokenType.DATE_REL.equals(drOuter.getType()))) { if (drInner == null) { dateRelative = drOuter; } else { @@ -2034,11 +2017,6 @@ public class SHEFParser { // * E Record specific methods. // ********************************* - /** - * - * @param record - * @return - */ private ShefRecord parseERecord(ShefRecord record) { reportLead = null; @@ -2047,13 +2025,15 @@ public class SHEFParser { correctMissingDelimiters(); StringBuilder sb = new StringBuilder(); - PRIMARY: for (int i = 0; i < parts.size();) { - ParserToken t = parts.remove(i); + boolean exit = false; + while (!parts.isEmpty() && !exit) { + ParserToken t = parts.remove(0); sb.append(t.getRawToken()); switch (t.getType()) { case TIMEZONE: { - break PRIMARY; + exit = true; + break; } case OBS_DATE_4: case OBS_DATE_6: @@ -2200,11 +2180,12 @@ public class SHEFParser { case PEDTSEP: { if (!inData) { String s = token.getToken(); - int currError = ShefUtil.validatePEDTSEP(s); + int currError = ShefUtil.validatePEDTSEP(s, + shefParm); if (currError == 0) { - PhysicalElement pe = PhysicalElement.getEnum(s - .substring(0, 2)); + PhysicalElement pe = PhysicalElement + .getEnum(s.substring(0, 2)); if (!PhysicalElement.UNKNOWN.equals(pe)) { pedtsep = s; } @@ -2214,7 +2195,8 @@ public class SHEFParser { // do we have a variable duration // defined? if (!"Z".equals(currentDuration)) { - if ("Z".equals(currentDurationOverride)) { + if ("Z".equals( + currentDurationOverride)) { currError = SHEFErrorCodes.LOG_032; } } else { @@ -2405,7 +2387,7 @@ public class SHEFParser { } } // switch if ((pedtsep != null) && (value != null)) { - ShefData data = new ShefData(); + ShefData data = new ShefData(shefParm); data.setParameterCodeString(pedtsep, currentDuration); data.setLocationId(getLocationId()); data.setObservationTime(record.getRecordDate()); @@ -2532,8 +2514,8 @@ public class SHEFParser { break; } case QUAL_CODE: { - isValid = isValidQualityCode(token.getRawToken().substring( - 2)); + isValid = isValidQualityCode( + token.getRawToken().substring(2)); error = SHEFErrorCodes.LOG_021; break; } @@ -2592,7 +2574,7 @@ public class SHEFParser { TokenType COMMA = TokenType.COMMA; TokenType SPACE = TokenType.SPACE; - if ((parts != null) && (parts.size() > 0)) { + if ((parts != null) && (!parts.isEmpty())) { ParserToken last = null; // First pass through we are going to look for possible commas in // the data. @@ -2612,8 +2594,8 @@ public class SHEFParser { } else { if ((last != null) && (last.getType() != null)) { if (last.isValueToken()) { - parts.set(i - 1, new ParserToken("", - TokenType.EMPTY)); + parts.set(i - 1, + new ParserToken("", TokenType.EMPTY)); t = getToken(parts, i); if (t.isValueToken()) { parts.remove(i); @@ -2661,7 +2643,7 @@ public class SHEFParser { * A list of tokens to process. * @return The Interval Code token found, or null if none were found. */ - private static ParserToken findInterval(List tokens) { + private ParserToken findInterval(List tokens) { ParserToken token = null; int prevInt = -1; for (int i = 0; i < tokens.size();) { @@ -2698,147 +2680,116 @@ public class SHEFParser { * Tokenize the current message. * * @param message - * @return + * @return List of ParserToken objects */ - private static List tokenize(String message) { - List tokens = new ArrayList(); + List tokenize(String message) { + List tokens = null; + String type = message.substring(0, message.indexOf(' ')); + + if (type.startsWith(".A")) { + tokens = parseAData(message); + } else if (type.startsWith(".B")) { + tokens = parseBData(message); + } else if (type.startsWith(".E")) { + tokens = parseEData(message); + } else { + ERR_LOGGER.error(getClass(), "Invalid record type: " + type); + } + + tokens = identifyEmpty(tokens); + + return tokens; + } + + private List parseAData(String message) { + List tokens = new ArrayList<>(); + if (message == null) { + return tokens; + } tokens.add(new ParserToken("", TokenType.START)); - if (message != null) { - StringTokenizer st = new StringTokenizer(message, "/" - + CARRIAGECONTROL + ",", true); - TokenType last = null; - while (st.hasMoreTokens()) { - String currToken = st.nextToken(); - // Constructor will attempt to determine the token type - ParserToken t = new ParserToken(currToken.trim()); - if (TokenType.COMMA.equals(last) && currToken.startsWith(" ")) { - tokens.add(new ParserToken(" ", TokenType.SPACE)); - } - if (TokenType.UNKNOWN.equals(t.getType()) - || TokenType.SPACEINMIDDLE.equals(t.getType())) { - // check possible failures - List subList = subTokenize(currToken); - if (subList != null) { - tokens.addAll(subList); + String[] lines = message.split("(?<=[\\r\\n])|(?=[\\r\\n])"); + for (String line : lines) { + String[] parts = line.split("\\s+"); + // Format specifier + ParserToken t = new ParserToken(parts[0]); + tokens.add(t); + + // Location ID + t = new ParserToken(parts[1], TokenType.LOC_ID); + tokens.add(t); + + // Date + t = new ParserToken(parts[2]); + tokens.add(t); + + for (int i = 3; i < parts.length; i++) { + String s = parts[i]; + if (s.indexOf('/') > -1) { + String[] sa = s.split("(?<=/)|(?=/)"); + for (String s1 : sa) { + t = new ParserToken(s1); + tokens.add(t); } } else { + tokens.add(new ParserToken(s)); + } + } + } + + return tokens; + } + + private List parseBData(String message) { + List tokens = new ArrayList<>(); + if (message == null) { + return tokens; + } + tokens.add(new ParserToken("", TokenType.START)); + String[] lines = message.split("(?<=[\\r\\n])|(?=[\\r\\n])"); + String bLine = lines[0]; + String[] parts = bLine.split("\\s+"); + // Format specifier + ParserToken t = new ParserToken(parts[0]); + tokens.add(t); + + // Message Source + t = new ParserToken(parts[1]); + tokens.add(t); + + // Date + t = new ParserToken(parts[2]); + tokens.add(t); + + for (int i = 3; i < parts.length; i++) { + String s = parts[i]; + if (s.indexOf('/') > -1) { + String[] sa = s.split("(?<=/)|(?=/)"); + for (String ss : sa) { + t = new ParserToken(ss); tokens.add(t); } - if (tokens.size() > 0) { - last = tokens.get(tokens.size() - 1).getType(); - } + } else { + t = new ParserToken(s); + tokens.add(t); } - tokens = identifyEmpty(collapseSpaces(tokens)); } - return tokens; - } - - /** - * - * @param token - * @return - */ - private static List subTokenize(String token) { - TokenType UNKNOWN = TokenType.UNKNOWN; - List tokens = new ArrayList(); - - // Tokenize by spaces - StringTokenizer st = new StringTokenizer(token, " ", true); - String lastToken = null; - while (st.hasMoreTokens()) { - String currToken = st.nextToken(); - if ((" ".equals(currToken) && " ".equals(lastToken))) { + // Data lines + for (int i = 1; i < lines.length; i++) { + if (NEWLINE.equals(lines[i])) { + tokens.add(new ParserToken(lines[i])); continue; } - ParserToken tt = new ParserToken(currToken); - if (TokenType.UNKNOWN.equals(tt.getType())) { - tt = tt.check_D_Directives(); - } - tokens.add(tt); - lastToken = currToken; - } - - // Make a pass through the tokens to see if there are any - // ill-formed retained comments - for (int i = 0; i < tokens.size(); i++) { - ParserToken t = tokens.get(i); - if (t != null) { - String s = t.getToken(); - int pos = s.indexOf("\""); - if (pos < 0) { - pos = s.indexOf("\'"); - } - // Do we need to split this token? - if (pos > 0) { - // Let the system try to reidentify the token - ParserToken tt = new ParserToken(s.substring(0, pos)); - tokens.set(i, tt); - tt = new ParserToken(s.substring(pos), - TokenType.RETAINEDCOMMENT); - i++; - tokens.add(i, tt); - } - } - } - - // We may have split retained comments so attempt to put them - // back together - for (int i = 0; i < tokens.size(); i++) { - ParserToken t = tokens.get(i); - String quoteType = null; - if (UNKNOWN.equals(t.getType())) { - int startIndex = -1; - int endIndex = -1; - String s = t.getToken(); - if ((s != null) && (s.length() > 0)) { - if (s.startsWith("\"")) { - if (s.endsWith("\"")) { - continue; - } else { - quoteType = "\""; - } - } else if (s.startsWith("\'")) { - if (s.endsWith("\'")) { - continue; - } else { - quoteType = "\'"; - } - } - // we found the beginning of a comment, but the end wasn't - // in the same token so we need to find it. - if (quoteType != null) { - startIndex = i; - // move to the next token - i++; - for (; i < tokens.size(); i++) { - s = tokens.get(i).getToken(); - if (s.endsWith(quoteType)) { - endIndex = i + 1; - break; - } - } - // Check if we found an ending quote. End of line ends - // a comment so its legal to not find one. - if (endIndex < 0) { - endIndex = i + 1; - } - StringBuilder sb = new StringBuilder(); - t = tokens.get(startIndex); - sb.append(t.getToken().substring(1)); - for (int ii = startIndex + 1; ii < endIndex; endIndex--) { - ParserToken tt = tokens.remove(ii); - s = tt.getToken(); - if (s.endsWith("\"")) { - sb.append(s.substring(0, s.length() - 1)); - } else { - sb.append(s); - } - } - ParserToken newToken = new ParserToken(sb.toString() - .trim(), TokenType.RETAINEDCOMMENT); - tokens.set(startIndex, newToken); + String[] sa = lines[i].split("\\s+"); + for (String ss : sa) { + if (ss.contains("/")) { + String[] qq = ss.split("(?<=/)|(?=/)"); + for (String q : qq) { + tokens.add(new ParserToken(q)); } + } else { + t = new ParserToken(ss); + tokens.add(t); } } } @@ -2846,8 +2797,30 @@ public class SHEFParser { return tokens; } - private static List identifyNeededSlashes( - List tokens) { + private List parseEData(String message) { + List tokens = new ArrayList<>(); + if (message == null) { + return tokens; + } + tokens.add(new ParserToken("", TokenType.START)); + String[] lines = message.split("[\\r\\n]+"); + String eLine = lines[0]; + String[] parts = eLine.split("\\s+"); + for (String part : parts) { + if (part.contains("/")) { + String[] sa = part.split("(?<=/)|(?=/)"); + for (String s : sa) { + tokens.add(new ParserToken(s)); + } + } else { + tokens.add(new ParserToken(part)); + } + } + + return tokens; + } + + private List identifyNeededSlashes(List tokens) { TokenType SLASH = TokenType.SLASH; TokenType UNKNOWN = TokenType.UNKNOWN; TokenType NUMERIC = TokenType.NUMERIC; @@ -2893,12 +2866,7 @@ public class SHEFParser { return tokens; } - /** - * - * @param tokens - * @return - */ - private static List identifyEmpty(List tokens) { + private List identifyEmpty(List tokens) { TokenType SLASH = TokenType.SLASH; TokenType SPACE = TokenType.SPACE; @@ -2906,7 +2874,7 @@ public class SHEFParser { TokenType NIL = TokenType.NIL; - List newTokens = new ArrayList(); + List newTokens = new ArrayList<>(); TokenType last = TokenType.NIL; @@ -2956,10 +2924,12 @@ public class SHEFParser { last = t.getType(); } } - } // for + } + if (SLASH.equals(last)) { newTokens.add(new ParserToken("", TokenType.EMPTY)); } + return newTokens; } @@ -2971,7 +2941,8 @@ public class SHEFParser { * @param isBData * Is the list the data sublist from a B record? */ - private void identifyUnknownToken(List tokens, boolean isBData) { + private void identifyUnknownToken(List tokens, + boolean isBData) { if (tokens != null) { for (int i = 0; i < tokens.size();) { ParserToken t = tokens.get(i); @@ -2988,20 +2959,22 @@ public class SHEFParser { } else { if (i > 0) { ParserToken tt = tokens.get(i - 1); - if (TokenType.PEDTSEP.equals(tt.getType())) { - // if the previous token is a PE token, - // then - // treat as missing + if (TokenType.PEDTSEP + .equals(tt.getType())) { + /* + * if the previous token is a PE token, + * then treat as missing + */ tt = new ParserToken( ShefConstants.SHEF_MISSING, TokenType.NUMERIC); tokens.set(i, tt); - } else if (TokenType.SLASH.equals(tt - .getType())) { - // if the previous token is a SLASH - // token, - // then - // treat as a PE + } else if (TokenType.SLASH + .equals(tt.getType())) { + /* + * if the previous token is a SLASH then + * treat as a PE + */ tt = new ParserToken(s, TokenType.PEDTSEP); tokens.set(i, tt); @@ -3009,16 +2982,16 @@ public class SHEFParser { } } } else { - PhysicalElement pe = PhysicalElement.getEnum(s - .substring(0, 2)); + PhysicalElement pe = PhysicalElement + .getEnum(s.substring(0, 2)); if (!PhysicalElement.UNKNOWN.equals(pe)) { int error = SHEFErrorCodes.LOG_000; String sendCode = null; - String trans = ShefParm - .getSendCodeDurationDefaults(pe - .getCode()); + String trans = shefParm + .getSendCodeDurationDefaults( + pe.getCode()); if (trans != null) { if (trans.length() > 3) { // Handle the send code translation @@ -3072,16 +3045,13 @@ public class SHEFParser { tt.setTrace(true); tokens.set(i, tt); } else { - // With the - // We have a problem! log.error(traceId + "- Could not identify token " + t); } i++; } else if (TokenType.SPACE.equals(t.getType())) { if (isBData) { - if ((i < tokens.size() - 2) - && (TokenType.SLASH.equals(tokens.get(i + 1) - .getType()))) { + if ((i < tokens.size() - 2) && (TokenType.SLASH + .equals(tokens.get(i + 1).getType()))) { tokens.set(i, new ParserToken("", TokenType.EMPTY)); } else { tokens.remove(i); @@ -3090,8 +3060,8 @@ public class SHEFParser { tokens.remove(i); } } else if (TokenType.DECIMAL.equals(t.getType())) { - ParserToken tt = new ParserToken( - ShefConstants.SHEF_MISSING, TokenType.NUMERIC); + ParserToken tt = new ParserToken(ShefConstants.SHEF_MISSING, + TokenType.NUMERIC); tokens.set(i, tt); } else { i++; @@ -3114,8 +3084,8 @@ public class SHEFParser { if (movePastSpaces()) { ParserToken t = parts.get(partsIndex); - if (ShefUtil.between(ShefConstants.LOWER_LID_LIMIT, t.getToken() - .length(), ShefConstants.UPPER_LID_LIMIT)) { + if (ShefUtil.between(ShefConstants.LOWER_LID_LIMIT, + t.getToken().length(), ShefConstants.UPPER_LID_LIMIT)) { setLocationId(t.getToken()); t = ParserToken.createLocIdToken(getLocationId()); @@ -3127,7 +3097,8 @@ public class SHEFParser { partsIndex++; movePastSpaces(); if (partsIndex < parts.size()) { - t = parts.get(partsIndex++); + t = parts.get(partsIndex); + partsIndex++; // This should be the observation time int obsTimeIndex = -1; if (TokenType.NUMERIC.equals(t.getType())) { @@ -3233,7 +3204,7 @@ public class SHEFParser { movePastSpaces(); } return foundPositionalData; - } // + } /** * Move past any SPACE tokens in the data list. @@ -3243,7 +3214,7 @@ public class SHEFParser { private boolean movePastSpaces() { boolean moreData = true; ParserToken t = null; - while ((parts.size() > 0) && (partsIndex < parts.size())) { + while ((!parts.isEmpty()) && (partsIndex < parts.size())) { t = parts.get(partsIndex); if (TokenType.SPACE.equals(t.getType())) { parts.remove(partsIndex); @@ -3251,33 +3222,10 @@ public class SHEFParser { break; } } - moreData = parts.size() > 0; + moreData = !parts.isEmpty(); return moreData; } - /** - * Collapse runs of spaces into a single space. - * - * @param tokenList - * @return - */ - private static List collapseSpaces(List tokenList) { - TokenType SPACE = TokenType.SPACE; - List newList = new ArrayList(); - TokenType lastToken = null; - for (ParserToken t : tokenList) { - if (SPACE.equals(t.getType())) { - if (!SPACE.equals(lastToken)) { - newList.add(t); - } - } else { - newList.add(t); - } - lastToken = t.getType(); - } - return newList; - } - /** * Adjust all date values in a token list to correspond to the given * timezone information. @@ -3287,7 +3235,7 @@ public class SHEFParser { * @param tz * The timezone to adjust time to. */ - private static void fixupDates(List tokens, TimeZone tz) { + private void fixupDates(List tokens, TimeZone tz) { for (ParserToken t : tokens) { switch (t.getType()) { case DATE_CREATE: @@ -3298,11 +3246,12 @@ public class SHEFParser { t.adjustToTimezone(tz); t.getDateData().validate(); } + break; } default: { // nothing } - } // end switch + } } } @@ -3408,7 +3357,7 @@ public class SHEFParser { * @param units * @return */ - private static Short getDuration(ParserToken token, String units) { + private Short getDuration(ParserToken token, String units) { Short duration = null; try { Short value = Short.parseShort(token.getToken().substring(3)); @@ -3435,7 +3384,7 @@ public class SHEFParser { * The value to check. * @return Is this value a possible missing value? */ - private static boolean isMissingValue(String value) { + private boolean isMissingValue(String value) { boolean isMissing = false; if ("+".equals(value)) { isMissing = true; @@ -3457,7 +3406,7 @@ public class SHEFParser { return isMissing; } - private static String getMissingQualifier(String value) { + private String getMissingQualifier(String value) { String qualifier = null; if ("mm".equals(value)) { qualifier = "M"; @@ -3474,35 +3423,19 @@ public class SHEFParser { * The value to check. * @return Is this value a possible trace value? */ - private static boolean isTraceValue(String value) { - boolean isTrace = false; - if ("T".equals(value)) { - isTrace = true; - } else if ("t".equals(value)) { - isTrace = true; - } - return isTrace; + private boolean isTraceValue(String value) { + return "T".equals(value.toUpperCase()); } - /** - * - * @param pe - * @return - */ - private static boolean legalTraceValue(PhysicalElement pe) { + private boolean legalTraceValue(PhysicalElement pe) { return VALID_TRACE_PE.contains(pe); } - /** - * - * @param qualCode - * @return - */ private boolean isValidQualityCode(String qualCode) { // Set to false by exception boolean isValid = true; if (qualCode != null) { - isValid = (ShefParm.getDataQualifierCodes(qualCode) != null); + isValid = (shefParm.getDataQualifierCodes(qualCode) != null); } else { isValid = false; } @@ -3515,7 +3448,7 @@ public class SHEFParser { * @param unitsCode * @return */ - private static boolean isValidUnits(String unitsCode) { + private boolean isValidUnits(String unitsCode) { // Set to false by exception boolean isValid = true; if (unitsCode != null) { @@ -3526,13 +3459,7 @@ public class SHEFParser { return isValid; } - /** - * - * @param list - * @param i - * @return - */ - private static ParserToken getToken(List list, int i) { + private ParserToken getToken(List list, int i) { ParserToken t = null; if ((list != null) && (i < list.size())) { t = list.get(i); @@ -3544,39 +3471,7 @@ public class SHEFParser { return t; } - /** - * - * @param msg - * The message data being parsed. - * @param index - * The current position of the message pointer. - * @param comment - * A not-null buffer to receive the comment. - * @return The modified index position. - */ - private static int getRetainedComment(String msg, int index, - StringBuilder comment) { - char retainedChar = msg.charAt(index++); - for (; index < msg.length();) { - if (retainedChar == msg.charAt(index)) { - // move past the terminating quote - index++; - break; - } - if ('\n' == msg.charAt(index) || '\r' == msg.charAt(index)) { - break; - } - comment.append(msg.charAt(index++)); - } - return index; - } - - /** - * - * @param rec - * @return - */ - private static String createRecordHeader(ShefRecord rec, String reportLead) { + private String createRecordHeader(ShefRecord rec, String reportLead) { StringBuilder recData = new StringBuilder("."); if (rec != null) { recData.append(rec.getShefType().name()); @@ -3589,89 +3484,29 @@ public class SHEFParser { return recData.toString(); } - /** - * - * @param p - * @return - */ - private static String createDataLine(List p) { + private String createDataLine(List tokens) { StringBuilder sb = new StringBuilder(); - Iterator it = p.iterator(); - while (it.hasNext()) { - ParserToken t = it.next(); - if (t.getSendCode() != null) { - sb.append(t.getSendCode()); + for (ParserToken token : tokens) { + if (token.getSendCode() != null) { + sb.append(token.getSendCode()); } else { - sb.append(t.getRawToken()); - } - if (it.hasNext()) { - sb.append(" "); + sb.append(token.getRawToken()); } + sb.append(" "); } - return sb.toString(); + + return sb.toString().trim(); } - private static String clearTabs(String message) { + private String clearTabs(String message) { if (message != null) { - StringBuilder sb = new StringBuilder(message); - for (int i = 0; i < sb.length(); i++) { - if ('\t' == sb.charAt(i)) { - sb.setCharAt(i, ' '); - } - } - message = sb.toString(); + message = message.replaceAll("\\t", " "); } + return message; } - /** - * - */ - public void dumpTokens() { - for (ParserToken t : parts) { - System.out.println(t); - } + private ShefParm getShefParm() { + return this.shefParm; } - - /** - * @param args - */ - public static void main(String[] args) { - - // List list = - // tokenize(".E EE0165 0323 Z DH01/HGI/DIH1 /\n" + - // ".E1 1.0 2..0 3+0 \"comment 3\" 4.0 \"comment 4\" 5.0 6.0"); - - // .A AA0447I 991216 Z DH09/ TX 10 \"Comm\" / - // .A AA0447J 991216 Z DH09/ TX 30\"Comm\" / - // .A AA0447K 991216 Z DH09/ TX 40M \"comment\" / - // .A AA0447L 991216 Z DH09/ TX 20M\"comment\" / - // .A AA0447M 991216 Z DH09/ TX 20A \"comment\" / - // .A AA0447N 991216 Z DH09/ TX 20A\"comment\" / - // .A AA0447P 991216 Z DH09/ TX 20R\'comment\' / - - // tokenize(".A AA0447L 991216 Z DH09/ TX 20M\"comment\""); - - // List list = - // tokenize(".E1 1.0 2..0 3+0 \"comment 3\" 4.0 \"comment 4\" 5.0 6.0 \"comment 5\"\n"); - - // System.out - // .println("------------------------------------------------------------"); - // for (ParserToken t : list) { - // System.out.println(t); - // } - - List list = tokenize(".E1 1.0 2..0 3+0 \"comment 3\" 4.0 \"comment 4\" 5.0 6.0 \"comment 5 \""); - - System.out - .println("------------------------------------------------------------"); - for (ParserToken t : list) { - System.out.println(t); - } - - ParserToken t = new ParserToken("HY"); - System.out.println(t + " " + t.getError()); - - } - } diff --git a/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/ShefDecoder.java b/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/ShefDecoder.java index 19c3ca09f7..273da815b1 100644 --- a/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/ShefDecoder.java +++ b/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/ShefDecoder.java @@ -26,6 +26,7 @@ import com.raytheon.edex.plugin.shef.ShefSeparator.ShefDecoderInput; import com.raytheon.edex.plugin.shef.data.ShefRecord; import com.raytheon.edex.plugin.shef.database.PostShef; import com.raytheon.edex.plugin.shef.database.PurgeText; +import com.raytheon.edex.plugin.shef.util.ShefParm; import com.raytheon.uf.common.dataplugin.PluginDataObject; import com.raytheon.uf.common.ohd.AppsDefaults; import com.raytheon.uf.common.status.IUFStatusHandler; @@ -39,8 +40,8 @@ import com.raytheon.uf.edex.decodertools.core.DecoderTools; * * SOFTWARE HISTORY * - * Date Ticket# Engineer Description - * ------------ ---------- ----------- -------------------------- + * Date Ticket# Engineer Description + * ------------ ---------- ----------- -------------------------- * 03/19/2008 387 M. Duff Initial creation. * 06/02/2008 1166 M. Duff Added checks for null data objects. * 10/16/2008 1548 jelkins Integrated ParameterCode Types @@ -55,6 +56,7 @@ import com.raytheon.uf.edex.decodertools.core.DecoderTools; * 12/--/2009 jkorman Major refactor - split into ShefDecoder/SHEFParser * 03/07/2013 15071 W. Kwock Skip empty data files. * 04/28/2014 3088 mpduff Use UFStatus logging, various cleanup. + * 01/10/2018 5049 mduff Pass ShefParm to the ShefParser. * */ public class ShefDecoder { @@ -66,6 +68,8 @@ public class ShefDecoder { // here. private PluginDataObject[] records = new PluginDataObject[0]; + private ShefParm shefParm; + /** * Constructor */ @@ -79,6 +83,8 @@ public class ShefDecoder { * @param name */ public ShefDecoder(String name) { + shefParm = new ShefParm(); + shefParm.populate(); } /** @@ -91,8 +97,8 @@ public class ShefDecoder { * @return PluginDataObject[] of decoded data */ public PluginDataObject[] decode(byte[] data, Headers headers) { - boolean archiveMode = AppsDefaults.getInstance().getBoolean( - "ALLOW_ARCHIVE_DATA", false); + boolean archiveMode = AppsDefaults.getInstance() + .getBoolean("ALLOW_ARCHIVE_DATA", false); String traceId = null; @@ -123,7 +129,7 @@ public class ShefDecoder { postDate = getPostTime(startTime); } - PostShef postShef = new PostShef(postDate); + PostShef postShef = new PostShef(postDate, shefParm); if (separator.hasNext()) { PurgeText pText = new PurgeText(postDate); pText.storeTextProduct(separator); @@ -138,12 +144,6 @@ public class ShefDecoder { return records; } - /** - * - * @param data - * @param headers - * @return - */ public PluginDataObject[] decodeNoWMOHeader(byte[] data, Headers headers) { String traceId = null; @@ -169,7 +169,7 @@ public class ShefDecoder { PostShef postShef = null; try { - postShef = new PostShef(postDate); + postShef = new PostShef(postDate, shefParm); } catch (Exception e) { logger.error("Could not create PostShef", e); } @@ -199,7 +199,8 @@ public class ShefDecoder { while (separator.hasNext()) { ShefDecoderInput sdi = separator.next(); try { - SHEFParser parser = new SHEFParser(sdi); + SHEFParser parser = new SHEFParser(sdi, appDefaults, + shefParm); ShefRecord shefRecord = parser.decode(); if (shefRecord != null) { if (shefRecord.getDataValues() != null) { @@ -209,8 +210,9 @@ public class ShefDecoder { } postShef.post(shefRecord); } catch (Throwable tt) { - logger.error(traceId - + "- Could not post record.", tt); + logger.error( + traceId + "- Could not post record.", + tt); } } else { logger.info(traceId + "- No data records in file."); @@ -219,39 +221,21 @@ public class ShefDecoder { logger.info(traceId + "- No records in file."); } } catch (Exception ee) { - logger.error(traceId + "- Could not parse SHEF report.", ee); + logger.error(traceId + "- Could not parse SHEF report.", + ee); } } // while() if (dataProcessed) { - postShef.logStats(traceId, System.currentTimeMillis() - - startTime); + postShef.logStats(traceId, + System.currentTimeMillis() - startTime); } } finally { postShef.close(); } } - /** - * - * @param startTime - * @return - */ - private static Date getPostTime(long startTime) { + private Date getPostTime(long startTime) { // Force time to nearest second. return new Date(startTime - (startTime % 1000)); } - - /* - * - */ - public static final void main(String[] args) { - - long t = System.currentTimeMillis(); - Date postDateA = new Date(t); - t = t - (t % 1000); - Date postDateB = new Date(t); - - System.out.println(postDateA.getTime()); - System.out.println(postDateB.getTime()); - } } diff --git a/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/data/ShefData.java b/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/data/ShefData.java index 949baffe4c..28720ce370 100644 --- a/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/data/ShefData.java +++ b/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/data/ShefData.java @@ -20,8 +20,6 @@ package com.raytheon.edex.plugin.shef.data; import java.util.Date; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import com.raytheon.edex.plugin.shef.util.SHEFDate; import com.raytheon.edex.plugin.shef.util.ShefParm; @@ -34,7 +32,8 @@ import com.raytheon.uf.common.dataplugin.shef.util.ParameterCode.Probability; import com.raytheon.uf.common.dataplugin.shef.util.ParameterCode.TypeSource; import com.raytheon.uf.common.dataplugin.shef.util.SHEFTimezone; import com.raytheon.uf.common.dataplugin.shef.util.ShefConstants; -import com.raytheon.uf.common.serialization.ISerializableObject; +import com.raytheon.uf.common.status.IUFStatusHandler; +import com.raytheon.uf.common.status.UFStatus; /** * SHEF Data parent object @@ -43,16 +42,21 @@ import com.raytheon.uf.common.serialization.ISerializableObject; * * SOFTWARE HISTORY * - * Date Ticket# Engineer Description - * ------------ ---------- ----------- -------------------------- + * Date Ticket# Engineer Description + * ------------ --------- ----------- -------------------------- * 03/19/08 387 M. Duff Initial creation. * 10/16/2008 1548 jelkins Integrated ParameterCode Types * 04/29/2014 3088 mpduff cleanup. * 06/26/2014 3321 mpduff Added ingestfilter primary key getter. + * 01/10/2018 5049 mduff ShefParm is now provided to this class. * * */ -public class ShefData implements ISerializableObject { +public class ShefData { + + private final IUFStatusHandler logger = UFStatus.getHandler(ShefData.class); + + private ShefParm shefParm; private String stringValue = null; @@ -109,10 +113,12 @@ public class ShefData implements ISerializableObject { private boolean revisedRecord = false; /** - * Empty constructor + * Constructor. + * + * @param ShefParm */ - public ShefData() { - + public ShefData(ShefParm shefParm) { + this.shefParm = shefParm; } /** @@ -128,6 +134,11 @@ public class ShefData implements ISerializableObject { */ public void setStringValue(String stringValue) { this.stringValue = stringValue; + if (stringValue == null) { + value = null; + return; + } + try { boolean neg = false; int negPos = stringValue.indexOf('-'); @@ -140,8 +151,7 @@ public class ShefData implements ISerializableObject { value *= -1.0; } } catch (NumberFormatException nfe) { - value = null; - } catch (NullPointerException npe) { + logger.warn("Invalid value: " + stringValue, nfe); value = null; } } @@ -279,15 +289,13 @@ public class ShefData implements ISerializableObject { // check to see if this is a valid typesource String key = paramType + paramSource; - Integer n = ShefParm.getTypeSourceCode(key); + Integer n = shefParm.getTypeSourceCode(key); if ((n != null) && (n == 1)) { TypeSource ts = TypeSource.getEnum(key); dataTypeCode = paramType; dataSourceCode = paramSource; setTypeSource(ts); - } else { - } Duration duration = Duration.INSTANTENOUS; @@ -524,10 +532,7 @@ public class ShefData implements ISerializableObject { * the creationDateObj to set */ public void setCreationDateObj(Date creationDate) { - SHEFDate d = new SHEFDate(creationDate, SHEFTimezone.GMT_TIMEZONE); - if (d != null) { - createTime = d; - } + createTime = new SHEFDate(creationDate, SHEFTimezone.GMT_TIMEZONE); } /** @@ -586,10 +591,7 @@ public class ShefData implements ISerializableObject { * the observationTimeObj to set */ public void setObservationTimeObj(Date observationTime) { - SHEFDate d = new SHEFDate(observationTime, SHEFTimezone.GMT_TIMEZONE); - if (d != null) { - obsTime = d; - } + obsTime = new SHEFDate(observationTime, SHEFTimezone.GMT_TIMEZONE); } public void setObsTime(SHEFDate date) { @@ -669,7 +671,7 @@ public class ShefData implements ISerializableObject { if ("S".equals(unitsCode)) { if (physicalElement != null) { String key = physicalElement.getCode(); - Double cf = ShefParm.getPhysicalElementConversionFactor(key); + Double cf = shefParm.getPhysicalElementConversionFactor(key); Double n = doConversion(physicalElement, unitsCode, value); if (n == null) { if (cf != null) { @@ -775,8 +777,8 @@ public class ShefData implements ISerializableObject { receiver.append((revisedRecord) ? " 1" : " 0"); receiver.append(" "); // Data source - receiver.append(String.format("%-8s", (dataSource != null) ? dataSource - : " ")); + receiver.append( + String.format("%-8s", (dataSource != null) ? dataSource : " ")); receiver.append(" "); // Time series indicator receiver.append(String.format("%3d", timeSeriesId)); @@ -868,39 +870,4 @@ public class ShefData implements ISerializableObject { } return dValue; } - - /** - * - * @param args - */ - public static final void main(String[] args) { - - // ShefData d = new ShefData(); - // - // d.setParameterCodeString("AD","Z"); - // - // System.out.println(d); - // - // double dv = 0.04; - // - // System.out.println(String.format("[%.3f]",dv)); - // - - double adjustedValue = 10; - double divisor = 1; - double base = 0; - double multiplier = 1000; - double adder = 0; - - double n = (adjustedValue / divisor + base) * multiplier + adder; - - System.out.println(n); - - Pattern Q_CODES = Pattern.compile("Q[^BEF]"); - Matcher m = Q_CODES.matcher("QI"); - if (m.matches()) { - System.out.println("found"); - } - - } } diff --git a/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/database/PostShef.java b/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/database/PostShef.java index 0247a76236..6c05cdcdaf 100644 --- a/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/database/PostShef.java +++ b/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/database/PostShef.java @@ -1,25 +1,24 @@ /** * This software was developed and / or modified by Raytheon Company, * pursuant to Contract DG133W-05-CQ-1067 with the US Government. - * + * * U.S. EXPORT CONTROLLED TECHNICAL DATA * This software product contains export-restricted data whose * export/transfer/disclosure is restricted by U.S. law. Dissemination * to non-U.S. persons whether in the United States or abroad requires * an export license or other authorization. - * + * * Contractor Name: Raytheon Company * Contractor Address: 6825 Pine Street, Suite 340 * Mail Stop B8 * Omaha, NE 68106 * 402.291.0100 - * + * * See the AWIPS II Master Rights File ("Master Rights File.pdf") for * further licensing information. **/ package com.raytheon.edex.plugin.shef.database; -import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; @@ -41,6 +40,7 @@ import com.raytheon.edex.plugin.shef.data.ShefRecord; import com.raytheon.edex.plugin.shef.data.ShefRecord.ShefType; import com.raytheon.edex.plugin.shef.util.BitUtils; import com.raytheon.edex.plugin.shef.util.ShefAdjustFactor; +import com.raytheon.edex.plugin.shef.util.ShefParm; import com.raytheon.edex.plugin.shef.util.ShefStats; import com.raytheon.edex.plugin.shef.util.ShefUtil; import com.raytheon.edex.plugin.shef.util.StoreDisposition; @@ -76,7 +76,6 @@ import com.raytheon.uf.common.dataplugin.shef.util.ParameterCode.Extremum; import com.raytheon.uf.common.dataplugin.shef.util.ParameterCode.PhysicalElement; import com.raytheon.uf.common.dataplugin.shef.util.ParameterCode.PhysicalElementCategory; import com.raytheon.uf.common.dataplugin.shef.util.ParameterCode.TypeSource; -import com.raytheon.uf.common.dataplugin.shef.util.SHEFTimezone; import com.raytheon.uf.common.dataplugin.shef.util.ShefConstants; import com.raytheon.uf.common.dataplugin.shef.util.ShefConstants.IngestSwitch; import com.raytheon.uf.common.dataplugin.shef.util.ShefQC; @@ -84,13 +83,15 @@ import com.raytheon.uf.common.ohd.AppsDefaults; import com.raytheon.uf.common.status.IUFStatusHandler; import com.raytheon.uf.common.status.UFStatus; import com.raytheon.uf.common.time.util.TimeUtil; +import com.raytheon.uf.edex.database.DataAccessLayerException; import com.raytheon.uf.edex.database.dao.CoreDao; import com.raytheon.uf.edex.database.dao.DaoConfig; +import com.raytheon.uf.edex.database.query.DatabaseQuery; import com.raytheon.uf.edex.decodertools.time.TimeTools; /** * Post the SHEF Data to the IHFS DB. - * + * * 
  * SOFTWARE HISTORY
  * Date         Ticket#    Engineer    Description
@@ -106,18 +107,18 @@ import com.raytheon.uf.edex.decodertools.time.TimeTools;
  *                                      Used dataValue as one of the params for populateDataObj
  * 01/13/2009   1802       askripsk    Fixed the reject_type for rejecteddata.
  * 01/26/2009   1927       J. Sanchez  Removed 'final' from postDate.
- * 02/02/2009   1943       J. Sanchez  Post fcst data to riverstatus table. 
+ * 02/02/2009   1943       J. Sanchez  Post fcst data to riverstatus table.
  *                                      Changed default value for alertAlarm
  * 05/28/2009   2410       J. Sanchez  Posted data for unknstnvalue.
  * 12/11/2009   2488       M. Duff     Fixed problem with storing text products.
  * 03/07/2013   15545      w. kwock    Added Observe time to log
  * 03/21/2013   15967      w. kwock    Fix the error in buildTsFcstRiv riverstatus table issue
- * 04/05/2013   16036      w. kwock    Fixed no ts=RZ in ingestfilter table but posted to height table 
+ * 04/05/2013   16036      w. kwock    Fixed no ts=RZ in ingestfilter table but posted to height table
  * 10/28/2013   16711      lbousaidi   if the id is not in location table,but defined in geoarea table
- *                                     data can be posted to appropriate pe-based tables only if the data 
+ *                                     data can be posted to appropriate pe-based tables only if the data
  *                                     type is not READING like in A1 code.
  * 02/18/2014   16572      l. Bousaidi only apply adjust factor to non missing values.
- * 04/24/2014   16904      lbousaidi   gross check should be applied to adjusted value.  
+ * 04/24/2014   16904      lbousaidi   gross check should be applied to adjusted value.
  * 04/29/2014   3088       mpduff      Change logging class, clean up/optimization.
  *                                     Updated with more performance fixes.
  * May 14, 2014 2536       bclement    removed TimeTools usage
@@ -134,11 +135,12 @@ import com.raytheon.uf.edex.decodertools.time.TimeTools;
  * Aug 05, 2015 4486       rjpeter     Changed Timestamp to Date.
  * Dec 16, 2015 5166       kbisanz     Update logging to use SLF4J
  * 07/26/2016   19001      ksteinfeld  Remove a restriction that prevents the calling of the method checkIngest() when ShefData
- *                                     object data type is CONTINGENCY.  
+ *                                     object data type is CONTINGENCY.
+ * 07/07/2017   6344       mapeters    Remove alarm/alert data when overridden by new data
+ * 01/10/2018   5049       mduff       ShefParm is now provided to this class.
  *
- * + * * @author mduff - * @version 1.0 */ public class PostShef { /** The logger */ @@ -150,14 +152,35 @@ public class PostShef { */ private enum Location { LOC_LOCATION, LOC_GEOAREA, LOC_NO_POST, LOC_UNDEFINED - }; + } /** * Quality Control Code Enum */ private enum QualityControlCode { - QC_DEFAULT, QC_GROSSRANGE_FAILED, QC_REASONRANGE_FAILED, QC_ROC_FAILED, QC_ROC_PASSED, QC_OUTLIER_FAILED, QC_OUTLIER_PASSED, QC_SCC_FAILED, QC_SCC_PASSED, QC_MSC_FAILED, QC_MSC_PASSED, QC_EXTERN_FAILED, QC_EXTERN_QUEST, QC_MANUAL_PASSED, QC_MANUAL_QUEST, QC_MANUAL_FAILED, QC_MANUAL_NEW, QC_PASSED, QC_QUESTIONABLE, QC_FAILED, QC_NOT_PASSED, QC_NOT_FAILED - }; + QC_DEFAULT, + QC_GROSSRANGE_FAILED, + QC_REASONRANGE_FAILED, + QC_ROC_FAILED, + QC_ROC_PASSED, + QC_OUTLIER_FAILED, + QC_OUTLIER_PASSED, + QC_SCC_FAILED, + QC_SCC_PASSED, + QC_MSC_FAILED, + QC_MSC_PASSED, + QC_EXTERN_FAILED, + QC_EXTERN_QUEST, + QC_MANUAL_PASSED, + QC_MANUAL_QUEST, + QC_MANUAL_FAILED, + QC_MANUAL_NEW, + QC_PASSED, + QC_QUESTIONABLE, + QC_FAILED, + QC_NOT_PASSED, + QC_NOT_FAILED + } /** Log entry separator */ private static final String LOG_SEP = "========================================"; @@ -171,14 +194,11 @@ public class PostShef { private static final int MISSING = -999; /** Questionable/bad threshold value */ - private static final int QUESTIONABLE_BAD_THRESHOLD = 1073741824; + private static final int QUESTIONABLE_BAD_THRESHOLD = 1_073_741_824; /** Map of value to duration character */ private static final Map DURATION_MAP; - /** The time this class is created and the shef file is processed. */ - private final long currentTime = System.currentTimeMillis(); - static { DURATION_MAP = Collections.unmodifiableMap(buildDurationMap()); } @@ -222,7 +242,7 @@ public class PostShef { private long basishrs = 72; /** Map of location identifiers to Location Objects */ - private final HashMap idLocations = new HashMap(); + private final Map idLocations = new HashMap<>(); /** number of milliseconds back for data to be considered valid */ private long lookbackMillis; @@ -237,10 +257,10 @@ public class PostShef { private final PostTables postTables; /** Map of adjustment factors for eacy data type */ - private final Map adjustmentMap = new HashMap(); + private final Map adjustmentMap = new HashMap<>(); /** Map of location identifier to IngestSwitch */ - private final Map ingestSwitchMap = new HashMap(); + private final Map ingestSwitchMap = new HashMap<>(); // AppsDefaults tokens private String undefStation; @@ -274,20 +294,13 @@ public class PostShef { private boolean perfLog; /** Type Source list */ - private final List tsList = new ArrayList(); + private final List tsList = new ArrayList<>(); /** Use latest value flag */ private int useLatest = MISSING; - /** Begin basis time */ - private final long basisBeginTime = currentTime - - (basishrs * ShefConstants.MILLIS_PER_HOUR); - - /** Basis time */ - private final Date basisTimeAnsi = new Date(basisBeginTime); - - /** river status update query value */ - private boolean riverStatusUpdateValueFlag; + private final Date basisTimeAnsi = new Date(System.currentTimeMillis() + - (basishrs * ShefConstants.MILLIS_PER_HOUR)); /** Type Source to use */ private String useTs = null; @@ -302,17 +315,20 @@ public class PostShef { private Object[] queryForecastResults; /** Cache of data limits and loc data limits */ - private final Map dataRangeMap = new HashMap(); + private final Map dataRangeMap = new HashMap<>(); /** Valid date range flag */ private boolean validDateRange = false; + private ShefParm shefParm; + /** - * + * * @param date */ - public PostShef(Date date) { + public PostShef(Date date, ShefParm shefParm) { postDate = date; + this.shefParm = shefParm; getAppsDefaults(); createConnection(); postTables = new PostTables(); @@ -339,11 +355,12 @@ public class PostShef { postLatest = appDefaults.getToken(ShefConstants.SHEF_POST_LATEST); - loadMaxFcst = appDefaults.getToken(ShefConstants.SHEF_LOAD_MAXFCST, - SHEF_ON).equals(SHEF_ON); + loadMaxFcst = appDefaults + .getToken(ShefConstants.SHEF_LOAD_MAXFCST, SHEF_ON) + .equals(SHEF_ON); - postBadData = appDefaults.getToken(ShefConstants.SHEF_POST_BADDATA, - "REJECT").equalsIgnoreCase("REJECT"); + postBadData = "REJECT".equalsIgnoreCase(appDefaults + .getToken(ShefConstants.SHEF_POST_BADDATA, "REJECT")); String basis_hours_str = appDefaults .getToken(ShefConstants.BASIS_HOURS_FILTER); @@ -353,7 +370,7 @@ public class PostShef { } } catch (NumberFormatException e) { log.info(ShefConstants.BASIS_HOURS_FILTER - + " not set, using default value of 72"); + + " invalid, using default value of 72", e); } loadIngest = appDefaults.getBoolean(ShefConstants.SHEF_LOAD_INGEST, @@ -372,7 +389,7 @@ public class PostShef { } private static Map buildDurationMap() { - Map map = new HashMap(); + Map map = new HashMap<>(); map.put(0, "I"); map.put(1, "U"); map.put(5, "E"); @@ -407,9 +424,9 @@ public class PostShef { /** * Builds the DB Objects for storage in the IHFS - * + * * The following tokens are used: - * + * * location.ipost: <= 0 - no post > 0 - post to appropriate table * ingestfilter.ingest: T - process this PEDTSE F - do not process this * PEDTSE post_unk : 0 - ignore unknowns completely 1 - store station ids @@ -419,7 +436,7 @@ public class PostShef { * not msg 3 - post latest data only if passed quality check or msg proc_obs * : 0 - treat Processed data as Processed 1 - treat Processed data as * Observed - * + * * @param shefRecord * - The ShefRecord object containing all the data */ @@ -450,14 +467,14 @@ public class PostShef { prodId = identifier; String locId = shefRecord.getLocationId(); - prodTime = shefRecord.getProductTime(); - - if ((locId == null) || (dataValues == null)) { + if (locId == null) { // Check for bad data log.warn("No data stored for " + prodId); return; } + prodTime = shefRecord.getProductTime(); + try { /* @@ -470,12 +487,13 @@ public class PostShef { for (ShefData data : dataValues) { if (data.getObsTime() == null) { log.error(data.toString()); - log.error("Not posted:Record does not contain an observation time"); + log.error( + "Not posted:Record does not contain an observation time"); return; } - postDate.setTime(getToNearestSecond(TimeUtil - .currentTimeMillis())); + postDate.setTime( + getToNearestSecond(TimeUtil.currentTimeMillis())); boolean same_lid_product = false; String dataValue = data.getStringValue(); @@ -522,7 +540,8 @@ public class PostShef { String dataQualifier = data.getQualifier(); TypeSource typeSource = data.getTypeSource(); - if ((typeSource == null) || (typeSource == TypeSource.UNKNOWN)) { + if ((typeSource == null) + || (typeSource == TypeSource.UNKNOWN)) { log.error("Unknown typesource code in data [" + data + "]"); continue; } @@ -530,8 +549,8 @@ public class PostShef { // Don't use the TypeSource directly because there are some // cases // where the "type" defaults. - DataType dataType = ParameterCode.DataType.getDataType( - typeSource, procObs); + DataType dataType = ParameterCode.DataType + .getDataType(typeSource, procObs); /* * if the station_id exists in location table and the data type @@ -559,13 +578,12 @@ public class PostShef { sMsg.append("LID [").append(locId) .append("] not defined; no data posted"); } else if ("IDS_ONLY".equalsIgnoreCase(undefStation)) { - sMsg.append("LID [") - .append(locId) - .append("] not defined; station info posting to UnkStn"); + sMsg.append("LID [").append(locId).append( + "] not defined; station info posting to UnkStn"); } else if ("IDS_AND_DATA" .equalsIgnoreCase(undefStation)) { - sMsg.append("LID [") - .append("] not defined; data posting to UnkStnValue"); + sMsg.append("LID [").append( + "] not defined; data posting to UnkStnValue"); } if (sMsg.length() > 0) { log.info(sMsg.toString()); @@ -626,9 +644,11 @@ public class PostShef { * be posted. */ ShefConstants.IngestSwitch ingestSwitch = ShefConstants.IngestSwitch.POST_PE_ONLY; - if (Location.LOC_LOCATION.equals(postLocData) || (Location.LOC_GEOAREA.equals(postLocData))) { + if (Location.LOC_LOCATION.equals(postLocData) + || (Location.LOC_GEOAREA.equals(postLocData))) { ingestSwitch = checkIngest(locId, data); - if (ShefConstants.IngestSwitch.POST_PE_OFF.equals(ingestSwitch)) { + if (ShefConstants.IngestSwitch.POST_PE_OFF + .equals(ingestSwitch)) { stats.incrementNoPost(); } } @@ -643,8 +663,9 @@ public class PostShef { boolean postPeOffSwitch = ShefConstants.IngestSwitch.POST_PE_OFF .equals(ingestSwitch); - if ((!Location.LOC_LOCATION.equals(postLocData) && !Location.LOC_GEOAREA - .equals(postLocData)) || postPeOffSwitch) { + if ((!Location.LOC_LOCATION.equals(postLocData) + && !Location.LOC_GEOAREA.equals(postLocData)) + || postPeOffSwitch) { /* * set the prev info for the next pass through this * function. this is info is used for to prevent redundant @@ -701,15 +722,18 @@ public class PostShef { || TypeSource.PROCESSED_MEAN_AREAL_DATA .equals(typeSource)) { - if (((postDate.getTime() - obsTime.getTime()) > lookbackMillis) - && (!Duration._1_MONTH.equals(data.getDuration()))) { + if (((postDate.getTime() + - obsTime.getTime()) > lookbackMillis) + && (!Duration._1_MONTH + .equals(data.getDuration()))) { stats.incrementWarningMessages(); stats.incrementOutsideWindow(); log.warn(locId + " " + data.getObsTime() + " obs time > " + lookBackDays + " days old; data not posted"); continue; - } else if ((obsTime.getTime() - postDate.getTime()) > lookfwdMillis) { + } else if ((obsTime.getTime() + - postDate.getTime()) > lookfwdMillis) { stats.incrementWarningMessages(); stats.incrementOutsideWindow(); log.warn(locId + " obs time (" + data.getObsTime() @@ -771,7 +795,7 @@ public class PostShef { * uniformity, most of these functions have the same argument * list even though some of the arguments are not used by some * functions - * + * * if instructed, post to the product link table, but only if * the info has changed */ @@ -779,8 +803,8 @@ public class PostShef { start = System.currentTimeMillis(); // Identifier has been set from the awipsHeader. postProductLink(locId, identifier, obsTime); - stats.addElapsedTimeIngest(System.currentTimeMillis() - - start); + stats.addElapsedTimeIngest( + System.currentTimeMillis() - start); if (dataLog) { log.info("Posted product link [" + identifier @@ -792,7 +816,7 @@ public class PostShef { * Check the quality of the data if observed or forecast. note * the posting may treat processed data as observed, including * this manner. - * + * * the quality_code defined contains information from two * 'sources'. one, the qc checks performed by shef, and two, * certain shef qualifier codes reflect the quality of the data. @@ -822,8 +846,8 @@ public class PostShef { if (DataType.READING.equals(dataType)) { if (SHEF_ON.equalsIgnoreCase(postLatest) || (ShefConstants.VALID_ONLY - .equalsIgnoreCase(postLatest) && valueOk && (data - .getStringValue() != ShefConstants.SHEF_MISSING)) + .equalsIgnoreCase(postLatest) && valueOk + && (data.getStringValue() != ShefConstants.SHEF_MISSING)) || (ShefConstants.VALID_OR_MISSING .equalsIgnoreCase(postLatest) && valueOk)) { @@ -836,7 +860,8 @@ public class PostShef { + data.getObservationTimeObj().toString() + "] for LID [" + locId + "] posted to the latestObsValue for PE [" - + data.getPhysicalElement().getCode() + "]"); + + data.getPhysicalElement().getCode() + + "]"); } } } @@ -889,11 +914,11 @@ public class PostShef { if (dataLog) { log.info("Posting data [" + data.getStringValue() - + "] for LID [" - + locId + + "] for LID [" + locId + "] for PE [" + data.getPhysicalElement() - .getCode() + "]"); + .getCode() + + "]"); } } } else if (DataType.AREAL_PROCESSED.equals(dataType)) { @@ -966,11 +991,10 @@ public class PostShef { * treating the processed data as observed. */ if (checkIfPaired(data)) { - postTables - .postPairedData(shefRecord, data, locId, - dataValue, dataQualifier, qualityCode, - prodId, prodTime, shefPostDuplicateDef, - stats, postDate); + postTables.postPairedData(shefRecord, data, locId, + dataValue, dataQualifier, qualityCode, prodId, + prodTime, shefPostDuplicateDef, stats, + postDate); if (dataLog) { log.info("Posting contingency data [" + dataValue + "] for LID [" + locId @@ -991,14 +1015,13 @@ public class PostShef { } } break; - } // case CONTINGENCY: + } case PROCESSED: { if (checkIfPaired(data)) { - postTables - .postPairedData(shefRecord, data, locId, - dataValue, dataQualifier, qualityCode, - prodId, prodTime, shefPostDuplicateDef, - stats, postDate); + postTables.postPairedData(shefRecord, data, locId, + dataValue, dataQualifier, qualityCode, prodId, + prodTime, shefPostDuplicateDef, stats, + postDate); if (dataLog) { log.info("Posting processed data [" + dataValue + "] for LID [" + locId @@ -1018,23 +1041,16 @@ public class PostShef { } } break; - } // case PROCESSED: + } } // switch /* - * post alertalarm data as necessary. Don't perform the - * alert/alarm post if the data is a ContingencyValue + * Update alert/alarm data as necessary. Don't update if the + * data is a ContingencyValue */ - if (!DataType.CONTINGENCY.equals(dataType) && shefAlertAlarm - && (alertAlarm != ShefConstants.NO_ALERTALARM)) { - // TODO: Ensure what is to be saved here! - post_alertalarm(data, locId, dataValue, dataQualifier, + if (!DataType.CONTINGENCY.equals(dataType) && shefAlertAlarm) { + updateAlertAlarm(data, locId, dataValue, dataQualifier, qualityCode); - stats.incrementAlertAlarm(); - if (dataLog) { - log.info("Posting data [" + dataValue + "] for LID [" - + locId + "] to alertAlarmVal table"); - } } /* @@ -1064,12 +1080,12 @@ public class PostShef { if (!dataValues.isEmpty()) { ShefData data = dataValues.get(0); - DataType dataType = ParameterCode.DataType.getDataType( - data.getTypeSource(), procObs); - if ((DataType.FORECAST.equals(dataType)) - && loadMaxFcst - && (data.getPhysicalElement().getCode().startsWith("H") || data - .getPhysicalElement().getCode().startsWith("Q"))) { + DataType dataType = ParameterCode.DataType + .getDataType(data.getTypeSource(), procObs); + if ((DataType.FORECAST.equals(dataType)) && loadMaxFcst + && (data.getPhysicalElement().getCode().startsWith("H") + || data.getPhysicalElement().getCode() + .startsWith("Q"))) { postRiverStatus(data, locId); log.info("Update RiverStatus for: " + locId + " " + data.getPhysicalElement().getCode()); @@ -1091,7 +1107,7 @@ public class PostShef { /** * Log the summary stats. - * + * * @param traceId * @param totalTime */ @@ -1111,7 +1127,8 @@ public class PostShef { logIt(perfLog, stats.getArealFcstOverwrite(), " Areal Fcst Overwrite"); logIt(perfLog, stats.getArealFcstValues(), " Areal Fcst Values"); - logIt(perfLog, stats.getArealObsOverwrite(), " Areal Obs Overwrite"); + logIt(perfLog, stats.getArealObsOverwrite(), + " Areal Obs Overwrite"); logIt(perfLog, stats.getArealValues(), " Areal Obs Values"); logIt(perfLog, stats.getCommentOverwrite(), " Comments Overwrite"); logIt(perfLog, stats.getContingencyOverwrite(), @@ -1138,7 +1155,8 @@ public class PostShef { logIt(perfLog, stats.getObsHeight(), " Obs Height"); logIt(perfLog, stats.getObsPe(), " Obs PE"); logIt(perfLog, stats.getObsPrecip(), " Obs Precip"); - logIt(perfLog, stats.getOutsideTimeWindow(), " Outside Time Window"); + logIt(perfLog, stats.getOutsideTimeWindow(), + " Outside Time Window"); logIt(perfLog, stats.getPaired(), " Paired"); logIt(perfLog, stats.getPairedOver(), " Paired Overwrite"); logIt(perfLog, stats.getPostProcessedOverwrite(), @@ -1164,42 +1182,74 @@ public class PostShef { } /** - * Post data to the alertalarmval data table. If duplicate found and new - * value should be overwritten, then overwrite the data without saving the - * replaced data. See post_tables.c: post_alertalarm + * Update the alertalarmval table based on the new shef data. + * + * @param data + * @param locId + * @param value + * @param qualifier + * @param qualityCode */ - private void post_alertalarm(ShefData data, String locId, Object value, + private void updateAlertAlarm(ShefData data, String locId, String value, String qualifier, long qualityCode) { String aaCategory = null; String aaCheck = null; - /* - * these fields are particular to the AlertAlarm operations - */ - if (alertAlarm == ShefConstants.ALERT_UPPER_DETECTED) { - aaCategory = ShefConstants.ALERT_CATEGSTR; - aaCheck = ShefConstants.UPPER_CHECKSTR; - } else if (alertAlarm == ShefConstants.ALARM_UPPER_DETECTED) { - aaCategory = ShefConstants.ALARM_CATEGSTR; - aaCheck = ShefConstants.UPPER_CHECKSTR; - } else if (alertAlarm == ShefConstants.ALERT_LOWER_DETECTED) { - aaCategory = ShefConstants.ALERT_CATEGSTR; - aaCheck = ShefConstants.LOWER_CHECKSTR; - } else if (alertAlarm == ShefConstants.ALARM_LOWER_DETECTED) { - aaCategory = ShefConstants.ALARM_CATEGSTR; - aaCheck = ShefConstants.LOWER_CHECKSTR; - } - - PersistableDataObject aaValue = populateDataObj(shefRecord, data, + PersistableDataObject dataObj = populateDataObj(shefRecord, data, locId, ShefConstants.ALERTALARM_VALUE, data.getStringValue(), qualifier, qualityCode); + Alertalarmval aaVal = (Alertalarmval) dataObj; - ((Alertalarmval) aaValue).getId().setAaCateg(aaCategory); - ((Alertalarmval) aaValue).getId().setAaCheck(aaCheck); + if (alertAlarm == ShefConstants.NO_ALERTALARM) { + /* + * If no alert/alarm, delete any matching entries from alertalarmval + * table since they are no longer valid + */ + DatabaseQuery dbQuery = new DatabaseQuery(Alertalarmval.class); + dbQuery.addQueryParam("id.lid", locId); + dbQuery.addQueryParam("id.pe", aaVal.getId().getPe()); + dbQuery.addQueryParam("id.dur", aaVal.getId().getDur()); + dbQuery.addQueryParam("id.ts", aaVal.getId().getTs()); + dbQuery.addQueryParam("id.extremum", aaVal.getId().getExtremum()); + dbQuery.addQueryParam("id.validtime", aaVal.getId().getValidtime()); + try { + dao.deleteByCriteria(dbQuery); + } catch (DataAccessLayerException e) { + log.error("Failed to delete " + ShefConstants.ALERTALARM_VALUE + + " entries matching criteria: " + dbQuery, e); + } + } else { + /* + * An alert/alarm has occurred, determine what type and post it to + * the alertalarmval table. If duplicate found and new value should + * be overwritten, then overwrite the data without saving the + * replaced data. See post_tables.c: post_alertalarm + */ + if (alertAlarm == ShefConstants.ALERT_UPPER_DETECTED) { + aaCategory = ShefConstants.ALERT_CATEGSTR; + aaCheck = ShefConstants.UPPER_CHECKSTR; + } else if (alertAlarm == ShefConstants.ALARM_UPPER_DETECTED) { + aaCategory = ShefConstants.ALARM_CATEGSTR; + aaCheck = ShefConstants.UPPER_CHECKSTR; + } else if (alertAlarm == ShefConstants.ALERT_LOWER_DETECTED) { + aaCategory = ShefConstants.ALERT_CATEGSTR; + aaCheck = ShefConstants.LOWER_CHECKSTR; + } else if (alertAlarm == ShefConstants.ALARM_LOWER_DETECTED) { + aaCategory = ShefConstants.ALARM_CATEGSTR; + aaCheck = ShefConstants.LOWER_CHECKSTR; + } - postTables.postAAData(aaValue, ShefConstants.ALERTALARM_VALUE, - shefPostDuplicate, stats, aaCategory, aaCheck); + aaVal.getId().setAaCateg(aaCategory); + aaVal.getId().setAaCheck(aaCheck); + postTables.postAAData(aaVal, ShefConstants.ALERTALARM_VALUE, + shefPostDuplicate, stats, aaCategory, aaCheck); + + if (dataLog) { + log.info("Posting data [" + value + "] for LID [" + locId + + "] to " + ShefConstants.ALERTALARM_VALUE + " table"); + } + } } /** @@ -1278,7 +1328,8 @@ public class PostShef { /** * Process forecast data for the given tableName. */ - private void loadMaxFcstData_lidpe(String tableName, String locId, String pe) { + private void loadMaxFcstData_lidpe(String tableName, String locId, + String pe) { Object[] oa = null; if ((tableName != null) && (locId != null) && (pe != null)) { if (shefRecord.getShefType() == ShefType.E) { @@ -1336,7 +1387,7 @@ public class PostShef { /** * Loads the max fcst info into the RiverStatus table for the current * location and pe. - * */ + */ private void loadMaxFcstItem(String lid, String pe, String ts) { Object[] oa = null; int qcFilter = 1; @@ -1368,8 +1419,9 @@ public class PostShef { } } catch (Exception e) { log.error("Query = [" + riverStatQuery + "]"); - log.error(shefRecord.getTraceId() - + " - PostgresSQL error loading max forecast item", + log.error( + shefRecord.getTraceId() + + " - PostgresSQL error loading max forecast item", e); } } @@ -1394,8 +1446,10 @@ public class PostShef { } } catch (Exception e) { log.error("Query = [" + riverStatQuery + "]"); - log.error(shefRecord.getTraceId() - + " - PostgresSQL error loading max forecast item", e); + log.error( + shefRecord.getTraceId() + + " - PostgresSQL error loading max forecast item", + e); } } @@ -1408,10 +1462,10 @@ public class PostShef { * This code sets the time values */ shefList = buildTsFcstRiv(lid, pe, ts, qcFilter, useLatest); - if ((shefList != null) && (shefList.size() > 0)) { + if (shefList != null && !shefList.isEmpty()) { ShefData maxShefDataValue = findMaxFcst(shefList); - riverStatusUpdateValueFlag = updateRiverStatus(lid, pe, ts); + boolean riverStatusUpdateValueFlag = updateRiverStatus(lid, pe, ts); postTables.postRiverStatus(shefRecord, maxShefDataValue, riverStatusUpdateValueFlag); } else { @@ -1451,30 +1505,30 @@ public class PostShef { } /** - * + * * This function assembles a forecast time series for a given location and * pe. The data are retrieved for: 1) either the specified type-source or * for the type-source defined in the ingest filter as the one to use, based * on its rank; and for 2) either all forecast values regardless of basis * time or only those forecasts with the latest basis time. 3) for * non-probabilistic values only. - * + * * It returns a times series of values in an array of structures, and also * returns the count of values. - * + * * Presuming that the duration and extremum values in the forecast table * never yield duplicates, then there can only be duplicates for the same * validtime due to multiple basis times. - * + * * There is a passed in limit regarding how far in the future data is * considered, and how old the forecast (basistime) can be. - * + * * This function is needed since some locations have short-term forecasts * and long-term forecasts, both of which are valid and do not prempt the * other. This avoids problems with the previous method where the software * always used the forecast with the latest creation time and ignored all * other forecasts, for certain purposes. - * + * * The approach herein does NOT assume that the creation data corresponds to * the valid time covered - i.e. it does NOT require that long-term forecast * have the latest creation time. The heart of the logic for this function @@ -1488,18 +1542,19 @@ public class PostShef { StringBuilder queryForecast = null; boolean[] doKeep = null; - Object[] row = null; + Fcstheight[] fcstHead = null; Fcstheight fcstHght = null; - List shefList = new ArrayList(); + List shefList = new ArrayList<>(); ShefData shefDataValue = null; if (shefRecord.getShefType() != ShefType.E) { useTs = null; basisTimeValues = null; } - if ((tsFilter == null) || ((tsFilter.length() == 0) && (useTs == null))) { + if ((tsFilter == null) + || ((tsFilter.length() == 0) && (useTs == null))) { useTs = getBestTs(lid, pe, "F%", 0); if (useTs == null) { return null; @@ -1525,14 +1580,14 @@ public class PostShef { + "' and " + "ts = '" + useTs + "' and " + "validtime >= CURRENT_TIMESTAMP and " + "basistime >= '" + basisTimeAnsi + "' and " - + "value != " + ShefConstants.SHEF_MISSING_INT - + " and " + "quality_code >= " - + QUESTIONABLE_BAD_THRESHOLD + " " + + "value != " + ShefConstants.SHEF_MISSING_INT + " and " + + "quality_code >= " + QUESTIONABLE_BAD_THRESHOLD + " " + "ORDER BY basistime DESC "; basisTimeValues = dao.executeSQLQuery(query); - if ((basisTimeValues == null) || (basisTimeValues.length <= 0)) { + if ((basisTimeValues == null) + || (basisTimeValues.length <= 0)) { return null; } } @@ -1548,11 +1603,11 @@ public class PostShef { .append(" WHERE lid = '").append(lid); queryForecast.append("' AND pe = '").append(pe) .append("' AND ts = '").append(useTs); - queryForecast - .append("' AND validtime >= CURRENT_TIMESTAMP AND probability < 0.0 AND "); + queryForecast.append( + "' AND validtime >= CURRENT_TIMESTAMP AND probability < 0.0 AND "); - if ((useLatest == 1) - || ((basisTimeValues != null) && (basisTimeValues.length == 1))) { + if ((useLatest == 1) || ((basisTimeValues != null) + && (basisTimeValues.length == 1))) { Date tempStamp = null; tempStamp = (Date) basisTimeValues[0]; queryForecast.append("basistime >= '").append(tempStamp) @@ -1562,19 +1617,18 @@ public class PostShef { .append("' AND "); } - queryForecast.append("value != ") - .append(ShefConstants.SHEF_MISSING) + queryForecast.append("value != ").append(ShefConstants.SHEF_MISSING) .append(" AND quality_code >= "); - queryForecast.append(ShefConstants.SHEF_MISSING).append( - " ORDER BY validtime ASC"); + queryForecast.append(ShefConstants.SHEF_MISSING) + .append(" ORDER BY validtime ASC"); if (!queryForecast.toString().equals(previousQueryForecast)) { previousQueryForecast = queryForecast.toString(); - queryForecastResults = dao.executeSQLQuery(queryForecast - .toString()); + queryForecastResults = dao + .executeSQLQuery(queryForecast.toString()); } - row = null; + Object[] row = null; if ((queryForecastResults != null) && (queryForecastResults.length > 0)) { fcstHead = new Fcstheight[queryForecastResults.length]; @@ -1638,7 +1692,7 @@ public class PostShef { */ for (int y = 0; y < fcstCount; y++) { - shefDataValue = new ShefData(); + shefDataValue = new ShefData(shefParm); if (doKeep[y]) { shefDataValue.setLocationId(fcstHead[y].getId().getLid()); @@ -1647,15 +1701,15 @@ public class PostShef { convertDur(fcstHead[y].getId().getDur(), shefDataValue); - shefDataValue.setTypeSource(TypeSource.getEnum(fcstHead[y] - .getId().getTs())); + shefDataValue.setTypeSource( + TypeSource.getEnum(fcstHead[y].getId().getTs())); - shefDataValue.setExtremum(Extremum.getEnum(fcstHead[y] - .getId().getExtremum())); - shefDataValue.setObservationTimeObj(fcstHead[y].getId() - .getValidtime()); - shefDataValue.setCreationDateObj(fcstHead[y].getId() - .getBasistime()); + shefDataValue.setExtremum(Extremum + .getEnum(fcstHead[y].getId().getExtremum())); + shefDataValue.setObservationTimeObj( + fcstHead[y].getId().getValidtime()); + shefDataValue.setCreationDateObj( + fcstHead[y].getId().getBasistime()); shefDataValue.setValue(fcstHead[y].getValue()); shefList.add(shefDataValue); } @@ -1671,7 +1725,7 @@ public class PostShef { /** * Convert duration int to String character. - * + * * @param dur * The duration value */ @@ -1731,12 +1785,13 @@ public class PostShef { /* find out which basis time's time series this value belongs to */ - fcstBasisTime = new Date(fcstHead[i].getId().getBasistime() - .getTime()); + fcstBasisTime = new Date( + fcstHead[i].getId().getBasistime().getTime()); basisIndex[i] = MISSING; - for (int j = 0; ((j < ulCount) && (basisIndex[i] == MISSING)); j++) { + for (int j = 0; ((j < ulCount) + && (basisIndex[i] == MISSING)); j++) { row = (Date) ulHead[j]; ulBasisTime = row; @@ -1793,8 +1848,8 @@ public class PostShef { * if it lies between the start and end time for this basis time */ for (int i = 0; i < fcstCount; i++) { - fcstValidTime = new Date(fcstHead[i].getId().getValidtime() - .getTime()); + fcstValidTime = new Date( + fcstHead[i].getId().getValidtime().getTime()); if ((fcstValidTime.compareTo(startTime[basisIndex[i]]) >= 0) && (fcstValidTime.compareTo(endTime[basisIndex[i]]) <= 0)) { doKeep[i] = true; @@ -1888,8 +1943,10 @@ public class PostShef { * is handled below. */ - if ((startValidTime[currentIndex].compareTo(fullStartValidTime) >= 0) - && (endValidTime[currentIndex].compareTo(fullEndValidTime) <= 0)) { + if ((startValidTime[currentIndex] + .compareTo(fullStartValidTime) >= 0) + && (endValidTime[currentIndex] + .compareTo(fullEndValidTime) <= 0)) { /* * if the basis time series being considered is fully within the * time of the already existing time series, then ignore it @@ -1899,7 +1956,8 @@ public class PostShef { endValidTime[currentIndex] = zero; } else if ((startValidTime[currentIndex] .compareTo(fullStartValidTime) <= 0) - && (endValidTime[currentIndex].compareTo(fullEndValidTime) >= 0)) { + && (endValidTime[currentIndex] + .compareTo(fullEndValidTime) >= 0)) { /* * if the basis time series being considered covers time both * before and after the existing time series, use the portion of @@ -1915,7 +1973,8 @@ public class PostShef { } else if ((startValidTime[currentIndex] .compareTo(fullStartValidTime) <= 0) - && (endValidTime[currentIndex].compareTo(fullEndValidTime) <= 0)) { + && (endValidTime[currentIndex] + .compareTo(fullEndValidTime) <= 0)) { /* * if the basis time series being considered straddles the * beginning or is completely before the existing time series, @@ -1926,7 +1985,8 @@ public class PostShef { fullStartValidTime = startValidTime[currentIndex]; } else if ((startValidTime[currentIndex] .compareTo(fullStartValidTime) >= 0) - && (endValidTime[currentIndex].compareTo(fullEndValidTime) >= 0)) { + && (endValidTime[currentIndex] + .compareTo(fullEndValidTime) >= 0)) { /* * if the basis time series being considered straddles the end * or is completely after the existing time series, then use the @@ -1937,7 +1997,7 @@ public class PostShef { fullEndValidTime = endValidTime[currentIndex]; } - } /* end for loop on the unique ordered basis times */ + } // Need to find a better way to do this rval[0] = startValidTime; @@ -1972,16 +2032,17 @@ public class PostShef { * requested, then the highest rank (1st) is returned. The type-source * prefix is normally given as a one-character string, R for observed data * and F for forecast data. - * + * * The function argument returns a status variable indicating whether the * request was satisfied. - * + * */ - private String getBestTs(String lid, String pe, String tsPrefix, int ordinal) { + private String getBestTs(String lid, String pe, String tsPrefix, + int ordinal) { int count = 0; String tsFound = null; - String query = "SELECT ts_rank,ts FROM ingestfilter WHERE lid = '" - + lid + "' AND pe = '" + pe + "' AND ts like '" + tsPrefix + String query = "SELECT ts_rank,ts FROM ingestfilter WHERE lid = '" + lid + + "' AND pe = '" + pe + "' AND ts like '" + tsPrefix + "' AND ingest = 'T' ORDER BY ts_rank, ts"; Object[] oa = null; try { @@ -2019,8 +2080,10 @@ public class PostShef { } } catch (Exception e) { log.error("Query = [" + query + "]"); - log.error(shefRecord.getTraceId() - + " - PostgresSQL error retrieving from ingestfilter", e); + log.error( + shefRecord.getTraceId() + + " - PostgresSQL error retrieving from ingestfilter", + e); } return tsFound; } @@ -2029,7 +2092,7 @@ public class PostShef { * Checks if location data should be posted. 4 possible return values: * Location defined as location - 0 Location defined as geoarea - 1 Location * defined but don't post - 2 Location undefined - 3 - * + * * @param locId * - Location Id to check * @return Location corresponding to 1 of 4 return values @@ -2059,7 +2122,8 @@ public class PostShef { } } catch (Exception e) { log.error("Query = [" + sql + "]"); - log.error(shefRecord.getTraceId() + " - Error checking location", e); + log.error(shefRecord.getTraceId() + " - Error checking location", + e); } return retVal; } @@ -2068,10 +2132,10 @@ public class PostShef { * Check whether this lid-PEDTSE combination has an entry in the * IngestFilter table which specifies it to be processed. This is checked * to: - * + * * 1) allow issuance of an error message if an entry is not found, and 2) if * the load_ingest flag is set, then it will insert an entry into the table. - * + * * @param locId * - location id * @param data @@ -2130,8 +2194,7 @@ public class PostShef { if (pe.equals(data.getPhysicalElement().getCode()) && ts.equals(data.getTypeSource().getCode()) - && extremum - .equals(data.getExtremum().getCode()) + && extremum.equals(data.getExtremum().getCode()) && (dur == data.getDurationValue())) { if ("T".equals(ingest)) { if ("T".equals(stg2_input)) { @@ -2178,7 +2241,8 @@ public class PostShef { ingestFilter.setIngest("T"); ingestFilter.setOfsInput("F"); - if ((data.getPhysicalElement() == PhysicalElement.PRECIPITATION_ACCUMULATOR) + if ((data + .getPhysicalElement() == PhysicalElement.PRECIPITATION_ACCUMULATOR) || (data.getPhysicalElement() == PhysicalElement.PRECIPITATION_INCREMENT)) { ingestSwitch = ShefConstants.IngestSwitch.POST_PE_AND_HOURLY; ingestFilter.setStg2Input("T"); @@ -2189,7 +2253,8 @@ public class PostShef { /* insert the record */ errorMsg.setLength(0); - errorMsg.append("PostgreSQL error putting data into IngestFilter"); + errorMsg.append( + "PostgreSQL error putting data into IngestFilter"); dao.saveOrUpdate(ingestFilter); /* @@ -2367,14 +2432,14 @@ public class PostShef { stnClass.setLid(locId); - List fields = new ArrayList(1); - List values = new ArrayList(1); + List fields = new ArrayList<>(1); + List values = new ArrayList<>(1); fields.add("lid"); values.add(locId); List queryResult = locDao.queryByCriteria(fields, values); com.raytheon.uf.common.dataplugin.shef.tables.Location loc = null; - if (queryResult.size() > 0) { + if (!queryResult.isEmpty()) { loc = (com.raytheon.uf.common.dataplugin.shef.tables.Location) queryResult .get(0); } @@ -2409,7 +2474,7 @@ public class PostShef { /** * Retrieves the number of records in the table based on the where clause - * + * * @param table * - table to search * @param where @@ -2437,7 +2502,7 @@ public class PostShef { /** * Determines if a PE or PE category code is in the array passed in - * + * * @param oa * - array to search * @param pe @@ -2456,7 +2521,7 @@ public class PostShef { /** * Adjust the data value using the adjustfactor table, if matched - * + * * @param locId * - location id of the record * @param data @@ -2476,21 +2541,24 @@ public class PostShef { StringBuilder sql = new StringBuilder(); try { - sql.append("select divisor, base, multiplier, adder from adjustfactor "); + sql.append( + "select divisor, base, multiplier, adder from adjustfactor "); sql.append("where lid = '").append(locId) .append("' and pe = '"); - sql.append(data.getPhysicalElement().getCode()).append( - "' and dur = "); + sql.append(data.getPhysicalElement().getCode()) + .append("' and dur = "); sql.append(data.getDurationValue()).append(" and ts = '"); - sql.append(data.getTypeSource().getCode()).append( - "' and extremum = '"); + sql.append(data.getTypeSource().getCode()) + .append("' and extremum = '"); sql.append(data.getExtremum().getCode()).append("'"); Object[] oa = dao.executeSQLQuery(sql.toString()); if (oa.length > 0) { Object[] oa2 = (Object[]) oa[0]; - /* if Correction Factor divisor value is NULL, set it to 1.0 */ + /* + * if Correction Factor divisor value is NULL, set it to 1.0 + */ divisor = ShefUtil.getDouble(oa2[0], 1.0); /* * if divisor is ZERO, set it to 1.0, DON'T WANT TO DIVIDE @@ -2535,7 +2603,7 @@ public class PostShef { /** * Insert the data into the table. if entry already exists then do nothing - * + * * @param locId * - the location id * @param productId @@ -2549,8 +2617,8 @@ public class PostShef { postDate.setTime(getToNearestSecond(TimeUtil.currentTimeMillis())); try { /* Get a Data Access Object */ - link = new Productlink(new ProductlinkId(locId, productId, obsTime, - postDate)); + link = new Productlink( + new ProductlinkId(locId, productId, obsTime, postDate)); dao.saveOrUpdate(link); } catch (Exception e) { @@ -2562,15 +2630,15 @@ public class PostShef { /** * Checks the quality of the data - * + * * Note: - * + * * When checking a station's qc and alert/alarm limits, this function uses * the full set of location limits if they exist in LocDataLimits, even if * for example, only the alert/alarm limits are defined and the qc limits * are not set. It will NOT attempt to get values for those null limits from * the general DataLimits tables. - * + * * @param lid * @param dataQualifier * @param dataValue @@ -2615,8 +2683,7 @@ public class PostShef { locLimitSql.append(sqlStart); locLimitSql.append("locdatalimits where "); - locLimitSql.append("lid = '").append(lid) - .append("' and pe = '") + locLimitSql.append("lid = '").append(lid).append("' and pe = '") .append(data.getPhysicalElement().getCode()) .append("' and dur = ").append(data.getDurationValue()); @@ -2663,22 +2730,22 @@ public class PostShef { * flag */ ShefRangeData rangeData = new ShefRangeData(); - rangeData.setGrossRangeMin(ShefUtil.getDouble(oa2[2], - missing)); - rangeData.setGrossRangeMax(ShefUtil.getDouble(oa2[3], - missing)); - rangeData.setReasonRangeMin(ShefUtil.getDouble(oa2[4], - missing)); - rangeData.setReasonRangeMax(ShefUtil.getDouble(oa2[5], - missing)); - rangeData.setAlarmLowerLimit(ShefUtil.getDouble( - oa2[12], missing)); - rangeData.setAlarmUpperLimit(ShefUtil.getDouble(oa2[9], - missing)); - rangeData.setAlertLowerLimit(ShefUtil.getDouble( - oa2[11], missing)); - rangeData.setAlertUpperLimit(ShefUtil.getDouble(oa2[7], - missing)); + rangeData.setGrossRangeMin( + ShefUtil.getDouble(oa2[2], missing)); + rangeData.setGrossRangeMax( + ShefUtil.getDouble(oa2[3], missing)); + rangeData.setReasonRangeMin( + ShefUtil.getDouble(oa2[4], missing)); + rangeData.setReasonRangeMax( + ShefUtil.getDouble(oa2[5], missing)); + rangeData.setAlarmLowerLimit( + ShefUtil.getDouble(oa2[12], missing)); + rangeData.setAlarmUpperLimit( + ShefUtil.getDouble(oa2[9], missing)); + rangeData.setAlertLowerLimit( + ShefUtil.getDouble(oa2[11], missing)); + rangeData.setAlertUpperLimit( + ShefUtil.getDouble(oa2[7], missing)); this.dataRangeMap.put(key, rangeData); break; } @@ -2690,10 +2757,10 @@ public class PostShef { /* * if a range is found, then check the value and set the flag */ - if (((rangeData.getGrossRangeMin() != missing) && (dValue < rangeData - .getGrossRangeMin())) - || ((rangeData.getGrossRangeMax() != missing) && (dValue > rangeData - .getGrossRangeMax()))) { + if (((rangeData.getGrossRangeMin() != missing) + && (dValue < rangeData.getGrossRangeMin())) + || ((rangeData.getGrossRangeMax() != missing) + && (dValue > rangeData.getGrossRangeMax()))) { qualityCode = ShefQC.setQcCode( (int) ShefConstants.QC_GROSSRANGE_FAILED, qualityCode); @@ -2709,10 +2776,11 @@ public class PostShef { * don't do anything if it fails the gross range check */ } else { - if (((rangeData.getReasonRangeMin() != missing) && (dValue < rangeData - .getReasonRangeMin())) - || ((rangeData.getReasonRangeMax() != missing) && (dValue > rangeData - .getReasonRangeMax()))) { + if (((rangeData.getReasonRangeMin() != missing) + && (dValue < rangeData.getReasonRangeMin())) + || ((rangeData.getReasonRangeMax() != missing) + && (dValue > rangeData + .getReasonRangeMax()))) { qualityCode = ShefQC.setQcCode( (int) ShefConstants.QC_REASONRANGE_FAILED, qualityCode); @@ -2760,14 +2828,14 @@ public class PostShef { * have on the quality code. for qualifiers G and M, do nothing, * since the default code embodies these values. */ - if ((dataQualifier != null) && !"Z".equalsIgnoreCase(dataQualifier)) { + if ((dataQualifier != null) + && !"Z".equalsIgnoreCase(dataQualifier)) { if ("Q".equalsIgnoreCase(dataQualifier) || "F".equalsIgnoreCase(dataQualifier)) { qualityCode = ShefQC.setQcCode( (int) ShefConstants.QC_EXTERN_QUEST, qualityCode); - } else if ((dataQualifier != null) - && ("R".equalsIgnoreCase(dataQualifier) || "B" - .equalsIgnoreCase(dataQualifier))) { + } else if (("R".equalsIgnoreCase(dataQualifier) + || "B".equalsIgnoreCase(dataQualifier))) { qualityCode = ShefQC.setQcCode( (int) ShefConstants.QC_EXTERN_FAILED, qualityCode); } @@ -2789,14 +2857,15 @@ public class PostShef { /** * Determine if the qualityCode passed in is of "Higher" quality than the * checkCode passed in - * + * * @param checkCode * - code to check against * @param qualityCode * - code to check * @return true if the qualityCode is of "Higher" quality */ - private boolean checkQcCode(QualityControlCode checkCode, long qualityCode) { + private boolean checkQcCode(QualityControlCode checkCode, + long qualityCode) { boolean returnValue = false; switch (checkCode) { case QC_DEFAULT: @@ -2806,23 +2875,24 @@ public class PostShef { returnValue = (qualityCode > ShefConstants.GOOD_QUESTIONABLE_THRESHOLD); break; case QC_QUESTIONABLE: - returnValue = ((qualityCode >= ShefConstants.QUESTIONABLE_BAD_THRESHOLD) && (qualityCode < ShefConstants.GOOD_QUESTIONABLE_THRESHOLD)); + returnValue = ((qualityCode >= ShefConstants.QUESTIONABLE_BAD_THRESHOLD) + && (qualityCode < ShefConstants.GOOD_QUESTIONABLE_THRESHOLD)); break; case QC_ROC_PASSED: - returnValue = BitUtils - .checkQcBit(ShefConstants.ROC_QC, qualityCode); + returnValue = BitUtils.checkQcBit(ShefConstants.ROC_QC, + qualityCode); break; case QC_OUTLIER_PASSED: returnValue = BitUtils.checkQcBit(ShefConstants.OUTLIER_QC, qualityCode); break; case QC_SCC_PASSED: - returnValue = BitUtils - .checkQcBit(ShefConstants.SCC_QC, qualityCode); + returnValue = BitUtils.checkQcBit(ShefConstants.SCC_QC, + qualityCode); break; case QC_MSC_PASSED: - returnValue = BitUtils - .checkQcBit(ShefConstants.MSC_QC, qualityCode); + returnValue = BitUtils.checkQcBit(ShefConstants.MSC_QC, + qualityCode); break; case QC_FAILED: returnValue = qualityCode < ShefConstants.QUESTIONABLE_BAD_THRESHOLD; @@ -2843,7 +2913,7 @@ public class PostShef { /** * check_if_paired() - * + * * Check if the SHEF record being processed is for a physical element that * has data for a special paired-and-dependent set of data. */ @@ -2872,7 +2942,7 @@ public class PostShef { /** * Check if the data value's time is within the given day-of-the-year * window. - * + * * @param obsTime * - Data time * @param monthDayStart @@ -2888,12 +2958,12 @@ public class PostShef { && (monthDayEnd != null)) { if ((monthDayStart.length() == 5) && (monthDayEnd.length() == 5)) { - int rangeStartDate = Integer.parseInt(monthDayStart.substring( - 0, 2)) * 100; + int rangeStartDate = Integer + .parseInt(monthDayStart.substring(0, 2)) * 100; rangeStartDate += Integer.parseInt(monthDayStart.substring(3)); - int rangeEndDate = Integer - .parseInt(monthDayEnd.substring(0, 2)) * 100; + int rangeEndDate = Integer.parseInt(monthDayEnd.substring(0, 2)) + * 100; rangeEndDate += Integer.parseInt(monthDayEnd.substring(3)); Calendar date = TimeUtil.newGmtCalendar(obsTime); @@ -2902,7 +2972,8 @@ public class PostShef { dataDate += date.get(Calendar.DAY_OF_MONTH); /* Compare the dates, don't check for straddling the year */ - valid = ((dataDate >= rangeStartDate) && (dataDate <= rangeEndDate)); + valid = ((dataDate >= rangeStartDate) + && (dataDate <= rangeEndDate)); } } return valid; @@ -2910,7 +2981,7 @@ public class PostShef { /** * Populates the data object for storage to the database. - * + * * @param shefRecord * - the shef record to store * @param data @@ -2940,7 +3011,7 @@ public class PostShef { basisTime = new Date(); } - if (dataValue == "") { + if (dataValue.isEmpty()) { dataValue = ShefConstants.SHEF_MISSING; } short revision = 0; @@ -2962,12 +3033,13 @@ public class PostShef { comment.setTraceId(shefRecord.getTraceId()); comment.getId().setTs(data.getTypeSource().getCode()); comment.setValue(Double.parseDouble(dataValue)); - comment.getId().setProbability( - (float) data.getProbability().getValue()); + comment.getId() + .setProbability((float) data.getProbability().getValue()); comment.getId().setBasistime(basisTime); comment.setShefComment(data.getRetainedComment()); dataObj = comment; - } else if (ShefConstants.CONTINGENCY_VALUE.equalsIgnoreCase(tableName)) { + } else if (ShefConstants.CONTINGENCY_VALUE + .equalsIgnoreCase(tableName)) { Contingencyvalue contingency = new Contingencyvalue( new ContingencyvalueId()); Date basisTime = data.getCreationDateObj(); @@ -3180,10 +3252,10 @@ public class PostShef { /** * Convert the provided millisecond value to the nearest second. - * + * * @param time * time in milliseconds - * + * * @return milliseconds rounded to the nearest second. */ private long getToNearestSecond(long time) { @@ -3194,52 +3266,4 @@ public class PostShef { public void close() { postTables.close(); } - - public static final void main(String[] args) { - - Calendar postDate = TimeUtil.newGmtCalendar(2011, 1, 12); - postDate.set(Calendar.HOUR_OF_DAY, 17); - postDate.set(Calendar.MINUTE, 25); - - Calendar obsTimef = TimeUtil.newGmtCalendar(2011, 1, 12); - obsTimef.set(Calendar.HOUR_OF_DAY, 17); - obsTimef.set(Calendar.MINUTE, 25); - obsTimef.add(Calendar.DAY_OF_MONTH, -30); - - Calendar obsTimeb = TimeUtil.newGmtCalendar(2011, 1, 12); - obsTimeb.set(Calendar.HOUR_OF_DAY, 17); - obsTimeb.set(Calendar.MINUTE, 25); - obsTimeb.add(Calendar.MINUTE, 10); - obsTimeb.set(Calendar.SECOND, 1); - - long lookbackMillis = 30 * ShefConstants.MILLIS_PER_DAY; - - long lookfwdMillis = 10 * ShefConstants.MILLIS_PER_MINUTE; - - long difff = postDate.getTimeInMillis() - obsTimef.getTimeInMillis(); - long diffb = obsTimeb.getTimeInMillis() - postDate.getTimeInMillis(); - - System.out.println(difff + " " + lookbackMillis); - System.out.println(difff > lookbackMillis); - - System.out.println(diffb + " " + lookfwdMillis); - System.out.println(diffb > lookfwdMillis); - - SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMDDhhmmssZ"); - sdf.setTimeZone(SHEFTimezone.GMT_TIMEZONE); - try { - Date d = sdf.parse("20110228102100-0000"); - - System.out.println(sdf.format(d)); - System.out.println(checkRangeDate(d, "01-01", "12-31") - + " expected true"); - System.out.println(checkRangeDate(d, "03-01", "10-01") - + " expected false"); - System.out.println(checkRangeDate(d, "99-99", "00-00") - + " expected false"); - - } catch (ParseException e) { - e.printStackTrace(); - } - } } diff --git a/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/database/PostTables.java b/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/database/PostTables.java index f6e3794ac9..f6cc31f989 100644 --- a/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/database/PostTables.java +++ b/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/database/PostTables.java @@ -19,6 +19,7 @@ **/ package com.raytheon.edex.plugin.shef.database; +import java.io.Serializable; import java.sql.CallableStatement; import java.sql.Connection; import java.sql.PreparedStatement; @@ -39,6 +40,10 @@ import com.raytheon.edex.plugin.shef.ohdlib.GagePP; import com.raytheon.edex.plugin.shef.ohdlib.GagePPOptions; import com.raytheon.edex.plugin.shef.ohdlib.GagePPOptions.shef_dup; import com.raytheon.edex.plugin.shef.ohdlib.PrecipUtils; +import com.raytheon.edex.plugin.shef.util.FcstPEPDOLookupUtil; +import com.raytheon.edex.plugin.shef.util.ObsPEPDOLookupUtil; +import com.raytheon.edex.plugin.shef.util.ObsPrecipPDOLookupUtil; +import com.raytheon.edex.plugin.shef.util.PDOLookupReturn; import com.raytheon.edex.plugin.shef.util.PrecipitationUtils; import com.raytheon.edex.plugin.shef.util.ShefStats; import com.raytheon.uf.common.dataplugin.persist.PersistableDataObject; @@ -47,6 +52,9 @@ import com.raytheon.uf.common.dataplugin.shef.tables.Arealfcst; import com.raytheon.uf.common.dataplugin.shef.tables.Arealobs; import com.raytheon.uf.common.dataplugin.shef.tables.Commentvalue; import com.raytheon.uf.common.dataplugin.shef.tables.Contingencyvalue; +import com.raytheon.uf.common.dataplugin.shef.tables.ICheckValue; +import com.raytheon.uf.common.dataplugin.shef.tables.Latestobsvalue; +import com.raytheon.uf.common.dataplugin.shef.tables.LatestobsvalueId; import com.raytheon.uf.common.dataplugin.shef.tables.Pairedvalue; import com.raytheon.uf.common.dataplugin.shef.tables.PairedvalueId; import com.raytheon.uf.common.dataplugin.shef.tables.Procvalue; @@ -89,10 +97,11 @@ import com.raytheon.uf.edex.database.dao.DaoConfig; * 09/03/2014 mpduff postRiverStatus() writes directly, not via batch * 10/16/2014 3454 bphillip Upgrading to Hibernate 4 * Aug 05, 2015 4486 rjpeter Changed Timestamp to Date. + * Dec 18, 2017 6554 bkowal Added checkForAndHandleDuplicate to avoid running SQL procedures that are guaranteed + * to fail because the configuration does not allow record updates. * * * @author mduff - * @version 1.0 */ public class PostTables { @@ -106,13 +115,8 @@ public class PostTables { private static final String RIVER_STATUS_INSERT_STATEMENT = "INSERT INTO riverstatus values(?,?,?,?,?,?,?,?,?)"; private static final String RIVER_STATUS_UPDATE_STATEMENT = "UPDATE riverstatus SET lid = ? , " - + "pe = ? , " - + "dur = ? , " - + "ts = ? , " - + "extremum = ? ," - + "probability = ? , " - + "validtime = ? , " - + "basistime = ? , " + + "pe = ? , " + "dur = ? , " + "ts = ? , " + "extremum = ? ," + + "probability = ? , " + "validtime = ? , " + "basistime = ? , " + "value = ? " + "WHERE lid= ? AND pe= ? AND ts= ?"; private static GagePPOptions gagePPOptions; @@ -190,20 +194,39 @@ public class PostTables { basisTime.setTime(0); } - String procName = "latestobs"; - if (dataValue.equals("")) { - dataValue = "-9999.0"; - } + final String procName = "latestobs"; /* now call the PostgreSQL function */ start = System.currentTimeMillis(); - execFunction(procName, record, shefData, locId, dataValue, qualifier, - qualityCode, productId, productTime, postTime, duplicateOption, - stats); - end = System.currentTimeMillis(); - end = System.currentTimeMillis(); - stats.addElapsedTimeIngest(end - start); - stats.incrementLatestObs(); + LatestobsvalueId id = new LatestobsvalueId(); + id.setLid(locId); + id.setPe(shefData.getPhysicalElement().getCode()); + id.setDur(shefData.getDurationValue()); + id.setTs(shefData.getTypeSource().getCode()); + id.setExtremum(shefData.getExtremum().getCode()); + id.setObstime(shefData.getObservationTimeObj()); + if (dataValue.isEmpty()) { + dataValue = ShefConstants.SHEF_MISSING; + } + id.setValue(Double.parseDouble(dataValue)); + id.setRevision((record.isRevisedRecord() ? (short) 1 : (short) 0)); + id.setShefQualCode(qualifier); + id.setQualityCode((int) qualityCode); + id.setProductId(productId); + id.setProducttime(productTime); + id.setPostingtime(postTime); + + if (checkForAndHandleDuplicate(shefData, locId, dataValue, + duplicateOption, Latestobsvalue.class, id, false)) { + stats.incrementIgnored(); + } else { + execFunction(procName, record, shefData, locId, dataValue, + qualifier, qualityCode, productId, productTime, postTime, + duplicateOption, stats); + end = System.currentTimeMillis(); + stats.addElapsedTimeIngest(end - start); + stats.incrementLatestObs(); + } } /** @@ -231,8 +254,8 @@ public class PostTables { short dur = Short.parseShort(shefData.getDuration().getValue() + ""); String ts = shefData.getTypeSource().getCode(); String extremum = shefData.getExtremum().getCode(); - float probability = Float.parseFloat(shefData.getProbability() - .getValue() + ""); + float probability = Float + .parseFloat(shefData.getProbability().getValue() + ""); Date validTime = shefData.getObservationTimeObj(); // CONCERN : Is this the correct behavior when the basisTime is missing? @@ -299,8 +322,8 @@ public class PostTables { id.setExtremum(shefData.getExtremum().getCode()); id.setLid(locId); id.setPe(shefData.getPhysicalElement().getCode()); - id.setProbability(Float.parseFloat(shefData.getProbability().getValue() - + "")); + id.setProbability( + Float.parseFloat(shefData.getProbability().getValue() + "")); pairedValue.setPostingtime(postTime); pairedValue.setProductId(productId); @@ -321,8 +344,8 @@ public class PostTables { // lid, pe, dur, ts, extremum, probability, validtime, basistime, // ref_value sql = "select value from pairedvalue where lid = '" + locId - + "' and pe = '" + pe + "' and dur = " + dur - + " and ts = '" + ts + "' and extremum = '" + extremum + + "' and pe = '" + pe + "' and dur = " + dur + " and ts = '" + + ts + "' and extremum = '" + extremum + "' and probability = " + probability + " and validtime = '" + ShefConstants.POSTGRES_DATE_FORMAT.format(validTime) @@ -356,8 +379,8 @@ public class PostTables { /* data was properly added to table */ stats.incrementRejected(); } else { - if (AppsDefaults.getInstance().getBoolean( - ShefConstants.DUP_MESSAGE, false)) { + if (AppsDefaults.getInstance() + .getBoolean(ShefConstants.DUP_MESSAGE, false)) { log.info("Ignoring duplicate PairedValue for " + locId + ", " + productId + ", " + shefData.getObservationTime()); @@ -397,19 +420,31 @@ public class PostTables { Date validTime, Date postTime, DataType type) { String procName = null; + Class daoClass = null; + Serializable id = null; if (DataType.READING.equals(type)) { /* see if the data is a precip report of some kind */ - int precipIndex = PrecipitationUtils.getPrecipitationIndex(shefData - .getPhysicalElement()); + int precipIndex = PrecipitationUtils + .getPrecipitationIndex(shefData.getPhysicalElement()); /* post all non-precip observed data */ if (precipIndex == ShefConstants.NOT_PRECIP) { procName = "obs_pe"; + PDOLookupReturn pdoLookupReturn = ObsPEPDOLookupUtil.lookupPDO( + shefData.getPhysicalElement().getCode(), locId, + shefData); + daoClass = pdoLookupReturn.getDaoClass(); + id = pdoLookupReturn.getId(); /* now call the PostgreSQL function */ } else { procName = "obs_precip"; + PDOLookupReturn pdoLookupReturn = ObsPrecipPDOLookupUtil + .lookupPDO(shefData.getPhysicalElement().getCode(), + locId, shefData); + daoClass = pdoLookupReturn.getDaoClass(); + id = pdoLookupReturn.getId(); /* * if gpp is enabled, and the switch for this record dictates, @@ -417,12 +452,11 @@ public class PostTables { * a file that will be sent to the gpp server after the product * is fully processed. if PP, only consider hourly data. */ - boolean gage_pp_enable = AppsDefaults.getInstance().getBoolean( - "gage_pp_enable", false); + boolean gage_pp_enable = AppsDefaults.getInstance() + .getBoolean("gage_pp_enable", false); if (gage_pp_enable && (ingestSwitch == ShefConstants.IngestSwitch.POST_PE_AND_HOURLY)) { - PrecipRecord precip = new PrecipRecord(shefData); precip.setPostingTime(postTime); precip.setQualCode(qualityCode); @@ -433,9 +467,10 @@ public class PostTables { if ((PhysicalElement.PRECIPITATION_INCREMENT.equals(pe)) && ((shefData.getDuration() == Duration._1_DAY) - || (shefData.getDuration() == Duration._1_PERIOD) || (shefData - .getDuration() == Duration._6_HOUR))) { - + || (shefData + .getDuration() == Duration._1_PERIOD) + || (shefData + .getDuration() == Duration._6_HOUR))) { PrecipitationUtils.writePrecipGpp(shefData, record, qualityCode, productId, productTime, postTime, locId, qualifier, dataValue); @@ -444,15 +479,17 @@ public class PostTables { } if ((PhysicalElement.PRECIPITATION_ACCUMULATOR.equals(pe)) || ((PhysicalElement.PRECIPITATION_INCREMENT - .equals(pe)) && ((shefData.getDuration() == Duration._60_MINUTES) || (shefData - .getDuration() == Duration._1_HOUR)))) { - - if (dataValue.equals("")) { + .equals(pe)) + && ((shefData + .getDuration() == Duration._60_MINUTES) + || (shefData + .getDuration() == Duration._1_HOUR)))) { + if (dataValue.isEmpty()) { dataValue = ShefConstants.SHEF_MISSING; } - if (PrecipUtils.checkPrecipWindow( - shefData.getObsTime(), pe, gagePPOptions)) { + if (PrecipUtils.checkPrecipWindow(shefData.getObsTime(), + pe, gagePPOptions)) { PrecipitationUtils.writePrecipGpp(shefData, record, qualityCode, productId, productTime, postTime, locId, qualifier, dataValue); @@ -462,63 +499,81 @@ public class PostTables { } } } - } else if (DataType.FORECAST.equals(type)) { procName = "fcst_pe"; + PDOLookupReturn pdoLookupReturn = FcstPEPDOLookupUtil.lookupPDO( + shefData.getPhysicalElement().getCode(), locId, shefData, + validTime); + daoClass = pdoLookupReturn.getDaoClass(); + id = pdoLookupReturn.getId(); } long start = System.currentTimeMillis(); int status = -1; - if (DataType.FORECAST.equals(type)) { - status = execFcstFunc(procName, record, shefData, locId, dataValue, - qualifier, qualityCode, productId, productTime, postTime, - duplicateOption, ingestSwitch, stats, validTime); + if (checkForAndHandleDuplicate(shefData, locId, dataValue, + duplicateOption, daoClass, id, true)) { + stats.incrementIgnored(); } else { - /* now call the PostgreSQL function */ - status = execFunction(procName, record, shefData, locId, dataValue, - qualifier, qualityCode, productId, productTime, postTime, - duplicateOption, ingestSwitch, stats); - } + if (DataType.FORECAST.equals(type)) { + status = execFcstFunc(procName, record, shefData, locId, + dataValue, qualifier, qualityCode, productId, + productTime, postTime, duplicateOption, ingestSwitch, + stats, validTime); + } else { + /* now call the PostgreSQL function */ + status = execFunction(procName, record, shefData, locId, + dataValue, qualifier, qualityCode, productId, + productTime, postTime, duplicateOption, ingestSwitch, + stats); + } - long end = System.currentTimeMillis(); + long end = System.currentTimeMillis(); - if (status < 0) { - log.error(record.getTraceId() + " - PostgresSQL error " + status - + " executing " + procName + " function for " + locId - + ", " + shefData.getObservationTimeObj().toString() + ", " - + productTime.toString() + ", " + productId + ", " - + postTime.toString()); - stats.incrementErrorMessages(); - } else { - if ((DataType.READING.equals(type)) - || (DataType.PROCESSED.equals(type))) { - stats.incrementObsPe(); - if (shefData.getPhysicalElement().getCategory() == PhysicalElementCategory.HEIGHT) { - stats.incrementObsHeight(); - stats.setElapsedTimeHeightIngest(end - start); + if (status < 0) { + log.error(record.getTraceId() + " - PostgresSQL error " + status + + " executing " + procName + " function for " + locId + + ", " + shefData.getObservationTimeObj().toString() + + ", " + productTime.toString() + ", " + productId + + ", " + postTime.toString()); + stats.incrementErrorMessages(); + } else { + if ((DataType.READING.equals(type)) + || (DataType.PROCESSED.equals(type))) { + stats.incrementObsPe(); + if (shefData.getPhysicalElement() + .getCategory() == PhysicalElementCategory.HEIGHT) { + stats.incrementObsHeight(); + stats.setElapsedTimeHeightIngest(end - start); - } else if ((shefData.getPhysicalElement() != PhysicalElement.PRESSURE_ATMOSPHERIC) - && (shefData.getPhysicalElement() != PhysicalElement.PRESSURE_ATMOSPHERIC_3HR) - && (shefData.getPhysicalElement() != PhysicalElement.PRESSURE_CHARACTERISTIC) - && (shefData.getPhysicalElement() != PhysicalElement.PRESSURE_SEA_LEVEL)) { - stats.incrementObsPrecip(); - stats.addElapsedTimePrecipIngest(end - start); - } else { - stats.addElapsedTimeOtherIngest(end - start); - } - } else if (DataType.FORECAST.equals(type)) { - stats.incrementForecastPe(); - stats.addElapsedTimeForecastIngest(end - start); + } else if ((shefData + .getPhysicalElement() != PhysicalElement.PRESSURE_ATMOSPHERIC) + && (shefData + .getPhysicalElement() != PhysicalElement.PRESSURE_ATMOSPHERIC_3HR) + && (shefData + .getPhysicalElement() != PhysicalElement.PRESSURE_CHARACTERISTIC) + && (shefData + .getPhysicalElement() != PhysicalElement.PRESSURE_SEA_LEVEL)) { + stats.incrementObsPrecip(); + stats.addElapsedTimePrecipIngest(end - start); + } else { + stats.addElapsedTimeOtherIngest(end - start); + } + } else if (DataType.FORECAST.equals(type)) { + stats.incrementForecastPe(); + stats.addElapsedTimeForecastIngest(end - start); - /* - * track which lid/pe combinations have been processed for - * forecast river data. - */ - if ((shefData.getPhysicalElement().getCategory() == PhysicalElementCategory.HEIGHT) - || (shefData.getPhysicalElement().getCategory() == PhysicalElementCategory.DISCHARGE)) { - loadForecastInfo(locId, shefData.getPhysicalElement(), - stats); + /* + * track which lid/pe combinations have been processed for + * forecast river data. + */ + if ((shefData.getPhysicalElement() + .getCategory() == PhysicalElementCategory.HEIGHT) + || (shefData.getPhysicalElement() + .getCategory() == PhysicalElementCategory.DISCHARGE)) { + loadForecastInfo(locId, shefData.getPhysicalElement(), + stats); + } } } } @@ -591,10 +646,10 @@ public class PostTables { probability = value.getId().getProbability(); dataValue = value.getValue(); revision = value.getRevision(); - validTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId() - .getValidtime()); - basisTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId() - .getBasistime()); + validTime = ShefConstants.POSTGRES_DATE_FORMAT + .format(value.getId().getValidtime()); + basisTime = ShefConstants.POSTGRES_DATE_FORMAT + .format(value.getId().getBasistime()); appendStr = "probability = " + probability + " and " + "validtime = '" + validTime + "' and " + "basistime = '" @@ -609,10 +664,10 @@ public class PostTables { probability = value.getId().getProbability(); dataValue = value.getValue(); revision = value.getRevision(); - validTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId() - .getValidtime()); - basisTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId() - .getBasistime()); + validTime = ShefConstants.POSTGRES_DATE_FORMAT + .format(value.getId().getValidtime()); + basisTime = ShefConstants.POSTGRES_DATE_FORMAT + .format(value.getId().getBasistime()); appendStr = "probability = " + probability + " and " + "validtime = '" + validTime + "' and " + "basistime = '" @@ -626,8 +681,8 @@ public class PostTables { extremum = value.getId().getExtremum(); dataValue = value.getValue(); revision = value.getRevision(); - validTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId() - .getObstime()); + validTime = ShefConstants.POSTGRES_DATE_FORMAT + .format(value.getId().getObstime()); appendStr = "obstime = '" + validTime + "'"; } else if (dataObj instanceof Rejecteddata) { @@ -641,10 +696,10 @@ public class PostTables { probability = value.getId().getProbability(); dataValue = value.getValue(); revision = value.getRevision(); - validTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId() - .getValidtime()); - basisTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId() - .getBasistime()); + validTime = ShefConstants.POSTGRES_DATE_FORMAT + .format(value.getId().getValidtime()); + basisTime = ShefConstants.POSTGRES_DATE_FORMAT + .format(value.getId().getBasistime()); String postingTime = ShefConstants.POSTGRES_DATE_FORMAT .format(value.getId().getPostingtime()); @@ -662,10 +717,10 @@ public class PostTables { probability = value.getId().getProbability(); dataValue = value.getValue(); revision = value.getRevision(); - validTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId() - .getValidtime()); - basisTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId() - .getBasistime()); + validTime = ShefConstants.POSTGRES_DATE_FORMAT + .format(value.getId().getValidtime()); + basisTime = ShefConstants.POSTGRES_DATE_FORMAT + .format(value.getId().getBasistime()); appendStr = "probability = " + probability + " and " + "validtime = '" + validTime + "' and " + "basistime = '" @@ -681,8 +736,8 @@ public class PostTables { extremum = value.getId().getExtremum(); dataValue = value.getValue(); revision = value.getRevision(); - validTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId() - .getObstime()); + validTime = ShefConstants.POSTGRES_DATE_FORMAT + .format(value.getId().getObstime()); appendStr = "obstime = '" + validTime + "'"; } else if (dataObj instanceof Arealfcst) { @@ -695,10 +750,10 @@ public class PostTables { dataValue = value.getValue(); revision = value.getRevision(); probability = value.getId().getProbability(); - validTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId() - .getValidtime()); - basisTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId() - .getBasistime()); + validTime = ShefConstants.POSTGRES_DATE_FORMAT + .format(value.getId().getValidtime()); + basisTime = ShefConstants.POSTGRES_DATE_FORMAT + .format(value.getId().getBasistime()); appendStr = "probability = " + probability + " and " + "validtime = '" + validTime + "' and " + "basistime = '" @@ -712,8 +767,8 @@ public class PostTables { extremum = value.getId().getExtremum(); dataValue = value.getId().getValue(); revision = value.getId().getRevision(); - validTime = ShefConstants.POSTGRES_DATE_FORMAT.format(value.getId() - .getObstime()); + validTime = ShefConstants.POSTGRES_DATE_FORMAT + .format(value.getId().getObstime()); appendStr = "obstime = '" + validTime + "'"; @@ -792,8 +847,8 @@ public class PostTables { /* * don't perform the overwrite since conditions were not met */ - if (AppsDefaults.getInstance().getBoolean( - ShefConstants.DUP_MESSAGE, false)) { + if (AppsDefaults.getInstance() + .getBoolean(ShefConstants.DUP_MESSAGE, false)) { log.info("Ignoring duplicate " + tableName + " for " + locId + ", " + validTime); } @@ -806,8 +861,10 @@ public class PostTables { } else { log.error("Query = [" + sql + "]"); } - log.error(dataObj.getTraceId() + " - PostgresSQL error updating " - + tableName + " for " + locId + ", " + validTime, e); + log.error( + dataObj.getTraceId() + " - PostgresSQL error updating " + + tableName + " for " + locId + ", " + validTime, + e); stats.incrementErrorMessages(); } } @@ -820,8 +877,8 @@ public class PostTables { public void postUnknownStation(Unkstn unkstn, ShefStats stats) { /* Build the sql query string */ StringBuilder sql = new StringBuilder(); - sql.append("select lid from unkstn where lid = '" + unkstn.getLid() - + "'"); + sql.append( + "select lid from unkstn where lid = '" + unkstn.getLid() + "'"); try { Object[] result = dao.executeSQLQuery(sql.toString()); @@ -836,12 +893,11 @@ public class PostTables { } } catch (Exception e) { log.error("Query = [" + sql.toString() + "]"); - log.error( - unkstn.getTraceId() - + " - PostgresSQL error updating UnkStn for " - + unkstn.getLid() + ", " - + unkstn.getProducttime().toString() + ", " - + unkstn.getPostingtime().toString(), e); + log.error(unkstn.getTraceId() + + " - PostgresSQL error updating UnkStn for " + + unkstn.getLid() + ", " + + unkstn.getProducttime().toString() + ", " + + unkstn.getPostingtime().toString(), e); stats.incrementErrorMessages(); } } @@ -938,13 +994,12 @@ public class PostTables { } private int execFunction(String functionName, ShefRecord record, - ShefData shefData, String locId, String dataValue, - String qualifier, long qualityCode, String productId, - Date productTime, Date postTime, String duplicateOption, - ShefStats stats) { + ShefData shefData, String locId, String dataValue, String qualifier, + long qualityCode, String productId, Date productTime, Date postTime, + String duplicateOption, ShefStats stats) { CallableStatement cs = null; int status = -1; - if (dataValue.equals("")) { + if (dataValue.isEmpty()) { dataValue = ShefConstants.SHEF_MISSING; } try { @@ -960,8 +1015,8 @@ public class PostTables { cs.setInt(3, shefData.getDurationValue()); cs.setString(4, shefData.getTypeSource().getCode()); cs.setString(5, shefData.getExtremum().getCode()); - cs.setTimestamp(6, new Timestamp(shefData.getObservationTimeObj() - .getTime())); + cs.setTimestamp(6, + new Timestamp(shefData.getObservationTimeObj().getTime())); cs.setDouble(7, Double.parseDouble(dataValue)); cs.setString(8, qualifier); cs.setInt(9, (int) qualityCode); @@ -986,17 +1041,16 @@ public class PostTables { if (status == 0) { conn.commit(); } else { - throw new Exception("PostgresSQL error executing function " - + functionName); + throw new Exception( + "PostgresSQL error executing function " + functionName); } } catch (Exception e) { log.error("Error updating/committing PE insert for PE " + shefData.getPhysicalElement()); log.error("Record Data: " + record); - log.error( - record.getTraceId() - + " - PostgresSQL error executing function " - + functionName, e); + log.error(record.getTraceId() + + " - PostgresSQL error executing function " + functionName, + e); } return status; } @@ -1009,13 +1063,13 @@ public class PostTables { * @return - status of action, 1 is good, 0 is bad */ private int execFunction(String functionName, ShefRecord record, - ShefData shefData, String locId, String dataValue, - String qualifier, long qualityCode, String productId, - Date productTime, Date postTime, String duplicateOption, - ShefConstants.IngestSwitch ingestSwitch, ShefStats stats) { + ShefData shefData, String locId, String dataValue, String qualifier, + long qualityCode, String productId, Date productTime, Date postTime, + String duplicateOption, ShefConstants.IngestSwitch ingestSwitch, + ShefStats stats) { CallableStatement cs = null; int status = -1; - if (dataValue.equals("")) { + if (dataValue.isEmpty()) { dataValue = ShefConstants.SHEF_MISSING; } try { @@ -1031,8 +1085,8 @@ public class PostTables { cs.setInt(3, shefData.getDurationValue()); cs.setString(4, shefData.getTypeSource().getCode()); cs.setString(5, shefData.getExtremum().getCode()); - cs.setTimestamp(6, new java.sql.Timestamp(shefData - .getObservationTimeObj().getTime())); + cs.setTimestamp(6, new java.sql.Timestamp( + shefData.getObservationTimeObj().getTime())); cs.setDouble(7, Double.parseDouble(dataValue)); cs.setString(8, qualifier); cs.setInt(9, (int) qualityCode); @@ -1058,16 +1112,16 @@ public class PostTables { if (status == 0) { conn.commit(); } else { - throw new Exception("PostgresSQL error executing function " - + functionName); + throw new Exception( + "PostgresSQL error executing function " + functionName); } } catch (Exception e) { log.error("Record Data: " + record); log.error(record.getTraceId() - + " - PostgresSQL error executing function " + functionName); - log.error( - "Error updating/committing PE insert for PE " - + shefData.getPhysicalElement(), e); + + " - PostgresSQL error executing function " + + functionName); + log.error("Error updating/committing PE insert for PE " + + shefData.getPhysicalElement(), e); } return status; } @@ -1080,19 +1134,19 @@ public class PostTables { * @return - status of action, 1 is good, 0 is bad */ private int execFcstFunc(String functionName, ShefRecord record, - ShefData shefData, String locId, String dataValue, - String qualifier, long qualityCode, String productId, - Date productTime, Date postTime, String duplicateOption, - ShefConstants.IngestSwitch ingestSwitch, ShefStats stats, - Date validTime) { - + ShefData shefData, String locId, String dataValue, String qualifier, + long qualityCode, String productId, Date productTime, Date postTime, + String duplicateOption, ShefConstants.IngestSwitch ingestSwitch, + ShefStats stats, Date validTime) { CallableStatement cs = null; java.sql.Timestamp timeStamp = null; int status = -1; - if (dataValue.equals("")) { + if (dataValue.isEmpty()) { dataValue = ShefConstants.SHEF_MISSING; } + int doOverwrite = determineUpdateAction(duplicateOption, + shefData.isRevisedRecord()); try { conn = getConnection(); cs = statementMap.get(functionName); @@ -1140,11 +1194,6 @@ public class PostTables { timeStamp = new java.sql.Timestamp(postTime.getTime()); cs.setTimestamp(15, timeStamp); - - int doOverwrite = 0; - - doOverwrite = determineUpdateAction(duplicateOption, - shefData.isRevisedRecord()); cs.setInt(16, doOverwrite); cs.registerOutParameter(17, java.sql.Types.INTEGER); @@ -1155,10 +1204,10 @@ public class PostTables { } catch (Exception e) { log.error("Record Data: " + record); log.error(record.getTraceId() - + " - PostgresSQL error executing function " + functionName); - log.error( - "Error updating/committing PE insert for PE " - + shefData.getPhysicalElement(), e); + + " - PostgresSQL error executing function " + + functionName); + log.error("Error updating/committing PE insert for PE " + + shefData.getPhysicalElement(), e); stats.incrementErrorMessages(); } @@ -1173,16 +1222,20 @@ public class PostTables { StringBuilder message = new StringBuilder("shef_duplicate : "); if ("ALWAYS_OVERWRITE".equals(token)) { gagePPOptions.setShef_duplicate(shef_dup.ALWAYS_OVERWRITE); - message.append("ALWAYS_OVERWRITE (always process duplicate reports)"); + message.append( + "ALWAYS_OVERWRITE (always process duplicate reports)"); } else if ("USE_REVCODE".equals(token)) { gagePPOptions.setShef_duplicate(shef_dup.USE_REVCODE); - message.append("USE_REVCODE (only process duplicate reports if they are revisions"); + message.append( + "USE_REVCODE (only process duplicate reports if they are revisions"); } else if ("IF_DIFFERENT_AND_REVCODE".equals(token)) { gagePPOptions.setShef_duplicate(shef_dup.IF_DIFFERENT_AND_REVCODE); - message.append("IF_DIFFERENT_OR_REVCODE (only process duplicate reports if they are revisions or their values are different"); + message.append( + "IF_DIFFERENT_OR_REVCODE (only process duplicate reports if they are revisions or their values are different"); } else { gagePPOptions.setShef_duplicate(shef_dup.IF_DIFFERENT); - message.append("IF_DIFFERENT (only process duplicate reports if their values are different"); + message.append( + "IF_DIFFERENT (only process duplicate reports if their values are different"); } log.info(message.toString()); @@ -1242,8 +1295,8 @@ public class PostTables { .floatValue(); ps.setFloat(6, probability); - timeStamp = new java.sql.Timestamp(shefDataValue - .getObservationTimeObj().getTime()); + timeStamp = new java.sql.Timestamp( + shefDataValue.getObservationTimeObj().getTime()); ps.setTimestamp(7, timeStamp); Date basisDate = shefDataValue.getCreationDateObj(); @@ -1269,7 +1322,8 @@ public class PostTables { } catch (Exception e) { if (updateFlag) { log.error(String.format( - "Error updating into RiverStatus with [%s]", record), e); + "Error updating into RiverStatus with [%s]", record), + e); } else { log.error(String.format( "Error inserting into RiverStatus with [%s]", record), @@ -1290,8 +1344,8 @@ public class PostTables { return status; } - private PreparedStatement getRiverStatusPreparedStatement(boolean updateFlag) - throws SQLException { + private PreparedStatement getRiverStatusPreparedStatement( + boolean updateFlag) throws SQLException { if (updateFlag) { PreparedStatement riverStatusUpdateStatement = conn .prepareCall(RIVER_STATUS_UPDATE_STATEMENT); @@ -1351,4 +1405,81 @@ public class PostTables { } } } -} + + private boolean checkForAndHandleDuplicate(final ShefData shefData, + final String locId, final String dataValue, + final String duplicateOption, final Class daoClass, + final Serializable id, final boolean alwaysUpdateIfMissing) { + if (daoClass == null || id == null) { + /* + * The Stored Procedure will not recognize the associated shef data. + */ + return false; + } + final CoreDao lookupDao = new CoreDao( + DaoConfig.forClass("ihfs", daoClass)); + PersistableDataObject pdo = lookupDao.queryById(id); + if (pdo == null) { + /* + * No duplicate records exist. + */ + return false; + } + /* + * Are updates allowed? + */ + final int doOverwrite = determineUpdateAction(duplicateOption, + shefData.isRevisedRecord()); + if (doOverwrite == ShefConstants.UPDATE_ACTION) { + /* + * Record will always be updated no matter what. + */ + return false; + } + if (pdo instanceof ICheckValue) { + /* + * Two possibilities here. (1) the value will always be updated if + * it is currently "missing" regardless of how the update action has + * been defined in Apps Defaults (2) the record will be updated if + * the new value is different per the update action specified in + * Apps Defaults. + */ + ICheckValue checkValue = (ICheckValue) pdo; + final String compareValue = checkValue.getCompareValue(); + if (compareValue == null) { + /* + * This should never happen because the current hydro ihfs has + * the concept of a Missing value filler. + */ + throw new IllegalStateException( + "Encountered unexpected null data value for " + + daoClass.getSimpleName() + + " with identifier: " + id.toString() + "."); + } + if ((alwaysUpdateIfMissing + && compareValue.startsWith(ShefConstants.SHEF_MISSING)) + || (doOverwrite == ShefConstants.IF_DIFFERENT_UPDATE_ACTION + && !compareValue.equals(dataValue))) { + return false; + } else { + /* + * The value is a duplicate and there is no path to update it. + * Ensure that no SQL procedures are executed beyond this point + * because they are guaranteed to fail. + */ + if (AppsDefaults.getInstance() + .getBoolean(ShefConstants.DUP_MESSAGE, false)) { + log.info("Ignoring duplicate " + daoClass.getSimpleName() + + " for: " + id.toString() + "."); + } + return true; + } + } + + /* + * Nothing to indicate that they record may not be a candidate for + * updates. + */ + return false; + } +} \ No newline at end of file diff --git a/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/util/ParserToken.java b/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/util/ParserToken.java index 267fadee53..6828ab1fee 100644 --- a/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/util/ParserToken.java +++ b/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/util/ParserToken.java @@ -21,15 +21,15 @@ package com.raytheon.edex.plugin.shef.util; import java.util.GregorianCalendar; import java.util.HashMap; +import java.util.Map; import java.util.TimeZone; import java.util.regex.Matcher; -import java.util.regex.Pattern; import com.raytheon.uf.common.dataplugin.shef.util.SHEFErrorCodes; import com.raytheon.uf.common.dataplugin.shef.util.ShefConstants; /** - * TODO + * Class that holds individual pieces of data for the SHEF Decoder. * * 
  *
@@ -38,19 +38,16 @@ import com.raytheon.uf.common.dataplugin.shef.util.ShefConstants;
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * Dec 17, 2009            jkorman     Initial creation
+ * Jan 10, 2018  5049      mduff       Changed how ShefParm is used in this class.
  *
  *
* * @author jkorman - * @version 1.0 */ public class ParserToken { - // Check for a qualifier that may be embedded in the data. - private static final Pattern BAD_DATA_1 = Pattern.compile("\\d*\\.{1}\\d*([A-Z]\\d{1,})*"); - - public static final HashMap DATE_TYPES = new HashMap(); + protected static final Map DATE_TYPES = new HashMap<>(); static { DATE_TYPES.put(TokenType.DATE_SEC, 0); DATE_TYPES.put(TokenType.DATE_MIN, 0); @@ -61,81 +58,92 @@ public class ParserToken { DATE_TYPES.put(TokenType.DATE_DATE, 0); DATE_TYPES.put(TokenType.DATE_CREATE, 0); DATE_TYPES.put(TokenType.DATE_JUL, 0); -// DATE_TYPES.put(TokenType.DATE_REL, 1); -// DATE_TYPES.put(TokenType.INT_CODE, 1); DATE_TYPES.put(TokenType.OBS_DATE_4, 4); DATE_TYPES.put(TokenType.OBS_DATE_6, 6); DATE_TYPES.put(TokenType.OBS_DATE_8, 8); } - + public static final int ERR_NO_ERROR = 0; - + public static final int ERR_INVALID_QUAL = -1; - + public static final int ERR_INV_JUL_DATE = -2; public static final int ERR_INV_CREATE_DATE = -3; - public static final int ERR_INV_SECONDS = -4; + public static final int ERR_INV_MINUTES = -5; + public static final int ERR_INV_HOURS = -6; + public static final int ERR_INV_DAY = -7; + public static final int ERR_INV_MONTH = -8; public static final int ERR_LOCID_NULL = -50; + public static final int ERR_LOCID_SHORT = -51; - public static final int ERR_LOCID_LONG = -52; + + public static final int ERR_LOCID_LONG = -52; + public static final int ERR_LOCID_INVCHAR = -53; - public static final int ERR_LOG035 = -135; + public static final int ERR_LOG044 = -144; + public static final int ERR_LOG079 = -179; - - + private static volatile ShefParm shefParm; + private final String rawToken; - + private String token; private String sendCode; - + private final TokenType type; - + private final boolean numeric; private final boolean qualifiedNumeric; private Double numericValue = null; - + private String qualifier = null; - + private boolean isTrace = false; - + // This value will be set for DT, DY, DM, DD, DH, DN, DS, and DC private SHEFDate dateData = null; - + private int error = ERR_NO_ERROR; - - private ParserToken(String token, TokenType type, boolean numeric, boolean qnumeric) { + + private ParserToken(String token, TokenType type, boolean numeric, + boolean qnumeric) { rawToken = token; this.token = token; this.type = type; this.numeric = numeric; this.qualifiedNumeric = qnumeric; + if (shefParm == null) { + shefParm = new ShefParm(); + shefParm.populate(); + } } - + /** - * Construct a ParserToken with a given token string. The TokenType - * is determined + * Construct a ParserToken with a given token string. The TokenType is + * determined + * * @param token */ public ParserToken(String token) { rawToken = token; this.token = token; - type = TokenType.getToken(token); - if(TokenType.ERROR.equals(type)) { + type = TokenType.getToken(token); + if (TokenType.ERROR.equals(type)) { error = ERR_INVALID_QUAL; } numeric = TokenType.NUMERIC.equals(type); @@ -145,8 +153,9 @@ public class ParserToken { } /** - * Construct a ParserToken with a given token string and an - * explicit TokenType. + * Construct a ParserToken with a given token string and an explicit + * TokenType. + * * @param token * @param type */ @@ -161,8 +170,9 @@ public class ParserToken { } /** - * Construct a ParserToken with a given token string and an - * explicit TokenType. + * Construct a ParserToken with a given token string and an explicit + * TokenType. + * * @param token * @param type */ @@ -171,7 +181,7 @@ public class ParserToken { this.token = token; this.type = type; error = errCode; - if(error == ERR_NO_ERROR) { + if (error == ERR_NO_ERROR) { numeric = TokenType.NUMERIC.equals(type); qualifiedNumeric = TokenType.QNUMERIC.equals(type); setDataValues(); @@ -181,7 +191,7 @@ public class ParserToken { qualifiedNumeric = false; } } - + /** * @return the rawToken */ @@ -196,7 +206,7 @@ public class ParserToken { public String getToken() { return token; } - + /** * @return the sendCode */ @@ -205,7 +215,8 @@ public class ParserToken { } /** - * @param sendCode the sendCode to set + * @param sendCode + * the sendCode to set */ public void setSendCode(String sendCode) { this.sendCode = sendCode; @@ -216,21 +227,22 @@ public class ParserToken { } /** - * Is this token numeric. Checks are made for - * integer and simple decimal numbers. + * Is this token numeric. Checks are made for integer and simple decimal + * numbers. + * * @return Is this a numeric token. */ public boolean isNumeric() { return numeric; } - + /** * @return the qualifiedNumeric */ public boolean isQualifiedNumeric() { return qualifiedNumeric; } - + /** * @return the numericValue */ @@ -239,7 +251,8 @@ public class ParserToken { } /** - * @param numericValue the numericValue to set + * @param numericValue + * the numericValue to set */ public void setNumericValue(Double numericValue) { this.numericValue = numericValue; @@ -253,7 +266,8 @@ public class ParserToken { } /** - * @param qualifier the qualifier to set + * @param qualifier + * the qualifier to set */ public void setQualifier(String qualifier) { this.qualifier = qualifier; @@ -267,14 +281,16 @@ public class ParserToken { } /** - * @param dateData the dateData to set + * @param dateData + * the dateData to set */ public void setDateData(SHEFDate dateData) { this.dateData = dateData; } - + /** * Was the numeric value derived from T (trace) + * * @return the isTrace */ public boolean isTrace() { @@ -282,7 +298,8 @@ public class ParserToken { } /** - * @param isTrace the isTrace to set + * @param isTrace + * the isTrace to set */ public void setTrace(boolean isTrace) { this.isTrace = isTrace; @@ -295,45 +312,47 @@ public class ParserToken { public int getError() { return error; } - + public void setError(int errorCode) { error = errorCode; } /** * Is this token a value type. + * * @return */ public boolean isValueToken() { - return (TokenType.NUMERIC.equals(type) || TokenType.QNUMERIC.equals(type)); + return (TokenType.NUMERIC.equals(type) + || TokenType.QNUMERIC.equals(type)); } - + public void adjustToTimezone(TimeZone tz) { - if(dateData != null) { + if (dateData != null) { dateData.setTimeZone(tz); dateData.adjustToTimezone(); token = dateData.toLocal(); - + // Did we set a julian day? int ddd = getDateData().getJulian(); - if(ddd > 0) { + if (ddd > 0) { GregorianCalendar c = new GregorianCalendar(tz); int year = getDateData().getYear(); - if(ddd > 365) { - if(!c.isLeapYear(year)) { + if (ddd > 365) { + if (!c.isLeapYear(year)) { error = ERR_INV_JUL_DATE; } } } error = dateData.getError(); - if(dateData.isDSTExclusion()) { + if (dateData.isDSTExclusion()) { error = ERR_LOG044; } } } - + private void setDateValues() { - if(DATE_TYPES.containsKey(type)) { + if (DATE_TYPES.containsKey(type)) { int ddd = 0; int second = 0; int minute = 0; @@ -345,101 +364,100 @@ public class ParserToken { // 2 digit century int cc = 0; StringBuilder dateData = null; - if(token.startsWith("D")) { + if (token.startsWith("D")) { dateData = new StringBuilder(token.substring(2)); } else { dateData = new StringBuilder(token); } boolean fallThrough = false; - switch(type) { - case DATE_YEAR : { + switch (type) { + case DATE_YEAR: { cc = -1; ly = getDateItem(dateData); } - case DATE_MON : { + case DATE_MON: { month = getDateItem(dateData); } - case DATE_DAY : { + case DATE_DAY: { day = getDateItem(dateData); hour = getDateItem(dateData); minute = getDateItem(dateData); second = getDateItem(dateData); - break; } - case DATE_HOUR : { + case DATE_HOUR: { hour = getDateItem(dateData); } - case DATE_MIN : { + case DATE_MIN: { minute = getDateItem(dateData); - if((minute < 0)&&(!fallThrough)) { + if ((minute < 0) && (!fallThrough)) { minute = 0; } } - case DATE_SEC : { + case DATE_SEC: { second = getDateItem(dateData); - if((second < 0)&&(!fallThrough)) { + if ((second < 0) && (!fallThrough)) { second = 0; } break; } - case DATE_DATE : { + case DATE_DATE: { cc = getDateItem(dateData); ly = getDateItem(dateData); month = getDateItem(dateData); day = getDateItem(dateData); - if(second == -1) { + if (second == -1) { second = 0; } hour = getDateItem(dateData); minute = getDateItem(dateData); - if(minute == -1) { + if (minute == -1) { minute = 0; } second = getDateItem(dateData); - if(second == -1) { + if (second == -1) { second = 0; } break; } - case DATE_JUL : { - switch(dateData.length()) { - case 7 : { + case DATE_JUL: { + switch (dateData.length()) { + case 7: { cc = getDateItem(dateData); } - case 5 : { + case 5: { ly = getDateItem(dateData); } - case 1 : - case 2 : - case 3 : { + case 1: + case 2: + case 3: { ddd = Integer.parseInt(dateData.toString()); break; } - default : { + default: { error = ERR_LOG079; } } // switch() break; } - case OBS_DATE_8 : - case OBS_DATE_6 : - case OBS_DATE_4 : { + case OBS_DATE_8: + case OBS_DATE_6: + case OBS_DATE_4: { fallThrough = false; - switch(dateData.length()) { - case 8 : { + switch (dateData.length()) { + case 8: { fallThrough = true; } - case 6 : { - if(fallThrough) { + case 6: { + if (fallThrough) { cc = getDateItem(dateData); } else { cc = -1; } fallThrough = true; } - case 4 : { - if(fallThrough) { + case 4: { + if (fallThrough) { ly = getDateItem(dateData); } else { cc = -1; @@ -449,25 +467,25 @@ public class ParserToken { day = getDateItem(dateData); hour = -1; } - } + } break; } - case DATE_CREATE : { + case DATE_CREATE: { fallThrough = false; - switch(dateData.length()) { - case 12 : { + switch (dateData.length()) { + case 12: { fallThrough = true; } - case 10 : { - if(fallThrough) { + case 10: { + if (fallThrough) { cc = getDateItem(dateData); } else { cc = -1; } fallThrough = true; } - case 8 : { - if(fallThrough) { + case 8: { + if (fallThrough) { ly = getDateItem(dateData); } else { cc = -1; @@ -480,7 +498,7 @@ public class ParserToken { second = 0; break; } - case 6 : { // MMDDHH + case 6: { // MMDDHH cc = -1; ly = -1; month = getDateItem(dateData); @@ -490,7 +508,7 @@ public class ParserToken { second = 0; break; } - case 4 : { // MMDD + case 4: { // MMDD cc = -1; ly = -1; month = getDateItem(dateData); @@ -500,7 +518,7 @@ public class ParserToken { second = 0; break; } - default : { + default: { error = ERR_INV_CREATE_DATE; } } @@ -508,8 +526,9 @@ public class ParserToken { } } // Check for a julian day (day of year). - if(ddd == 0) { - setDateData(new SHEFDate(cc,ly,month,day,hour,minute,second)); + if (ddd == 0) { + setDateData( + new SHEFDate(cc, ly, month, day, hour, minute, second)); } else { SHEFDate d = new SHEFDate(); d.setCentury(cc); @@ -517,23 +536,23 @@ public class ParserToken { d.setJulian(ddd); setDateData(d); } - - if(month > 12) { + + if (month > 12) { error = ERR_INV_MONTH; - } else if((month > 0) && (day > SHEFDate.DAYS_MONTH[month])) { + } else if ((month > 0) && (day > SHEFDate.DAYS_MONTH[month])) { error = ERR_INV_DAY; - } else if(hour > 24) { + } else if (hour > 24) { error = ERR_INV_HOURS; - } else if(minute > 59) { + } else if (minute > 59) { error = ERR_INV_MINUTES; - } else if(second > 59) { + } else if (second > 59) { error = ERR_INV_SECONDS; } - + token = getDateData().toString(); } } - + /** * * @param data @@ -541,26 +560,27 @@ public class ParserToken { */ private static int getDateItem(StringBuilder data) { int intData = -1; - if(data.length() >= 2) { - intData = Integer.parseInt(data.substring(0,2)); - data.delete(0,2); + if (data.length() >= 2) { + intData = Integer.parseInt(data.substring(0, 2)); + data.delete(0, 2); } return intData; } /** - * Use the derived type information to populate the numeric data - * for this token if it is numeric or qualified numeric. + * Use the derived type information to populate the numeric data for this + * token if it is numeric or qualified numeric. */ private void setDataValues() { - if(numeric) { + if (numeric) { numericValue = Double.parseDouble(token); } else if (qualifiedNumeric) { - numericValue = Double.parseDouble(token.substring(0,token.length()-1)); - qualifier = token.substring(token.length()-1); + numericValue = Double + .parseDouble(token.substring(0, token.length() - 1)); + qualifier = token.substring(token.length() - 1); } } - + /** * * @param locId @@ -568,15 +588,15 @@ public class ParserToken { */ public static final ParserToken createLocIdToken(String locId) { ParserToken t = null; - if(locId != null) { + if (locId != null) { Matcher m = TokenType.LOC_ID.getPattern().matcher(locId); - if(m.matches()) { + if (m.matches()) { t = new ParserToken(locId, TokenType.LOC_ID); } else { - t = new ParserToken(locId,TokenType.ERROR); + t = new ParserToken(locId, TokenType.ERROR); // locId is no good, find out why int n = locId.length(); - if(n < 3) { + if (n < 3) { t.error = ERR_LOCID_SHORT; } else if (n > 8) { t.error = ERR_LOCID_LONG; @@ -585,16 +605,15 @@ public class ParserToken { } } } else { - t = new ParserToken("",TokenType.ERROR); + t = new ParserToken("", TokenType.ERROR); t.error = ERR_LOCID_NULL; } return t; } - - + /** - * This is a factory method that takes a ParserToken with the type set - * to unknown and attempts to do further analysis of the unknown text. + * This is a factory method that takes a ParserToken with the type set to + * unknown and attempts to do further analysis of the unknown text. * * @return */ @@ -602,114 +621,115 @@ public class ParserToken { ParserToken pToken = null; TokenType type = TokenType.UNKNOWN; int error = ERR_NO_ERROR; - - if(token != null) { - if(rawToken.length() > 2) { - String s = token.substring(0,2); + + if (token != null) { + if (rawToken.length() > 2) { + String s = token.substring(0, 2); String ss = token.substring(2); - - if("DS".equals(s)) { -// DATE_SEC("DS\\d{2}"), // SS + + if ("DS".equals(s)) { + // DATE_SEC("DS\\d{2}"), // SS type = TokenType.DATE_SEC; - if(ss.length() != 2) { + if (ss.length() != 2) { error = SHEFErrorCodes.LOG_016; } else { error = evaluateDateTime(ss); } - } else if("DN".equals(s)) { -// DATE_MIN("DN\\d{2,4}"), // NNSS + } else if ("DN".equals(s)) { + // DATE_MIN("DN\\d{2,4}"), // NNSS type = TokenType.DATE_MIN; - if((ss.length() < 2)||(ss.length() > 4)) { + if ((ss.length() < 2) || (ss.length() > 4)) { error = SHEFErrorCodes.LOG_016; } else { error = evaluateDateTime(ss); } - } else if("DH".equals(s)) { -// DATE_HOUR("DH\\d{2,6}"), // HHNNSS + } else if ("DH".equals(s)) { + // DATE_HOUR("DH\\d{2,6}"), // HHNNSS type = TokenType.DATE_HOUR; - if((ss.length() < 2)||(ss.length() > 6)) { + if ((ss.length() < 2) || (ss.length() > 6)) { error = SHEFErrorCodes.LOG_016; } else { error = evaluateDateTime(ss); } - } else if("DD".equals(s)) { -// DATE_DAY("DD\\d{2,8}"), // DDHHNNSS + } else if ("DD".equals(s)) { + // DATE_DAY("DD\\d{2,8}"), // DDHHNNSS type = TokenType.DATE_DAY; - if((ss.length() < 2)||(ss.length() > 8)) { + if ((ss.length() < 2) || (ss.length() > 8)) { error = SHEFErrorCodes.LOG_016; } else { error = evaluateDateTime(ss); } error = evaluateDateTime(ss); - } else if("DM".equals(s)) { -// DATE_MON("DM\\d{2,10}"), // MMDDHHNNSS + } else if ("DM".equals(s)) { + // DATE_MON("DM\\d{2,10}"), // MMDDHHNNSS type = TokenType.DATE_MON; - if((ss.length() < 2)||(ss.length() > 10)) { + if ((ss.length() < 2) || (ss.length() > 10)) { error = SHEFErrorCodes.LOG_016; } else { error = evaluateDateTime(ss); } - } else if("DY".equals(s)) { -// DATE_YEAR("DY\\d{2,12}"), // YYMMDDHHNNSS + } else if ("DY".equals(s)) { + // DATE_YEAR("DY\\d{2,12}"), // YYMMDDHHNNSS type = TokenType.DATE_YEAR; - if((ss.length() < 2)||(ss.length() > 12)) { + if ((ss.length() < 2) || (ss.length() > 12)) { error = SHEFErrorCodes.LOG_016; } else { error = evaluateDateTime(ss); } - } else if("DT".equals(s)) { -// DATE_DATE("DT\\d{2,14}"), // CCYYMMDDHHNNSS + } else if ("DT".equals(s)) { + // DATE_DATE("DT\\d{2,14}"), // CCYYMMDDHHNNSS type = TokenType.DATE_DATE; - if((ss.length() < 2)||(ss.length() > 14)) { + if ((ss.length() < 2) || (ss.length() > 14)) { error = SHEFErrorCodes.LOG_016; } else { error = evaluateDateTime(ss); } - } else if("DJ".equals(s)) { -// DATE_JUL("DJ\\d{3,7}"), // CCYYDDD | YYDDD | DDD + } else if ("DJ".equals(s)) { + // DATE_JUL("DJ\\d{3,7}"), // CCYYDDD | YYDDD | DDD type = TokenType.DATE_JUL; error = evaluateDateTime(ss); - } else if("DC".equals(s)) { -// DATE_CREATE("DC\\d{4,12}"), // CCYYMMDDHHNN + } else if ("DC".equals(s)) { + // DATE_CREATE("DC\\d{4,12}"), // CCYYMMDDHHNN type = TokenType.DATE_CREATE; - if((ss.length() < 2)||(ss.length() > 12)) { + if ((ss.length() < 2) || (ss.length() > 12)) { error = SHEFErrorCodes.LOG_016; } else { error = evaluateDateTime(ss); } -//---------------------- -//done with date codes -//---------------------- - } else if("DU".equals(s)) { -// UNITS_CODE("DU[ES]"), // DUu + // ---------------------- + // done with date codes + // ---------------------- + } else if ("DU".equals(s)) { + // UNITS_CODE("DU[ES]"), // DUu type = TokenType.UNITS_CODE; error = SHEFErrorCodes.LOG_022; - } else if("DQ".equals(s)) { -// QUAL_CODE("DQ[A-Z]"), // DQq + } else if ("DQ".equals(s)) { + // QUAL_CODE("DQ[A-Z]"), // DQq type = TokenType.QUAL_CODE; - if(ss.length() == 1) { - if(!isValidQualifier(ss)) { + if (ss.length() == 1) { + if (!isValidQualifier(ss)) { error = SHEFErrorCodes.LOG_084; } } else { error = SHEFErrorCodes.LOG_021; } - } else if("DR".equals(s)) { -// DATE_REL("(DR)([SNHDMEY])([\\+-]?)(\\d{1,2})"), // DRtXX + } else if ("DR".equals(s)) { + // DATE_REL("(DR)([SNHDMEY])([\\+-]?)(\\d{1,2})"), // DRtXX type = TokenType.DATE_REL; type = TokenType.DUR_CODE; - if(ss.length() == 0) { + if (ss.length() == 0) { error = SHEFErrorCodes.LOG_027; } else { - if(isValidIntervalCode(ss.charAt(0))) { + if (isValidIntervalCode(ss.charAt(0))) { ss = ss.substring(1); - if((ss.length() > 0) && isSign(ss.substring(0,1))) { + if ((ss.length() > 0) + && isSign(ss.substring(0, 1))) { ss = ss.substring(1); } - if(!isAllDigits(ss)) { + if (!isAllDigits(ss)) { error = SHEFErrorCodes.LOG_028; } else { - if(ss.length() > 3) { + if (ss.length() > 3) { error = SHEFErrorCodes.LOG_103; } } @@ -717,17 +737,17 @@ public class ParserToken { error = SHEFErrorCodes.LOG_027; } } - } else if("DV".equals(s)) { -// DUR_CODE("DV[SNHDMYZ]\\d{1,2}"), // DVvXX + } else if ("DV".equals(s)) { + // DUR_CODE("DV[SNHDMYZ]\\d{1,2}"), // DVvXX type = TokenType.DUR_CODE; - if(ss.length() == 0) { + if (ss.length() == 0) { error = SHEFErrorCodes.LOG_023; } else { - if(isValidDurationCode(ss.charAt(0))) { - if(!isAllDigits(ss)) { + if (isValidDurationCode(ss.charAt(0))) { + if (!isAllDigits(ss)) { error = SHEFErrorCodes.LOG_024; } else { - if(ss.length() > 2) { + if (ss.length() > 2) { error = SHEFErrorCodes.LOG_104; } } @@ -735,21 +755,22 @@ public class ParserToken { error = SHEFErrorCodes.LOG_023; } } - } else if("DI".equals(s)) { -// INT_CODE("(DI)([SNHDMEY])([\\+-]?)(\\d{1,2})"), // DIvXX + } else if ("DI".equals(s)) { + // INT_CODE("(DI)([SNHDMEY])([\\+-]?)(\\d{1,2})"), // DIvXX type = TokenType.INT_CODE; - if(ss.length() == 0) { + if (ss.length() == 0) { error = SHEFErrorCodes.LOG_025; } else { - if(isValidIntervalCode(ss.charAt(0))) { + if (isValidIntervalCode(ss.charAt(0))) { ss = ss.substring(1); - if((ss.length() > 0) && isSign(ss.substring(0,1))) { + if ((ss.length() > 0) + && isSign(ss.substring(0, 1))) { ss = ss.substring(1); } - if(!isAllDigits(ss)) { + if (!isAllDigits(ss)) { error = SHEFErrorCodes.LOG_026; } else { - if(ss.length() > 2) { + if (ss.length() > 2) { error = SHEFErrorCodes.LOG_102; } } @@ -757,8 +778,6 @@ public class ParserToken { error = SHEFErrorCodes.LOG_025; } } - } else { - } } } @@ -767,52 +786,6 @@ public class ParserToken { return pToken; } - /** - * Enter with a unknown parser token to analyze. This should only be - * used once the positional data has been established. Gets us past - * the location identifier, timezone issues. - * @param token - * @return - */ - public ParserToken analyzeUnknown(String token) { - ParserToken t = null; - if(token != null) { - if((token.length() == 1) || (token.length() == 2)) { - // the only legal 1 character are timezones - // and single digits. These will have already - // been identified, so just set up an error. - t = new ParserToken(token,TokenType.UNKNOWN); - t.setError(64); - } else if (token.length() > 2) { - if(token.indexOf(',') > 0) { - // some sort of numeric that isn't right. - t = new ParserToken(token,TokenType.UNKNOWN); - t.setError(65); - } else if(isAllDigitPunctuation(token)) { - // some sort of numeric that isn't right. - t = new ParserToken(token,TokenType.UNKNOWN); - t.setError(78); - } else { - Matcher m = BAD_DATA_1.matcher(token); - if(m.matches()) { - t = new ParserToken(token,TokenType.UNKNOWN); - t.setError(78); - } - } - } - } - if(t == null) { - t = new ParserToken(token,TokenType.UNKNOWN); - t.setError(1); - } - return t; - } - - /** - * - * @param t - * @return - */ public static ParserToken identifyUnknown(ParserToken t) { ParserToken newToken = null; Character PLUS = '+'; @@ -825,47 +798,49 @@ public class ParserToken { StringBuilder sb = new StringBuilder(); String qual = null; - boolean seenSign = false; boolean seenWhole = false; boolean seenFrac = false; - + int error = 0; if (target.length() > 0) { // eat any leading space - int index = -1; - Character c = getChar(target, ++index); + int index = 0; + Character c = getChar(target, index); if (c != null) { if (PLUS.equals(c)) { - seenSign = true; sb.append(c); - c = getChar(target, ++index); + ++index; + c = getChar(target, index); } else if (MINUS.equals(c)) { - seenSign = true; sb.append(c); - c = getChar(target, ++index); + ++index; + c = getChar(target, index); } if (c != null) { - if(Character.isDigit(c)) { + if (Character.isDigit(c)) { while (isDigit(c)) { sb.append(c); - c = getChar(target, ++index); - if(c == null) { + ++index; + c = getChar(target, index); + if (c == null) { break; } } seenWhole = true; } if (c != null) { - if(DECIMAL.equals(c)) { + if (DECIMAL.equals(c)) { sb.append(c); - c = getChar(target, ++index); + ++index; + c = getChar(target, index); if (c != null) { - if(Character.isDigit(c)) { + if (Character.isDigit(c)) { while (isDigit(c)) { sb.append(c); - c = getChar(target, ++index); - if(c == null) { + ++index; + c = getChar(target, index); + if (c == null) { break; } } @@ -873,43 +848,51 @@ public class ParserToken { } } } - if(c != null) { - if(seenFrac || seenWhole) { + if (c != null) { + if (seenFrac || seenWhole) { // From here we're looking for a qualifier qual = String.valueOf(c); - c = getChar(target, ++index); + ++index; + c = getChar(target, index); } // Shouldn't see another character after // the qualifier. - if(c != null) { + if (c != null) { error = -1; newToken = t; - if(seenFrac || seenWhole) { - newToken.setError(SHEFErrorCodes.LOG_084); + if (seenFrac || seenWhole) { + newToken.setError( + SHEFErrorCodes.LOG_084); } else { - newToken.setError(SHEFErrorCodes.LOG_078); + newToken.setError( + SHEFErrorCodes.LOG_078); } } } } - } } + } + } } - if(error == 0) { - if(seenFrac || seenWhole) { - if(qual != null) { - newToken = new ParserToken(sb.toString(), TokenType.QNUMERIC, false, true); + if (error == 0) { + if (seenFrac || seenWhole) { + if (qual != null) { + newToken = new ParserToken(sb.toString(), + TokenType.QNUMERIC, false, true); } else { - newToken = new ParserToken(sb.toString(), TokenType.NUMERIC, true, false); + newToken = new ParserToken(sb.toString(), + TokenType.NUMERIC, true, false); } newToken.setDataValues(); newToken.setError(ERR_NO_ERROR); } else { - if("+.".equals(sb.toString())) { - newToken = new ParserToken("-9999", TokenType.NUMERIC, true, false); + if ("+.".equals(sb.toString())) { + newToken = new ParserToken("-9999", TokenType.NUMERIC, + true, false); newToken.setDataValues(); newToken.setError(ERR_NO_ERROR); - } else if("-.".equals(sb.toString())) { - newToken = new ParserToken("-9999", TokenType.NUMERIC, true, false); + } else if ("-.".equals(sb.toString())) { + newToken = new ParserToken("-9999", TokenType.NUMERIC, + true, false); newToken.setDataValues(); newToken.setError(ERR_NO_ERROR); } @@ -918,7 +901,7 @@ public class ParserToken { } return newToken; } - + /** * * @param s @@ -939,16 +922,17 @@ public class ParserToken { /** * Is the target string all digits or punctuation? + * * @param s * @return */ public static final boolean isAllDigitPunctuation(String s) { boolean retValue = true; - if(s != null) { - for(int i = 0;i < s.length();i++) { + if (s != null) { + for (int i = 0; i < s.length(); i++) { char c = s.charAt(i); - if(!isDigit(c)) { - if(!isPunctuation(c)) { + if (!isDigit(c)) { + if (!isPunctuation(c)) { retValue = false; break; } @@ -959,11 +943,11 @@ public class ParserToken { } return retValue; } - + public static final boolean isPunctuation(char c) { - return ",.-_=+!@#$%^&*(){}[]|\\;<>".indexOf(c) > 0; + return ",.-_=+!@#$%^&*(){}[]|\\;<>".indexOf(c) >= 0; } - + /** * * @param c @@ -971,12 +955,12 @@ public class ParserToken { */ public static final boolean isSign(String s) { boolean isSign = false; - if(s != null) { + if (s != null) { isSign = (s.startsWith("+") || s.startsWith("-")); } return isSign; } - + /** * * @param c @@ -985,7 +969,7 @@ public class ParserToken { public static final boolean isDigit(char c) { return "0123456789".indexOf(c) > -1; } - + /** * * @param c @@ -994,16 +978,15 @@ public class ParserToken { public static final boolean isNotDigit(char c) { return !isDigit(c); } - + public static final boolean isValidQualifier(String s) { - return (ShefParm.getDataQualifierCodes(s) != null); + return (shefParm.getDataQualifierCodes(s) != null); } - + public static final boolean isValidQualifier(Character c) { - return (ShefParm.getDataQualifierCodes(String.valueOf(c)) != null); + return (shefParm.getDataQualifierCodes(String.valueOf(c)) != null); } - - + /** * * @param c @@ -1012,7 +995,7 @@ public class ParserToken { public static final boolean isValidDurationCode(char c) { return ShefConstants.DURATION_CODES.indexOf(c) > -1; } - + /** * * @param c @@ -1021,7 +1004,7 @@ public class ParserToken { public static final boolean isNotValidDurationCode(char c) { return !isValidDurationCode(c); } - + /** * * @param c @@ -1030,7 +1013,7 @@ public class ParserToken { public static final boolean isValidIntervalCode(char c) { return ShefConstants.DATE_INC_CODES.indexOf(c) > -1; } - + /** * * @param c @@ -1047,28 +1030,29 @@ public class ParserToken { */ public static final int evaluateDateTime(String s) { int error = SHEFErrorCodes.LOG_000; - if(s != null) { - if(!isAllDigits(s)) { + if (s != null) { + if (!isAllDigits(s)) { error = SHEFErrorCodes.LOG_002; } else { - if((s.length() % 2) == 1) { + if ((s.length() % 2) == 1) { error = SHEFErrorCodes.LOG_002; } } } return error; } - + /** * Is the target string all digits? + * * @param s * @return */ public static final boolean isAllDigits(String s) { boolean retValue = true; - if(s != null) { - for(int i = 0;i < s.length();i++) { - if(isNotDigit(s.charAt(i))) { + if (s != null) { + for (int i = 0; i < s.length(); i++) { + if (isNotDigit(s.charAt(i))) { retValue = false; break; } @@ -1078,8 +1062,7 @@ public class ParserToken { } return retValue; } - - + @Override public String toString() { StringBuilder sb = new StringBuilder("ParserToken:{"); @@ -1090,119 +1073,18 @@ public class ParserToken { sb.append((isNumeric() ? "numeric" : "nonnumeric")); sb.append("}:{"); sb.append((isQualifiedNumeric() ? "qnumeric}" : "nonqnumeric}")); - if(isNumeric() || isQualifiedNumeric()) { + if (isNumeric() || isQualifiedNumeric()) { sb.append("}:["); sb.append(numericValue); sb.append("]{"); sb.append(qualifier); sb.append("}"); } - if(dateData != null) { + if (dateData != null) { sb.append("["); sb.append(dateData.toString()); sb.append("]"); } return sb.toString(); } - - public static final void main(String [] args) { - -// ParserToken t = new ParserToken("001"); -// System.out.println(t.type + " " + t.isNumeric() + " " + t.isQualifiedNumeric()); -// -// t = new ParserToken("DH0311"); -// System.out.println(t.type + " " + t.isNumeric() + " " + t.isQualifiedNumeric()); -// -// t = new ParserToken("3.2M"); -// System.out.println(t.type + " " + t.isNumeric() + " " + t.isQualifiedNumeric()); - -// System.out.println(new SHEFDate(20,11,11,31,18,11,27)); -// -// ParserToken pt = new ParserToken("20111131000000"); -// System.out.println(pt); -// System.out.println(pt.getError()); -// -// pt = new ParserToken("DT20010812124530"); -// System.out.println(pt); -// System.out.println(pt.getError()); -// -// String[] locIds = { null, "", "TO", "TOP", "TO%40", "_TO", "TOP_5", -// "ABCDEFGH", "ABCDEFGHI", }; -// int[] locErrs = { ERR_LOCID_NULL, ERR_LOCID_SHORT, ERR_LOCID_SHORT, -// ERR_NO_ERROR, ERR_LOCID_INVCHAR, ERR_NO_ERROR, ERR_NO_ERROR, ERR_NO_ERROR, ERR_LOCID_LONG}; -// public static final int ERR_LOCID_NULL = -50; -// public static final int ERR_LOCID_SHORT = -51; -// public static final int ERR_LOCID_LONG = -52; -// public static final int ERR_LOCID_INVCHAR = -53; - -// int i = 0; -// for(String s : locIds) { -// ParserToken id = createLocIdToken(s); -// System.out.println(String.format("%10s %10b %3d %s",s, (id.getError() == locErrs[i++]), id.getError(), id.getType())); -// } - -// TimeZone tz = TimeZone.getTimeZone("US/Central"); -// ParserToken pt = new ParserToken("DJ84366"); -// System.out.println(pt); -// pt.adjustToTimezone(tz); -// System.out.println(pt); -// -// GregorianCalendar c = new GregorianCalendar(tz); -// System.out.println(c.isLeapYear(1984)); -// -// String s = "DI+I0"; -// -// String s1 = s.substring(0,2); -// -// String s2 = s.substring(2); -// -// System.out.println(String.format("\"%s\" [%s].%d [%s].%d", s, s1, s1.length(), s2, s2.length())); -// -// if (s2.startsWith("+") || s2.startsWith("-")) { -// System.out.println("Stripping sign"); -// s2 = s2.substring(1); -// } -// if (isAllDigits(s2)) { -// System.out.println("All is well"); -// } else { -// System.out.println("Bad date interval"); -// } -// -// System.out.println(String.format("\"%s\" [%s].%d [%s].%d", s, s1, s1.length(), s2, s2.length())); -// -// System.out.println("32".length() % 2); -// -// pt = new ParserToken("DM801"); -// System.out.println(pt); -// -// pt = new ParserToken("20110502", TokenType.OBS_DATE_6); -// pt.adjustToTimezone(SHEFTimezone.GMT_TIMEZONE); -// pt.dateData.applyData(new ParserToken("DH1200")); -// -// System.out.println(pt + " [Error = " + pt.getError() + "]"); -// - - - ParserToken t = new ParserToken("3.3A6"); - System.out.println(t + " error = " + t.getError()); - ParserToken tt = t.analyzeUnknown(t.getRawToken()); - System.out.println(tt + " error = " + tt.getError()); - - isAllDigitPunctuation("3.1+"); - -// .E EE0116 820101 Z DH12 -// .E1 Z DH12/HG/DIH06/2.1/2.2 -// ? -// ** ERROR 64 ** Parameter code too short or field misinterpreted as param-code - - t = new ParserToken("DH04"); - System.out.println(t); - - ParserToken p = new ParserToken("3.2+"); - System.out.println(p); - System.out.println(p.getError()); - - - - } } diff --git a/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/util/ShefParm.java b/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/util/ShefParm.java index a144d27ae9..054d711e87 100644 --- a/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/util/ShefParm.java +++ b/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/util/ShefParm.java @@ -45,51 +45,49 @@ import com.raytheon.uf.common.localization.PathManagerFactory; * ------------ ---------- ----------- -------------------------- * Apr 18, 2008 387 M. Duff Initial Version. * Dec 16, 2015 5166 kbisanz Update logging to use SLF4J + * Jan 10, 2018 5049 mduff Made class not so static. * * * * @author mduff - * @version 1.0 */ public class ShefParm { - static { - log = LoggerFactory - .getLogger(com.raytheon.edex.plugin.shef.util.ShefParm.class); - physicalElementConversion = new HashMap(); - durationCodeValues = new HashMap(); - typeSourceCodes = new HashMap(); - extremumCodes = new HashMap(); - probabilityCodeValues = new HashMap(); - sendCodesDuration = new HashMap(); - dataQualifierCodes = new HashMap(); - MAX_ERRORS = 0; - populate(); - } - - private static Map physicalElementConversion; - - private static Map durationCodeValues; - - private static Map typeSourceCodes; - - private static Map extremumCodes; - - private static Map probabilityCodeValues; - - private static Map sendCodesDuration; - - private static Map dataQualifierCodes; - - private static int MAX_ERRORS; - - private static final Logger log; - private static final String PLUGIN_NAME = "shef"; private static final String PROPFILE_NAME = "SHEFPARM"; - private static int fileSection = 0; + private Map physicalElementConversion; + + private Map durationCodeValues; + + private Map typeSourceCodes; + + private Map extremumCodes; + + private Map probabilityCodeValues; + + private Map sendCodesDuration; + + private Map dataQualifierCodes; + + private int MAX_ERRORS; + + private static final Logger log = LoggerFactory + .getLogger(com.raytheon.edex.plugin.shef.util.ShefParm.class);; + + private int fileSection = 0; + + public ShefParm() { + durationCodeValues = new HashMap<>(); + typeSourceCodes = new HashMap<>(); + extremumCodes = new HashMap<>(); + probabilityCodeValues = new HashMap<>(); + sendCodesDuration = new HashMap<>(); + dataQualifierCodes = new HashMap<>(); + physicalElementConversion = new HashMap<>(); + MAX_ERRORS = 0; + } /** * Get a Physical Element conversion factor @@ -98,7 +96,7 @@ public class ShefParm { * - Physical Element * @return - the conversion factor */ - public static Double getPhysicalElementConversionFactor(String key) { + public Double getPhysicalElementConversionFactor(String key) { return physicalElementConversion.get(key); } @@ -109,7 +107,7 @@ public class ShefParm { * - Duration Code * @return - the Duration Code value */ - public static Short getDurationCodeValue(String key) { + public Short getDurationCodeValue(String key) { return durationCodeValues.get(key); } @@ -120,7 +118,7 @@ public class ShefParm { * - Type Source * @return - 1 if valid code, null if invalid */ - public static Integer getTypeSourceCode(String key) { + public Integer getTypeSourceCode(String key) { return typeSourceCodes.get(key); } @@ -131,7 +129,7 @@ public class ShefParm { * - Extremum Code * @return - 1 if valid code, null if invalid */ - public static Integer getExtremumCode(String key) { + public Integer getExtremumCode(String key) { return extremumCodes.get(key); } @@ -142,7 +140,7 @@ public class ShefParm { * - Probability Code * @return - Probability Code's value */ - public static Float getProbabilityCodeValue(String key) { + public Float getProbabilityCodeValue(String key) { return probabilityCodeValues.get(key); } @@ -153,7 +151,7 @@ public class ShefParm { * - Code * @return - Default Values */ - public static String getSendCodeDurationDefaults(String key) { + public String getSendCodeDurationDefaults(String key) { return sendCodesDuration.get(key); } @@ -164,7 +162,7 @@ public class ShefParm { * - Extremum Code * @return - 1 if valid code, null if invalid */ - public static Integer getDataQualifierCodes(String key) { + public Integer getDataQualifierCodes(String key) { return dataQualifierCodes.get(key); } @@ -173,14 +171,14 @@ public class ShefParm { * * @return - the maximum number of errors */ - public static Integer getMaxErrors() { + public Integer getMaxErrors() { return MAX_ERRORS; } /** * Populate the values from the file. */ - private static void populate() { + public void populate() { PathManager pathMgr = (PathManager) PathManagerFactory.getPathManager(); @@ -189,43 +187,18 @@ public class ShefParm { File baseDir = pathMgr.getFile(ctx, PLUGIN_NAME); File srcFile = new File(baseDir, PROPFILE_NAME); - BufferedReader in = null; - try { - in = new BufferedReader(new FileReader(srcFile)); + try (BufferedReader in = new BufferedReader(new FileReader(srcFile))) { String line = null; while ((line = in.readLine()) != null) { processLine(line); } in.close(); } catch (IOException ioe) { - ioe.printStackTrace(); - log.error("Error loading " + PROPFILE_NAME); + log.error("Error loading " + PROPFILE_NAME, ioe); } } - private static String expandPE(String pe) { - // 0123456 - // ADZZZZZ - StringBuilder peCode = new StringBuilder("--IRZZ"); - if ((pe != null) && (pe.length() >= 2)) { - for (int i = 0; i < pe.length() && (i < peCode.length()); i++) { - peCode.setCharAt(i, pe.charAt(i)); - } - } - char z4 = peCode.charAt(3); - char z5 = peCode.charAt(5); - if ('Z' == z4) { - if ('Z' == z5) { - peCode.setCharAt(3, 'R'); - } else { - // FIXME: This is an error - } - } - - return peCode.toString(); - } - - private static void processLine(String line) { + private void processLine(String line) { String[] pair = null; if (line.startsWith("$")) { return; @@ -237,7 +210,7 @@ public class ShefParm { } return; } else { - if (line.equals("SHEFPARM")) { + if ("SHEFPARM".equals(line)) { return; } switch (fileSection) { @@ -276,10 +249,4 @@ public class ShefParm { } } } - - public static final void main(String[] args) { - - String s = expandPE("ADZZZZZ"); - System.out.println(s); - } } diff --git a/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/util/ShefUtil.java b/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/util/ShefUtil.java index ec94d18645..5c8b10b5cf 100644 --- a/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/util/ShefUtil.java +++ b/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/plugin/shef/util/ShefUtil.java @@ -37,13 +37,14 @@ import com.raytheon.uf.common.wmo.WMOHeader; * * SOFTWARE HISTORY * - * Date Ticket# Engineer Description - * ------------ ---------- ----------- -------------------------- + * Date Ticket# Engineer Description + * ------------ ---------- ----------- -------------------------- * 03/19/08 387 M. Duff Initial creation. * 12/12/08 1786 J. Sanchez Handled date parsing exceptions. * 01/12/08 1846 J. Sanchez Update parseCreationDate to use default year. * May 14, 2014 2536 bclement moved WMO Header to common, removed unused log * Aug 05, 2015 4486 rjpeter Changed Timestamp to Date. + * Jan 10, 2018 5049 mduff ShefParm is passed in when needed. * */ public final class ShefUtil { @@ -78,7 +79,7 @@ public final class ShefUtil { Calendar recordTime = Calendar.getInstance(); recordTime.setTime(recordDate); - if (time.indexOf("/") > -1) { + if (time.indexOf('/') > -1) { /* Split the string apart and set the individual values */ parts = time.split("/"); for (String part : parts) { @@ -105,7 +106,7 @@ public final class ShefUtil { * Put the pieces together to get a full time stamp Start with the * century and work to the right */ - StringBuffer sb = new StringBuffer(); + StringBuilder sb = new StringBuilder(); if ((dt != null) && (dy != null)) { sb.append(dt + dy); @@ -113,7 +114,7 @@ public final class ShefUtil { dy = String.valueOf(recordTime.get(Calendar.YEAR)).substring(0, 2); sb.append(dt + dy); - } else if (dy == null) { + } else { int year = recordTime.get(Calendar.YEAR); sb.append(year); } @@ -174,7 +175,7 @@ public final class ShefUtil { timeString = sb.toString(); } else { - StringBuffer sb = new StringBuffer(); + StringBuilder sb = new StringBuilder(); String year = String.valueOf(recordTime.get(Calendar.YEAR)); String month = String.valueOf(recordTime.get(Calendar.MONTH) + 1); String day = String.valueOf(recordTime.get(Calendar.DAY_OF_MONTH)); @@ -410,14 +411,14 @@ public final class ShefUtil { int year = 0; cal.setTimeInMillis(System.currentTimeMillis()); - year = cal.get(Calendar.YEAR); // retrieves current year + year = cal.get(Calendar.YEAR); currentDate = cal.getTime(); cal.setTime(creationDate); cal.set(Calendar.YEAR, year); if (cal.after(currentDate)) { - --year; // uses the previous year + --year; } return String.valueOf(year); @@ -446,7 +447,7 @@ public final class ShefUtil { positive = false; } String relativeAmount = null; - if (relativeValue.equals("DRE")) { + if ("DRE".equals(relativeValue)) { relativeAmount = relativeValue; } else { relativeAmount = relativeValue.substring(4); @@ -457,41 +458,41 @@ public final class ShefUtil { if (positive) { obsTimeCal.add(Calendar.SECOND, new Integer(relativeAmount)); } else { - obsTimeCal.add(Calendar.SECOND, (new Integer(relativeAmount)) - * -1); + obsTimeCal.add(Calendar.SECOND, + (new Integer(relativeAmount)) * -1); } break; case 'N': if (positive) { obsTimeCal.add(Calendar.MINUTE, new Integer(relativeAmount)); } else { - obsTimeCal.add(Calendar.MINUTE, (new Integer(relativeAmount)) - * -1); + obsTimeCal.add(Calendar.MINUTE, + (new Integer(relativeAmount)) * -1); } break; case 'H': if (positive) { obsTimeCal.add(Calendar.HOUR, new Integer(relativeAmount)); } else { - obsTimeCal.add(Calendar.HOUR, (new Integer(relativeAmount)) - * -1); + obsTimeCal.add(Calendar.HOUR, + (new Integer(relativeAmount)) * -1); } break; case 'D': if (positive) { - obsTimeCal.add(Calendar.DAY_OF_MONTH, new Integer( - relativeAmount)); + obsTimeCal.add(Calendar.DAY_OF_MONTH, + new Integer(relativeAmount)); } else { - obsTimeCal.add(Calendar.DAY_OF_MONTH, (new Integer( - relativeAmount)) * -1); + obsTimeCal.add(Calendar.DAY_OF_MONTH, + (new Integer(relativeAmount)) * -1); } break; case 'M': if (positive) { obsTimeCal.add(Calendar.MONTH, new Integer(relativeAmount)); } else { - obsTimeCal.add(Calendar.MONTH, (new Integer(relativeAmount)) - * -1); + obsTimeCal.add(Calendar.MONTH, + (new Integer(relativeAmount)) * -1); } break; case 'E': @@ -503,8 +504,8 @@ public final class ShefUtil { if (positive) { obsTimeCal.add(Calendar.YEAR, new Integer(relativeAmount)); } else { - obsTimeCal.add(Calendar.YEAR, (new Integer(relativeAmount)) - * -1); + obsTimeCal.add(Calendar.YEAR, + (new Integer(relativeAmount)) * -1); } break; } @@ -596,7 +597,7 @@ public final class ShefUtil { century--; } returnVal = (century * 100) + pYear; - } else { // recordDate length must be 8 + } else { date = ShefConstants.YYYYMMDD_FORMAT.parse(dateString); calendar.setTime(date); returnVal = calendar.get(Calendar.YEAR); @@ -612,19 +613,17 @@ public final class ShefUtil { * @return - the identifier */ public static synchronized String createIdentifier(WMOHeader wmoHeader) { - String retVal = null; - StringBuffer sb = new StringBuffer(); - sb.append(wmoHeader.getWmoHeader() + "::::"); - sb.append(wmoHeader.getA1() + "::::"); - sb.append(wmoHeader.getA2() + "::::"); - sb.append(wmoHeader.getIi() + "::::"); - sb.append(wmoHeader.getT1() + "::::"); - sb.append(wmoHeader.getT2() + "::::"); - sb.append(wmoHeader.getBBBIndicator() + "::::"); - sb.append(wmoHeader.getCccc() + "::::"); + StringBuilder sb = new StringBuilder(); + sb.append(wmoHeader.getWmoHeader()).append("::::"); + sb.append(wmoHeader.getA1()).append("::::"); + sb.append(wmoHeader.getA2()).append("::::"); + sb.append(wmoHeader.getIi()).append("::::"); + sb.append(wmoHeader.getT1()).append("::::"); + sb.append(wmoHeader.getT2()).append("::::"); + sb.append(wmoHeader.getBBBIndicator()).append("::::"); + sb.append(wmoHeader.getCccc()).append("::::"); sb.append(wmoHeader.getYYGGgg()); - retVal = sb.toString(); - return retVal; + return sb.toString(); } /** @@ -656,7 +655,7 @@ public final class ShefUtil { return ((lower < value) && (value < upper)); } - public static final int validatePEDTSEP(String pedtsep) { + public static final int validatePEDTSEP(String pedtsep, ShefParm shefParm) { // Set to non-error code for switch statement flowdown. int err = -1; @@ -671,47 +670,59 @@ public final class ShefUtil { switch (pedtsep.length()) { case 7: { - if (ShefParm.getProbabilityCodeValue(pedtsep.substring(6, 7)) == null) { - // "Non-existent value for given probability parameter code", + if (shefParm.getProbabilityCodeValue( + pedtsep.substring(6, 7)) == null) { + // "Non-existent value for given probability parameter + // code", err = SHEFErrorCodes.LOG_063; } else { err = 0; } + break; } case 6: { if (err <= 0) { - if (ShefParm.getExtremumCode(pedtsep.substring(5, 6)) == null) { - // "Non-existent value for given extremum parameter code" + if (shefParm + .getExtremumCode(pedtsep.substring(5, 6)) == null) { + // "Non-existent value for given extremum parameter + // code" err = SHEFErrorCodes.LOG_061; } else { err = 0; } } + break; } - case 5: // Type Source + case 5: if (err <= 0) { - if (ShefParm.getTypeSourceCode(pedtsep.substring(3, 5)) == null) { - // "Non-existent value for given type and source parameter code" + if (shefParm.getTypeSourceCode( + pedtsep.substring(3, 5)) == null) { + // "Non-existent value for given type and source + // parameter code" err = SHEFErrorCodes.LOG_034; } else { err = 0; } } + break; case 3: { // Duration if (err <= 0) { - if (ShefParm.getDurationCodeValue(pedtsep.substring(2, 3)) == null) { - // "Non-existent value for given duration parameter code" + if (shefParm.getDurationCodeValue( + pedtsep.substring(2, 3)) == null) { + // "Non-existent value for given duration parameter + // code" err = SHEFErrorCodes.LOG_060; } else { err = 0; } } + break; } case 2: { if (err <= 0) { - if (ShefParm.getPhysicalElementConversionFactor(pedtsep - .substring(0, 2)) == null) { + if (shefParm.getPhysicalElementConversionFactor( + pedtsep.substring(0, 2)) == null) { // "An expected parameter code is missing" err = SHEFErrorCodes.LOG_003; } else { @@ -722,12 +733,14 @@ public final class ShefUtil { } case 1: { // This is an error - // "Parameter code too short or field misinterpreted as parameter code", + // "Parameter code too short or field misinterpreted as + // parameter code", err = 64; break; } default: { - // "Parameter code too short or field misinterpreted as parameter code", + // "Parameter code too short or field misinterpreted as + // parameter code", err = SHEFErrorCodes.LOG_064; } } diff --git a/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/AbstractShefFilter.java b/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/AbstractShefFilter.java index d45de8e38b..5fbccb3051 100644 --- a/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/AbstractShefFilter.java +++ b/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/AbstractShefFilter.java @@ -1,36 +1,34 @@ /** * This software was developed and / or modified by Raytheon Company, * pursuant to Contract DG133W-05-CQ-1067 with the US Government. - * + * * U.S. EXPORT CONTROLLED TECHNICAL DATA * This software product contains export-restricted data whose * export/transfer/disclosure is restricted by U.S. law. Dissemination * to non-U.S. persons whether in the United States or abroad requires * an export license or other authorization. - * + * * Contractor Name: Raytheon Company * Contractor Address: 6825 Pine Street, Suite 340 * Mail Stop B8 * Omaha, NE 68106 * 402.291.0100 - * + * * See the AWIPS II Master Rights File ("Master Rights File.pdf") for * further licensing information. **/ package com.raytheon.edex.transform.shef; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; +import java.io.FileNotFoundException; +import java.io.InputStream; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.raytheon.uf.common.dataplugin.PluginDataObject; +import com.raytheon.uf.common.localization.ILocalizationFile; import com.raytheon.uf.common.localization.IPathManager; -import com.raytheon.uf.common.localization.LocalizationContext; -import com.raytheon.uf.common.localization.LocalizationContext.LocalizationLevel; import com.raytheon.uf.common.localization.LocalizationContext.LocalizationType; import com.raytheon.uf.common.localization.PathManagerFactory; import com.raytheon.uf.common.serialization.JAXBManager; @@ -44,28 +42,27 @@ import com.raytheon.uf.edex.decodertools.core.filterimpl.WMOHeaderFilterElement; /** * Abstraction of a Shef Filter. - * + * * 
- * 
+ *
  * SOFTWARE HISTORY
- * 
+ *
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * Oct 28, 2015 4783       bkowal      Initial creation
  * Dec 16, 2015 5166       kbisanz     Update logging to use SLF4J
  * Jul 13, 2016 5744       mapeters    Retrieve config files from common_static
  *                                     instead of edex_static
- * 
+ * Nov 02, 2017 6132       mapeters    Support localization overrides for filter file
+ *
  *
- * + * * @author bkowal */ public abstract class AbstractShefFilter { - private static final String ERROR_1_FMT = "Could not create {%s} context for file \"%s\""; - - private static final String ERROR_2_FMT = "File %s does not exist"; + private static final String ERROR_FMT = "File %s does not exist"; protected static final String METAR_CFG = "metar.cfg"; @@ -81,36 +78,27 @@ public abstract class AbstractShefFilter { this.init(); } - protected AbstractShefFilter(String configFile, String localContext, + protected AbstractShefFilter(String configFile, final Class shefFilterClass) { this.init(); this.filterConfigFile = configFile; - this.readConfig(localContext, shefFilterClass); + this.readConfig(shefFilterClass); } - private void readConfig(String localContext, + private void readConfig( final Class shefFilterClass) { try { /* * Retrieve and verify the existence of the configuration file. */ + String configPath = FILTERS_DIR + IPathManager.SEPARATOR + + filterConfigFile; IPathManager manager = PathManagerFactory.getPathManager(); - if (manager == null) { - throw new Exception("Failed to construct a Path Manager."); - } - LocalizationContext context = manager.getContext( - LocalizationType.COMMON_STATIC, - LocalizationLevel.valueOf(localContext)); - if (context == null) { - throw new Exception(String.format(ERROR_1_FMT, localContext, - this.filterConfigFile)); - } - Path configPath = Paths.get( - manager.getFile(context, FILTERS_DIR).getAbsolutePath()) - .resolve(filterConfigFile); - if (!Files.exists(configPath)) { - throw new Exception(String.format(ERROR_2_FMT, - configPath.toString())); + ILocalizationFile configFile = manager.getStaticLocalizationFile( + LocalizationType.COMMON_STATIC, configPath); + if (configFile == null || !configFile.exists()) { + throw new FileNotFoundException( + String.format(ERROR_FMT, configPath)); } /* @@ -120,8 +108,10 @@ public abstract class AbstractShefFilter { shefFilterClass, RadiusFilterElement.class, RectFilterElement.class, StationIdFilterElement.class, WMOHeaderFilterElement.class); - byte[] data = Files.readAllBytes(configPath); - Object obj = jaxb.unmarshalFromXml(new String(data)); + Object obj; + try (InputStream is = configFile.openInputStream()) { + obj = jaxb.unmarshalFromInputStream(is); + } if (shefFilterClass.isInstance(obj) || obj instanceof PluginDataObjectFilter) { this.buildRun(obj, this.filterConfigFile); @@ -147,19 +137,18 @@ public abstract class AbstractShefFilter { * Only allow keep to be set to false. Once false it stays that * way. */ - if (AbstractObsFilter.INCLUDE_TYPE.equals(element - .getFilterType())) { + if (AbstractObsFilter.INCLUDE_TYPE + .equals(element.getFilterType())) { // Did the filter pass? if (r == null) { // If we fail an element, exit now. keep = false; break; } - } else if (AbstractObsFilter.EXCLUDE_TYPE.equals(element - .getFilterType())) { + } else if (AbstractObsFilter.EXCLUDE_TYPE + .equals(element.getFilterType())) { if (r != null) { - // There was a match, so we want to remove this - // item. + // There was a match, so we want to remove this item. keep = false; // And there's no reason for further checks. break; diff --git a/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/MetarToShefFilter.java b/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/MetarToShefFilter.java index 5b9d713633..6c96b2966c 100644 --- a/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/MetarToShefFilter.java +++ b/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/MetarToShefFilter.java @@ -1,19 +1,19 @@ /** * This software was developed and / or modified by Raytheon Company, * pursuant to Contract DG133W-05-CQ-1067 with the US Government. - * + * * U.S. EXPORT CONTROLLED TECHNICAL DATA * This software product contains export-restricted data whose * export/transfer/disclosure is restricted by U.S. law. Dissemination * to non-U.S. persons whether in the United States or abroad requires * an export license or other authorization. - * + * * Contractor Name: Raytheon Company * Contractor Address: 6825 Pine Street, Suite 340 * Mail Stop B8 * Omaha, NE 68106 * 402.291.0100 - * + * * See the AWIPS II Master Rights File ("Master Rights File.pdf") for * further licensing information. **/ @@ -41,9 +41,9 @@ import com.raytheon.uf.edex.decodertools.core.filterimpl.PluginDataObjectFilter; * Use information in metarToShefFilter.xml, MetarToShefFilter filters out the * metar messages before send the message to MetarToShefTransformer to encode to * a SHEF message. - * + * * 
- * 
+ *
  * SOFTWARE HISTORY
  * Date       Ticket# Engineer Description
  * ---------- ------- -------- --------------------------
@@ -52,16 +52,17 @@ import com.raytheon.uf.edex.decodertools.core.filterimpl.PluginDataObjectFilter;
  * 08/08/2013 16408   wkwock   Use different metar.cfg file and options
  * 09/09/2014  3548   mapeters Replaced SerializationUtil usage with JAXBManager.
  * 10/28/2015  4783   bkowal   Refactored and abstracted into {@link AbstractShefFilter}.
- * 
+ * 11/02/2017 6132    mapeters Support localization overrides for filter file
+ *
  *
- * + * * @author wkwock - * @version 1.0 */ @XmlRootElement @XmlAccessorType(XmlAccessType.NONE) @DynamicSerialize public class MetarToShefFilter extends AbstractShefFilter { + @XmlElement @DynamicSerializeElement protected List metarToShefRun; @@ -75,8 +76,8 @@ public class MetarToShefFilter extends AbstractShefFilter { public MetarToShefFilter() { } - public MetarToShefFilter(String configFile, String localContext) { - super(configFile, localContext, MetarToShefFilter.class); + public MetarToShefFilter(String configFile) { + super(configFile, MetarToShefFilter.class); for (MetarToShefRun mtsr : metarToShefRun) { logger.info("Filter name = " + mtsr.getFilterName() @@ -109,12 +110,11 @@ public class MetarToShefFilter extends AbstractShefFilter { /** * Apply the list of filters against given input data. - * */ @Override public PluginDataObject[] filter(PluginDataObject[] reports) { Map matchList = new HashMap<>(); - List reportList = new ArrayList(); + List reportList = new ArrayList<>(); for (PluginDataObject report : reports) { for (MetarToShefRun mtsr : metarToShefRun) { PluginDataObject resultRpt = filterARun(report, @@ -126,10 +126,10 @@ public class MetarToShefFilter extends AbstractShefFilter { } } } - if (matchList.isEmpty() == false) { + if (!matchList.isEmpty()) { MetarToShefTransformer.setMatchList(matchList); } - return (PluginDataObject[]) reportList.toArray(new PluginDataObject[0]); + return reportList.toArray(new PluginDataObject[0]); } @Override @@ -155,18 +155,10 @@ public class MetarToShefFilter extends AbstractShefFilter { metarToShefRun.add(element); } - /** - * - * @return - */ public List getMetarToShefRun() { return metarToShefRun; } - /** - * - * @param elements - */ public void setMetarToShefRun(List elements) { metarToShefRun = elements; } diff --git a/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/SynopticToShefFilter.java b/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/SynopticToShefFilter.java index d0f60f4b82..40173e7875 100644 --- a/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/SynopticToShefFilter.java +++ b/edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/SynopticToShefFilter.java @@ -1,34 +1,34 @@ /** * This software was developed and / or modified by Raytheon Company, * pursuant to Contract DG133W-05-CQ-1067 with the US Government. - * + * * U.S. EXPORT CONTROLLED TECHNICAL DATA * This software product contains export-restricted data whose * export/transfer/disclosure is restricted by U.S. law. Dissemination * to non-U.S. persons whether in the United States or abroad requires * an export license or other authorization. - * + * * Contractor Name: Raytheon Company * Contractor Address: 6825 Pine Street, Suite 340 * Mail Stop B8 * Omaha, NE 68106 * 402.291.0100 - * + * * See the AWIPS II Master Rights File ("Master Rights File.pdf") for * further licensing information. **/ package com.raytheon.edex.transform.shef; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlRootElement; - import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + import com.raytheon.uf.common.dataplugin.PluginDataObject; import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; @@ -39,24 +39,25 @@ import com.raytheon.uf.edex.decodertools.core.filterimpl.PluginDataObjectFilter; /** * Used to filter synoptic messages before sending them to * {@link SMToShefTransformer}. - * + * * 
- * 
+ *
  * SOFTWARE HISTORY
- * 
+ *
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * Oct 28, 2015 4783       bkowal      Initial creation
- * 
+ * Nov 02, 2017 6132       mapeters    Support localization overrides for filter file
+ *
  *
- * + * * @author bkowal - * @version 1.0 */ @XmlRootElement @XmlAccessorType(XmlAccessType.NONE) @DynamicSerialize public class SynopticToShefFilter extends AbstractShefFilter { + @XmlElement @DynamicSerializeElement private List synopticToShefRun; @@ -68,8 +69,8 @@ public class SynopticToShefFilter extends AbstractShefFilter { this.synopticToShefRun = new ArrayList<>(); } - public SynopticToShefFilter(String configFile, String localContext) { - super(configFile, localContext, SynopticToShefFilter.class); + public SynopticToShefFilter(String configFile) { + super(configFile, SynopticToShefFilter.class); for (SynopticToShefRun run : synopticToShefRun) { logger.info("Filter name = " + run.getFilterName() @@ -82,13 +83,6 @@ public class SynopticToShefFilter extends AbstractShefFilter { this.synopticToShefRun = new ArrayList<>(); } - /* - * (non-Javadoc) - * - * @see - * com.raytheon.edex.transform.shef.AbstractShefFilter#buildRun(java.lang - * .Object, java.lang.String) - */ @Override protected void buildRun(Object obj, String configFile) { if (obj instanceof PluginDataObjectFilter) { @@ -127,7 +121,7 @@ public class SynopticToShefFilter extends AbstractShefFilter { @Override public PluginDataObject[] filter(PluginDataObject[] reports) { Map matchList = new HashMap<>(); - List reportList = new ArrayList(); + List reportList = new ArrayList<>(); for (PluginDataObject report : reports) { for (SynopticToShefRun run : this.synopticToShefRun) { PluginDataObject resultRpt = filterARun(report, @@ -139,10 +133,10 @@ public class SynopticToShefFilter extends AbstractShefFilter { } } } - if (matchList.isEmpty() == false) { + if (!matchList.isEmpty()) { SMToShefTransformer.setMatchMap(matchList); } - return (PluginDataObject[]) reportList.toArray(new PluginDataObject[0]); + return reportList.toArray(new PluginDataObject[0]); } /** @@ -156,7 +150,8 @@ public class SynopticToShefFilter extends AbstractShefFilter { * @param synopticToShefRun * the synopticToShefRun to set */ - public void setSynopticToShefRun(List synopticToShefRun) { + public void setSynopticToShefRun( + List synopticToShefRun) { this.synopticToShefRun = synopticToShefRun; } } \ No newline at end of file diff --git a/edexOsgi/com.raytheon.edex.plugin.shef/utility/common_static/base/distribution/shef.xml b/edexOsgi/com.raytheon.edex.plugin.shef/utility/common_static/base/distribution/shef.xml index b9a6462f30..bfcf79d729 100644 --- a/edexOsgi/com.raytheon.edex.plugin.shef/utility/common_static/base/distribution/shef.xml +++ b/edexOsgi/com.raytheon.edex.plugin.shef/utility/common_static/base/distribution/shef.xml @@ -2,22 +2,28 @@ + + ^[AF][BS].... (KOMA|KOAX|KLSE|KARX|KDSM|KDMX|KDVN|KMLI|KEAX|KMCI|KFSD|KGRI|KGID|KLBF|KSTL|KLSX|KMSP|KMPX|KTOP|KZMP|KPQR).* ^FGUS.. (KKRF|KMSR|KSTR ).* diff --git a/edexOsgi/com.raytheon.edex.plugin.shef/utility/common_static/base/hydro/shefGadTokens.xml b/edexOsgi/com.raytheon.edex.plugin.shef/utility/common_static/base/hydro/shefGadTokens.xml index 29b1b123df..226e2f705d 100644 --- a/edexOsgi/com.raytheon.edex.plugin.shef/utility/common_static/base/hydro/shefGadTokens.xml +++ b/edexOsgi/com.raytheon.edex.plugin.shef/utility/common_static/base/hydro/shefGadTokens.xml @@ -1,28 +1,53 @@ + + + - db_name - whfs_local_data_dir - dupmess - shef_post_unk - shef_duplicate - shef_alertalarm - locmess - shef_winpast - shef_winfuture - shef_post_link - shef_post_latest - shef_load_maxfcst - shef_post_baddata - basis_hours_filter - elgmess - shef_load_ingest - ingest_mess - shef_procobs - shef_data_log - shef_perflog - shef_storetext - gage_pp_enable - gage_pp_data - gage_pp_log - gage_pp_sleep - \ No newline at end of file + db_name + whfs_local_data_dir + dupmess + shef_post_unk + shef_duplicate + shef_alertalarm + locmess + shef_winpast + shef_winfuture + shef_post_link + shef_post_latest + shef_load_maxfcst + shef_post_baddata + basis_hours_filter + elgmess + shef_load_ingest + ingest_mess + shef_procobs + shef_data_log + shef_perflog + shef_storetext + gage_pp_enable + gage_pp_data + gage_pp_log + gage_pp_sleep + diff --git a/edexOsgi/com.raytheon.edex.plugin.shef/utility/common_static/base/shef/metar.cfg b/edexOsgi/com.raytheon.edex.plugin.shef/utility/common_static/base/shef/metar.cfg index 5592670a39..0c8e5bee51 100644 --- a/edexOsgi/com.raytheon.edex.plugin.shef/utility/common_static/base/shef/metar.cfg +++ b/edexOsgi/com.raytheon.edex.plugin.shef/utility/common_static/base/shef/metar.cfg @@ -1,3 +1,7 @@ +# This is an absolute override file, +# indicating that a higher priority version of the file +# will completely replace a lower priority version of the file. + /tmp/queue/metar/in /tmp/queue/metar/out /tmp/queue/metar/err