xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #2652. Fixed issue with edex-text-ingest log.

Browse source 
Former-commit-id: 7ca9607e68 [formerly ef75ea04c0 [formerly 62f19ca599740c580973e166e8053ba1c4800178]]
Former-commit-id: ef75ea04c0
Former-commit-id: 68a98f4d85
This commit is contained in:
Slav Korolev 2014-03-04 13:24:34 -05:00
parent b94ed41ba2
commit 70c7389edd
5 changed files with 92 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

63
edexOsgi/com.raytheon.edex.plugin.text/src/com/raytheon/edex/plugin/text/TextDecoder.java
View file

@ -79,6 +79,7 @@ import com.raytheon.uf.edex.wmo.message.WMOHeader;
* Dec 13, 2010 5805 cjeanbap Parse Report to get AFOS Product Id
* Jul 16, 2013 16323 D. Friedman Use concurrent map
* Aug 30, 2013 2298 rjpeter Make getPluginName abstract
* Mar 14, 2014 2652 skorolev Changed logging for skipped headers.
* </pre>
*
* @author
@ -339,62 +340,32 @@ public class TextDecoder extends AbstractDecoder {
msg.append("/");
msg.append(stored + separator.getMessagesSkipped());
msg.append("]");
List<WMOHeader> skippedSubHeaders = separator
Map<WMOHeader, String> skippedSubHeaders = separator
.getSubHeadersSkipped();
long curTime = System.currentTimeMillis();
if (skippedSubHeaders.size() > 0) {
Iterator<WMOHeader> iter = skippedSubHeaders.iterator();
while (iter.hasNext()) {
WMOHeader header = iter.next();
String key = null;
msg.append("\nSkipped WMO Sub Headers:");
for (WMOHeader header : skippedSubHeaders.keySet()) {
if (header.isValid()) {
key = header.getTtaaii() + " "
+ header.getCccc();
} else if (header.getOriginalMessage().length() > 11) {
key = header.getOriginalMessage().substring(0,
11);
msg.append("\n[");
msg.append(header.getTtaaii());
msg.append(" ");
msg.append(header.getCccc());
msg.append("] - ");
msg.append(skippedSubHeaders.get(header));
} else {
key = header.getOriginalMessage();
}
Long time = notMappedHeaders.get(key);
if ((time == null)
|| ((curTime - time) > NOT_FOUND_LOG_PERIOD)) {
notMappedHeaders.put(key, curTime);
} else {
iter.remove();
}
}
if (skippedSubHeaders.size() > 0) {
msg.append("\nSkipped WMO Sub Headers:");
for (WMOHeader header : skippedSubHeaders) {
if (header.isValid()) {
msg.append("\n[");
msg.append(header.getTtaaii());
msg.append(" ");
msg.append(header.getCccc());
msg.append("]");
msg.append("\n[");
if (header.getOriginalMessage().length() > 11) {
msg.append(header.getOriginalMessage()
.substring(0, 11));
} else {
msg.append("\n[");
if (header.getOriginalMessage().length() > 11) {
msg.append(header.getOriginalMessage()
.substring(0, 11));
} else {
msg.append(header.getOriginalMessage());
}
msg.append("]");
msg.append(header.getOriginalMessage());
}
}
if ((curTime - msgHdrLogTime) > MSG_HDR_LOG_PERIOD) {
msg.append("\nMsg for a given header will only be logged once in a "
+ (NOT_FOUND_LOG_PERIOD / MILLIS_PER_HOUR)
+ " hour period");
msgHdrLogTime = curTime;
msg.append("]");
}
}
logger.error(msg.toString());
logger.info(msg.toString());
saveFile = moveBadTxt;
}
}

66
edexOsgi/com.raytheon.edex.plugin.text/src/com/raytheon/edex/plugin/text/impl/separator/StdCollectiveSeparator.java
View file

@ -42,8 +42,10 @@ import com.raytheon.uf.edex.wmo.message.WMOHeader;
* Sep 3, 2008 jkorman Initial creation
* Jul 10, 2009 2191 rjpeter Reimplemented.
* Sep 22, 2010 6932 cjeanbap Added METAR/SPECI to product.
* Feb 18, 2014 2652 skorolev Fixed error in the makeCollId.
*
* Feb 18, 2014 2652 skorolev Fixed error in the makeCollId.
* Mar 06, 2014 2652 skorolev Corrected rawMsg extraction.
* Mar 14, 2014 2652 skorolev Changed logging for skipped headers.
* Fixed calculation of message end.
* </pre>
*
* @author jkorman
@ -133,12 +135,20 @@ public class StdCollectiveSeparator extends WMOMessageSeparator {
String productType = null;
int startIndex = wmoHdr.getMessageDataStart();
int endIndex = TextSeparatorFactory.findDataEnd(rawData);
if (endIndex <= startIndex) {
endIndex = rawData.length - 1;
}
String rawMsg = new String(rawData, startIndex, endIndex - startIndex);
StringBuilder sb = null;
// This is a fake line "METXXX" to permit skipping of NNNXXX pattern
// line.
if ((rawMsg.indexOf(METAR) == 0) || (rawMsg.indexOf(SPECI) == 0)) {
productType = (rawMsg.indexOf(METAR) == 0 ? METAR : SPECI);
sb = new StringBuilder(rawMsg);
sb.insert(0, "METXXX\n");
rawMsg = sb.toString();
}
StringBuilder sb = null;
if ("TAF".equals(afos_id.getNnn())) {
sb = new StringBuilder(rawMsg);
Matcher m = P_TAF.matcher(sb);
@ -238,14 +248,13 @@ public class StdCollectiveSeparator extends WMOMessageSeparator {
// during the AFOS to AWIPS transition so that products can
// still be stored.
if (!makeCollId(product_id, XXX_id, afos_id,
wmoHdr.getCccc())) {
wmoHdr.getCccc(), newWmoHdr)) {
if (NOAFOSPIL.equals(afos_id)) {
logger.info("No AFOS ID found; use TTAAii CCCC to store: "
+ dataDes + wmoHdr.getCccc());
product_id = NOAFOSPIL;
} else {
numSkipped++;
subHeadersSkipped.add(newWmoHdr);
/*
* if (moveBadTxt) { logger.error(
@ -389,7 +398,7 @@ public class StdCollectiveSeparator extends WMOMessageSeparator {
}
pirFlag = false;
}
blank = buffer.toString();
if (blank.startsWith("AMD") || blank.startsWith("COR")) {
if (safeStrpbrk(buffer, CSPC)) {
buffer.deleteCharAt(0);
@ -467,10 +476,11 @@ public class StdCollectiveSeparator extends WMOMessageSeparator {
* @param XXX_id
* @param afos_id
* @param origin
* @param newWmoHdr
* @return
*/
private boolean makeCollId(AFOSProductId product_id, StringBuilder XXX_id,
AFOSProductId afos_id, String origin) {
AFOSProductId afos_id, String origin, WMOHeader newWmoHdr) {
// /TextString CCC_id, newId;
String CCC_id;
String newId;
@ -487,33 +497,58 @@ public class StdCollectiveSeparator extends WMOMessageSeparator {
// If the XXX is 3 characters, and the origin starts with K, try
// prepending K or P (the latter for AK, HI products)
if ((trimmedXXX.length() == 3)
&& (origin.startsWith("K") && !trimmedXXX.equals("RMK"))) {
if (trimmedXXX.length() == 3 && origin.startsWith("K")) {
newId = "K" + trimmedXXX;
if ((CCC_id = staticData.mapICAOToCCC(newId)) == null) {
newId = "P" + trimmedXXX;
if ((CCC_id = staticData.mapICAOToCCC(newId)) == null) {
// logger.error("NCF_FAIL to map XXX to CCC: " +
// XXX_id);
subHeadersSkipped
.put(newWmoHdr,
"Product "
+ afos_id.toString()
+ " is excluded from storage due to "
+ newId
+ " not present in national_category_table.template");
return false;
}
}
}
// Otherwise, if the XXX is 3 characters, try prepending the first
// character of the origin except "RMK" which is remark code.
else if (trimmedXXX.length() == 3 && !trimmedXXX.equals("RMK")) {
// character of the origin.
else if (trimmedXXX.length() == 3) {
newId = origin.charAt(0) + trimmedXXX;
if ((CCC_id = staticData.mapICAOToCCC(newId)) == null) {
// logger.error("NCF_FAIL to map XXX to CCC: " + XXX_id);
subHeadersSkipped
.put(newWmoHdr,
"Product "
+ afos_id.toString()
+ " is excluded from storage due to "
+ newId
+ " not present in national_category_table.template");
return false;
}
} else {
// logger.error("NCF_FAIL to map XXX to CCC: " + XXX_id);
if (!checkCharNum(XXX_id.charAt(0))) {
// logger.error("bad XXX id");
subHeadersSkipped
.put(newWmoHdr,
"Product "
+ afos_id.toString()
+ " is excluded from storage due to incorrect xxxid"
+ XXX_id);
return false;
} else
return true;
// If trimmedXXX has 4 characters and not found in
// national_category_table.template.
subHeadersSkipped
.put(newWmoHdr,
"Product "
+ afos_id.toString()
+ " is excluded from storage due to "
+ trimmedXXX
+ " not present in national_category_table.template");
return false;
}
}
@ -528,7 +563,6 @@ public class StdCollectiveSeparator extends WMOMessageSeparator {
product_id.setXxx(trimmedXXX);
else
product_id.setXxx(XXX_id.substring(1, 4));
return true;
}
}

14
edexOsgi/com.raytheon.edex.plugin.text/src/com/raytheon/edex/plugin/text/impl/separator/StdTextSeparator.java
View file

@ -41,6 +41,7 @@ import com.raytheon.uf.edex.wmo.message.WMOHeader;
* Jul 10, 2009 2191 rjpeter Reimplemented.
* Jul 26, 2011 10043 rferrel Modified identifyReports to
* have checks like A1.
* Mar 13, 2014 2652 skorolev Fixed calculation of message end.
* </pre>
*
* @author jkorman
@ -108,6 +109,9 @@ public class StdTextSeparator extends WMOMessageSeparator {
}
int startIndex = wmoHeader.getMessageDataStart();
int endIndex = TextSeparatorFactory.findDataEnd(rawData);
if (endIndex <= startIndex) {
endIndex = rawData.length - 1;
}
StringBuilder buffer = new StringBuilder(new String(rawData,
startIndex, endIndex - startIndex));
if (!decodeStdMsg(buffer, ispanId, wmoHeader)) {
@ -176,7 +180,7 @@ public class StdTextSeparator extends WMOMessageSeparator {
}
StringBuilder newProductId = new StringBuilder();
if (!makeStdId(newProductId, nnnxxx, ispanId)) {
logger.debug("No AFOS ID found; use TTAAii CCCC to store: "
+ ispanId);
@ -374,7 +378,7 @@ public class StdTextSeparator extends WMOMessageSeparator {
return true;
}
}
// Try to map the nnnId with the bit table for a national bit product;
// otherwise, check the AFOS table for the ccc value of the origin.
newId = staticData.getSiteIdFromNNN(nnnId);
@ -383,9 +387,11 @@ public class StdTextSeparator extends WMOMessageSeparator {
cccId = staticData.getAFOSTableMap(origin);
if (cccId == null) {
if (nnnxxx.length() >= 6) {
cccId = staticData.getAFOSTableMap("K" + nnnxxx.substring(3, 6));
cccId = staticData.getAFOSTableMap("K"
+ nnnxxx.substring(3, 6));
} else if (nnnxxx.length() > 3) {
cccId = staticData.getAFOSTableMap("K" + nnnxxx.substring(3));
cccId = staticData.getAFOSTableMap("K"
+ nnnxxx.substring(3));
}
if (cccId == null) { // KWBC RCM,VER
// logger.error("Can't get ccc: " + origin);

4
edexOsgi/com.raytheon.edex.plugin.text/src/com/raytheon/edex/plugin/text/impl/separator/UACollectiveSeparator.java
View file

@ -42,6 +42,7 @@ import com.raytheon.uf.edex.wmo.message.WMOHeader;
* ------------ ---------- ----------- --------------------------
* Sep 3, 2008 jkorman Initial creation
* Jul 10, 2009 2191 rjpeter Reimplemented.
* Mar 13, 2014 2652 skorolev Fixed calculation of message end.
* </pre>
*
* @author jkorman
@ -113,6 +114,9 @@ public class UACollectiveSeparator extends WMOMessageSeparator {
WMOHeader wmoHdr = getWmoHeader();
int startIndex = wmoHdr.getMessageDataStart();
int endIndex = TextSeparatorFactory.findDataEnd(rawData);
if (endIndex <= startIndex) {
endIndex = rawData.length - 1;
}
String rawMsg = new String(rawData, startIndex, endIndex - startIndex);
Matcher nnnxxxMatcher = NNNXXX.matcher(rawMsg);
if (nnnxxxMatcher.find()) {

18
edexOsgi/com.raytheon.edex.plugin.text/src/com/raytheon/edex/plugin/text/impl/separator/WMOMessageSeparator.java
View file

@ -20,8 +20,9 @@
package com.raytheon.edex.plugin.text.impl.separator;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Matcher;
import java.util.Map;
import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
@ -46,6 +47,8 @@ import com.raytheon.uf.edex.wmo.message.WMOHeader;
* 20090327 2151 jkorman Modified separate to remove leading/trailing
* white space.
* Jul 10, 2009 2191 rjpeter Reimplemented.
* Mar 04, 2014 2652 skorolev Corrected NNNXXX pattern.
* Mar 14, 2014 2652 skorolev Changed logging for skipped headers.
* </pre>
*
* @author
@ -107,8 +110,9 @@ public abstract class WMOMessageSeparator extends AbstractRecordSeparator {
protected int numSkipped = 0;
// used for collective messages to show the skipped header
protected List<WMOHeader> subHeadersSkipped = new ArrayList<WMOHeader>();
// used for collective messages to show the skipped header and reason to
// skip.
protected Map<WMOHeader, String> subHeadersSkipped = new HashMap<WMOHeader, String>();
/**
*
@ -278,9 +282,6 @@ public abstract class WMOMessageSeparator extends AbstractRecordSeparator {
return numSkipped;
}
public List<WMOHeader> getSubHeadersSkipped() {
return subHeadersSkipped;
}
/*
* private AFOSProductId createProductId() { AFOSProductId productId = null;
@ -407,6 +408,10 @@ public abstract class WMOMessageSeparator extends AbstractRecordSeparator {
}
}
public Map<WMOHeader, String> getSubHeadersSkipped() {
return subHeadersSkipped;
}
public static final void main(String[] args) {
StringBuilder sb = new StringBuilder(
@ -426,5 +431,4 @@ public abstract class WMOMessageSeparator extends AbstractRecordSeparator {
// }
// }
}
}