>();
+ RadarTabularBlockParser.parseTabularBlock(tb,
+ record.getProductCode(), map, recordVals);
+ record.setProductVals(map);
+ record.setMapRecordVals(recordVals);
+ record.setTabularBlock(tb);
+ }
+ }
+
+ try {
+ finalizeRecord(record);
+ } catch (PluginException e) {
+ logger.error(e);
+ return new PluginDataObject[0];
+ }
+
+ timer.stop();
+ perfLog.logDuration("Time to Decode", timer.getElapsedTime());
+
+ recordList.add(record);
}
return recordList.toArray(new PluginDataObject[recordList.size()]);
diff --git a/edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/level3/Level3BaseRadar.java b/edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/level3/Level3BaseRadar.java
index 2c83e63566..7be200f199 100644
--- a/edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/level3/Level3BaseRadar.java
+++ b/edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/level3/Level3BaseRadar.java
@@ -40,6 +40,7 @@ import org.itadaki.bzip2.BZip2InputStream;
import com.raytheon.edex.esb.Headers;
import com.raytheon.edex.plugin.radar.util.RadarEdexTextProductUtil;
import com.raytheon.edex.plugin.radar.util.RadarSpatialUtil;
+import com.raytheon.uf.common.dataplugin.exception.MalformedDataException;
import com.raytheon.uf.common.dataplugin.radar.RadarStation;
import com.raytheon.uf.common.dataplugin.radar.level3.AlertAdaptationParameters;
import com.raytheon.uf.common.dataplugin.radar.level3.AlertMessage;
@@ -91,7 +92,16 @@ import com.raytheon.uf.edex.wmo.message.WMOHeader;
* graphics context and the extents of the Component used to create the Graphics
* object.
*
- * Copyright 2006 Raytheon Corporation
+ *
+ *
+ * SOFTWARE HISTORY
+ * Date Ticket# Engineer Description
+ * ------------- -------- ----------- --------------------------
+ * --/--/2006 brockwoo Initial creation
+ * Jan 21, 2014 2627 njensen Changed offset errors to MalformedDataException
+ *
+ *
+ *
*
* @author Bryan Rockwood
* @version 1.0
@@ -200,8 +210,10 @@ public class Level3BaseRadar {
* A java.io.File object containing a raw radar file
* @throws IOException
* If the radar head parsing fails, an IO exception is thrown
+ * @throws MalformedDataException
*/
- public Level3BaseRadar(File aRadar, Headers headers) throws IOException {
+ public Level3BaseRadar(File aRadar, Headers headers) throws IOException,
+ MalformedDataException {
int fileSize = (int) aRadar.length();
byte[] tempRawRadarByteArray = new byte[fileSize];
@@ -227,13 +239,15 @@ public class Level3BaseRadar {
* A byte array containing a raw radar file
* @throws IOException
* If the radar head parsing fails, an IO exception is thrown
+ * @throws MalformedDataException
*/
- public Level3BaseRadar(byte[] aRadar, Headers headers) throws IOException {
+ public Level3BaseRadar(byte[] aRadar, Headers headers) throws IOException,
+ MalformedDataException {
init(aRadar, headers);
}
public Level3BaseRadar(byte[] aRadar, Headers headers, RadarInfoDict dict)
- throws IOException {
+ throws IOException, MalformedDataException {
this.dict = dict;
init(aRadar, headers);
}
@@ -241,8 +255,10 @@ public class Level3BaseRadar {
/**
* @param aRadar
* @throws IOException
+ * @throws MalformedDataException
*/
- private void init(byte[] aRadar, Headers headers) throws IOException {
+ private void init(byte[] aRadar, Headers headers) throws IOException,
+ MalformedDataException {
// printPacketContents(aRadar);
int wmoHeaderSize;
@@ -512,7 +528,8 @@ public class Level3BaseRadar {
return dataLevelThresholds[code];
}
- private SymbologyBlock readSymbologyBlock(int offset) throws IOException {
+ private SymbologyBlock readSymbologyBlock(int offset) throws IOException,
+ MalformedDataException {
SymbologyBlock symBlock = null;
if (offset != 0) {
theRadarData.reset();
@@ -520,8 +537,8 @@ public class Level3BaseRadar {
int divider = theRadarData.readShort();
int blockId = theRadarData.readUnsignedShort();
if ((divider != -1) || (blockId != SymbologyBlock.getBlockId())) {
- throw new IOException(
- "This does not appear to be a symbology block");
+ throw new MalformedDataException("Symbology block offset "
+ + offset + " does not point to a symbology block");
}
int blockLen = theRadarData.readInt();
byte[] buf = RadarUtil.subArray(theRawRadarByteArray, offset,
@@ -537,7 +554,8 @@ public class Level3BaseRadar {
return symBlock;
}
- private GraphicBlock readGraphicBlock(int offset) throws IOException {
+ private GraphicBlock readGraphicBlock(int offset) throws IOException,
+ MalformedDataException {
GraphicBlock graphicBlock = null;
if (offset != 0) {
theRadarData.reset();
@@ -545,8 +563,8 @@ public class Level3BaseRadar {
int divider = theRadarData.readShort();
int blockId = theRadarData.readUnsignedShort();
if ((divider != -1) || (blockId != GraphicBlock.getBlockId())) {
- throw new IOException(
- "This does not appear to be a graphic block");
+ throw new MalformedDataException("Graphic block offset "
+ + offset + " does not point to a graphic block");
}
int blockLen = theRadarData.readInt();
byte[] buf = RadarUtil.subArray(theRawRadarByteArray, offset,
@@ -559,7 +577,8 @@ public class Level3BaseRadar {
return graphicBlock;
}
- private TabularBlock readTabularBlock(int offset) throws IOException {
+ private TabularBlock readTabularBlock(int offset) throws IOException,
+ MalformedDataException {
TabularBlock tabBlock = null;
if (offset != 0) {
theRadarData.reset();
@@ -567,8 +586,8 @@ public class Level3BaseRadar {
int divider = theRadarData.readShort();
int blockId = theRadarData.readUnsignedShort();
if ((divider != -1) || (blockId != TabularBlock.getBlockId())) {
- throw new IOException(
- "This does not appear to be a tabular block");
+ throw new MalformedDataException("Tabular block offset "
+ + offset + " does not point to a tabular block");
}
int blockLen = theRadarData.readInt();
byte[] buf = RadarUtil.subArray(theRawRadarByteArray, offset,
@@ -587,13 +606,14 @@ public class Level3BaseRadar {
* @return
* @throws IOException
*/
- private TabularBlock readStandaloneTabular(int offset) throws IOException {
+ private TabularBlock readStandaloneTabular(int offset) throws IOException,
+ MalformedDataException {
int divider = theRadarData.readShort();
TabularBlock tabBlock = new TabularBlock();
int numPages = theRadarData.readUnsignedShort();
if ((divider != -1)) {
- throw new IOException(
- "This does not appear to be a standalone tabular block");
+ throw new MalformedDataException("Standalone tabular block offset "
+ + offset + " does not point to a standalone tabular block");
}
List> pages = new ArrayList>();
for (int p = 0; p < numPages; p++) {
@@ -678,7 +698,8 @@ public class Level3BaseRadar {
}
- private void parseRadarMessage(Headers headers) throws IOException {
+ private void parseRadarMessage(Headers headers) throws IOException,
+ MalformedDataException {
// Product Description Block
theRadarData.skip(2);
theLatitude = theRadarData.readInt() * 0.001;
@@ -728,16 +749,21 @@ public class Level3BaseRadar {
byte[] msg = new byte[120];
InputStream byt;
if (uncompressedSize + msg.length != theRawRadarByteArray.length) {
+ InputStream ins = null;
try {
theRadarData.reset();
theRadarData.readFully(msg);
- InputStream ins = new BZip2InputStream(theRadarData, false);
+ ins = new BZip2InputStream(theRadarData, false);
uncompressed = new byte[uncompressedSize];
ins.read(uncompressed);
} catch (IOException e) {
theHandler.handle(Priority.ERROR,
"Error decompressing product: ", e);
return;
+ } finally {
+ if (ins != null) {
+ ins.close();
+ }
}
theRawRadarByteArray = new byte[120 + uncompressed.length];
System.arraycopy(msg, 0, theRawRadarByteArray, 0, 120);
@@ -842,9 +868,9 @@ public class Level3BaseRadar {
}
tabularBlock.setString(builder.toString());
-
+
lookupAfosId();
-
+
if (RadarTextProductUtil.radarTable.keySet().contains(theMessageCode)) {
byte[] wmoid = wmoHeader.getBytes();
WMOHeader header = new WMOHeader(wmoid, headers);
@@ -944,10 +970,11 @@ public class Level3BaseRadar {
}
}
- private void parseGeneralStatusMessage() throws IOException {
+ private void parseGeneralStatusMessage() throws IOException,
+ MalformedDataException {
int divider = theRadarData.readShort();
if ((divider != -1)) {
- throw new IOException("This does not appear to be a gsm block");
+ throw new MalformedDataException("This is not a gsm block");
}
int blockLen = theRadarData.readShort();
byte[] buf = RadarUtil.subArray(theRawRadarByteArray, 22, blockLen);
diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin/src/com/raytheon/uf/common/dataplugin/exception/MalformedDataException.java b/edexOsgi/com.raytheon.uf.common.dataplugin/src/com/raytheon/uf/common/dataplugin/exception/MalformedDataException.java
new file mode 100644
index 0000000000..159be20188
--- /dev/null
+++ b/edexOsgi/com.raytheon.uf.common.dataplugin/src/com/raytheon/uf/common/dataplugin/exception/MalformedDataException.java
@@ -0,0 +1,73 @@
+/**
+ * This software was developed and / or modified by Raytheon Company,
+ * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+ *
+ * U.S. EXPORT CONTROLLED TECHNICAL DATA
+ * This software product contains export-restricted data whose
+ * export/transfer/disclosure is restricted by U.S. law. Dissemination
+ * to non-U.S. persons whether in the United States or abroad requires
+ * an export license or other authorization.
+ *
+ * Contractor Name: Raytheon Company
+ * Contractor Address: 6825 Pine Street, Suite 340
+ * Mail Stop B8
+ * Omaha, NE 68106
+ * 402.291.0100
+ *
+ * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+ * further licensing information.
+ **/
+package com.raytheon.uf.common.dataplugin.exception;
+
+/**
+ * An exception for when data is encoded incorrectly and is malformed.
+ *
+ *
+ *
+ * SOFTWARE HISTORY
+ *
+ * Date Ticket# Engineer Description
+ * ------------ ---------- ----------- --------------------------
+ * Jan 21, 2014 njensen Initial creation
+ *
+ *
+ *
+ * @author njensen
+ * @version 1.0
+ */
+
+public class MalformedDataException extends BadDataException {
+
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Default Constructor
+ *
+ */
+ public MalformedDataException() {
+ super();
+ }
+
+ /**
+ * @param message
+ */
+ public MalformedDataException(String message) {
+ super(message);
+ }
+
+ /**
+ * @param message
+ * @param cause
+ */
+ public MalformedDataException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ /**
+ * @param cause
+ */
+ public MalformedDataException(Throwable cause) {
+ super(cause);
+ }
+
+}
diff --git a/edexOsgi/com.raytheon.uf.edex.esb.camel/src/com/raytheon/uf/edex/esb/camel/ProcessUtil.java b/edexOsgi/com.raytheon.uf.edex.esb.camel/src/com/raytheon/uf/edex/esb/camel/ProcessUtil.java
index b76f7b2f81..5df90f1dae 100644
--- a/edexOsgi/com.raytheon.uf.edex.esb.camel/src/com/raytheon/uf/edex/esb/camel/ProcessUtil.java
+++ b/edexOsgi/com.raytheon.uf.edex.esb.camel/src/com/raytheon/uf/edex/esb/camel/ProcessUtil.java
@@ -20,11 +20,13 @@
package com.raytheon.uf.edex.esb.camel;
import java.io.File;
+import java.io.IOException;
import java.text.DecimalFormat;
import java.util.Arrays;
import java.util.Iterator;
import java.util.Map;
+import org.apache.camel.Exchange;
import org.apache.camel.Header;
import org.apache.camel.Headers;
@@ -34,6 +36,8 @@ import com.raytheon.uf.common.stats.ProcessEvent;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
+import com.raytheon.uf.common.util.FileUtil;
+import com.raytheon.uf.edex.core.props.PropertiesFactory;
/**
* Provides logging and deletion services for camel
@@ -43,9 +47,10 @@ import com.raytheon.uf.common.status.UFStatus.Priority;
* SOFTWARE HISTORY
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
- * Dec 1, 2008 chammack Initial creation
- * Feb 05, 2013 1580 mpduff EventBus refactor.
- * Feb 12, 2013 1615 bgonzale Changed ProcessEvent pluginName to dataType.
+ * Dec 1, 2008 chammack Initial creation
+ * Feb 05, 2013 1580 mpduff EventBus refactor.
+ * Feb 12, 2013 1615 bgonzale Changed ProcessEvent pluginName to dataType.
+ * Jan 21, 2014 2627 njensen Added logFailedData() and logFailureAsInfo()
*
*
*
@@ -69,6 +74,26 @@ public class ProcessUtil {
};
+ private static final String FAILED_DIR;
+
+ protected static final boolean RETAIN_FAILED_DATA;
+
+ static {
+ // this will probably only ever be true on a testbed
+ RETAIN_FAILED_DATA = Boolean.getBoolean("retain.failed.data");
+ if (RETAIN_FAILED_DATA) {
+ FAILED_DIR = PropertiesFactory.getInstance().getEnvProperties()
+ .getEnvValue("DEFAULTDATADIR")
+ + File.separator + "failed";
+ File file = new File(FAILED_DIR);
+ if (!file.exists()) {
+ file.mkdir();
+ }
+ } else {
+ FAILED_DIR = null;
+ }
+ }
+
/**
* Get the value of a specified property if it exists.
*
@@ -118,11 +143,23 @@ public class ProcessUtil {
}
/**
+ * Logs the processing and latency time of a file
*
* @param headers
*/
public void log(@Headers Map headers) {
+ logInternal(headers, true);
+ }
+ /**
+ * Logs the processing and latency time of a file
+ *
+ * @param headers
+ * the headers associated with the ingest routes
+ * @param successful
+ * whether or not the file was successfully ingested
+ */
+ protected void logInternal(Map headers, boolean successful) {
long curTime = System.currentTimeMillis();
StringBuilder sb = new StringBuilder(128);
@@ -168,7 +205,7 @@ public class ProcessUtil {
// processing in less than 0 millis isn't trackable, usually due to an
// error occurred and statement logged incorrectly
- if ((processEvent.getProcessingLatency() > 0)
+ if (successful && (processEvent.getProcessingLatency() > 0)
&& (processEvent.getProcessingTime() > 0)) {
EventBus.publish(processEvent);
}
@@ -179,5 +216,66 @@ public class ProcessUtil {
} else {
handler.handle(Priority.INFO, "No logging information available");
}
+
}
+
+ /**
+ * Logs a failure to ingest a file. Potentially saves off the data that
+ * failed to ingest, based on a system property.
+ *
+ * @param ex
+ * the exchange that failed to ingest
+ */
+ public void logFailedData(Exchange ex) {
+ Exception e = ex.getException();
+ if (e == null) {
+ e = ex.getProperty(Exchange.EXCEPTION_CAUGHT, Exception.class);
+ }
+ Map headers = ex.getIn().getHeaders();
+ String fullpath = getHeaderProperty(headers, ("ingestFileName"));
+ handler.error("Failed to ingest " + fullpath, e);
+
+ if (RETAIN_FAILED_DATA) {
+ File badfile = new File(fullpath);
+ if (badfile.exists()) {
+ String filename = badfile.getName();
+ File keepfile = new File(FAILED_DIR + File.separator + filename);
+ try {
+ FileUtil.copyFile(badfile, keepfile);
+ handler.info("Copied failed data to " + keepfile.getPath());
+ } catch (IOException e1) {
+ handler.error(
+ "Unable to copy failed data for later analysis", e);
+ }
+ }
+ }
+
+ logInternal(headers, false);
+ }
+
+ /**
+ * Logs a failure to ingest data as an info message. This should only be
+ * used when the failure can be considered as expected given the input, such
+ * as submitting invalid/bad data to a decoder.
+ *
+ * @param ex
+ * the exchange that failed to ingest
+ */
+ public void logFailureAsInfo(Exchange ex) {
+ Exception e = ex.getException();
+ if (e == null) {
+ e = ex.getProperty(Exchange.EXCEPTION_CAUGHT, Exception.class);
+ }
+ Map headers = ex.getIn().getHeaders();
+ String fullpath = getHeaderProperty(headers, ("ingestFileName"));
+ String msg = "Discarding " + fullpath;
+ if (e != null) {
+ msg += " due to " + e.getClass().getSimpleName() + ": "
+ + e.getLocalizedMessage();
+ }
+ handler.info(msg);
+
+ logInternal(headers, false);
+ }
+
}
diff --git a/edexOsgi/com.raytheon.uf.edex.ingest/res/spring/persist-ingest.xml b/edexOsgi/com.raytheon.uf.edex.ingest/res/spring/persist-ingest.xml
index 988582f79e..e58c99323a 100644
--- a/edexOsgi/com.raytheon.uf.edex.ingest/res/spring/persist-ingest.xml
+++ b/edexOsgi/com.raytheon.uf.edex.ingest/res/spring/persist-ingest.xml
@@ -54,11 +54,22 @@
-
+
+
+
+
+
+
+
+
+
+
+
+