From edae6656694d643e3dc0fc9aab75b1a082bbd96f Mon Sep 17 00:00:00 2001 From: Nate Jensen Date: Wed, 22 Jan 2014 09:24:12 -0600 Subject: [PATCH] Issue #2627 configurable ability to retain failed data, log 'expected' failures as infos without stacktraces Change-Id: I199c2125bb6c35b703b71ac5e8342d7c0a4dbcb0 Former-commit-id: 50b6eced5d0ecb50d27add8670465e290b2ff330 [formerly 84f2de90fb307bead050de7eb3c1cbf7c5ee94ee [formerly 36ad567a6de562b8be183beecc80c30cc177efbd] [formerly 50b6eced5d0ecb50d27add8670465e290b2ff330 [formerly 0a1049cb785f782fe7bea8379f7df694dfb45e23]]] Former-commit-id: 84f2de90fb307bead050de7eb3c1cbf7c5ee94ee [formerly 36ad567a6de562b8be183beecc80c30cc177efbd] Former-commit-id: 84f2de90fb307bead050de7eb3c1cbf7c5ee94ee Former-commit-id: fe0af876a83084c2ea9ea0d1ea14558293a374e2 --- edexOsgi/build.edex/esb/conf/wrapper.conf | 6 +- .../res/spring/radar-ingest.xml | 6 +- .../edex/plugin/radar/RadarDecoder.java | 515 +++++++++--------- .../plugin/radar/level3/Level3BaseRadar.java | 73 ++- .../exception/MalformedDataException.java | 73 +++ .../uf/edex/esb/camel/ProcessUtil.java | 106 +++- .../res/spring/persist-ingest.xml | 13 +- 7 files changed, 498 insertions(+), 294 deletions(-) create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin/src/com/raytheon/uf/common/dataplugin/exception/MalformedDataException.java diff --git a/edexOsgi/build.edex/esb/conf/wrapper.conf b/edexOsgi/build.edex/esb/conf/wrapper.conf index 7d96702b5a..e8720b0848 100644 --- a/edexOsgi/build.edex/esb/conf/wrapper.conf +++ b/edexOsgi/build.edex/esb/conf/wrapper.conf @@ -142,14 +142,12 @@ wrapper.java.additional.log.5=-Djava.util.logging.config.file=${EDEX_HOME}/conf/ wrapper.java.additional.web.1=-Dweb.port=8080 wrapper.java.additional.web.2=-Dconfidential.port=8443 -# notifies SerializationManager to initialize hibernatables, can be removed IF Hibernatables code -# is removed from SerializationManager -wrapper.java.additional.misc.1=-DinitializeHibernatables=true - # the max size in MB of any stream sent to thrift, this prevents the OutOfMemory # errors reported by thrift sometimes when the stream is corrupt/incorrect wrapper.java.additional.thrift.maxStreamSize=-Dthrift.stream.maxsize=200 +wrapper.java.additional.edex.retain.failed.data=-Dretain.failed.data=${RETAIN_FAILED} + # enables yourkit profiling, determined by flag to start.sh wrapper.java.additional.profile.1=${PROFILER_PARAM_1} diff --git a/edexOsgi/com.raytheon.edex.plugin.radar/res/spring/radar-ingest.xml b/edexOsgi/com.raytheon.edex.plugin.radar/res/spring/radar-ingest.xml index 4099b9b31c..d5fd8f0fec 100644 --- a/edexOsgi/com.raytheon.edex.plugin.radar/res/spring/radar-ingest.xml +++ b/edexOsgi/com.raytheon.edex.plugin.radar/res/spring/radar-ingest.xml @@ -81,8 +81,12 @@ + com.raytheon.uf.common.dataplugin.exception.MalformedDataException + + + java.lang.Throwable - + diff --git a/edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/RadarDecoder.java b/edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/RadarDecoder.java index 586451a579..ce8eae0a71 100644 --- a/edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/RadarDecoder.java +++ b/edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/RadarDecoder.java @@ -97,6 +97,8 @@ import com.raytheon.uf.edex.wmo.message.WMOHeader; * added status to decode. * Aug 30, 2013 2298 rjpeter Make getPluginName abstract * Oct 09, 2013 2457 bsteffen Improve error message for missing icao. + * Jan 21, 2014 2627 njensen Removed decode()'s try/catch, camel route will do try/catch + * * * * @author bphillip @@ -172,7 +174,7 @@ public class RadarDecoder extends AbstractDecoder { * @throws DecoderException */ public PluginDataObject[] decode(byte[] messageData, Headers headers) - throws DecoderException { + throws Exception { if (headers != null) { traceId = (String) headers.get("traceId"); } @@ -181,270 +183,261 @@ public class RadarDecoder extends AbstractDecoder { // decode the product String arch = new String(messageData, 0, 4); - try { - ITimer timer = TimeUtil.getTimer(); - timer.start(); - // for level2 data, this does not happen very often - if (LEVEL_TWO_IDENTS.contains(arch)) { - decodeLevelTwoData(messageData, recordList); + ITimer timer = TimeUtil.getTimer(); + + timer.start(); + // for level2 data, this does not happen very often + if (LEVEL_TWO_IDENTS.contains(arch)) { + decodeLevelTwoData(messageData, recordList); + } + // for free text messages, which come in with the following wmo + else if (NOUS.equals(arch)) { + decodeFreeTextMessage(messageData, headers); + } else { + if (headers.get("header") != null) { + // handle an interesting special case + String wmoHeader = headers.get("header").toString(); + if (wmoHeader.contains("SDUS4")) { + WMOHeader header = new WMOHeader(wmoHeader.getBytes(), + headers); + String dataString = new String(messageData, 0, + messageData.length).substring(1, + messageData.length - 1); + String siteId = dataString.substring(0, 3); + AFOSProductId afos = new AFOSProductId("WSR", "ROB", siteId); + // store the product ROB that is barely do-able + Calendar cal = (TimeTools.allowArchive() ? header + .getHeaderDate() : Calendar.getInstance()); + RadarEdexTextProductUtil.storeTextProduct(afos, header, + dataString, true, cal); + return new PluginDataObject[0]; + } } - // for free text messages, which come in with the following wmo - else if (NOUS.equals(arch)) { - decodeFreeTextMessage(messageData, headers); + Level3BaseRadar l3Radar = new Level3BaseRadar(messageData, headers, + infoDict); + RadarRecord record = new RadarRecord(); + record.setProductCode(l3Radar.getMessageCode()); + record.setDataTime(new DataTime(l3Radar.getMessageTimestamp())); + RadarStation station = RadarSpatialUtil + .getRadarStationByRpgIdDec(l3Radar.getSourceId()); + if (station == null) { + record.setIcao("unkn"); + logger.error(headers.get("ingestfilename") + + " contains an rpg id(" + l3Radar.getSourceId() + + ") that is not in the radar_spatial table."); } else { - if (headers.get("header") != null) { - // handle an interesting special case - String wmoHeader = headers.get("header").toString(); - if (wmoHeader.contains("SDUS4")) { - WMOHeader header = new WMOHeader(wmoHeader.getBytes(), - headers); - String dataString = new String(messageData, 0, - messageData.length).substring(1, - messageData.length - 1); - String siteId = dataString.substring(0, 3); - AFOSProductId afos = new AFOSProductId("WSR", "ROB", - siteId); - // store the product ROB that is barely do-able - Calendar cal = (TimeTools.allowArchive() ? header - .getHeaderDate() : Calendar.getInstance()); - RadarEdexTextProductUtil.storeTextProduct(afos, header, - dataString, true, cal); - return new PluginDataObject[0]; - } - } - Level3BaseRadar l3Radar = new Level3BaseRadar(messageData, - headers, infoDict); - RadarRecord record = new RadarRecord(); - record.setProductCode(l3Radar.getMessageCode()); - record.setDataTime(new DataTime(l3Radar.getMessageTimestamp())); - RadarStation station = RadarSpatialUtil - .getRadarStationByRpgIdDec(l3Radar.getSourceId()); - if (station == null) { - record.setIcao("unkn"); - logger.error(headers.get("ingestfilename") - + " contains an rpg id(" + l3Radar.getSourceId() - + ") that is not in the radar_spatial table."); - } else { - record.setIcao(station.getRdaId().toLowerCase()); - } - - record.setLocation(station); - - RadarInfo info = infoDict.getInfo(record.getProductCode()); - if (info == null) { - theHandler.handle( - Priority.ERROR, - "Unknown radar product code: " - + record.getProductCode() + " for " - + headers.get("ingestfilename")); - return new PluginDataObject[0]; - } - record.setFormat(info.getFormat()); - record.setNumLevels(info.getNumLevels()); - record.setGateResolution(info.getResolution()); - record.setMnemonic(info.getMnemonic()); - record.setDisplayModes(info.getDisplayModes()); - record.setUnit(info.getUnit()); - - // -- some product specific decode functionality -- - // the general status message product - if (l3Radar.getMessageCode() == l3Radar.GSM_MESSAGE) { - record.setGsmMessage(l3Radar.getGsmBlock().getMessage()); - record.setPrimaryElevationAngle(0.0); - record.setTrueElevationAngle(0.0f); - handleRadarStatus(record); - } - // the product request response product - else if (l3Radar.getMessageCode() == l3Radar.PRODUCT_REQUEST_RESPONSE_MESSAGE) { - // do nothing with this, it will get excessive otherwise! - return new PluginDataObject[0]; - } - // the user alert message product - else if (l3Radar.getMessageCode() == USER_ALERT_MESSAGE) { - EDEXUtil.sendMessageAlertViz(Priority.VERBOSE, - RadarConstants.PLUGIN_ID, EDEX, RADAR, - record.getIcao() + ": User Alert Message Received", - l3Radar.getTabularBlock().toString(), null); - return new PluginDataObject[0]; - } - // handle the other case for free text message - else if (l3Radar.getMessageCode() == FREE_TEXT_MESSAGE) { - // product already stored to the text database, so just send - // to alertviz - String formattedMsg = l3Radar.getTabularBlock().toString() - .replace("Page 1\n\t", ""); - EDEXUtil.sendMessageAlertViz(Priority.SIGNIFICANT, - RadarConstants.PLUGIN_ID, EDEX, RADAR, - record.getIcao() + ": Free Text Message Received", - formattedMsg, null); - return new PluginDataObject[0]; - } - // the alert adaptations parameters product - else if (l3Radar.getMessageCode() == l3Radar.ALERT_ADAPTATION_PARAMETERS) { - record.setAapMessage(l3Radar.getAapMessage()); - record.setPrimaryElevationAngle(0.0); - record.setTrueElevationAngle(0.0f); - EDEXUtil.sendMessageAlertViz( - Priority.VERBOSE, - RadarConstants.PLUGIN_ID, - EDEX, - RADAR, - record.getIcao() - + ": Alert Adapation Parameter Message Received", - l3Radar.getAapMessage().toString(), null); - } - // the alert message product - else if (l3Radar.getMessageCode() == l3Radar.ALERT_MESSAGE) { - record.setPrimaryElevationAngle(0.0); - record.setTrueElevationAngle(0.0f); - AlertMessage msg = l3Radar.getAlertMessage(); - String details = "Alert Area : " + msg.getAlertAreaNum() - + "\n"; - details += "Position : " + msg.getGridBoxAz() - + " deg\nRange : " + msg.getGridBoxRange() + "nm\n"; - String category = AlertCategory.values()[msg - .getAlertCategory()].toString(); - category = category.substring(category.indexOf("_")); - category = category.replaceAll("_", " "); - details += "Alert Category : " + category + "\n"; - details += "Threshold : " + msg.getThresholdValue() + "\n"; - details += "Exceeding : " + msg.getExceedingValue() + "\n"; - details += "Storm Cell ID :" + msg.getStormId() + "\n"; - record.setAlertMessage(msg); - EDEXUtil.sendMessageAlertViz(Priority.SIGNIFICANT, - RadarConstants.PLUGIN_ID, EDEX, RADAR, - record.getIcao() + ": Alert Message Received", - details, null); - } else { - record.setLatitude((float) l3Radar.getLatitude()); - record.setLongitude((float) l3Radar.getLongitude()); - record.setElevation((float) l3Radar.getHeight()); - record.setVolumeCoveragePattern(l3Radar - .getVolumeCoveragePattern()); - record.setOperationalMode(l3Radar.getOperationalMode()); - - record.setElevationNumber(l3Radar.getElevationNumber()); - // some products don't have real elevation angles, 0 is a - // default value - if (record.getElevationNumber() == 0) { - record.setTrueElevationAngle(0f); - } else { - record.setTrueElevationAngle(l3Radar - .getProductDependentValue(2) * 0.1f); - } - - // determine to use the primary elevations or the elevation - // in the terminal radar configuration file - if (TerminalRadarUtils.isTerminalRadar(record.getIcao())) { - Double elevation = TerminalRadarUtils.getPrimarysMap( - record.getIcao()).get( - TiltAngleBin.getPrimaryElevationAngle(record - .getTrueElevationAngle())); - if (elevation != null) { - record.setPrimaryElevationAngle(elevation - .doubleValue()); - } else { - // fall back - record.setPrimaryElevationAngle(record - .getTrueElevationAngle().doubleValue()); - } - } else { - record.setPrimaryElevationAngle(TiltAngleBin - .getPrimaryElevationAngle(record - .getTrueElevationAngle())); - } - - // code specific for clutter filter control - if (record.getProductCode() == CLUTTER_FILTER_CONTROL) { - int segment = ((int) (Math.log(l3Radar - .getProductDependentValue(0)) / Math.log(2))); - record.setLayer((double) segment); - } - // code specific for user select accum - else if (record.getProductCode() == USER_SELECT_ACCUM) { - int layer = 0; // Default to zero - - int timeSpan = l3Radar.getProductDependentValue(1); - if (timeSpan == 60) { - layer = 1; - } else if (timeSpan == 120) { - layer = 2; - } else if (timeSpan == 180) { - layer = 3; - } else if (timeSpan == 360) { - layer = 4; - } else if (timeSpan == 720) { - layer = 5; - } else if (timeSpan == 1440) { - layer = 6; - } else { - layer = 0; - } - record.setLayer((double) layer); - } - - // handle times because radar times are sent out in batches - // (a volume scan) and we want the volume scan time to be - // part of the data uri - if (l3Radar.getVolumeScanTime() != null) { - record.setDataTime(new DataTime(l3Radar - .getVolumeScanTime().getTime())); - record.setVolScanTime(l3Radar - .getProductGenerationTime().getTime()); - } else { - record.setDataTime(new DataTime(l3Radar - .getMessageTimestamp().getTime())); - record.setVolScanTime(l3Radar.getMessageTimestamp() - .getTime()); - } - - // thresholds specific per product and site - if (l3Radar.getDataLevelThresholds() != null) { - for (int i = 0; i < 16; ++i) { - record.setThreshold(i, - l3Radar.getDataLevelThreshold(i)); - } - } - // values that are product dependent as defined in the ICD - record.setProductDependentValues(l3Radar - .getProductDependentValue()); - - processSymbologyBlock(record, l3Radar.getSymbologyBlock()); - - GraphicBlock gb = l3Radar.getGraphicBlock(); - record.setGraphicBlock(gb); - - // Tabular block is where most of the text values are... - TabularBlock tb = l3Radar.getTabularBlock(); - if (tb != null) { - // complicated, but makes for logical access to needed - // elements - HashMap>> map = new HashMap>>(); - HashMap> recordVals = new HashMap>(); - RadarTabularBlockParser.parseTabularBlock(tb, - record.getProductCode(), map, recordVals); - record.setProductVals(map); - record.setMapRecordVals(recordVals); - record.setTabularBlock(tb); - } - } - - try { - finalizeRecord(record); - } catch (PluginException e) { - logger.error(e); - return new PluginDataObject[0]; - } - - timer.stop(); - perfLog.logDuration("Time to Decode", timer.getElapsedTime()); - - recordList.add(record); + record.setIcao(station.getRdaId().toLowerCase()); } - } catch (Exception e) { - theHandler.handle(Priority.ERROR, "Couldn't properly handle " - + headers.get("ingestfilename"), e); + + record.setLocation(station); + + RadarInfo info = infoDict.getInfo(record.getProductCode()); + if (info == null) { + theHandler.handle( + Priority.ERROR, + "Unknown radar product code: " + + record.getProductCode() + " for " + + headers.get("ingestfilename")); + return new PluginDataObject[0]; + } + record.setFormat(info.getFormat()); + record.setNumLevels(info.getNumLevels()); + record.setGateResolution(info.getResolution()); + record.setMnemonic(info.getMnemonic()); + record.setDisplayModes(info.getDisplayModes()); + record.setUnit(info.getUnit()); + + // -- some product specific decode functionality -- + // the general status message product + if (l3Radar.getMessageCode() == l3Radar.GSM_MESSAGE) { + record.setGsmMessage(l3Radar.getGsmBlock().getMessage()); + record.setPrimaryElevationAngle(0.0); + record.setTrueElevationAngle(0.0f); + handleRadarStatus(record); + } + // the product request response product + else if (l3Radar.getMessageCode() == l3Radar.PRODUCT_REQUEST_RESPONSE_MESSAGE) { + // do nothing with this, it will get excessive otherwise! + return new PluginDataObject[0]; + } + // the user alert message product + else if (l3Radar.getMessageCode() == USER_ALERT_MESSAGE) { + EDEXUtil.sendMessageAlertViz(Priority.VERBOSE, + RadarConstants.PLUGIN_ID, EDEX, RADAR, record.getIcao() + + ": User Alert Message Received", l3Radar + .getTabularBlock().toString(), null); + return new PluginDataObject[0]; + } + // handle the other case for free text message + else if (l3Radar.getMessageCode() == FREE_TEXT_MESSAGE) { + // product already stored to the text database, so just send + // to alertviz + String formattedMsg = l3Radar.getTabularBlock().toString() + .replace("Page 1\n\t", ""); + EDEXUtil.sendMessageAlertViz(Priority.SIGNIFICANT, + RadarConstants.PLUGIN_ID, EDEX, RADAR, record.getIcao() + + ": Free Text Message Received", formattedMsg, + null); + return new PluginDataObject[0]; + } + // the alert adaptations parameters product + else if (l3Radar.getMessageCode() == l3Radar.ALERT_ADAPTATION_PARAMETERS) { + record.setAapMessage(l3Radar.getAapMessage()); + record.setPrimaryElevationAngle(0.0); + record.setTrueElevationAngle(0.0f); + EDEXUtil.sendMessageAlertViz( + Priority.VERBOSE, + RadarConstants.PLUGIN_ID, + EDEX, + RADAR, + record.getIcao() + + ": Alert Adapation Parameter Message Received", + l3Radar.getAapMessage().toString(), null); + } + // the alert message product + else if (l3Radar.getMessageCode() == l3Radar.ALERT_MESSAGE) { + record.setPrimaryElevationAngle(0.0); + record.setTrueElevationAngle(0.0f); + AlertMessage msg = l3Radar.getAlertMessage(); + String details = "Alert Area : " + msg.getAlertAreaNum() + "\n"; + details += "Position : " + msg.getGridBoxAz() + + " deg\nRange : " + msg.getGridBoxRange() + "nm\n"; + String category = AlertCategory.values()[msg.getAlertCategory()] + .toString(); + category = category.substring(category.indexOf("_")); + category = category.replaceAll("_", " "); + details += "Alert Category : " + category + "\n"; + details += "Threshold : " + msg.getThresholdValue() + "\n"; + details += "Exceeding : " + msg.getExceedingValue() + "\n"; + details += "Storm Cell ID :" + msg.getStormId() + "\n"; + record.setAlertMessage(msg); + EDEXUtil.sendMessageAlertViz(Priority.SIGNIFICANT, + RadarConstants.PLUGIN_ID, EDEX, RADAR, record.getIcao() + + ": Alert Message Received", details, null); + } else { + record.setLatitude((float) l3Radar.getLatitude()); + record.setLongitude((float) l3Radar.getLongitude()); + record.setElevation((float) l3Radar.getHeight()); + record.setVolumeCoveragePattern(l3Radar + .getVolumeCoveragePattern()); + record.setOperationalMode(l3Radar.getOperationalMode()); + + record.setElevationNumber(l3Radar.getElevationNumber()); + // some products don't have real elevation angles, 0 is a + // default value + if (record.getElevationNumber() == 0) { + record.setTrueElevationAngle(0f); + } else { + record.setTrueElevationAngle(l3Radar + .getProductDependentValue(2) * 0.1f); + } + + // determine to use the primary elevations or the elevation + // in the terminal radar configuration file + if (TerminalRadarUtils.isTerminalRadar(record.getIcao())) { + Double elevation = TerminalRadarUtils.getPrimarysMap( + record.getIcao()).get( + TiltAngleBin.getPrimaryElevationAngle(record + .getTrueElevationAngle())); + if (elevation != null) { + record.setPrimaryElevationAngle(elevation.doubleValue()); + } else { + // fall back + record.setPrimaryElevationAngle(record + .getTrueElevationAngle().doubleValue()); + } + } else { + record.setPrimaryElevationAngle(TiltAngleBin + .getPrimaryElevationAngle(record + .getTrueElevationAngle())); + } + + // code specific for clutter filter control + if (record.getProductCode() == CLUTTER_FILTER_CONTROL) { + int segment = ((int) (Math.log(l3Radar + .getProductDependentValue(0)) / Math.log(2))); + record.setLayer((double) segment); + } + // code specific for user select accum + else if (record.getProductCode() == USER_SELECT_ACCUM) { + int layer = 0; // Default to zero + + int timeSpan = l3Radar.getProductDependentValue(1); + if (timeSpan == 60) { + layer = 1; + } else if (timeSpan == 120) { + layer = 2; + } else if (timeSpan == 180) { + layer = 3; + } else if (timeSpan == 360) { + layer = 4; + } else if (timeSpan == 720) { + layer = 5; + } else if (timeSpan == 1440) { + layer = 6; + } else { + layer = 0; + } + record.setLayer((double) layer); + } + + // handle times because radar times are sent out in batches + // (a volume scan) and we want the volume scan time to be + // part of the data uri + if (l3Radar.getVolumeScanTime() != null) { + record.setDataTime(new DataTime(l3Radar.getVolumeScanTime() + .getTime())); + record.setVolScanTime(l3Radar.getProductGenerationTime() + .getTime()); + } else { + record.setDataTime(new DataTime(l3Radar + .getMessageTimestamp().getTime())); + record.setVolScanTime(l3Radar.getMessageTimestamp() + .getTime()); + } + + // thresholds specific per product and site + if (l3Radar.getDataLevelThresholds() != null) { + for (int i = 0; i < 16; ++i) { + record.setThreshold(i, l3Radar.getDataLevelThreshold(i)); + } + } + // values that are product dependent as defined in the ICD + record.setProductDependentValues(l3Radar + .getProductDependentValue()); + + processSymbologyBlock(record, l3Radar.getSymbologyBlock()); + + GraphicBlock gb = l3Radar.getGraphicBlock(); + record.setGraphicBlock(gb); + + // Tabular block is where most of the text values are... + TabularBlock tb = l3Radar.getTabularBlock(); + if (tb != null) { + // complicated, but makes for logical access to needed + // elements + HashMap>> map = new HashMap>>(); + HashMap> recordVals = new HashMap>(); + RadarTabularBlockParser.parseTabularBlock(tb, + record.getProductCode(), map, recordVals); + record.setProductVals(map); + record.setMapRecordVals(recordVals); + record.setTabularBlock(tb); + } + } + + try { + finalizeRecord(record); + } catch (PluginException e) { + logger.error(e); + return new PluginDataObject[0]; + } + + timer.stop(); + perfLog.logDuration("Time to Decode", timer.getElapsedTime()); + + recordList.add(record); } return recordList.toArray(new PluginDataObject[recordList.size()]); diff --git a/edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/level3/Level3BaseRadar.java b/edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/level3/Level3BaseRadar.java index 2c83e63566..7be200f199 100644 --- a/edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/level3/Level3BaseRadar.java +++ b/edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/level3/Level3BaseRadar.java @@ -40,6 +40,7 @@ import org.itadaki.bzip2.BZip2InputStream; import com.raytheon.edex.esb.Headers; import com.raytheon.edex.plugin.radar.util.RadarEdexTextProductUtil; import com.raytheon.edex.plugin.radar.util.RadarSpatialUtil; +import com.raytheon.uf.common.dataplugin.exception.MalformedDataException; import com.raytheon.uf.common.dataplugin.radar.RadarStation; import com.raytheon.uf.common.dataplugin.radar.level3.AlertAdaptationParameters; import com.raytheon.uf.common.dataplugin.radar.level3.AlertMessage; @@ -91,7 +92,16 @@ import com.raytheon.uf.edex.wmo.message.WMOHeader; * graphics context and the extents of the Component used to create the Graphics * object. * - * Copyright 2006 Raytheon Corporation + * + * 
+ * SOFTWARE HISTORY
+ * Date          Ticket#  Engineer    Description
+ * ------------- -------- ----------- --------------------------
+ * --/--/2006             brockwoo    Initial creation
+ * Jan 21, 2014  2627     njensen     Changed offset errors to MalformedDataException
+ * 
+ *
+ * * * @author Bryan Rockwood * @version 1.0 @@ -200,8 +210,10 @@ public class Level3BaseRadar { * A java.io.File object containing a raw radar file * @throws IOException * If the radar head parsing fails, an IO exception is thrown + * @throws MalformedDataException */ - public Level3BaseRadar(File aRadar, Headers headers) throws IOException { + public Level3BaseRadar(File aRadar, Headers headers) throws IOException, + MalformedDataException { int fileSize = (int) aRadar.length(); byte[] tempRawRadarByteArray = new byte[fileSize]; @@ -227,13 +239,15 @@ public class Level3BaseRadar { * A byte array containing a raw radar file * @throws IOException * If the radar head parsing fails, an IO exception is thrown + * @throws MalformedDataException */ - public Level3BaseRadar(byte[] aRadar, Headers headers) throws IOException { + public Level3BaseRadar(byte[] aRadar, Headers headers) throws IOException, + MalformedDataException { init(aRadar, headers); } public Level3BaseRadar(byte[] aRadar, Headers headers, RadarInfoDict dict) - throws IOException { + throws IOException, MalformedDataException { this.dict = dict; init(aRadar, headers); } @@ -241,8 +255,10 @@ public class Level3BaseRadar { /** * @param aRadar * @throws IOException + * @throws MalformedDataException */ - private void init(byte[] aRadar, Headers headers) throws IOException { + private void init(byte[] aRadar, Headers headers) throws IOException, + MalformedDataException { // printPacketContents(aRadar); int wmoHeaderSize; @@ -512,7 +528,8 @@ public class Level3BaseRadar { return dataLevelThresholds[code]; } - private SymbologyBlock readSymbologyBlock(int offset) throws IOException { + private SymbologyBlock readSymbologyBlock(int offset) throws IOException, + MalformedDataException { SymbologyBlock symBlock = null; if (offset != 0) { theRadarData.reset(); @@ -520,8 +537,8 @@ public class Level3BaseRadar { int divider = theRadarData.readShort(); int blockId = theRadarData.readUnsignedShort(); if ((divider != -1) || (blockId != SymbologyBlock.getBlockId())) { - throw new IOException( - "This does not appear to be a symbology block"); + throw new MalformedDataException("Symbology block offset " + + offset + " does not point to a symbology block"); } int blockLen = theRadarData.readInt(); byte[] buf = RadarUtil.subArray(theRawRadarByteArray, offset, @@ -537,7 +554,8 @@ public class Level3BaseRadar { return symBlock; } - private GraphicBlock readGraphicBlock(int offset) throws IOException { + private GraphicBlock readGraphicBlock(int offset) throws IOException, + MalformedDataException { GraphicBlock graphicBlock = null; if (offset != 0) { theRadarData.reset(); @@ -545,8 +563,8 @@ public class Level3BaseRadar { int divider = theRadarData.readShort(); int blockId = theRadarData.readUnsignedShort(); if ((divider != -1) || (blockId != GraphicBlock.getBlockId())) { - throw new IOException( - "This does not appear to be a graphic block"); + throw new MalformedDataException("Graphic block offset " + + offset + " does not point to a graphic block"); } int blockLen = theRadarData.readInt(); byte[] buf = RadarUtil.subArray(theRawRadarByteArray, offset, @@ -559,7 +577,8 @@ public class Level3BaseRadar { return graphicBlock; } - private TabularBlock readTabularBlock(int offset) throws IOException { + private TabularBlock readTabularBlock(int offset) throws IOException, + MalformedDataException { TabularBlock tabBlock = null; if (offset != 0) { theRadarData.reset(); @@ -567,8 +586,8 @@ public class Level3BaseRadar { int divider = theRadarData.readShort(); int blockId = theRadarData.readUnsignedShort(); if ((divider != -1) || (blockId != TabularBlock.getBlockId())) { - throw new IOException( - "This does not appear to be a tabular block"); + throw new MalformedDataException("Tabular block offset " + + offset + " does not point to a tabular block"); } int blockLen = theRadarData.readInt(); byte[] buf = RadarUtil.subArray(theRawRadarByteArray, offset, @@ -587,13 +606,14 @@ public class Level3BaseRadar { * @return * @throws IOException */ - private TabularBlock readStandaloneTabular(int offset) throws IOException { + private TabularBlock readStandaloneTabular(int offset) throws IOException, + MalformedDataException { int divider = theRadarData.readShort(); TabularBlock tabBlock = new TabularBlock(); int numPages = theRadarData.readUnsignedShort(); if ((divider != -1)) { - throw new IOException( - "This does not appear to be a standalone tabular block"); + throw new MalformedDataException("Standalone tabular block offset " + + offset + " does not point to a standalone tabular block"); } List> pages = new ArrayList>(); for (int p = 0; p < numPages; p++) { @@ -678,7 +698,8 @@ public class Level3BaseRadar { } - private void parseRadarMessage(Headers headers) throws IOException { + private void parseRadarMessage(Headers headers) throws IOException, + MalformedDataException { // Product Description Block theRadarData.skip(2); theLatitude = theRadarData.readInt() * 0.001; @@ -728,16 +749,21 @@ public class Level3BaseRadar { byte[] msg = new byte[120]; InputStream byt; if (uncompressedSize + msg.length != theRawRadarByteArray.length) { + InputStream ins = null; try { theRadarData.reset(); theRadarData.readFully(msg); - InputStream ins = new BZip2InputStream(theRadarData, false); + ins = new BZip2InputStream(theRadarData, false); uncompressed = new byte[uncompressedSize]; ins.read(uncompressed); } catch (IOException e) { theHandler.handle(Priority.ERROR, "Error decompressing product: ", e); return; + } finally { + if (ins != null) { + ins.close(); + } } theRawRadarByteArray = new byte[120 + uncompressed.length]; System.arraycopy(msg, 0, theRawRadarByteArray, 0, 120); @@ -842,9 +868,9 @@ public class Level3BaseRadar { } tabularBlock.setString(builder.toString()); - + lookupAfosId(); - + if (RadarTextProductUtil.radarTable.keySet().contains(theMessageCode)) { byte[] wmoid = wmoHeader.getBytes(); WMOHeader header = new WMOHeader(wmoid, headers); @@ -944,10 +970,11 @@ public class Level3BaseRadar { } } - private void parseGeneralStatusMessage() throws IOException { + private void parseGeneralStatusMessage() throws IOException, + MalformedDataException { int divider = theRadarData.readShort(); if ((divider != -1)) { - throw new IOException("This does not appear to be a gsm block"); + throw new MalformedDataException("This is not a gsm block"); } int blockLen = theRadarData.readShort(); byte[] buf = RadarUtil.subArray(theRawRadarByteArray, 22, blockLen); diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin/src/com/raytheon/uf/common/dataplugin/exception/MalformedDataException.java b/edexOsgi/com.raytheon.uf.common.dataplugin/src/com/raytheon/uf/common/dataplugin/exception/MalformedDataException.java new file mode 100644 index 0000000000..159be20188 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin/src/com/raytheon/uf/common/dataplugin/exception/MalformedDataException.java @@ -0,0 +1,73 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.common.dataplugin.exception; + +/** + * An exception for when data is encoded incorrectly and is malformed. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Jan 21, 2014            njensen     Initial creation
+ * 
+ *
+ * + * @author njensen + * @version 1.0 + */ + +public class MalformedDataException extends BadDataException { + + private static final long serialVersionUID = 1L; + + /** + * Default Constructor + * + */ + public MalformedDataException() { + super(); + } + + /** + * @param message + */ + public MalformedDataException(String message) { + super(message); + } + + /** + * @param message + * @param cause + */ + public MalformedDataException(String message, Throwable cause) { + super(message, cause); + } + + /** + * @param cause + */ + public MalformedDataException(Throwable cause) { + super(cause); + } + +} diff --git a/edexOsgi/com.raytheon.uf.edex.esb.camel/src/com/raytheon/uf/edex/esb/camel/ProcessUtil.java b/edexOsgi/com.raytheon.uf.edex.esb.camel/src/com/raytheon/uf/edex/esb/camel/ProcessUtil.java index b76f7b2f81..5df90f1dae 100644 --- a/edexOsgi/com.raytheon.uf.edex.esb.camel/src/com/raytheon/uf/edex/esb/camel/ProcessUtil.java +++ b/edexOsgi/com.raytheon.uf.edex.esb.camel/src/com/raytheon/uf/edex/esb/camel/ProcessUtil.java @@ -20,11 +20,13 @@ package com.raytheon.uf.edex.esb.camel; import java.io.File; +import java.io.IOException; import java.text.DecimalFormat; import java.util.Arrays; import java.util.Iterator; import java.util.Map; +import org.apache.camel.Exchange; import org.apache.camel.Header; import org.apache.camel.Headers; @@ -34,6 +36,8 @@ import com.raytheon.uf.common.stats.ProcessEvent; import com.raytheon.uf.common.status.IUFStatusHandler; import com.raytheon.uf.common.status.UFStatus; import com.raytheon.uf.common.status.UFStatus.Priority; +import com.raytheon.uf.common.util.FileUtil; +import com.raytheon.uf.edex.core.props.PropertiesFactory; /** * Provides logging and deletion services for camel @@ -43,9 +47,10 @@ import com.raytheon.uf.common.status.UFStatus.Priority; * SOFTWARE HISTORY * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- - * Dec 1, 2008 chammack Initial creation - * Feb 05, 2013 1580 mpduff EventBus refactor. - * Feb 12, 2013 1615 bgonzale Changed ProcessEvent pluginName to dataType. + * Dec 1, 2008 chammack Initial creation + * Feb 05, 2013 1580 mpduff EventBus refactor. + * Feb 12, 2013 1615 bgonzale Changed ProcessEvent pluginName to dataType. + * Jan 21, 2014 2627 njensen Added logFailedData() and logFailureAsInfo() * * * @@ -69,6 +74,26 @@ public class ProcessUtil { }; + private static final String FAILED_DIR; + + protected static final boolean RETAIN_FAILED_DATA; + + static { + // this will probably only ever be true on a testbed + RETAIN_FAILED_DATA = Boolean.getBoolean("retain.failed.data"); + if (RETAIN_FAILED_DATA) { + FAILED_DIR = PropertiesFactory.getInstance().getEnvProperties() + .getEnvValue("DEFAULTDATADIR") + + File.separator + "failed"; + File file = new File(FAILED_DIR); + if (!file.exists()) { + file.mkdir(); + } + } else { + FAILED_DIR = null; + } + } + /** * Get the value of a specified property if it exists. * @@ -118,11 +143,23 @@ public class ProcessUtil { } /** + * Logs the processing and latency time of a file * * @param headers */ public void log(@Headers Map headers) { + logInternal(headers, true); + } + /** + * Logs the processing and latency time of a file + * + * @param headers + * the headers associated with the ingest routes + * @param successful + * whether or not the file was successfully ingested + */ + protected void logInternal(Map headers, boolean successful) { long curTime = System.currentTimeMillis(); StringBuilder sb = new StringBuilder(128); @@ -168,7 +205,7 @@ public class ProcessUtil { // processing in less than 0 millis isn't trackable, usually due to an // error occurred and statement logged incorrectly - if ((processEvent.getProcessingLatency() > 0) + if (successful && (processEvent.getProcessingLatency() > 0) && (processEvent.getProcessingTime() > 0)) { EventBus.publish(processEvent); } @@ -179,5 +216,66 @@ public class ProcessUtil { } else { handler.handle(Priority.INFO, "No logging information available"); } + } + + /** + * Logs a failure to ingest a file. Potentially saves off the data that + * failed to ingest, based on a system property. + * + * @param ex + * the exchange that failed to ingest + */ + public void logFailedData(Exchange ex) { + Exception e = ex.getException(); + if (e == null) { + e = ex.getProperty(Exchange.EXCEPTION_CAUGHT, Exception.class); + } + Map headers = ex.getIn().getHeaders(); + String fullpath = getHeaderProperty(headers, ("ingestFileName")); + handler.error("Failed to ingest " + fullpath, e); + + if (RETAIN_FAILED_DATA) { + File badfile = new File(fullpath); + if (badfile.exists()) { + String filename = badfile.getName(); + File keepfile = new File(FAILED_DIR + File.separator + filename); + try { + FileUtil.copyFile(badfile, keepfile); + handler.info("Copied failed data to " + keepfile.getPath()); + } catch (IOException e1) { + handler.error( + "Unable to copy failed data for later analysis", e); + } + } + } + + logInternal(headers, false); + } + + /** + * Logs a failure to ingest data as an info message. This should only be + * used when the failure can be considered as expected given the input, such + * as submitting invalid/bad data to a decoder. + * + * @param ex + * the exchange that failed to ingest + */ + public void logFailureAsInfo(Exchange ex) { + Exception e = ex.getException(); + if (e == null) { + e = ex.getProperty(Exchange.EXCEPTION_CAUGHT, Exception.class); + } + Map headers = ex.getIn().getHeaders(); + String fullpath = getHeaderProperty(headers, ("ingestFileName")); + String msg = "Discarding " + fullpath; + if (e != null) { + msg += " due to " + e.getClass().getSimpleName() + ": " + + e.getLocalizedMessage(); + } + handler.info(msg); + + logInternal(headers, false); + } + } diff --git a/edexOsgi/com.raytheon.uf.edex.ingest/res/spring/persist-ingest.xml b/edexOsgi/com.raytheon.uf.edex.ingest/res/spring/persist-ingest.xml index 988582f79e..e58c99323a 100644 --- a/edexOsgi/com.raytheon.uf.edex.ingest/res/spring/persist-ingest.xml +++ b/edexOsgi/com.raytheon.uf.edex.ingest/res/spring/persist-ingest.xml @@ -54,11 +54,22 @@ - + + + + + + + + + + + +