xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #2627 configurable ability to retain failed data, log 'expected'

Browse source 
failures as infos without stacktraces

Change-Id: I199c2125bb6c35b703b71ac5e8342d7c0a4dbcb0

Former-commit-id: 36ad567a6d [formerly 0a1049cb785f782fe7bea8379f7df694dfb45e23]
Former-commit-id: 50b6eced5d
This commit is contained in:
Nate Jensen 2014-01-22 09:24:12 -06:00
parent 252c7ee40a
commit a021248cf3
7 changed files with 498 additions and 294 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
edexOsgi/build.edex/esb/conf/wrapper.conf
View file

@ -142,14 +142,12 @@ wrapper.java.additional.log.5=-Djava.util.logging.config.file=${EDEX_HOME}/conf/
wrapper.java.additional.web.1=-Dweb.port=8080
wrapper.java.additional.web.2=-Dconfidential.port=8443
# notifies SerializationManager to initialize hibernatables, can be removed IF Hibernatables code
# is removed from SerializationManager
wrapper.java.additional.misc.1=-DinitializeHibernatables=true
# the max size in MB of any stream sent to thrift, this prevents the OutOfMemory
# errors reported by thrift sometimes when the stream is corrupt/incorrect
wrapper.java.additional.thrift.maxStreamSize=-Dthrift.stream.maxsize=200
wrapper.java.additional.edex.retain.failed.data=-Dretain.failed.data=${RETAIN_FAILED}
# enables yourkit profiling, determined by flag to start.sh
wrapper.java.additional.profile.1=${PROFILER_PARAM_1}

6
edexOsgi/com.raytheon.edex.plugin.radar/res/spring/radar-ingest.xml
View file

@ -81,8 +81,12 @@
<to uri="direct-vm:persistIndexAlert" />
</pipeline>
<doCatch>
<exception>com.raytheon.uf.common.dataplugin.exception.MalformedDataException</exception>
<to uri="direct-vm:logFailureAsInfo" />
</doCatch>
<doCatch>
<exception>java.lang.Throwable</exception>
<to uri="log:radar?level=ERROR" />
<to uri="direct-vm:logFailedData" />
</doCatch>
</doTry>
</route>

515
edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/RadarDecoder.java
View file

@ -97,6 +97,8 @@ import com.raytheon.uf.edex.wmo.message.WMOHeader;
* added status to decode.
* Aug 30, 2013 2298 rjpeter Make getPluginName abstract
* Oct 09, 2013 2457 bsteffen Improve error message for missing icao.
* Jan 21, 2014 2627 njensen Removed decode()'s try/catch, camel route will do try/catch
*
* </pre>
*
* @author bphillip
@ -172,7 +174,7 @@ public class RadarDecoder extends AbstractDecoder {
* @throws DecoderException
*/
public PluginDataObject[] decode(byte[] messageData, Headers headers)
throws DecoderException {
throws Exception {
if (headers != null) {
traceId = (String) headers.get("traceId");
}
@ -181,270 +183,261 @@ public class RadarDecoder extends AbstractDecoder {
// decode the product
String arch = new String(messageData, 0, 4);
try {
ITimer timer = TimeUtil.getTimer();
timer.start();
// for level2 data, this does not happen very often
if (LEVEL_TWO_IDENTS.contains(arch)) {
decodeLevelTwoData(messageData, recordList);
ITimer timer = TimeUtil.getTimer();
timer.start();
// for level2 data, this does not happen very often
if (LEVEL_TWO_IDENTS.contains(arch)) {
decodeLevelTwoData(messageData, recordList);
}
// for free text messages, which come in with the following wmo
else if (NOUS.equals(arch)) {
decodeFreeTextMessage(messageData, headers);
} else {
if (headers.get("header") != null) {
// handle an interesting special case
String wmoHeader = headers.get("header").toString();
if (wmoHeader.contains("SDUS4")) {
WMOHeader header = new WMOHeader(wmoHeader.getBytes(),
headers);
String dataString = new String(messageData, 0,
messageData.length).substring(1,
messageData.length - 1);
String siteId = dataString.substring(0, 3);
AFOSProductId afos = new AFOSProductId("WSR", "ROB", siteId);
// store the product ROB that is barely do-able
Calendar cal = (TimeTools.allowArchive() ? header
.getHeaderDate() : Calendar.getInstance());
RadarEdexTextProductUtil.storeTextProduct(afos, header,
dataString, true, cal);
return new PluginDataObject[0];
}
}
// for free text messages, which come in with the following wmo
else if (NOUS.equals(arch)) {
decodeFreeTextMessage(messageData, headers);
Level3BaseRadar l3Radar = new Level3BaseRadar(messageData, headers,
infoDict);
RadarRecord record = new RadarRecord();
record.setProductCode(l3Radar.getMessageCode());
record.setDataTime(new DataTime(l3Radar.getMessageTimestamp()));
RadarStation station = RadarSpatialUtil
.getRadarStationByRpgIdDec(l3Radar.getSourceId());
if (station == null) {
record.setIcao("unkn");
logger.error(headers.get("ingestfilename")
+ " contains an rpg id(" + l3Radar.getSourceId()
+ ") that is not in the radar_spatial table.");
} else {
if (headers.get("header") != null) {
// handle an interesting special case
String wmoHeader = headers.get("header").toString();
if (wmoHeader.contains("SDUS4")) {
WMOHeader header = new WMOHeader(wmoHeader.getBytes(),
headers);
String dataString = new String(messageData, 0,
messageData.length).substring(1,
messageData.length - 1);
String siteId = dataString.substring(0, 3);
AFOSProductId afos = new AFOSProductId("WSR", "ROB",
siteId);
// store the product ROB that is barely do-able
Calendar cal = (TimeTools.allowArchive() ? header
.getHeaderDate() : Calendar.getInstance());
RadarEdexTextProductUtil.storeTextProduct(afos, header,
dataString, true, cal);
return new PluginDataObject[0];
}
}
Level3BaseRadar l3Radar = new Level3BaseRadar(messageData,
headers, infoDict);
RadarRecord record = new RadarRecord();
record.setProductCode(l3Radar.getMessageCode());
record.setDataTime(new DataTime(l3Radar.getMessageTimestamp()));
RadarStation station = RadarSpatialUtil
.getRadarStationByRpgIdDec(l3Radar.getSourceId());
if (station == null) {
record.setIcao("unkn");
logger.error(headers.get("ingestfilename")
+ " contains an rpg id(" + l3Radar.getSourceId()
+ ") that is not in the radar_spatial table.");
} else {
record.setIcao(station.getRdaId().toLowerCase());
}
record.setLocation(station);
RadarInfo info = infoDict.getInfo(record.getProductCode());
if (info == null) {
theHandler.handle(
Priority.ERROR,
"Unknown radar product code: "
+ record.getProductCode() + " for "
+ headers.get("ingestfilename"));
return new PluginDataObject[0];
}
record.setFormat(info.getFormat());
record.setNumLevels(info.getNumLevels());
record.setGateResolution(info.getResolution());
record.setMnemonic(info.getMnemonic());
record.setDisplayModes(info.getDisplayModes());
record.setUnit(info.getUnit());
// -- some product specific decode functionality --
// the general status message product
if (l3Radar.getMessageCode() == l3Radar.GSM_MESSAGE) {
record.setGsmMessage(l3Radar.getGsmBlock().getMessage());
record.setPrimaryElevationAngle(0.0);
record.setTrueElevationAngle(0.0f);
handleRadarStatus(record);
}
// the product request response product
else if (l3Radar.getMessageCode() == l3Radar.PRODUCT_REQUEST_RESPONSE_MESSAGE) {
// do nothing with this, it will get excessive otherwise!
return new PluginDataObject[0];
}
// the user alert message product
else if (l3Radar.getMessageCode() == USER_ALERT_MESSAGE) {
EDEXUtil.sendMessageAlertViz(Priority.VERBOSE,
RadarConstants.PLUGIN_ID, EDEX, RADAR,
record.getIcao() + ": User Alert Message Received",
l3Radar.getTabularBlock().toString(), null);
return new PluginDataObject[0];
}
// handle the other case for free text message
else if (l3Radar.getMessageCode() == FREE_TEXT_MESSAGE) {
// product already stored to the text database, so just send
// to alertviz
String formattedMsg = l3Radar.getTabularBlock().toString()
.replace("Page 1\n\t", "");
EDEXUtil.sendMessageAlertViz(Priority.SIGNIFICANT,
RadarConstants.PLUGIN_ID, EDEX, RADAR,
record.getIcao() + ": Free Text Message Received",
formattedMsg, null);
return new PluginDataObject[0];
}
// the alert adaptations parameters product
else if (l3Radar.getMessageCode() == l3Radar.ALERT_ADAPTATION_PARAMETERS) {
record.setAapMessage(l3Radar.getAapMessage());
record.setPrimaryElevationAngle(0.0);
record.setTrueElevationAngle(0.0f);
EDEXUtil.sendMessageAlertViz(
Priority.VERBOSE,
RadarConstants.PLUGIN_ID,
EDEX,
RADAR,
record.getIcao()
+ ": Alert Adapation Parameter Message Received",
l3Radar.getAapMessage().toString(), null);
}
// the alert message product
else if (l3Radar.getMessageCode() == l3Radar.ALERT_MESSAGE) {
record.setPrimaryElevationAngle(0.0);
record.setTrueElevationAngle(0.0f);
AlertMessage msg = l3Radar.getAlertMessage();
String details = "Alert Area : " + msg.getAlertAreaNum()
+ "\n";
details += "Position : " + msg.getGridBoxAz()
+ " deg\nRange : " + msg.getGridBoxRange() + "nm\n";
String category = AlertCategory.values()[msg
.getAlertCategory()].toString();
category = category.substring(category.indexOf("_"));
category = category.replaceAll("_", " ");
details += "Alert Category : " + category + "\n";
details += "Threshold : " + msg.getThresholdValue() + "\n";
details += "Exceeding : " + msg.getExceedingValue() + "\n";
details += "Storm Cell ID :" + msg.getStormId() + "\n";
record.setAlertMessage(msg);
EDEXUtil.sendMessageAlertViz(Priority.SIGNIFICANT,
RadarConstants.PLUGIN_ID, EDEX, RADAR,
record.getIcao() + ": Alert Message Received",
details, null);
} else {
record.setLatitude((float) l3Radar.getLatitude());
record.setLongitude((float) l3Radar.getLongitude());
record.setElevation((float) l3Radar.getHeight());
record.setVolumeCoveragePattern(l3Radar
.getVolumeCoveragePattern());
record.setOperationalMode(l3Radar.getOperationalMode());
record.setElevationNumber(l3Radar.getElevationNumber());
// some products don't have real elevation angles, 0 is a
// default value
if (record.getElevationNumber() == 0) {
record.setTrueElevationAngle(0f);
} else {
record.setTrueElevationAngle(l3Radar
.getProductDependentValue(2) * 0.1f);
}
// determine to use the primary elevations or the elevation
// in the terminal radar configuration file
if (TerminalRadarUtils.isTerminalRadar(record.getIcao())) {
Double elevation = TerminalRadarUtils.getPrimarysMap(
record.getIcao()).get(
TiltAngleBin.getPrimaryElevationAngle(record
.getTrueElevationAngle()));
if (elevation != null) {
record.setPrimaryElevationAngle(elevation
.doubleValue());
} else {
// fall back
record.setPrimaryElevationAngle(record
.getTrueElevationAngle().doubleValue());
}
} else {
record.setPrimaryElevationAngle(TiltAngleBin
.getPrimaryElevationAngle(record
.getTrueElevationAngle()));
}
// code specific for clutter filter control
if (record.getProductCode() == CLUTTER_FILTER_CONTROL) {
int segment = ((int) (Math.log(l3Radar
.getProductDependentValue(0)) / Math.log(2)));
record.setLayer((double) segment);
}
// code specific for user select accum
else if (record.getProductCode() == USER_SELECT_ACCUM) {
int layer = 0; // Default to zero
int timeSpan = l3Radar.getProductDependentValue(1);
if (timeSpan == 60) {
layer = 1;
} else if (timeSpan == 120) {
layer = 2;
} else if (timeSpan == 180) {
layer = 3;
} else if (timeSpan == 360) {
layer = 4;
} else if (timeSpan == 720) {
layer = 5;
} else if (timeSpan == 1440) {
layer = 6;
} else {
layer = 0;
}
record.setLayer((double) layer);
}
// handle times because radar times are sent out in batches
// (a volume scan) and we want the volume scan time to be
// part of the data uri
if (l3Radar.getVolumeScanTime() != null) {
record.setDataTime(new DataTime(l3Radar
.getVolumeScanTime().getTime()));
record.setVolScanTime(l3Radar
.getProductGenerationTime().getTime());
} else {
record.setDataTime(new DataTime(l3Radar
.getMessageTimestamp().getTime()));
record.setVolScanTime(l3Radar.getMessageTimestamp()
.getTime());
}
// thresholds specific per product and site
if (l3Radar.getDataLevelThresholds() != null) {
for (int i = 0; i < 16; ++i) {
record.setThreshold(i,
l3Radar.getDataLevelThreshold(i));
}
}
// values that are product dependent as defined in the ICD
record.setProductDependentValues(l3Radar
.getProductDependentValue());
processSymbologyBlock(record, l3Radar.getSymbologyBlock());
GraphicBlock gb = l3Radar.getGraphicBlock();
record.setGraphicBlock(gb);
// Tabular block is where most of the text values are...
TabularBlock tb = l3Radar.getTabularBlock();
if (tb != null) {
// complicated, but makes for logical access to needed
// elements
HashMap<RadarConstants.MapValues, Map<String, Map<RadarConstants.MapValues, String>>> map = new HashMap<RadarConstants.MapValues, Map<String, Map<RadarConstants.MapValues, String>>>();
HashMap<RadarConstants.MapValues, Map<RadarConstants.MapValues, String>> recordVals = new HashMap<RadarConstants.MapValues, Map<RadarConstants.MapValues, String>>();
RadarTabularBlockParser.parseTabularBlock(tb,
record.getProductCode(), map, recordVals);
record.setProductVals(map);
record.setMapRecordVals(recordVals);
record.setTabularBlock(tb);
}
}
try {
finalizeRecord(record);
} catch (PluginException e) {
logger.error(e);
return new PluginDataObject[0];
}
timer.stop();
perfLog.logDuration("Time to Decode", timer.getElapsedTime());
recordList.add(record);
record.setIcao(station.getRdaId().toLowerCase());
}
} catch (Exception e) {
theHandler.handle(Priority.ERROR, "Couldn't properly handle "
+ headers.get("ingestfilename"), e);
record.setLocation(station);
RadarInfo info = infoDict.getInfo(record.getProductCode());
if (info == null) {
theHandler.handle(
Priority.ERROR,
"Unknown radar product code: "
+ record.getProductCode() + " for "
+ headers.get("ingestfilename"));
return new PluginDataObject[0];
}
record.setFormat(info.getFormat());
record.setNumLevels(info.getNumLevels());
record.setGateResolution(info.getResolution());
record.setMnemonic(info.getMnemonic());
record.setDisplayModes(info.getDisplayModes());
record.setUnit(info.getUnit());
// -- some product specific decode functionality --
// the general status message product
if (l3Radar.getMessageCode() == l3Radar.GSM_MESSAGE) {
record.setGsmMessage(l3Radar.getGsmBlock().getMessage());
record.setPrimaryElevationAngle(0.0);
record.setTrueElevationAngle(0.0f);
handleRadarStatus(record);
}
// the product request response product
else if (l3Radar.getMessageCode() == l3Radar.PRODUCT_REQUEST_RESPONSE_MESSAGE) {
// do nothing with this, it will get excessive otherwise!
return new PluginDataObject[0];
}
// the user alert message product
else if (l3Radar.getMessageCode() == USER_ALERT_MESSAGE) {
EDEXUtil.sendMessageAlertViz(Priority.VERBOSE,
RadarConstants.PLUGIN_ID, EDEX, RADAR, record.getIcao()
+ ": User Alert Message Received", l3Radar
.getTabularBlock().toString(), null);
return new PluginDataObject[0];
}
// handle the other case for free text message
else if (l3Radar.getMessageCode() == FREE_TEXT_MESSAGE) {
// product already stored to the text database, so just send
// to alertviz
String formattedMsg = l3Radar.getTabularBlock().toString()
.replace("Page 1\n\t", "");
EDEXUtil.sendMessageAlertViz(Priority.SIGNIFICANT,
RadarConstants.PLUGIN_ID, EDEX, RADAR, record.getIcao()
+ ": Free Text Message Received", formattedMsg,
null);
return new PluginDataObject[0];
}
// the alert adaptations parameters product
else if (l3Radar.getMessageCode() == l3Radar.ALERT_ADAPTATION_PARAMETERS) {
record.setAapMessage(l3Radar.getAapMessage());
record.setPrimaryElevationAngle(0.0);
record.setTrueElevationAngle(0.0f);
EDEXUtil.sendMessageAlertViz(
Priority.VERBOSE,
RadarConstants.PLUGIN_ID,
EDEX,
RADAR,
record.getIcao()
+ ": Alert Adapation Parameter Message Received",
l3Radar.getAapMessage().toString(), null);
}
// the alert message product
else if (l3Radar.getMessageCode() == l3Radar.ALERT_MESSAGE) {
record.setPrimaryElevationAngle(0.0);
record.setTrueElevationAngle(0.0f);
AlertMessage msg = l3Radar.getAlertMessage();
String details = "Alert Area : " + msg.getAlertAreaNum() + "\n";
details += "Position : " + msg.getGridBoxAz()
+ " deg\nRange : " + msg.getGridBoxRange() + "nm\n";
String category = AlertCategory.values()[msg.getAlertCategory()]
.toString();
category = category.substring(category.indexOf("_"));
category = category.replaceAll("_", " ");
details += "Alert Category : " + category + "\n";
details += "Threshold : " + msg.getThresholdValue() + "\n";
details += "Exceeding : " + msg.getExceedingValue() + "\n";
details += "Storm Cell ID :" + msg.getStormId() + "\n";
record.setAlertMessage(msg);
EDEXUtil.sendMessageAlertViz(Priority.SIGNIFICANT,
RadarConstants.PLUGIN_ID, EDEX, RADAR, record.getIcao()
+ ": Alert Message Received", details, null);
} else {
record.setLatitude((float) l3Radar.getLatitude());
record.setLongitude((float) l3Radar.getLongitude());
record.setElevation((float) l3Radar.getHeight());
record.setVolumeCoveragePattern(l3Radar
.getVolumeCoveragePattern());
record.setOperationalMode(l3Radar.getOperationalMode());
record.setElevationNumber(l3Radar.getElevationNumber());
// some products don't have real elevation angles, 0 is a
// default value
if (record.getElevationNumber() == 0) {
record.setTrueElevationAngle(0f);
} else {
record.setTrueElevationAngle(l3Radar
.getProductDependentValue(2) * 0.1f);
}
// determine to use the primary elevations or the elevation
// in the terminal radar configuration file
if (TerminalRadarUtils.isTerminalRadar(record.getIcao())) {
Double elevation = TerminalRadarUtils.getPrimarysMap(
record.getIcao()).get(
TiltAngleBin.getPrimaryElevationAngle(record
.getTrueElevationAngle()));
if (elevation != null) {
record.setPrimaryElevationAngle(elevation.doubleValue());
} else {
// fall back
record.setPrimaryElevationAngle(record
.getTrueElevationAngle().doubleValue());
}
} else {
record.setPrimaryElevationAngle(TiltAngleBin
.getPrimaryElevationAngle(record
.getTrueElevationAngle()));
}
// code specific for clutter filter control
if (record.getProductCode() == CLUTTER_FILTER_CONTROL) {
int segment = ((int) (Math.log(l3Radar
.getProductDependentValue(0)) / Math.log(2)));
record.setLayer((double) segment);
}
// code specific for user select accum
else if (record.getProductCode() == USER_SELECT_ACCUM) {
int layer = 0; // Default to zero
int timeSpan = l3Radar.getProductDependentValue(1);
if (timeSpan == 60) {
layer = 1;
} else if (timeSpan == 120) {
layer = 2;
} else if (timeSpan == 180) {
layer = 3;
} else if (timeSpan == 360) {
layer = 4;
} else if (timeSpan == 720) {
layer = 5;
} else if (timeSpan == 1440) {
layer = 6;
} else {
layer = 0;
}
record.setLayer((double) layer);
}
// handle times because radar times are sent out in batches
// (a volume scan) and we want the volume scan time to be
// part of the data uri
if (l3Radar.getVolumeScanTime() != null) {
record.setDataTime(new DataTime(l3Radar.getVolumeScanTime()
.getTime()));
record.setVolScanTime(l3Radar.getProductGenerationTime()
.getTime());
} else {
record.setDataTime(new DataTime(l3Radar
.getMessageTimestamp().getTime()));
record.setVolScanTime(l3Radar.getMessageTimestamp()
.getTime());
}
// thresholds specific per product and site
if (l3Radar.getDataLevelThresholds() != null) {
for (int i = 0; i < 16; ++i) {
record.setThreshold(i, l3Radar.getDataLevelThreshold(i));
}
}
// values that are product dependent as defined in the ICD
record.setProductDependentValues(l3Radar
.getProductDependentValue());
processSymbologyBlock(record, l3Radar.getSymbologyBlock());
GraphicBlock gb = l3Radar.getGraphicBlock();
record.setGraphicBlock(gb);
// Tabular block is where most of the text values are...
TabularBlock tb = l3Radar.getTabularBlock();
if (tb != null) {
// complicated, but makes for logical access to needed
// elements
HashMap<RadarConstants.MapValues, Map<String, Map<RadarConstants.MapValues, String>>> map = new HashMap<RadarConstants.MapValues, Map<String, Map<RadarConstants.MapValues, String>>>();
HashMap<RadarConstants.MapValues, Map<RadarConstants.MapValues, String>> recordVals = new HashMap<RadarConstants.MapValues, Map<RadarConstants.MapValues, String>>();
RadarTabularBlockParser.parseTabularBlock(tb,
record.getProductCode(), map, recordVals);
record.setProductVals(map);
record.setMapRecordVals(recordVals);
record.setTabularBlock(tb);
}
}
try {
finalizeRecord(record);
} catch (PluginException e) {
logger.error(e);
return new PluginDataObject[0];
}
timer.stop();
perfLog.logDuration("Time to Decode", timer.getElapsedTime());
recordList.add(record);
}
return recordList.toArray(new PluginDataObject[recordList.size()]);

73
edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/level3/Level3BaseRadar.java
View file

@ -40,6 +40,7 @@ import org.itadaki.bzip2.BZip2InputStream;
import com.raytheon.edex.esb.Headers;
import com.raytheon.edex.plugin.radar.util.RadarEdexTextProductUtil;
import com.raytheon.edex.plugin.radar.util.RadarSpatialUtil;
import com.raytheon.uf.common.dataplugin.exception.MalformedDataException;
import com.raytheon.uf.common.dataplugin.radar.RadarStation;
import com.raytheon.uf.common.dataplugin.radar.level3.AlertAdaptationParameters;
import com.raytheon.uf.common.dataplugin.radar.level3.AlertMessage;
@ -91,7 +92,16 @@ import com.raytheon.uf.edex.wmo.message.WMOHeader;
* graphics context and the extents of the Component used to create the Graphics
* object.
*
* Copyright 2006 Raytheon Corporation
*
* <pre>
* SOFTWARE HISTORY
* Date Ticket# Engineer Description
* ------------- -------- ----------- --------------------------
* --/--/2006 brockwoo Initial creation
* Jan 21, 2014 2627 njensen Changed offset errors to MalformedDataException
*
* </pre>
*
*
* @author Bryan Rockwood
* @version 1.0
@ -200,8 +210,10 @@ public class Level3BaseRadar {
* A java.io.File object containing a raw radar file
* @throws IOException
* If the radar head parsing fails, an IO exception is thrown
* @throws MalformedDataException
*/
public Level3BaseRadar(File aRadar, Headers headers) throws IOException {
public Level3BaseRadar(File aRadar, Headers headers) throws IOException,
MalformedDataException {
int fileSize = (int) aRadar.length();
byte[] tempRawRadarByteArray = new byte[fileSize];
@ -227,13 +239,15 @@ public class Level3BaseRadar {
* A byte array containing a raw radar file
* @throws IOException
* If the radar head parsing fails, an IO exception is thrown
* @throws MalformedDataException
*/
public Level3BaseRadar(byte[] aRadar, Headers headers) throws IOException {
public Level3BaseRadar(byte[] aRadar, Headers headers) throws IOException,
MalformedDataException {
init(aRadar, headers);
}
public Level3BaseRadar(byte[] aRadar, Headers headers, RadarInfoDict dict)
throws IOException {
throws IOException, MalformedDataException {
this.dict = dict;
init(aRadar, headers);
}
@ -241,8 +255,10 @@ public class Level3BaseRadar {
/**
* @param aRadar
* @throws IOException
* @throws MalformedDataException
*/
private void init(byte[] aRadar, Headers headers) throws IOException {
private void init(byte[] aRadar, Headers headers) throws IOException,
MalformedDataException {
// printPacketContents(aRadar);
int wmoHeaderSize;
@ -512,7 +528,8 @@ public class Level3BaseRadar {
return dataLevelThresholds[code];
}
private SymbologyBlock readSymbologyBlock(int offset) throws IOException {
private SymbologyBlock readSymbologyBlock(int offset) throws IOException,
MalformedDataException {
SymbologyBlock symBlock = null;
if (offset != 0) {
theRadarData.reset();
@ -520,8 +537,8 @@ public class Level3BaseRadar {
int divider = theRadarData.readShort();
int blockId = theRadarData.readUnsignedShort();
if ((divider != -1) || (blockId != SymbologyBlock.getBlockId())) {
throw new IOException(
"This does not appear to be a symbology block");
throw new MalformedDataException("Symbology block offset "
+ offset + " does not point to a symbology block");
}
int blockLen = theRadarData.readInt();
byte[] buf = RadarUtil.subArray(theRawRadarByteArray, offset,
@ -537,7 +554,8 @@ public class Level3BaseRadar {
return symBlock;
}
private GraphicBlock readGraphicBlock(int offset) throws IOException {
private GraphicBlock readGraphicBlock(int offset) throws IOException,
MalformedDataException {
GraphicBlock graphicBlock = null;
if (offset != 0) {
theRadarData.reset();
@ -545,8 +563,8 @@ public class Level3BaseRadar {
int divider = theRadarData.readShort();
int blockId = theRadarData.readUnsignedShort();
if ((divider != -1) || (blockId != GraphicBlock.getBlockId())) {
throw new IOException(
"This does not appear to be a graphic block");
throw new MalformedDataException("Graphic block offset "
+ offset + " does not point to a graphic block");
}
int blockLen = theRadarData.readInt();
byte[] buf = RadarUtil.subArray(theRawRadarByteArray, offset,
@ -559,7 +577,8 @@ public class Level3BaseRadar {
return graphicBlock;
}
private TabularBlock readTabularBlock(int offset) throws IOException {
private TabularBlock readTabularBlock(int offset) throws IOException,
MalformedDataException {
TabularBlock tabBlock = null;
if (offset != 0) {
theRadarData.reset();
@ -567,8 +586,8 @@ public class Level3BaseRadar {
int divider = theRadarData.readShort();
int blockId = theRadarData.readUnsignedShort();
if ((divider != -1) || (blockId != TabularBlock.getBlockId())) {
throw new IOException(
"This does not appear to be a tabular block");
throw new MalformedDataException("Tabular block offset "
+ offset + " does not point to a tabular block");
}
int blockLen = theRadarData.readInt();
byte[] buf = RadarUtil.subArray(theRawRadarByteArray, offset,
@ -587,13 +606,14 @@ public class Level3BaseRadar {
* @return
* @throws IOException
*/
private TabularBlock readStandaloneTabular(int offset) throws IOException {
private TabularBlock readStandaloneTabular(int offset) throws IOException,
MalformedDataException {
int divider = theRadarData.readShort();
TabularBlock tabBlock = new TabularBlock();
int numPages = theRadarData.readUnsignedShort();
if ((divider != -1)) {
throw new IOException(
"This does not appear to be a standalone tabular block");
throw new MalformedDataException("Standalone tabular block offset "
+ offset + " does not point to a standalone tabular block");
}
List<List<String>> pages = new ArrayList<List<String>>();
for (int p = 0; p < numPages; p++) {
@ -678,7 +698,8 @@ public class Level3BaseRadar {
}
private void parseRadarMessage(Headers headers) throws IOException {
private void parseRadarMessage(Headers headers) throws IOException,
MalformedDataException {
// Product Description Block
theRadarData.skip(2);
theLatitude = theRadarData.readInt() * 0.001;
@ -728,16 +749,21 @@ public class Level3BaseRadar {
byte[] msg = new byte[120];
InputStream byt;
if (uncompressedSize + msg.length != theRawRadarByteArray.length) {
InputStream ins = null;
try {
theRadarData.reset();
theRadarData.readFully(msg);
InputStream ins = new BZip2InputStream(theRadarData, false);
ins = new BZip2InputStream(theRadarData, false);
uncompressed = new byte[uncompressedSize];
ins.read(uncompressed);
} catch (IOException e) {
theHandler.handle(Priority.ERROR,
"Error decompressing product: ", e);
return;
} finally {
if (ins != null) {
ins.close();
}
}
theRawRadarByteArray = new byte[120 + uncompressed.length];
System.arraycopy(msg, 0, theRawRadarByteArray, 0, 120);
@ -842,9 +868,9 @@ public class Level3BaseRadar {
}
tabularBlock.setString(builder.toString());
lookupAfosId();
if (RadarTextProductUtil.radarTable.keySet().contains(theMessageCode)) {
byte[] wmoid = wmoHeader.getBytes();
WMOHeader header = new WMOHeader(wmoid, headers);
@ -944,10 +970,11 @@ public class Level3BaseRadar {
}
}
private void parseGeneralStatusMessage() throws IOException {
private void parseGeneralStatusMessage() throws IOException,
MalformedDataException {
int divider = theRadarData.readShort();
if ((divider != -1)) {
throw new IOException("This does not appear to be a gsm block");
throw new MalformedDataException("This is not a gsm block");
}
int blockLen = theRadarData.readShort();
byte[] buf = RadarUtil.subArray(theRawRadarByteArray, 22, blockLen);

73
edexOsgi/com.raytheon.uf.common.dataplugin/src/com/raytheon/uf/common/dataplugin/exception/MalformedDataException.java Normal file
View file

@ -0,0 +1,73 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.dataplugin.exception;
/**
* An exception for when data is encoded incorrectly and is malformed.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 21, 2014 njensen Initial creation
*
* </pre>
*
* @author njensen
* @version 1.0
*/
public class MalformedDataException extends BadDataException {
private static final long serialVersionUID = 1L;
/**
* Default Constructor
*
*/
public MalformedDataException() {
super();
}
/**
* @param message
*/
public MalformedDataException(String message) {
super(message);
}
/**
* @param message
* @param cause
*/
public MalformedDataException(String message, Throwable cause) {
super(message, cause);
}
/**
* @param cause
*/
public MalformedDataException(Throwable cause) {
super(cause);
}
}

106
edexOsgi/com.raytheon.uf.edex.esb.camel/src/com/raytheon/uf/edex/esb/camel/ProcessUtil.java
View file

@ -20,11 +20,13 @@
package com.raytheon.uf.edex.esb.camel;
import java.io.File;
import java.io.IOException;
import java.text.DecimalFormat;
import java.util.Arrays;
import java.util.Iterator;
import java.util.Map;
import org.apache.camel.Exchange;
import org.apache.camel.Header;
import org.apache.camel.Headers;
@ -34,6 +36,8 @@ import com.raytheon.uf.common.stats.ProcessEvent;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.util.FileUtil;
import com.raytheon.uf.edex.core.props.PropertiesFactory;
/**
* Provides logging and deletion services for camel
@ -43,9 +47,10 @@ import com.raytheon.uf.common.status.UFStatus.Priority;
* SOFTWARE HISTORY
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Dec 1, 2008 chammack Initial creation
* Feb 05, 2013 1580 mpduff EventBus refactor.
* Feb 12, 2013 1615 bgonzale Changed ProcessEvent pluginName to dataType.
* Dec 1, 2008 chammack Initial creation
* Feb 05, 2013 1580 mpduff EventBus refactor.
* Feb 12, 2013 1615 bgonzale Changed ProcessEvent pluginName to dataType.
* Jan 21, 2014 2627 njensen Added logFailedData() and logFailureAsInfo()
*
* </pre>
*
@ -69,6 +74,26 @@ public class ProcessUtil {
};
private static final String FAILED_DIR;
protected static final boolean RETAIN_FAILED_DATA;
static {
// this will probably only ever be true on a testbed
RETAIN_FAILED_DATA = Boolean.getBoolean("retain.failed.data");
if (RETAIN_FAILED_DATA) {
FAILED_DIR = PropertiesFactory.getInstance().getEnvProperties()
.getEnvValue("DEFAULTDATADIR")
+ File.separator + "failed";
File file = new File(FAILED_DIR);
if (!file.exists()) {
file.mkdir();
}
} else {
FAILED_DIR = null;
}
}
/**
* Get the value of a specified property if it exists.
*
@ -118,11 +143,23 @@ public class ProcessUtil {
}
/**
* Logs the processing and latency time of a file
*
* @param headers
*/
public void log(@Headers Map<?, ?> headers) {
logInternal(headers, true);
}
/**
* Logs the processing and latency time of a file
*
* @param headers
* the headers associated with the ingest routes
* @param successful
* whether or not the file was successfully ingested
*/
protected void logInternal(Map<?, ?> headers, boolean successful) {
long curTime = System.currentTimeMillis();
StringBuilder sb = new StringBuilder(128);
@ -168,7 +205,7 @@ public class ProcessUtil {
// processing in less than 0 millis isn't trackable, usually due to an
// error occurred and statement logged incorrectly
if ((processEvent.getProcessingLatency() > 0)
if (successful && (processEvent.getProcessingLatency() > 0)
&& (processEvent.getProcessingTime() > 0)) {
EventBus.publish(processEvent);
}
@ -179,5 +216,66 @@ public class ProcessUtil {
} else {
handler.handle(Priority.INFO, "No logging information available");
}
}
/**
* Logs a failure to ingest a file. Potentially saves off the data that
* failed to ingest, based on a system property.
*
* @param ex
* the exchange that failed to ingest
*/
public void logFailedData(Exchange ex) {
Exception e = ex.getException();
if (e == null) {
e = ex.getProperty(Exchange.EXCEPTION_CAUGHT, Exception.class);
}
Map<?, ?> headers = ex.getIn().getHeaders();
String fullpath = getHeaderProperty(headers, ("ingestFileName"));
handler.error("Failed to ingest " + fullpath, e);
if (RETAIN_FAILED_DATA) {
File badfile = new File(fullpath);
if (badfile.exists()) {
String filename = badfile.getName();
File keepfile = new File(FAILED_DIR + File.separator + filename);
try {
FileUtil.copyFile(badfile, keepfile);
handler.info("Copied failed data to " + keepfile.getPath());
} catch (IOException e1) {
handler.error(
"Unable to copy failed data for later analysis", e);
}
}
}
logInternal(headers, false);
}
/**
* Logs a failure to ingest data as an info message. This should only be
* used when the failure can be considered as expected given the input, such
* as submitting invalid/bad data to a decoder.
*
* @param ex
* the exchange that failed to ingest
*/
public void logFailureAsInfo(Exchange ex) {
Exception e = ex.getException();
if (e == null) {
e = ex.getProperty(Exchange.EXCEPTION_CAUGHT, Exception.class);
}
Map<?, ?> headers = ex.getIn().getHeaders();
String fullpath = getHeaderProperty(headers, ("ingestFileName"));
String msg = "Discarding " + fullpath;
if (e != null) {
msg += " due to " + e.getClass().getSimpleName() + ": "
+ e.getLocalizedMessage();
}
handler.info(msg);
logInternal(headers, false);
}
}

13
edexOsgi/com.raytheon.uf.edex.ingest/res/spring/persist-ingest.xml
View file

@ -54,11 +54,22 @@
<route id="notificationAggregation">
<from uri="direct-vm:stageNotification"/>
<bean ref="pluginNotifier" method="notify" />
</route>
</route>
<route id="notificationTimer">
<from uri="timer://notificationTimer?fixedRate=true&amp;period=1000" />
<bean ref="pluginNotifier" method="sendQueuedNotifications" />
</route>
<route id="logFailedData">
<from uri="direct-vm:logFailedData" />
<bean ref="processUtil" method="logFailedData" />
</route>
<route id="logFailureAsInfo">
<from uri="direct-vm:logFailureAsInfo" />
<bean ref="processUtil" method="logFailureAsInfo" />
</route>
</camelContext>
</beans>