Omaha #3226 moved ost lightning code to new plugin
fixed NPE when missing encryption config Change-Id: I8ef6004f44b686918220e7729a499e456787a834 Former-commit-id: 5421b666fc52e849e4ad5b39e2328a5c90a42408
This commit is contained in:
Brian Clements
parent
e5ffd0f40c
commit
664d9282c3
14 changed files with 399 additions and 220 deletions
|
|
@ -1,7 +1,11 @@
|
|||
#Thu Mar 26 10:16:39 CDT 2009
|
||||
eclipse.preferences.version=1
|
||||
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
|
||||
org.eclipse.jdt.core.compiler.compliance=1.6
|
||||
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
|
||||
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
|
||||
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
|
||||
org.eclipse.jdt.core.compiler.compliance=1.7
|
||||
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
|
||||
org.eclipse.jdt.core.compiler.debug.localVariable=generate
|
||||
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
|
||||
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
|
||||
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
|
||||
org.eclipse.jdt.core.compiler.source=1.6
|
||||
org.eclipse.jdt.core.compiler.source=1.7
|
||||
|
|
|
|||
|
|
@ -2,14 +2,16 @@ Manifest-Version: 1.0
|
|||
Bundle-ManifestVersion: 2
|
||||
Bundle-Name: Binlightning Plug-in
|
||||
Bundle-SymbolicName: com.raytheon.edex.plugin.binlightning
|
||||
Bundle-Version: 1.12.1174.qualifier
|
||||
Bundle-Version: 1.14.0.qualifier
|
||||
Bundle-Vendor: RAYTHEON
|
||||
Bundle-RequiredExecutionEnvironment: JavaSE-1.6
|
||||
Bundle-RequiredExecutionEnvironment: JavaSE-1.7
|
||||
Export-Package: com.raytheon.edex.plugin.binlightning.dao
|
||||
Import-Package: com.raytheon.edex.esb,
|
||||
com.raytheon.edex.exception,
|
||||
com.raytheon.edex.plugin,
|
||||
com.raytheon.uf.common.status,
|
||||
com.raytheon.uf.common.wmo,
|
||||
gov.noaa.nws.ost.edex.plugin.binlightning,
|
||||
org.apache.commons.logging
|
||||
Require-Bundle: com.raytheon.uf.common.dataplugin.binlightning;bundle-version="1.12.1174",
|
||||
com.raytheon.uf.common.dataplugin;bundle-version="1.12.1174",
|
||||
|
|
|
|||
|
|
@ -19,30 +19,41 @@
|
|||
**/
|
||||
package com.raytheon.edex.plugin.binlightning;
|
||||
|
||||
import gov.noaa.nws.ost.edex.plugin.binlightning.BinLigntningDecoderUtil;
|
||||
import gov.noaa.nws.ost.edex.plugin.binlightning.BinLightningAESKey;
|
||||
import gov.noaa.nws.ost.edex.plugin.binlightning.BinLightningDataDecryptionException;
|
||||
import gov.noaa.nws.ost.edex.plugin.binlightning.BinLightningDecoderUtil;
|
||||
import gov.noaa.nws.ost.edex.plugin.binlightning.EncryptedBinLightningCipher;
|
||||
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Calendar;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.TimeZone;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import javax.crypto.BadPaddingException;
|
||||
import javax.crypto.IllegalBlockSizeException;
|
||||
|
||||
import com.raytheon.edex.esb.Headers;
|
||||
import com.raytheon.edex.exception.DecoderException;
|
||||
import com.raytheon.edex.plugin.AbstractDecoder;
|
||||
import com.raytheon.edex.plugin.binlightning.impl.BinLightningFactory;
|
||||
import com.raytheon.edex.plugin.binlightning.impl.IBinLightningDecoder;
|
||||
import com.raytheon.edex.plugin.binlightning.impl.LightningDataSource;
|
||||
import com.raytheon.uf.common.dataplugin.PluginDataObject;
|
||||
import com.raytheon.uf.common.dataplugin.PluginException;
|
||||
import com.raytheon.uf.common.dataplugin.binlightning.BinLightningRecord;
|
||||
import com.raytheon.uf.common.dataplugin.binlightning.impl.LightningStrikePoint;
|
||||
import com.raytheon.uf.common.dataplugin.binlightning.impl.LtgStrikeType;
|
||||
import com.raytheon.uf.common.status.IUFStatusHandler;
|
||||
import com.raytheon.uf.common.status.UFStatus;
|
||||
import com.raytheon.uf.common.time.DataTime;
|
||||
import com.raytheon.uf.common.time.TimeRange;
|
||||
import com.raytheon.uf.common.time.util.TimeUtil;
|
||||
import com.raytheon.uf.common.wmo.WMOHeader;
|
||||
import com.raytheon.uf.common.wmo.WMOTimeParser;
|
||||
import com.raytheon.uf.edex.decodertools.core.DecoderTools;
|
||||
import com.raytheon.uf.edex.decodertools.time.TimeTools;
|
||||
import com.raytheon.uf.edex.decodertools.core.IBinDataSource;
|
||||
|
||||
/**
|
||||
* AWIPS decoder adapter strategy for binary lightning data.<br/>
|
||||
|
|
@ -84,6 +95,9 @@ import com.raytheon.uf.edex.decodertools.time.TimeTools;
|
|||
* and to used WMO header to distinguish bit-shifted
|
||||
* GLD360 and NLDN data.
|
||||
* May 14, 2014 2536 bclement moved WMO Header to common
|
||||
* Jun 03, 2014 3226 bclement removed unused WMO patterns, switched to UFStatus
|
||||
* removed TimeTools usage, removed constructDataURI() call
|
||||
* added decodeBinLightningData() and decodeBitShiftedBinLightningData() from BinLightningDecoderUtil
|
||||
*
|
||||
* </pre>
|
||||
*
|
||||
|
|
@ -91,16 +105,14 @@ import com.raytheon.uf.edex.decodertools.time.TimeTools;
|
|||
* @version 1.0
|
||||
*/
|
||||
public class BinLightningDecoder extends AbstractDecoder {
|
||||
private static final String SFUS_PATTERN = "SFUS41 KWBC \\d{6}[^\\r\\n]*[\\r\\n]+";
|
||||
|
||||
private static final String SFPA_PATTERN = "SFPA41 KWBC \\d{6}[^\\r\\n]*[\\r\\n]+";
|
||||
|
||||
// Allow ingest up to 10 minutes into the future.
|
||||
private static final long TEN_MINUTES = 10 * 60 * 1000L;
|
||||
|
||||
private final SimpleDateFormat SDF;
|
||||
|
||||
private final Log logger = LogFactory.getLog(getClass());
|
||||
private static final IUFStatusHandler logger = UFStatus
|
||||
.getHandler(BinLightningDecoder.class);
|
||||
|
||||
/**
|
||||
* Default lightning strike type for FLASH messages. RT_FLASH documents
|
||||
|
|
@ -166,7 +178,8 @@ public class BinLightningDecoder extends AbstractDecoder {
|
|||
// both encrypted data and legacy data
|
||||
//
|
||||
|
||||
List<LightningStrikePoint> strikes = BinLigntningDecoderUtil.decodeBinLightningData(data, pdata, traceId, wmoHdr, baseTime.getTime());
|
||||
List<LightningStrikePoint> strikes = decodeBinLightningData(
|
||||
data, pdata, traceId, wmoHdr, baseTime.getTime());
|
||||
|
||||
if (strikes == null) { // keep-alive record, log and return
|
||||
logger.info(traceId + " - found keep-alive record. ignore for now.");
|
||||
|
|
@ -189,7 +202,7 @@ public class BinLightningDecoder extends AbstractDecoder {
|
|||
return new PluginDataObject[0];
|
||||
}
|
||||
|
||||
Calendar c = TimeTools.copy(baseTime);
|
||||
Calendar c = TimeUtil.newCalendar(baseTime);
|
||||
if (c == null) {
|
||||
throw new DecoderException(traceId + " - Error decoding times");
|
||||
}
|
||||
|
|
@ -212,14 +225,7 @@ public class BinLightningDecoder extends AbstractDecoder {
|
|||
|
||||
if (report != null) {
|
||||
report.setTraceId(traceId);
|
||||
//report.setPluginName("binlightning"); // line disappear in OB15.5.3
|
||||
try {
|
||||
report.constructDataURI();
|
||||
reports = new PluginDataObject[] { report };
|
||||
} catch (PluginException e) {
|
||||
logger.error("Error constructing datauri", e);
|
||||
throw new DecoderException("Error constructing datauri", e);
|
||||
}
|
||||
reports = new PluginDataObject[] { report };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -229,6 +235,261 @@ public class BinLightningDecoder extends AbstractDecoder {
|
|||
return reports;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode bin lightning data, able to handle both legacy bit-shifted and new
|
||||
* encryted data
|
||||
*
|
||||
* The BinLightningDecoder.decode() method will use this method to decode
|
||||
* data, which will try to decrypt first, and decode the old fashioned way
|
||||
* when decryption fails
|
||||
*
|
||||
* @author Wufeng Zhou
|
||||
*
|
||||
* @param data
|
||||
* - data content from file, including WMO header section
|
||||
* @param pdata
|
||||
* - data with WMO header stripped, optional, if null, will strip
|
||||
* WMO header internally from passed in data parameter
|
||||
* @param traceId
|
||||
* - the file name of the data to be deoced
|
||||
* @param wmoHdr
|
||||
* - WMOHeader, added 12/24/2013 to help distinguish bit-shifted
|
||||
* NLDN and GLD360 data (GLD data will have header starts like
|
||||
* SFPA)
|
||||
* @param dataDate
|
||||
* - date of the data, optional, used as a hint to find
|
||||
* appropriate encryption key faster
|
||||
* @return null if keep-alive record, otherwise a list (could be empty) of
|
||||
* LightningStrikePoint
|
||||
*/
|
||||
public static List<LightningStrikePoint> decodeBinLightningData(
|
||||
byte[] data, byte[] pdata, String traceId, WMOHeader wmoHdr,
|
||||
Date dataDate) {
|
||||
if (pdata == null) { // if data without header not passed, we'll strip
|
||||
// the WMO header here
|
||||
WMOHeader header = new WMOHeader(data);
|
||||
if (header.isValid() && header.getMessageDataStart() > 0) {
|
||||
pdata = new byte[data.length - header.getMessageDataStart()];
|
||||
System.arraycopy(data, header.getMessageDataStart(), pdata, 0,
|
||||
data.length - header.getMessageDataStart());
|
||||
}
|
||||
}
|
||||
|
||||
List<LightningStrikePoint> strikes = new ArrayList<LightningStrikePoint>();
|
||||
boolean needDecrypt = true; // set as default unless clear evidence says
|
||||
// otherwise
|
||||
boolean decodeDone = false;
|
||||
EncryptedBinLightningCipher cipher = new EncryptedBinLightningCipher();
|
||||
|
||||
//
|
||||
// Using different WMO headers to indicate whether the data is encrypted
|
||||
// or not would be a nice option.
|
||||
// However, that idea has been discussed but not adopted.
|
||||
// If in the future, WMO header can be different for legacy and
|
||||
// encrypted data, or some other metadata can be used to decide
|
||||
// whether deceyption is needed, logic can be added here.
|
||||
//
|
||||
// Before that happens, we'll use hints and trial & error to decode the
|
||||
// data
|
||||
// Hints: Per lightning data format spec, there are 3 bytes in the WMO
|
||||
// header starting line that indicates the size of the encrypted block
|
||||
// or the ASCII sequence # for legacy bit-shifted data
|
||||
// However, the starting line is optional and AWIPS decode may not see
|
||||
// it at all because TG will strip that starting line away
|
||||
// We'll try to use this hint first, if is is not found, then trial and
|
||||
// error way to decrypt and decode
|
||||
//
|
||||
// As of 11/05/2013, There is change in data spec. that the 3-bytes will
|
||||
// not be encoded as encrypted block size anymore (it will always be
|
||||
// transmission sequence # if present)
|
||||
// So there should have some minor changes in the logic below for
|
||||
// decoding the data.
|
||||
// However, as reading into the
|
||||
// com.raytheon.edex.plugin.binlightning.impl.BinLightningFactory.getDecoder()
|
||||
// and follow-on code, we see the following data patterns
|
||||
// for legacy bit-shifted data, which could be used to reduce guess-work
|
||||
// in data decryption:
|
||||
// The bit-shifted data will have multiple groups of the following
|
||||
// patterns:
|
||||
// 1-byte (unsigned byte): for size count
|
||||
// 1-byte (unsigned byte): for flash type:
|
||||
// 0x96 for FLASH_RPT (message size is 6 bytes each)
|
||||
// 0x97 for RT_FLASH_RPT (message size is 8 bytes each)
|
||||
// 0xd0 for OTHER_RPT (The D2D decoders declare but do not define this
|
||||
// message, so unimplemented decoder)
|
||||
// 0xd1 for COMM_RPT (The D2D decoders declare but do not define this
|
||||
// message, so unimplemented decoder)
|
||||
// 4-bytes: date time
|
||||
// multiple of 6 or 8 bytes (as determined by 2nd byte flash type) with
|
||||
// count indicated in 1st byte
|
||||
//
|
||||
// So this is be used to determine whether the data need to be
|
||||
// decrypted.
|
||||
|
||||
/*
|
||||
* // looks like previous assumption on block size bytes are not valid
|
||||
* any more. 11/20/2013 if (data != null) { byte[] sizeSeqBytes =
|
||||
* BinLigntningDecoderUtil.findSizeOrSeqBytesFromWMOHeader(data); if
|
||||
* (sizeSeqBytes != null) { // if this is in the header (which may not),
|
||||
* use that as a hint to determine which decoding route to go if
|
||||
* (BinLigntningDecoderUtil
|
||||
* .isPossibleWMOHeaderSequenceNumber(sizeSeqBytes) &&
|
||||
* BinLigntningDecoderUtil
|
||||
* .getEncryptedBlockSizeFromWMOHeader(sizeSeqBytes) != pdata.length) {
|
||||
* // looks like a sequence #, and if treat as size, it does not equal
|
||||
* to the data block size, so most likely legacy data needDecrypt =
|
||||
* false; } } }
|
||||
*/
|
||||
|
||||
if (needDecrypt) {
|
||||
try {
|
||||
// NOTE: 11/14/2013 WZ:
|
||||
// encrypted test data on TNCF (got from Melissa Porricelli)
|
||||
// seems to have extra 4 bytes (0x0d 0x0d 0x0a 0x03) at the end,
|
||||
// making the data size not a multiple of 16. However, original
|
||||
// test data do not have this trailing bytes. while NCEP test
|
||||
// data has extra 8 trailing bytes.
|
||||
// Brain Rapp's email on 11/13/2013 confirms that Unidata LDM
|
||||
// software used by AWIPS II will strips off all SBN protocol
|
||||
// headers
|
||||
// that precede the WMO header and adds its own 11 byte header
|
||||
// like this: "soh cr cr nl 2 5 4 sp cr cr nl". It
|
||||
// also adds a four byte trailer consisting of "cr cr nl etx"
|
||||
// (0x0d 0x0d 0x0a 0x03)
|
||||
// So, it seems necessary to trim trailing bytes if it is not
|
||||
// multiple of 16, warning messages will be logged though
|
||||
int dataLengthToBeDecrypted = pdata.length;
|
||||
if (pdata.length % 16 != 0) {
|
||||
dataLengthToBeDecrypted = pdata.length
|
||||
- (pdata.length % 16);
|
||||
logger.warn(traceId + " - Data length from file " + traceId
|
||||
+ " is " + pdata.length + " bytes, trailing "
|
||||
+ (pdata.length - dataLengthToBeDecrypted)
|
||||
+ " bytes has been trimmed to "
|
||||
+ dataLengthToBeDecrypted
|
||||
+ " bytes for decryption.");
|
||||
}
|
||||
byte[] encryptedData = new byte[dataLengthToBeDecrypted];
|
||||
encryptedData = Arrays.copyOfRange(pdata, 0,
|
||||
dataLengthToBeDecrypted);
|
||||
|
||||
byte[] decryptedData = cipher.decryptData(encryptedData,
|
||||
dataDate);
|
||||
// decrypt ok, then decode, first check if keep-alive record
|
||||
if (BinLightningDecoderUtil.isKeepAliveRecord(decryptedData)) {
|
||||
logger.info(traceId
|
||||
+ " - Keep-alive record detected, ignore for now.");
|
||||
decodeDone = true;
|
||||
return null;
|
||||
}
|
||||
// not keep-alive record, then check data validity and decode
|
||||
// into an ArrayList<LightningStrikePoint> of strikes
|
||||
if (BinLightningDecoderUtil
|
||||
.isLightningDataRecords(decryptedData)) {
|
||||
strikes = BinLightningDecoderUtil
|
||||
.decodeDecryptedBinLightningData(decryptedData);
|
||||
decodeDone = true;
|
||||
} else {
|
||||
logger.info(traceId
|
||||
+ " - Failed data validity check of the decrypted data, will try decode the old-fashioned way.");
|
||||
decodeDone = false;
|
||||
}
|
||||
} catch (IllegalBlockSizeException e) {
|
||||
logger.info(traceId
|
||||
+ " - "
|
||||
+ e.getMessage()
|
||||
+ ": Decryption failed, will try decode the old-fashioned way.");
|
||||
decodeDone = false;
|
||||
} catch (BadPaddingException e) {
|
||||
logger.info(traceId
|
||||
+ " - "
|
||||
+ e.getMessage()
|
||||
+ ": Decryption failed, will try decode the old-fashioned way.");
|
||||
decodeDone = false;
|
||||
} catch (BinLightningDataDecryptionException e) {
|
||||
logger.info(traceId
|
||||
+ " - "
|
||||
+ e.getMessage()
|
||||
+ ": Decryption failed, will try decode the old-fashioned way.");
|
||||
decodeDone = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (decodeDone == false) { // not decoded through decrypt->decode
|
||||
// process, try the legacy decoder
|
||||
logger.info(traceId + " - decoding as bit-shifted data");
|
||||
// bit-shifting data format check call here will get us some more
|
||||
// information on the data, also can compare the strikes with the
|
||||
// decoder result
|
||||
int estimatedStrikes = BinLightningDecoderUtil
|
||||
.getBitShiftedDataStrikeCount(pdata);
|
||||
strikes = decodeBitShiftedBinLightningData(pdata, wmoHdr);
|
||||
if (estimatedStrikes != strikes.size()) {
|
||||
logger.warn(traceId
|
||||
+ ": bit-shifted decoder found "
|
||||
+ strikes
|
||||
+ " strikes, which is different from estimate from data pattern examination: "
|
||||
+ estimatedStrikes);
|
||||
}
|
||||
}
|
||||
|
||||
return strikes;
|
||||
}
|
||||
|
||||
/**
|
||||
* extracted from the original {@link #decode(byte[], Headers)} method then
|
||||
* modified by Wufeng Zhou
|
||||
*
|
||||
* @param pdata
|
||||
* @param wmoHdr
|
||||
* - WMOHeader, added 12/24/2013 to help distinguish bit-shifted
|
||||
* NLDN and GLD360 data (GLD data will have header starts like
|
||||
* SFPA)
|
||||
* @return
|
||||
*/
|
||||
public static List<LightningStrikePoint> decodeBitShiftedBinLightningData(
|
||||
byte[] pdata, WMOHeader wmoHdr) {
|
||||
List<LightningStrikePoint> strikes = new ArrayList<LightningStrikePoint>();
|
||||
|
||||
IBinDataSource msgData = new LightningDataSource(pdata);
|
||||
|
||||
boolean continueDecode = true;
|
||||
while (continueDecode) {
|
||||
IBinLightningDecoder decoder = BinLightningFactory
|
||||
.getDecoder(msgData);
|
||||
|
||||
switch (decoder.getError()) {
|
||||
case IBinLightningDecoder.NO_ERROR: {
|
||||
for (LightningStrikePoint strike : decoder) {
|
||||
// use WMO Header to distinguish NLDN or GLD360 data because
|
||||
// no bit-shifted data spec available for GLD360.
|
||||
// 12/24/2013, WZ
|
||||
// The WMO header start string is defined in
|
||||
// BinLightningAESKey.properties file (normally, GLD360 data
|
||||
// will have WMO header
|
||||
// starts with SFPA41, or SFPA99 for test data.)
|
||||
String gld360WMOHeaderString = BinLightningAESKey
|
||||
.getProps().getProperty(
|
||||
"binlightning.gld360WMOHeaderStartString",
|
||||
"");
|
||||
if (gld360WMOHeaderString.trim().equals("") == false
|
||||
&& wmoHdr.getWmoHeader().startsWith(
|
||||
gld360WMOHeaderString)) {
|
||||
// GLD360 data based on the setup
|
||||
strike.setLightSource("GLD");
|
||||
}
|
||||
strikes.add(strike);
|
||||
}
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
continueDecode = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return strikes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set a trace identifier for the source data.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -24,4 +24,11 @@
|
|||
version="0.0.0"
|
||||
unpack="false"/>
|
||||
|
||||
<plugin
|
||||
id="gov.noaa.nws.ost.edex.plugin.binlightning"
|
||||
download-size="0"
|
||||
install-size="0"
|
||||
version="0.0.0"
|
||||
unpack="false"/>
|
||||
|
||||
</feature>
|
||||
|
|
|
|||
7
ost/gov.noaa.nws.ost.edex.plugin.binlightning/.classpath
Normal file
7
ost/gov.noaa.nws.ost.edex.plugin.binlightning/.classpath
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<classpath>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7"/>
|
||||
<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
|
||||
<classpathentry kind="src" path="src"/>
|
||||
<classpathentry kind="output" path="bin"/>
|
||||
</classpath>
|
||||
28
ost/gov.noaa.nws.ost.edex.plugin.binlightning/.project
Normal file
28
ost/gov.noaa.nws.ost.edex.plugin.binlightning/.project
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<projectDescription>
|
||||
<name>gov.noaa.nws.ost.edex.plugin.binlightning</name>
|
||||
<comment></comment>
|
||||
<projects>
|
||||
</projects>
|
||||
<buildSpec>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.jdt.core.javabuilder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.pde.ManifestBuilder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.pde.SchemaBuilder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
</buildSpec>
|
||||
<natures>
|
||||
<nature>org.eclipse.pde.PluginNature</nature>
|
||||
<nature>org.eclipse.jdt.core.javanature</nature>
|
||||
</natures>
|
||||
</projectDescription>
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
eclipse.preferences.version=1
|
||||
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
|
||||
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
|
||||
org.eclipse.jdt.core.compiler.compliance=1.7
|
||||
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
|
||||
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
|
||||
org.eclipse.jdt.core.compiler.source=1.7
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
Manifest-Version: 1.0
|
||||
Bundle-ManifestVersion: 2
|
||||
Bundle-Name: OST Binlightning
|
||||
Bundle-SymbolicName: gov.noaa.nws.ost.edex.plugin.binlightning
|
||||
Bundle-Version: 1.14.0.qualifier
|
||||
Bundle-RequiredExecutionEnvironment: JavaSE-1.7
|
||||
Import-Package: com.raytheon.uf.common.dataplugin.binlightning,
|
||||
com.raytheon.uf.common.dataplugin.binlightning.dataaccess,
|
||||
com.raytheon.uf.common.dataplugin.binlightning.impl,
|
||||
com.raytheon.uf.common.status,
|
||||
com.raytheon.uf.common.wmo,
|
||||
com.raytheon.uf.edex.decodertools.core
|
||||
Export-Package: gov.noaa.nws.ost.edex.plugin.binlightning
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
source.. = src/
|
||||
output.. = bin/
|
||||
bin.includes = META-INF/,\
|
||||
.
|
||||
|
|
@ -27,13 +27,14 @@ import java.util.TreeMap;
|
|||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import com.raytheon.uf.common.status.IUFStatusHandler;
|
||||
import com.raytheon.uf.common.status.UFStatus;
|
||||
|
||||
/**
|
||||
* BinLightningAESKey
|
||||
*
|
||||
* Simple representation of bin lightning AES encryption key and its associated key aliases in the keystore
|
||||
* Simple representation of bin lightning AES encryption key and its associated
|
||||
* key aliases in the keystore
|
||||
*
|
||||
* <pre>
|
||||
*
|
||||
|
|
@ -42,6 +43,7 @@ import org.apache.commons.logging.LogFactory;
|
|||
* Date Ticket# Engineer Description
|
||||
* ------------ ---------- ----------- --------------------------
|
||||
* 20130503 DCS 112 Wufeng Zhou To handle both the new encrypted data and legacy bit-shifted data
|
||||
* Jun 03, 2014 3226 bclement moved from com.raytheon.edex.plugin.binlightning to gov.noaa.nws.ost.edex.plugin.binlightning
|
||||
*
|
||||
* </pre>
|
||||
*
|
||||
|
|
@ -63,7 +65,8 @@ public class BinLightningAESKey {
|
|||
private static final Pattern KEY_ALIAS_PREFIX_PATTERN = Pattern.compile(KEY_ALIAS_PREFIX);
|
||||
private static final SimpleDateFormat KEY_ALIAS_DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd");
|
||||
|
||||
private static Log logger = LogFactory.getLog(BinLightningAESKey.class);
|
||||
private static IUFStatusHandler logger = UFStatus
|
||||
.getHandler(BinLightningAESKey.class);
|
||||
|
||||
private static Properties props = new Properties();
|
||||
private static KeyStore keystore;
|
||||
|
|
@ -9,24 +9,14 @@ import java.nio.ByteOrder;
|
|||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Calendar;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
import javax.crypto.BadPaddingException;
|
||||
import javax.crypto.IllegalBlockSizeException;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
import com.raytheon.edex.plugin.binlightning.impl.BinLightningFactory;
|
||||
import com.raytheon.edex.plugin.binlightning.impl.IBinLightningDecoder;
|
||||
import com.raytheon.edex.plugin.binlightning.impl.LightningDataSource;
|
||||
import com.raytheon.uf.common.dataplugin.binlightning.impl.LightningStrikePoint;
|
||||
import com.raytheon.uf.common.dataplugin.binlightning.impl.LtgMsgType;
|
||||
import com.raytheon.uf.common.dataplugin.binlightning.impl.LtgStrikeType;
|
||||
import com.raytheon.uf.common.wmo.WMOHeader;
|
||||
import com.raytheon.uf.common.status.IUFStatusHandler;
|
||||
import com.raytheon.uf.common.status.UFStatus;
|
||||
import com.raytheon.uf.edex.decodertools.core.BasePoint;
|
||||
import com.raytheon.uf.edex.decodertools.core.IBinDataSource;
|
||||
|
||||
/**
|
||||
* BinLigntningDecoderUtil
|
||||
|
|
@ -44,12 +34,16 @@ import com.raytheon.uf.edex.decodertools.core.IBinDataSource;
|
|||
* ------------ ---------- ----------- --------------------------
|
||||
* 20130503 DCS 112 Wufeng Zhou To handle both the new encrypted data and legacy bit-shifted data
|
||||
* 20140501 Wufeng Zhou Fix the encrypted decoding with correct offset
|
||||
* Jun 03, 2014 3226 bclement renamed from BinLigntningDecoderUtil to BinLightningDecoderUtil
|
||||
* moved from com.raytheon.edex.plugin.binlightning to gov.noaa.nws.ost.edex.plugin.binlightning
|
||||
* moved decodeBinLightningData() and decodeBitShiftedBinLightningData()
|
||||
* to BinLightningDecoder to solve circular dependency
|
||||
* </pre>
|
||||
*
|
||||
* @author Wufeng Zhou
|
||||
*
|
||||
*/
|
||||
public class BinLigntningDecoderUtil {
|
||||
public class BinLightningDecoderUtil {
|
||||
|
||||
/**
|
||||
* Message type for keep alive data records.
|
||||
|
|
@ -84,47 +78,10 @@ public class BinLigntningDecoderUtil {
|
|||
static final byte OTHER_RPT = (byte)0xD0;
|
||||
static final byte COMM_RPT = (byte)0xD1;
|
||||
|
||||
private static Log logger = LogFactory.getLog(BinLigntningDecoderUtil.class);
|
||||
private static IUFStatusHandler logger = UFStatus
|
||||
.getHandler(BinLightningDecoderUtil.class);
|
||||
|
||||
/**
|
||||
* extracted from the decode() of the original
|
||||
* com.raytheon.edex.plugin.binlightning.BinLightningDecoder class
|
||||
*
|
||||
* @param pdata
|
||||
* @param wmoHdr - WMOHeader, added 12/24/2013 to help distinguish bit-shifted NLDN and GLD360 data (GLD data will have header starts like SFPA)
|
||||
* @return
|
||||
*/
|
||||
public static List<LightningStrikePoint> decodeBitShiftedBinLightningData(byte[] pdata, WMOHeader wmoHdr) {
|
||||
List<LightningStrikePoint> strikes = new ArrayList<LightningStrikePoint>();
|
||||
|
||||
IBinDataSource msgData = new LightningDataSource(pdata);
|
||||
|
||||
boolean continueDecode = true;
|
||||
while (continueDecode) {
|
||||
IBinLightningDecoder decoder = BinLightningFactory.getDecoder(msgData);
|
||||
|
||||
switch (decoder.getError()) {
|
||||
case IBinLightningDecoder.NO_ERROR: {
|
||||
for (LightningStrikePoint strike : decoder) {
|
||||
// use WMO Header to distinguish NLDN or GLD360 data because no bit-shifted data spec available for GLD360. 12/24/2013, WZ
|
||||
// The WMO header start string is defined in BinLightningAESKey.properties file (normally, GLD360 data will have WMO header
|
||||
// starts with SFPA41, or SFPA99 for test data.)
|
||||
String gld360WMOHeaderString = BinLightningAESKey.getProps().getProperty("binlightning.gld360WMOHeaderStartString", "");
|
||||
if (gld360WMOHeaderString.trim().equals("") == false && wmoHdr.getWmoHeader().startsWith(gld360WMOHeaderString)) {
|
||||
// GLD360 data based on the setup
|
||||
strike.setLightSource("GLD");
|
||||
}
|
||||
strikes.add(strike);
|
||||
}
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
continueDecode = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return strikes;
|
||||
}
|
||||
|
||||
/**
|
||||
* decode the new bin lightning data, after the data record is decrypted, and it is not keep-alive record
|
||||
|
|
@ -170,8 +127,9 @@ public class BinLigntningDecoderUtil {
|
|||
int strokeType = buffer.getShort() & 0xffff; // 0x0000 for cloud-to-ground, 0x00ff for cloud-to-cloud, 0xffff for total flash
|
||||
short strokeKiloAmps = buffer.getShort(); // valid range: -254 to 254, specifically 16 bit signed integer
|
||||
int strokeMultiplicity = buffer.getShort() & 0xffff; // i.e. stroke count, valid range: 0 to 15
|
||||
int strokeDuration = buffer.getShort() & 0xffff; // valid range: 0 to 65535 (i.e., looks like unsigned short)
|
||||
int reserved = buffer.getShort() & 0xffff;
|
||||
// int strokeDuration = buffer.getShort() & 0xffff; // valid range:
|
||||
// 0 to 65535 (i.e., looks like unsigned short)
|
||||
// int reserved = buffer.getShort() & 0xffff;
|
||||
|
||||
// Create the strike record from the report info and base time information.
|
||||
BasePoint base = new BasePoint(lat, lon);
|
||||
|
|
@ -236,134 +194,7 @@ public class BinLigntningDecoderUtil {
|
|||
return strikes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode bin lightning data, able to handle both legacy bit-shifted and new encryted data
|
||||
*
|
||||
* The modified BinLightningDecoder.decode() method will use this method to decode data, which
|
||||
* will try to decrypt first, and decode the old fashioned way when decryption fails
|
||||
*
|
||||
* @param data - data content from file, including WMO header section
|
||||
* @param pdata - data with WMO header stripped, optional, if null, will strip WMO header internally from passed in data parameter
|
||||
* @param traceId - the file name of the data to be deoced
|
||||
* @param wmoHdr - WMOHeader, added 12/24/2013 to help distinguish bit-shifted NLDN and GLD360 data (GLD data will have header starts like SFPA)
|
||||
* @param dataDate - date of the data, optional, used as a hint to find appropriate encryption key faster
|
||||
* @return null if keep-alive record, otherwise a list (could be empty) of LightningStrikePoint
|
||||
*/
|
||||
public static List<LightningStrikePoint> decodeBinLightningData(byte[] data, byte[] pdata, String traceId, WMOHeader wmoHdr, Date dataDate) {
|
||||
if (pdata == null) { // if data without header not passed, we'll strip the WMO header here
|
||||
WMOHeader header = new WMOHeader(data);
|
||||
if (header.isValid() && header.getMessageDataStart() > 0) {
|
||||
pdata = new byte[data.length - header.getMessageDataStart()];
|
||||
System.arraycopy(data, header.getMessageDataStart(), pdata, 0, data.length - header.getMessageDataStart());
|
||||
}
|
||||
}
|
||||
|
||||
List<LightningStrikePoint> strikes = new ArrayList<LightningStrikePoint>();
|
||||
boolean needDecrypt = true; // set as default unless clear evidence says otherwise
|
||||
boolean decodeDone = false;
|
||||
EncryptedBinLightningCipher cipher = new EncryptedBinLightningCipher();
|
||||
|
||||
//
|
||||
// Using different WMO headers to indicate whether the data is encrypted or not would be a nice option.
|
||||
// However, that idea has been discussed but not adopted.
|
||||
// If in the future, WMO header can be different for legacy and encrypted data, or some other metadata can be used to decide
|
||||
// whether deceyption is needed, logic can be added here.
|
||||
//
|
||||
// Before that happens, we'll use hints and trial & error to decode the data
|
||||
// Hints: Per lightning data format spec, there are 3 bytes in the WMO header starting line that indicates the size of the encrypted block
|
||||
// or the ASCII sequence # for legacy bit-shifted data
|
||||
// However, the starting line is optional and AWIPS decode may not see it at all because TG will strip that starting line away
|
||||
// We'll try to use this hint first, if is is not found, then trial and error way to decrypt and decode
|
||||
//
|
||||
// As of 11/05/2013, There is change in data spec. that the 3-bytes will not be encoded as encrypted block size anymore (it will always be transmission sequence # if present)
|
||||
// So there should have some minor changes in the logic below for decoding the data.
|
||||
// However, as reading into the com.raytheon.edex.plugin.binlightning.impl.BinLightningFactory.getDecoder() and follow-on code, we see the following data patterns
|
||||
// for legacy bit-shifted data, which could be used to reduce guess-work in data decryption:
|
||||
// The bit-shifted data will have multiple groups of the following patterns:
|
||||
// 1-byte (unsigned byte): for size count
|
||||
// 1-byte (unsigned byte): for flash type:
|
||||
// 0x96 for FLASH_RPT (message size is 6 bytes each)
|
||||
// 0x97 for RT_FLASH_RPT (message size is 8 bytes each)
|
||||
// 0xd0 for OTHER_RPT (The D2D decoders declare but do not define this message, so unimplemented decoder)
|
||||
// 0xd1 for COMM_RPT (The D2D decoders declare but do not define this message, so unimplemented decoder)
|
||||
// 4-bytes: date time
|
||||
// multiple of 6 or 8 bytes (as determined by 2nd byte flash type) with count indicated in 1st byte
|
||||
//
|
||||
// So this is be used to determine whether the data need to be decrypted.
|
||||
|
||||
/*
|
||||
// looks like previous assumption on block size bytes are not valid any more. 11/20/2013
|
||||
if (data != null) {
|
||||
byte[] sizeSeqBytes = BinLigntningDecoderUtil.findSizeOrSeqBytesFromWMOHeader(data);
|
||||
if (sizeSeqBytes != null) {
|
||||
// if this is in the header (which may not), use that as a hint to determine which decoding route to go
|
||||
if (BinLigntningDecoderUtil.isPossibleWMOHeaderSequenceNumber(sizeSeqBytes)
|
||||
&& BinLigntningDecoderUtil.getEncryptedBlockSizeFromWMOHeader(sizeSeqBytes) != pdata.length) {
|
||||
// looks like a sequence #, and if treat as size, it does not equal to the data block size, so most likely legacy data
|
||||
needDecrypt = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
if (needDecrypt) {
|
||||
try {
|
||||
// NOTE: 11/14/2013 WZ:
|
||||
// encrypted test data on TNCF (got from Melissa Porricelli) seems to have extra 4 bytes (0x0d 0x0d 0x0a 0x03) at the end,
|
||||
// making the data size not a multiple of 16. However, original test data do not have this trailing bytes. while NCEP test
|
||||
// data has extra 8 trailing bytes.
|
||||
// Brain Rapp's email on 11/13/2013 confirms that Unidata LDM software used by AWIPS II will strips off all SBN protocol headers
|
||||
// that precede the WMO header and adds its own 11 byte header like this: "soh cr cr nl 2 5 4 sp cr cr nl". It
|
||||
// also adds a four byte trailer consisting of "cr cr nl etx" (0x0d 0x0d 0x0a 0x03)
|
||||
// So, it seems necessary to trim trailing bytes if it is not multiple of 16, warning messages will be logged though
|
||||
int dataLengthToBeDecrypted = pdata.length;
|
||||
if (pdata.length % 16 != 0) {
|
||||
dataLengthToBeDecrypted = pdata.length - (pdata.length % 16);
|
||||
logger.warn(traceId + " - Data length from file " + traceId + " is " + pdata.length + " bytes, trailing " +
|
||||
(pdata.length - dataLengthToBeDecrypted) + " bytes has been trimmed to " + dataLengthToBeDecrypted + " bytes for decryption.");
|
||||
}
|
||||
byte[] encryptedData = new byte[dataLengthToBeDecrypted];
|
||||
encryptedData = Arrays.copyOfRange(pdata, 0, dataLengthToBeDecrypted);
|
||||
|
||||
byte[] decryptedData = cipher.decryptData(encryptedData, dataDate);
|
||||
// decrypt ok, then decode, first check if keep-alive record
|
||||
if (BinLigntningDecoderUtil.isKeepAliveRecord(decryptedData)) {
|
||||
logger.info(traceId + " - Keep-alive record detected, ignore for now.");
|
||||
decodeDone = true;
|
||||
return null;
|
||||
}
|
||||
// not keep-alive record, then check data validity and decode into an ArrayList<LightningStrikePoint> of strikes
|
||||
if (BinLigntningDecoderUtil.isLightningDataRecords(decryptedData)) {
|
||||
strikes = BinLigntningDecoderUtil.decodeDecryptedBinLightningData(decryptedData);
|
||||
decodeDone = true;
|
||||
} else {
|
||||
logger.info(traceId + " - Failed data validity check of the decrypted data, will try decode the old-fashioned way.");
|
||||
decodeDone = false;
|
||||
}
|
||||
} catch (IllegalBlockSizeException e) {
|
||||
logger.info(traceId + " - " + e.getMessage() + ": Decryption failed, will try decode the old-fashioned way.");
|
||||
decodeDone = false;
|
||||
} catch (BadPaddingException e) {
|
||||
logger.info(traceId + " - " + e.getMessage() + ": Decryption failed, will try decode the old-fashioned way.");
|
||||
decodeDone = false;
|
||||
} catch (BinLightningDataDecryptionException e) {
|
||||
logger.info(traceId + " - " + e.getMessage() + ": Decryption failed, will try decode the old-fashioned way.");
|
||||
decodeDone = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (decodeDone == false) { // not decoded through decrypt->decode process, try the legacy decoder
|
||||
logger.info(traceId + " - decoding as bit-shifted data");
|
||||
// bit-shifting data format check call here will get us some more information on the data, also can compare the strikes with the decoder result
|
||||
int estimatedStrikes = getBitShiftedDataStrikeCount(pdata);
|
||||
strikes = BinLigntningDecoderUtil.decodeBitShiftedBinLightningData(pdata, wmoHdr);
|
||||
if (estimatedStrikes != strikes.size()) {
|
||||
logger.warn(traceId + ": bit-shifted decoder found " + strikes + " strikes, which is different from estimate from data pattern examination: " + estimatedStrikes);
|
||||
}
|
||||
}
|
||||
|
||||
return strikes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines if the bytes passed are a standard "NWS Keep Alive" message.
|
||||
|
|
@ -6,6 +6,7 @@ package gov.noaa.nws.ost.edex.plugin.binlightning;
|
|||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
|
|
@ -15,13 +16,14 @@ import javax.crypto.Cipher;
|
|||
import javax.crypto.IllegalBlockSizeException;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import com.raytheon.uf.common.status.IUFStatusHandler;
|
||||
import com.raytheon.uf.common.status.UFStatus;
|
||||
|
||||
/**
|
||||
* EncryptedBinLightningCipher
|
||||
*
|
||||
* Use AES secret keys found in configured keystore to decrypt bin lightning data
|
||||
* Use AES secret keys found in configured keystore to decrypt bin lightning
|
||||
* data
|
||||
*
|
||||
* <pre>
|
||||
*
|
||||
|
|
@ -30,6 +32,8 @@ import org.apache.commons.logging.LogFactory;
|
|||
* Date Ticket# Engineer Description
|
||||
* ------------ ---------- ----------- --------------------------
|
||||
* 20130503 DCS 112 Wufeng Zhou To handle both the new encrypted data and legacy bit-shifted data
|
||||
* Jun 03, 2014 3226 bclement moved from com.raytheon.edex.plugin.binlightning to gov.noaa.nws.ost.edex.plugin.binlightning
|
||||
* handled null return from BinLightningAESKey.getBinLightningAESKeys()
|
||||
*
|
||||
* </pre>
|
||||
*
|
||||
|
|
@ -51,6 +55,9 @@ public class EncryptedBinLightningCipher {
|
|||
protected HashMap<String, Cipher> initialValue() {
|
||||
// get AES keys from keystore and create encryption and decryption ciphers from them
|
||||
BinLightningAESKey[] keys = BinLightningAESKey.getBinLightningAESKeys();
|
||||
if (keys == null) {
|
||||
keys = new BinLightningAESKey[0];
|
||||
}
|
||||
HashMap<String, Cipher> cipherMap = new HashMap<String, Cipher>();
|
||||
for (BinLightningAESKey key : keys) {
|
||||
try {
|
||||
|
|
@ -67,7 +74,8 @@ public class EncryptedBinLightningCipher {
|
|||
}
|
||||
};
|
||||
|
||||
private static Log logger = LogFactory.getLog(EncryptedBinLightningCipher.class);
|
||||
private static IUFStatusHandler logger = UFStatus
|
||||
.getHandler(EncryptedBinLightningCipher.class);
|
||||
|
||||
public EncryptedBinLightningCipher() {
|
||||
|
||||
|
|
@ -122,7 +130,7 @@ public class EncryptedBinLightningCipher {
|
|||
|
||||
// wrong key will decrypt data into random noise/garbage, so we need to do a sanity check to make sure
|
||||
// we are decrypting with the right key
|
||||
if ( BinLigntningDecoderUtil.isKeepAliveRecord(decryptedData) == false && BinLigntningDecoderUtil.isLightningDataRecords(decryptedData) == false) {
|
||||
if ( BinLightningDecoderUtil.isKeepAliveRecord(decryptedData) == false && BinLightningDecoderUtil.isLightningDataRecords(decryptedData) == false) {
|
||||
//if (BinLigntningDecoderUtil.isValidMixedRecordData(decryptedData) == false) { // use this only if keep-alive record could be mixed with lightning records
|
||||
throw new BinLightningDataDecryptionException("Decrypted data (" + decryptedData.length + " bytes) with key "
|
||||
+ preferredKeyList.get(i).getAlias() + " is not valid keep-alive or binLightning records.", decryptedData);
|
||||
|
|
@ -163,7 +171,11 @@ public class EncryptedBinLightningCipher {
|
|||
* @return preferred key list order
|
||||
*/
|
||||
private List<BinLightningAESKey> findPreferredKeyOrderForData(Date dataDate) {
|
||||
List<BinLightningAESKey> defKeyList = Arrays.asList(BinLightningAESKey.getBinLightningAESKeys());
|
||||
BinLightningAESKey[] binLightningAESKeys = BinLightningAESKey.getBinLightningAESKeys();
|
||||
if (binLightningAESKeys == null || binLightningAESKeys.length < 1) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
List<BinLightningAESKey> defKeyList = Arrays.asList(binLightningAESKeys);
|
||||
if (dataDate == null) {
|
||||
return defKeyList; // use default order
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue