xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #131

Browse source 
Former-commit-id: dea5f087d3 [formerly bd5db64b5b] [formerly 35b25ee2a9 [formerly 4167f33801ee2450502231438955e72c4428fbcc]]
Former-commit-id: 35b25ee2a9
Former-commit-id: 4701088670
This commit is contained in:
Brian.Dyke 2014-02-04 16:36:11 -05:00
parent cdb408daf8
commit 6882fc8d52
5 changed files with 190 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

85
edexOsgi/com.raytheon.edex.plugin.binlightning/src/com/raytheon/edex/plugin/binlightning/BinLightningDecoder.java
View file

@ -79,6 +79,9 @@ import com.raytheon.uf.edex.wmo.message.WMOHeader;
* new encrypted data and legacy bit-shifted
* data
* Aug 30, 2013 2298 rjpeter Make getPluginName abstract
* Jan 24, 2014 DR 16774 Wufeng Zhou Modified for updated Bin-lightning data spec,
* and to used WMO header to distinguish bit-shifted
* GLD360 and NLDN data.
*
* </pre>
*
@ -104,7 +107,7 @@ public class BinLightningDecoder extends AbstractDecoder {
public LtgStrikeType DEFAULT_FLASH_TYPE = LtgStrikeType.STRIKE_CG;
private String traceId = null;
/**
* Construct a BinLightning decoder. Calling hasNext() after construction
* will return false, decode() will return a null.
@ -121,10 +124,9 @@ public class BinLightningDecoder extends AbstractDecoder {
* @throws DecoderException
* Thrown if no data is available.
*/
public PluginDataObject[] decode(byte[] data, Headers headers)
throws DecoderException {
public PluginDataObject[] decode(byte[] data, Headers headers) throws DecoderException {
// String traceId = null;
//String traceId = null;
PluginDataObject[] reports = new PluginDataObject[0];
if (data != null) {
@ -135,56 +137,43 @@ public class BinLightningDecoder extends AbstractDecoder {
Calendar baseTime = TimeTools.findDataTime(wmoHdr.getYYGGgg(),
headers);
// Because binary nature of the encrypted data, the string
// created with its byte[] array may not have the same length of
// the byte[] array length
// So when DecoderTools.stripWMOHeader() assumes byte[] length
// == String length in it logic, it is observed that it may
// return a shorter byte[] than
// the real data array. (Looks like a bug???)
// byte[] pdata = DecoderTools.stripWMOHeader(data,
// SFUS_PATTERN);
// if (pdata == null) {
// pdata = DecoderTools.stripWMOHeader(data, SFPA_PATTERN);
// }
// instead the following is used to strip WMO header a little
// more safely.
// Because binary nature of the encrypted data, the string created with its byte[] array may not have the same length of the byte[] array length
// So when DecoderTools.stripWMOHeader() assumes byte[] length == String length in its logic, it is observed that it may return a shorter byte[] than
// the real data array. (Looks like a bug???)
// byte[] pdata = DecoderTools.stripWMOHeader(data, SFUS_PATTERN);
// if (pdata == null) {
// pdata = DecoderTools.stripWMOHeader(data, SFPA_PATTERN);
// }
// instead the following is used to strip WMO header a little more safely.
byte[] pdata = null;
if (wmoHdr.isValid() && (wmoHdr.getMessageDataStart() > 0)) {
pdata = new byte[data.length - wmoHdr.getMessageDataStart()];
System.arraycopy(data, wmoHdr.getMessageDataStart(), pdata,
0, data.length - wmoHdr.getMessageDataStart());
}
if (wmoHdr.isValid() && wmoHdr.getMessageDataStart() > 0) {
pdata = new byte[data.length - wmoHdr.getMessageDataStart()];
System.arraycopy(data, wmoHdr.getMessageDataStart(), pdata, 0, data.length - wmoHdr.getMessageDataStart());
}
if ((pdata == null) || (pdata.length == 0)) {
return new PluginDataObject[0];
}
//
// Modified by Wufeng Zhou to handle both legacy bit-shifted and
// new encryted data
// Modified by Wufeng Zhou to handle both legacy bit-shifted and new encryted data
//
// Preserved the legacy decoding in
// BinLigntningDecoderUtil.decodeBitShiftedBinLightningData(),
// and added logic to process
// both encrypted data and legacy data
//
List<LightningStrikePoint> strikes = BinLigntningDecoderUtil
.decodeBinLightningData(data, pdata, traceId,
baseTime.getTime());
// Preserved the legacy decoding in BinLigntningDecoderUtil.decodeBitShiftedBinLightningData(), and added logic to process
// both encrypted data and legacy data
//
List<LightningStrikePoint> strikes = BinLigntningDecoderUtil.decodeBinLightningData(data, pdata, traceId, wmoHdr, baseTime.getTime());
if (strikes == null) { // keep-alive record, log and return
logger.info(traceId
+ " - found keep-alive record. ignore for now.");
return reports;
logger.info(traceId + " - found keep-alive record. ignore for now.");
return reports;
}
//
// Done MOD by Wufeng Zhou
//
// post processing data - if not keep-alive record
BinLightningRecord report = null;
if (strikes.size() > 0) {
@ -199,11 +188,9 @@ public class BinLightningDecoder extends AbstractDecoder {
Calendar c = TimeTools.copy(baseTime);
if (c == null) {
throw new DecoderException(traceId
+ " - Error decoding times");
throw new DecoderException(traceId + " - Error decoding times");
}
// report.setInsertTime(c); // OB13.4 source code does not have
// this line anymore, WZ 05/03/2013
//report.setInsertTime(c); // OB13.4 source code does not have this line anymore, WZ 05/03/2013
Calendar cStart = report.getStartTime();
if (cStart.getTimeInMillis() > (c.getTimeInMillis() + TEN_MINUTES)) {
@ -222,19 +209,19 @@ public class BinLightningDecoder extends AbstractDecoder {
if (report != null) {
report.setTraceId(traceId);
//report.setPluginName("binlightning"); // line disappear in OB15.5.3
try {
report.constructDataURI();
reports = new PluginDataObject[] { report };
} catch (PluginException e) {
logger.error("Error constructing datauri", e);
throw new DecoderException(
"Error constructing datauri", e);
logger.error("Error constructing datauri", e);
throw new DecoderException("Error constructing datauri", e);
}
}
}
}
} else {
logger.error("No WMOHeader found in data");
logger.error("No WMOHeader found in data");
}
return reports;
}

4
edexOsgi/com.raytheon.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningAESKey.java Normal file → Executable file
View file

@ -235,5 +235,9 @@ public class BinLightningAESKey {
public void setKeyDate(Date keyDate) {
this.keyDate = keyDate;
}
public static Properties getProps() {
return props;
}
}

8
edexOsgi/com.raytheon.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningAESKey.properties Normal file → Executable file
View file

@ -31,3 +31,11 @@ binlightning.AESKeystorePassword=testStorePass
#
#binlightning.AESKeystore=/usr/local/ldm/binLightningAESKeystore.jce
#binlightning.AESKeystorePassword=notShownHere
#
# The WMO Header string to determine if bit-shifted data is GLD360 data: binlightning.gld360WMOHeaderStartString
# By default or when this is not set, the bit-shifted data will not set the lightning source (which defaults to NLDN in database table column)
# Normally, SFPA41 denotes GLD360, SFPA99 is test GLD360 data
# Added 12/24/2013, WZ
#
binlightning.gld360WMOHeaderStartString=SFPA

144
edexOsgi/com.raytheon.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLigntningDecoderUtil.java
View file

@ -54,30 +54,36 @@ public class BinLigntningDecoderUtil {
/**
* Message type for keep alive data records.
*/
final static short KEEP_ALIVE_TYPE = 0x0000;
static final short KEEP_ALIVE_TYPE = 0x0000;
/**
* Message type for lightning data records.
*/
final static short LIGHTNING_TYPE = 0x00ff;
static final short LIGHTNING_TYPE = 0x00ff;
/**
* If there are more elements within the data record, this terminator is used.
*/
final static byte[] MORE_TERM_BYTES = {0x0d, 0x0d, 0x0a, 0x00};
static final byte[] MORE_TERM_BYTES = {0x0d, 0x0d, 0x0a, 0x00};
/**
* Last element within data records should be terminated by these 4 bytes.
*/
final static byte[] LAST_TERM_BYTES = {0x0d, 0x0d, 0x0a, 0x03};
static final byte[] LAST_TERM_BYTES = {0x0d, 0x0d, 0x0a, 0x03};
/**
* WMO header start bytes, optional (it is known that TG will strip this away)
*/
final static byte[] WMO_HEADER_START_BYTES = {0x01, 0x0d, 0x0d, 0x0a};
static final byte[] WMO_HEADER_START_BYTES = {0x01, 0x0d, 0x0d, 0x0a};
/* Size of binary NWS lightning data record. */
static final int BINLIGHTNING_RECORD_SIZE = 32;
/** legacy lightning data types */
static final byte FLASH_RPT = (byte)0x96;
static final byte RT_FLASH_RPT = (byte)0x97;
static final byte OTHER_RPT = (byte)0xD0;
static final byte COMM_RPT = (byte)0xD1;
private static Log logger = LogFactory.getLog(BinLigntningDecoderUtil.class);
/**
@ -85,9 +91,10 @@ public class BinLigntningDecoderUtil {
* com.raytheon.edex.plugin.binlightning.BinLightningDecoder class
*
* @param pdata
* @param wmoHdr - WMOHeader, added 12/24/2013 to help distinguish bit-shifted NLDN and GLD360 data (GLD data will have header starts like SFPA)
* @return
*/
public static List<LightningStrikePoint> decodeBitShiftedBinLightningData(byte[] pdata) {
public static List<LightningStrikePoint> decodeBitShiftedBinLightningData(byte[] pdata, WMOHeader wmoHdr) {
List<LightningStrikePoint> strikes = new ArrayList<LightningStrikePoint>();
IBinDataSource msgData = new LightningDataSource(pdata);
@ -99,6 +106,14 @@ public class BinLigntningDecoderUtil {
switch (decoder.getError()) {
case IBinLightningDecoder.NO_ERROR: {
for (LightningStrikePoint strike : decoder) {
// use WMO Header to distinguish NLDN or GLD360 data because no bit-shifted data spec available for GLD360. 12/24/2013, WZ
// The WMO header start string is defined in BinLightningAESKey.properties file (normally, GLD360 data will have WMO header
// starts with SFPA41, or SFPA99 for test data.)
String gld360WMOHeaderString = BinLightningAESKey.getProps().getProperty("binlightning.gld360WMOHeaderStartString", "");
if (gld360WMOHeaderString.trim().equals("") == false && wmoHdr.getWmoHeader().startsWith(gld360WMOHeaderString)) {
// GLD360 data based on the setup
strike.setLightSource("GLD");
}
strikes.add(strike);
}
break;
@ -229,10 +244,11 @@ public class BinLigntningDecoderUtil {
* @param data - data content from file, including WMO header section
* @param pdata - data with WMO header stripped, optional, if null, will strip WMO header internally from passed in data parameter
* @param traceId - the file name of the data to be deoced
* @param wmoHdr - WMOHeader, added 12/24/2013 to help distinguish bit-shifted NLDN and GLD360 data (GLD data will have header starts like SFPA)
* @param dataDate - date of the data, optional, used as a hint to find appropriate encryption key faster
* @return null if keep-alive record, otherwise a list (could be empty) of LightningStrikePoint
*/
public static List<LightningStrikePoint> decodeBinLightningData(byte[] data, byte[] pdata, String traceId, Date dataDate) {
public static List<LightningStrikePoint> decodeBinLightningData(byte[] data, byte[] pdata, String traceId, WMOHeader wmoHdr, Date dataDate) {
if (pdata == null) { // if data without header not passed, we'll strip the WMO header here
WMOHeader header = new WMOHeader(data);
if (header.isValid() && header.getMessageDataStart() > 0) {
@ -257,6 +273,25 @@ public class BinLigntningDecoderUtil {
// or the ASCII sequence # for legacy bit-shifted data
// However, the starting line is optional and AWIPS decode may not see it at all because TG will strip that starting line away
// We'll try to use this hint first, if is is not found, then trial and error way to decrypt and decode
//
// As of 11/05/2013, There is change in data spec. that the 3-bytes will not be encoded as encrypted block size anymore (it will always be transmission sequence # if present)
// So there should have some minor changes in the logic below for decoding the data.
// However, as reading into the com.raytheon.edex.plugin.binlightning.impl.BinLightningFactory.getDecoder() and follow-on code, we see the following data patterns
// for legacy bit-shifted data, which could be used to reduce guess-work in data decryption:
// The bit-shifted data will have multiple groups of the following patterns:
// 1-byte (unsigned byte): for size count
// 1-byte (unsigned byte): for flash type:
// 0x96 for FLASH_RPT (message size is 6 bytes each)
// 0x97 for RT_FLASH_RPT (message size is 8 bytes each)
// 0xd0 for OTHER_RPT (The D2D decoders declare but do not define this message, so unimplemented decoder)
// 0xd1 for COMM_RPT (The D2D decoders declare but do not define this message, so unimplemented decoder)
// 4-bytes: date time
// multiple of 6 or 8 bytes (as determined by 2nd byte flash type) with count indicated in 1st byte
//
// So this is be used to determine whether the data need to be decrypted.
/*
// looks like previous assumption on block size bytes are not valid any more. 11/20/2013
if (data != null) {
byte[] sizeSeqBytes = BinLigntningDecoderUtil.findSizeOrSeqBytesFromWMOHeader(data);
if (sizeSeqBytes != null) {
@ -268,19 +303,42 @@ public class BinLigntningDecoderUtil {
}
}
}
*/
if (needDecrypt) {
try {
byte[] decryptedData = cipher.decryptData(pdata, dataDate);
// NOTE: 11/14/2013 WZ:
// encrypted test data on TNCF (got from Melissa Porricelli) seems to have extra 4 bytes (0x0d 0x0d 0x0a 0x03) at the end,
// making the data size not a multiple of 16. However, original test data do not have this trailing bytes. while NCEP test
// data has extra 8 trailing bytes.
// Brain Rapp's email on 11/13/2013 confirms that Unidata LDM software used by AWIPS II will strips off all SBN protocol headers
// that precede the WMO header and adds its own 11 byte header like this: "soh cr cr nl 2 5 4 sp cr cr nl". It
// also adds a four byte trailer consisting of "cr cr nl etx" (0x0d 0x0d 0x0a 0x03)
// So, it seems necessary to trim trailing bytes if it is not multiple of 16, warning messages will be logged though
int dataLengthToBeDecrypted = pdata.length;
if (pdata.length % 16 != 0) {
dataLengthToBeDecrypted = pdata.length - (pdata.length % 16);
logger.warn(traceId + " - Data length from file " + traceId + " is " + pdata.length + " bytes, trailing " +
(pdata.length - dataLengthToBeDecrypted) + " bytes has been trimmed to " + dataLengthToBeDecrypted + " bytes for decryption.");
}
byte[] encryptedData = new byte[dataLengthToBeDecrypted];
encryptedData = Arrays.copyOfRange(pdata, 0, dataLengthToBeDecrypted);
byte[] decryptedData = cipher.decryptData(encryptedData, dataDate);
// decrypt ok, then decode, first check if keep-alive record
if (BinLigntningDecoderUtil.isKeepAliveRecord(decryptedData)) {
logger.info(traceId + " - Keep-alive record detected, ignore for now.");
decodeDone = true;
return null;
}
// not keep-alive record, then decode into an ArrayList<LightningStrikePoint> of strikes
strikes = BinLigntningDecoderUtil.decodeDecryptedBinLightningData(decryptedData);
decodeDone = true;
// not keep-alive record, then check data validity and decode into an ArrayList<LightningStrikePoint> of strikes
if (BinLigntningDecoderUtil.isLightningDataRecords(decryptedData)) {
strikes = BinLigntningDecoderUtil.decodeDecryptedBinLightningData(decryptedData);
decodeDone = true;
} else {
logger.info(traceId + " - Failed data validity check of the decrypted data, will try decode the old-fashioned way.");
decodeDone = false;
}
} catch (IllegalBlockSizeException e) {
logger.info(traceId + " - " + e.getMessage() + ": Decryption failed, will try decode the old-fashioned way.");
decodeDone = false;
@ -294,7 +352,13 @@ public class BinLigntningDecoderUtil {
}
if (decodeDone == false) { // not decoded through decrypt->decode process, try the legacy decoder
strikes = BinLigntningDecoderUtil.decodeBitShiftedBinLightningData(pdata);
logger.info(traceId + " - decoding as bit-shifted data");
// bit-shifting data format check call here will get us some more information on the data, also can compare the strikes with the decoder result
int estimatedStrikes = getBitShiftedDataStrikeCount(pdata);
strikes = BinLigntningDecoderUtil.decodeBitShiftedBinLightningData(pdata, wmoHdr);
if (estimatedStrikes != strikes.size()) {
logger.warn(traceId + ": bit-shifted decoder found " + strikes + " strikes, which is different from estimate from data pattern examination: " + estimatedStrikes);
}
}
return strikes;
@ -526,4 +590,58 @@ public class BinLigntningDecoderUtil {
return false;
}
/**
* Based on the data format described below, trying to get an estimate of bit-shifted data strike count without actually decoding the data
*
* Legacy bit-shifted data format has the following known data pattern:
* The bit-shifted data will have multiple groups of the following patterns:
* 1-byte (unsigned byte): for size count. Count be 0, in that case, will only have 4-bytes data-time following (sounds like keep-alive)
* 1-byte (unsigned byte): for flash type:
* 0x96 for FLASH_RPT (message size is 6 bytes each)
* 0x97 for RT_FLASH_RPT (message size is 8 bytes each)
* 0xd0 for OTHER_RPT (The D2D decoders declare but do not define this message, so unimplemented decoder)
* 0xd1 for COMM_RPT (The D2D decoders declare but do not define this message, so unimplemented decoder)
* 4-bytes: date time
* multiple of 6 or 8 bytes (as determined by 2nd byte flash type) with count indicated in 1st byte
*
* NOTE: as test file SFUS41_KWBC_241059_53594581.nldn.2013042411 revealed, it is bit-shifted data but somehow with 4 few extra bytes at
* the end of the file (0x0d 0x0d 0x0a 0x03). It still can be decoded as bit-shifted data.
* So this method of checking data format is not foolproof.
* - 11/07/2013 WZ
*
* @param pdata - binary data byte array
* @return true if conforms to bit-shifted data format, false otherwise
*/
public static int getBitShiftedDataStrikeCount(byte[] pdata) {
int pos = 0;
int strikeCount = 0;
int totalGroupCount = 0; // groups that conforms to the data pattern described above in comment
while (pos < pdata.length) {
int recCount = (int)(pdata[pos] & 0xff);
if ((pos+1) >= pdata.length) break; // end of data before even read the flash type
byte flashType = pdata[pos + 1];
if (flashType == FLASH_RPT) { // FLASH_RPT 0x96, record size is 6 bytes
totalGroupCount++;
pos = pos + 6 + 6*recCount; // next record start position
} else if (flashType == RT_FLASH_RPT) { // RT_FLASH_RPT 0x97, record size is 8 bytes
totalGroupCount++;
pos = pos + 6 + 8*recCount; // next record start position
} else {
break; // not a known type
}
// only when the next record start position is exactly right after the end of data, we get a well conformed data
if (pos <= pdata.length) {
strikeCount += recCount;
} else if (pos > pdata.length) {
break;
}
}
if (totalGroupCount == 0 && strikeCount == 0) {
logger.info("getBitShiftedDataStrikeCount(): no bit-shifted data pattern found, not likely to be bit-shifted data.");
return -1;
} else {
logger.info("getBitShiftedDataStrikeCount(): found " + totalGroupCount + " groups of bit-shifted data pattern, which contains " + strikeCount + " strikes.");
return strikeCount;
}
}
}

17
edexOsgi/com.raytheon.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/EncryptedBinLightningCipher.java Normal file → Executable file
View file

@ -124,8 +124,6 @@ public class EncryptedBinLightningCipher {
// we are decrypting with the right key
if ( BinLigntningDecoderUtil.isKeepAliveRecord(decryptedData) == false && BinLigntningDecoderUtil.isLightningDataRecords(decryptedData) == false) {
//if (BinLigntningDecoderUtil.isValidMixedRecordData(decryptedData) == false) { // use this only if keep-alive record could be mixed with lightning records
logger.info("Decrypted data (" + decryptedData.length + " bytes) with key " + preferredKeyList.get(i).getAlias()
+ " is not valid keep-alive or binLightning records. Try other key.");
throw new BinLightningDataDecryptionException("Decrypted data (" + decryptedData.length + " bytes) with key "
+ preferredKeyList.get(i).getAlias() + " is not valid keep-alive or binLightning records.", decryptedData);
}
@ -133,16 +131,23 @@ public class EncryptedBinLightningCipher {
break; // decrypt ok, break out
} catch (IllegalBlockSizeException e) {
// ignore exception if not the last, and try next cipher
logger.info("Fail to decrypt data (" + data.length + " bytes) with key: " + preferredKeyList.get(i).getAlias() + " - " + e.getMessage() + ", will try other keys");
logger.info("Fail to decrypt data (" + data.length + " bytes) with key: " + preferredKeyList.get(i).getAlias() + " - " + e.getMessage() + ", will try other available key");
if (i == (preferredKeyList.size() - 1)) {
logger.info("Fail to decrypt with all know keys, either data is not encrypted or is invalid.");
logger.error("Fail to decrypt with all known keys, either data is not encrypted or is invalid: " + e.getMessage());
throw e;
}
} catch (BadPaddingException e) {
// ignore exception if not the last, and try next cipher
logger.info("Fail to decrypt data (" + data.length + " bytes) with key: " + preferredKeyList.get(i).getAlias() + " - " + e.getMessage() + ", will try other keys");
logger.info("Fail to decrypt data (" + data.length + " bytes) with key: " + preferredKeyList.get(i).getAlias() + " - " + e.getMessage() + ", will try other available key");
if (i == (preferredKeyList.size() - 1)) {
logger.info("Fail to decrypt with all know keys, either data is not encrypted or is invalid.");
logger.error("Fail to decrypt with all known keys, either data is not encrypted or is invalid: " + e.getMessage());
throw e;
}
} catch (BinLightningDataDecryptionException e) {
// ignore exception if not the last, and try next cipher
logger.info("Fail to decrypt data (" + data.length + " bytes) with key: " + preferredKeyList.get(i).getAlias() + " - " + e.getMessage() + ", will try other available key");
if (i == (preferredKeyList.size() - 1)) {
logger.error("Fail to decrypt with all known keys, either data is not encrypted or is invalid: " + e.getMessage());
throw e;
}
}