From 0f029c301515697fafd1c8c5a842f3321c6d1927 Mon Sep 17 00:00:00 2001 From: Brian Clements Date: Mon, 2 Jun 2014 13:16:34 -0500 Subject: [PATCH] Omaha #3226 moved ost lightning code to new plugin fixed NPE when missing encryption config Change-Id: I8ef6004f44b686918220e7729a499e456787a834 Former-commit-id: 6b09d9b3c5e755cdd2eedb789ce7c23fa464ac73 [formerly 5421b666fc52e849e4ad5b39e2328a5c90a42408] Former-commit-id: 664d9282c3518be9578996b49c47dd8eab8022b7 --- .../.settings/org.eclipse.jdt.core.prefs | 12 +- .../META-INF/MANIFEST.MF | 6 +- .../binlightning/BinLightningDecoder.java | 299 ++++++++++++++++-- .../feature.xml | 7 + .../.classpath | 7 + .../.project | 28 ++ .../.settings/org.eclipse.jdt.core.prefs | 7 + .../META-INF/MANIFEST.MF | 13 + .../build.properties | 4 + .../binlightning/BinLightningAESKey.java | 15 +- .../BinLightningAESKey.properties | 0 .../BinLightningDataDecryptionException.java | 0 .../binlightning/BinLightningDecoderUtil.java | 193 +---------- .../EncryptedBinLightningCipher.java | 28 +- 14 files changed, 399 insertions(+), 220 deletions(-) create mode 100644 ost/gov.noaa.nws.ost.edex.plugin.binlightning/.classpath create mode 100644 ost/gov.noaa.nws.ost.edex.plugin.binlightning/.project create mode 100644 ost/gov.noaa.nws.ost.edex.plugin.binlightning/.settings/org.eclipse.jdt.core.prefs create mode 100644 ost/gov.noaa.nws.ost.edex.plugin.binlightning/META-INF/MANIFEST.MF create mode 100644 ost/gov.noaa.nws.ost.edex.plugin.binlightning/build.properties rename {edexOsgi/com.raytheon.edex.plugin.binlightning => ost/gov.noaa.nws.ost.edex.plugin.binlightning}/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningAESKey.java (95%) rename {edexOsgi/com.raytheon.edex.plugin.binlightning => ost/gov.noaa.nws.ost.edex.plugin.binlightning}/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningAESKey.properties (100%) rename {edexOsgi/com.raytheon.edex.plugin.binlightning => ost/gov.noaa.nws.ost.edex.plugin.binlightning}/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningDataDecryptionException.java (100%) rename edexOsgi/com.raytheon.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLigntningDecoderUtil.java => ost/gov.noaa.nws.ost.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningDecoderUtil.java (63%) rename {edexOsgi/com.raytheon.edex.plugin.binlightning => ost/gov.noaa.nws.ost.edex.plugin.binlightning}/src/gov/noaa/nws/ost/edex/plugin/binlightning/EncryptedBinLightningCipher.java (87%) diff --git a/edexOsgi/com.raytheon.edex.plugin.binlightning/.settings/org.eclipse.jdt.core.prefs b/edexOsgi/com.raytheon.edex.plugin.binlightning/.settings/org.eclipse.jdt.core.prefs index 0c77225be8..7341ab1683 100644 --- a/edexOsgi/com.raytheon.edex.plugin.binlightning/.settings/org.eclipse.jdt.core.prefs +++ b/edexOsgi/com.raytheon.edex.plugin.binlightning/.settings/org.eclipse.jdt.core.prefs @@ -1,7 +1,11 @@ -#Thu Mar 26 10:16:39 CDT 2009 eclipse.preferences.version=1 -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 -org.eclipse.jdt.core.compiler.compliance=1.6 +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 +org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve +org.eclipse.jdt.core.compiler.compliance=1.7 +org.eclipse.jdt.core.compiler.debug.lineNumber=generate +org.eclipse.jdt.core.compiler.debug.localVariable=generate +org.eclipse.jdt.core.compiler.debug.sourceFile=generate org.eclipse.jdt.core.compiler.problem.assertIdentifier=error org.eclipse.jdt.core.compiler.problem.enumIdentifier=error -org.eclipse.jdt.core.compiler.source=1.6 +org.eclipse.jdt.core.compiler.source=1.7 diff --git a/edexOsgi/com.raytheon.edex.plugin.binlightning/META-INF/MANIFEST.MF b/edexOsgi/com.raytheon.edex.plugin.binlightning/META-INF/MANIFEST.MF index 094f8a9bbe..cc77f6f3b2 100644 --- a/edexOsgi/com.raytheon.edex.plugin.binlightning/META-INF/MANIFEST.MF +++ b/edexOsgi/com.raytheon.edex.plugin.binlightning/META-INF/MANIFEST.MF @@ -2,14 +2,16 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: Binlightning Plug-in Bundle-SymbolicName: com.raytheon.edex.plugin.binlightning -Bundle-Version: 1.12.1174.qualifier +Bundle-Version: 1.14.0.qualifier Bundle-Vendor: RAYTHEON -Bundle-RequiredExecutionEnvironment: JavaSE-1.6 +Bundle-RequiredExecutionEnvironment: JavaSE-1.7 Export-Package: com.raytheon.edex.plugin.binlightning.dao Import-Package: com.raytheon.edex.esb, com.raytheon.edex.exception, com.raytheon.edex.plugin, + com.raytheon.uf.common.status, com.raytheon.uf.common.wmo, + gov.noaa.nws.ost.edex.plugin.binlightning, org.apache.commons.logging Require-Bundle: com.raytheon.uf.common.dataplugin.binlightning;bundle-version="1.12.1174", com.raytheon.uf.common.dataplugin;bundle-version="1.12.1174", diff --git a/edexOsgi/com.raytheon.edex.plugin.binlightning/src/com/raytheon/edex/plugin/binlightning/BinLightningDecoder.java b/edexOsgi/com.raytheon.edex.plugin.binlightning/src/com/raytheon/edex/plugin/binlightning/BinLightningDecoder.java index 9968912ed2..b291bddb3c 100644 --- a/edexOsgi/com.raytheon.edex.plugin.binlightning/src/com/raytheon/edex/plugin/binlightning/BinLightningDecoder.java +++ b/edexOsgi/com.raytheon.edex.plugin.binlightning/src/com/raytheon/edex/plugin/binlightning/BinLightningDecoder.java @@ -19,30 +19,41 @@ **/ package com.raytheon.edex.plugin.binlightning; -import gov.noaa.nws.ost.edex.plugin.binlightning.BinLigntningDecoderUtil; +import gov.noaa.nws.ost.edex.plugin.binlightning.BinLightningAESKey; +import gov.noaa.nws.ost.edex.plugin.binlightning.BinLightningDataDecryptionException; +import gov.noaa.nws.ost.edex.plugin.binlightning.BinLightningDecoderUtil; +import gov.noaa.nws.ost.edex.plugin.binlightning.EncryptedBinLightningCipher; import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Calendar; +import java.util.Date; import java.util.List; import java.util.TimeZone; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import javax.crypto.BadPaddingException; +import javax.crypto.IllegalBlockSizeException; import com.raytheon.edex.esb.Headers; import com.raytheon.edex.exception.DecoderException; import com.raytheon.edex.plugin.AbstractDecoder; +import com.raytheon.edex.plugin.binlightning.impl.BinLightningFactory; +import com.raytheon.edex.plugin.binlightning.impl.IBinLightningDecoder; +import com.raytheon.edex.plugin.binlightning.impl.LightningDataSource; import com.raytheon.uf.common.dataplugin.PluginDataObject; -import com.raytheon.uf.common.dataplugin.PluginException; import com.raytheon.uf.common.dataplugin.binlightning.BinLightningRecord; import com.raytheon.uf.common.dataplugin.binlightning.impl.LightningStrikePoint; import com.raytheon.uf.common.dataplugin.binlightning.impl.LtgStrikeType; +import com.raytheon.uf.common.status.IUFStatusHandler; +import com.raytheon.uf.common.status.UFStatus; import com.raytheon.uf.common.time.DataTime; import com.raytheon.uf.common.time.TimeRange; +import com.raytheon.uf.common.time.util.TimeUtil; import com.raytheon.uf.common.wmo.WMOHeader; import com.raytheon.uf.common.wmo.WMOTimeParser; import com.raytheon.uf.edex.decodertools.core.DecoderTools; -import com.raytheon.uf.edex.decodertools.time.TimeTools; +import com.raytheon.uf.edex.decodertools.core.IBinDataSource; /** * AWIPS decoder adapter strategy for binary lightning data.
@@ -84,6 +95,9 @@ import com.raytheon.uf.edex.decodertools.time.TimeTools; * and to used WMO header to distinguish bit-shifted * GLD360 and NLDN data. * May 14, 2014 2536 bclement moved WMO Header to common + * Jun 03, 2014 3226 bclement removed unused WMO patterns, switched to UFStatus + * removed TimeTools usage, removed constructDataURI() call + * added decodeBinLightningData() and decodeBitShiftedBinLightningData() from BinLightningDecoderUtil * * * @@ -91,16 +105,14 @@ import com.raytheon.uf.edex.decodertools.time.TimeTools; * @version 1.0 */ public class BinLightningDecoder extends AbstractDecoder { - private static final String SFUS_PATTERN = "SFUS41 KWBC \\d{6}[^\\r\\n]*[\\r\\n]+"; - - private static final String SFPA_PATTERN = "SFPA41 KWBC \\d{6}[^\\r\\n]*[\\r\\n]+"; // Allow ingest up to 10 minutes into the future. private static final long TEN_MINUTES = 10 * 60 * 1000L; private final SimpleDateFormat SDF; - private final Log logger = LogFactory.getLog(getClass()); + private static final IUFStatusHandler logger = UFStatus + .getHandler(BinLightningDecoder.class); /** * Default lightning strike type for FLASH messages. RT_FLASH documents @@ -166,7 +178,8 @@ public class BinLightningDecoder extends AbstractDecoder { // both encrypted data and legacy data // - List strikes = BinLigntningDecoderUtil.decodeBinLightningData(data, pdata, traceId, wmoHdr, baseTime.getTime()); + List strikes = decodeBinLightningData( + data, pdata, traceId, wmoHdr, baseTime.getTime()); if (strikes == null) { // keep-alive record, log and return logger.info(traceId + " - found keep-alive record. ignore for now."); @@ -189,7 +202,7 @@ public class BinLightningDecoder extends AbstractDecoder { return new PluginDataObject[0]; } - Calendar c = TimeTools.copy(baseTime); + Calendar c = TimeUtil.newCalendar(baseTime); if (c == null) { throw new DecoderException(traceId + " - Error decoding times"); } @@ -212,14 +225,7 @@ public class BinLightningDecoder extends AbstractDecoder { if (report != null) { report.setTraceId(traceId); - //report.setPluginName("binlightning"); // line disappear in OB15.5.3 - try { - report.constructDataURI(); - reports = new PluginDataObject[] { report }; - } catch (PluginException e) { - logger.error("Error constructing datauri", e); - throw new DecoderException("Error constructing datauri", e); - } + reports = new PluginDataObject[] { report }; } } } @@ -229,6 +235,261 @@ public class BinLightningDecoder extends AbstractDecoder { return reports; } + /** + * Decode bin lightning data, able to handle both legacy bit-shifted and new + * encryted data + * + * The BinLightningDecoder.decode() method will use this method to decode + * data, which will try to decrypt first, and decode the old fashioned way + * when decryption fails + * + * @author Wufeng Zhou + * + * @param data + * - data content from file, including WMO header section + * @param pdata + * - data with WMO header stripped, optional, if null, will strip + * WMO header internally from passed in data parameter + * @param traceId + * - the file name of the data to be deoced + * @param wmoHdr + * - WMOHeader, added 12/24/2013 to help distinguish bit-shifted + * NLDN and GLD360 data (GLD data will have header starts like + * SFPA) + * @param dataDate + * - date of the data, optional, used as a hint to find + * appropriate encryption key faster + * @return null if keep-alive record, otherwise a list (could be empty) of + * LightningStrikePoint + */ + public static List decodeBinLightningData( + byte[] data, byte[] pdata, String traceId, WMOHeader wmoHdr, + Date dataDate) { + if (pdata == null) { // if data without header not passed, we'll strip + // the WMO header here + WMOHeader header = new WMOHeader(data); + if (header.isValid() && header.getMessageDataStart() > 0) { + pdata = new byte[data.length - header.getMessageDataStart()]; + System.arraycopy(data, header.getMessageDataStart(), pdata, 0, + data.length - header.getMessageDataStart()); + } + } + + List strikes = new ArrayList(); + boolean needDecrypt = true; // set as default unless clear evidence says + // otherwise + boolean decodeDone = false; + EncryptedBinLightningCipher cipher = new EncryptedBinLightningCipher(); + + // + // Using different WMO headers to indicate whether the data is encrypted + // or not would be a nice option. + // However, that idea has been discussed but not adopted. + // If in the future, WMO header can be different for legacy and + // encrypted data, or some other metadata can be used to decide + // whether deceyption is needed, logic can be added here. + // + // Before that happens, we'll use hints and trial & error to decode the + // data + // Hints: Per lightning data format spec, there are 3 bytes in the WMO + // header starting line that indicates the size of the encrypted block + // or the ASCII sequence # for legacy bit-shifted data + // However, the starting line is optional and AWIPS decode may not see + // it at all because TG will strip that starting line away + // We'll try to use this hint first, if is is not found, then trial and + // error way to decrypt and decode + // + // As of 11/05/2013, There is change in data spec. that the 3-bytes will + // not be encoded as encrypted block size anymore (it will always be + // transmission sequence # if present) + // So there should have some minor changes in the logic below for + // decoding the data. + // However, as reading into the + // com.raytheon.edex.plugin.binlightning.impl.BinLightningFactory.getDecoder() + // and follow-on code, we see the following data patterns + // for legacy bit-shifted data, which could be used to reduce guess-work + // in data decryption: + // The bit-shifted data will have multiple groups of the following + // patterns: + // 1-byte (unsigned byte): for size count + // 1-byte (unsigned byte): for flash type: + // 0x96 for FLASH_RPT (message size is 6 bytes each) + // 0x97 for RT_FLASH_RPT (message size is 8 bytes each) + // 0xd0 for OTHER_RPT (The D2D decoders declare but do not define this + // message, so unimplemented decoder) + // 0xd1 for COMM_RPT (The D2D decoders declare but do not define this + // message, so unimplemented decoder) + // 4-bytes: date time + // multiple of 6 or 8 bytes (as determined by 2nd byte flash type) with + // count indicated in 1st byte + // + // So this is be used to determine whether the data need to be + // decrypted. + + /* + * // looks like previous assumption on block size bytes are not valid + * any more. 11/20/2013 if (data != null) { byte[] sizeSeqBytes = + * BinLigntningDecoderUtil.findSizeOrSeqBytesFromWMOHeader(data); if + * (sizeSeqBytes != null) { // if this is in the header (which may not), + * use that as a hint to determine which decoding route to go if + * (BinLigntningDecoderUtil + * .isPossibleWMOHeaderSequenceNumber(sizeSeqBytes) && + * BinLigntningDecoderUtil + * .getEncryptedBlockSizeFromWMOHeader(sizeSeqBytes) != pdata.length) { + * // looks like a sequence #, and if treat as size, it does not equal + * to the data block size, so most likely legacy data needDecrypt = + * false; } } } + */ + + if (needDecrypt) { + try { + // NOTE: 11/14/2013 WZ: + // encrypted test data on TNCF (got from Melissa Porricelli) + // seems to have extra 4 bytes (0x0d 0x0d 0x0a 0x03) at the end, + // making the data size not a multiple of 16. However, original + // test data do not have this trailing bytes. while NCEP test + // data has extra 8 trailing bytes. + // Brain Rapp's email on 11/13/2013 confirms that Unidata LDM + // software used by AWIPS II will strips off all SBN protocol + // headers + // that precede the WMO header and adds its own 11 byte header + // like this: "soh cr cr nl 2 5 4 sp cr cr nl". It + // also adds a four byte trailer consisting of "cr cr nl etx" + // (0x0d 0x0d 0x0a 0x03) + // So, it seems necessary to trim trailing bytes if it is not + // multiple of 16, warning messages will be logged though + int dataLengthToBeDecrypted = pdata.length; + if (pdata.length % 16 != 0) { + dataLengthToBeDecrypted = pdata.length + - (pdata.length % 16); + logger.warn(traceId + " - Data length from file " + traceId + + " is " + pdata.length + " bytes, trailing " + + (pdata.length - dataLengthToBeDecrypted) + + " bytes has been trimmed to " + + dataLengthToBeDecrypted + + " bytes for decryption."); + } + byte[] encryptedData = new byte[dataLengthToBeDecrypted]; + encryptedData = Arrays.copyOfRange(pdata, 0, + dataLengthToBeDecrypted); + + byte[] decryptedData = cipher.decryptData(encryptedData, + dataDate); + // decrypt ok, then decode, first check if keep-alive record + if (BinLightningDecoderUtil.isKeepAliveRecord(decryptedData)) { + logger.info(traceId + + " - Keep-alive record detected, ignore for now."); + decodeDone = true; + return null; + } + // not keep-alive record, then check data validity and decode + // into an ArrayList of strikes + if (BinLightningDecoderUtil + .isLightningDataRecords(decryptedData)) { + strikes = BinLightningDecoderUtil + .decodeDecryptedBinLightningData(decryptedData); + decodeDone = true; + } else { + logger.info(traceId + + " - Failed data validity check of the decrypted data, will try decode the old-fashioned way."); + decodeDone = false; + } + } catch (IllegalBlockSizeException e) { + logger.info(traceId + + " - " + + e.getMessage() + + ": Decryption failed, will try decode the old-fashioned way."); + decodeDone = false; + } catch (BadPaddingException e) { + logger.info(traceId + + " - " + + e.getMessage() + + ": Decryption failed, will try decode the old-fashioned way."); + decodeDone = false; + } catch (BinLightningDataDecryptionException e) { + logger.info(traceId + + " - " + + e.getMessage() + + ": Decryption failed, will try decode the old-fashioned way."); + decodeDone = false; + } + } + + if (decodeDone == false) { // not decoded through decrypt->decode + // process, try the legacy decoder + logger.info(traceId + " - decoding as bit-shifted data"); + // bit-shifting data format check call here will get us some more + // information on the data, also can compare the strikes with the + // decoder result + int estimatedStrikes = BinLightningDecoderUtil + .getBitShiftedDataStrikeCount(pdata); + strikes = decodeBitShiftedBinLightningData(pdata, wmoHdr); + if (estimatedStrikes != strikes.size()) { + logger.warn(traceId + + ": bit-shifted decoder found " + + strikes + + " strikes, which is different from estimate from data pattern examination: " + + estimatedStrikes); + } + } + + return strikes; + } + + /** + * extracted from the original {@link #decode(byte[], Headers)} method then + * modified by Wufeng Zhou + * + * @param pdata + * @param wmoHdr + * - WMOHeader, added 12/24/2013 to help distinguish bit-shifted + * NLDN and GLD360 data (GLD data will have header starts like + * SFPA) + * @return + */ + public static List decodeBitShiftedBinLightningData( + byte[] pdata, WMOHeader wmoHdr) { + List strikes = new ArrayList(); + + IBinDataSource msgData = new LightningDataSource(pdata); + + boolean continueDecode = true; + while (continueDecode) { + IBinLightningDecoder decoder = BinLightningFactory + .getDecoder(msgData); + + switch (decoder.getError()) { + case IBinLightningDecoder.NO_ERROR: { + for (LightningStrikePoint strike : decoder) { + // use WMO Header to distinguish NLDN or GLD360 data because + // no bit-shifted data spec available for GLD360. + // 12/24/2013, WZ + // The WMO header start string is defined in + // BinLightningAESKey.properties file (normally, GLD360 data + // will have WMO header + // starts with SFPA41, or SFPA99 for test data.) + String gld360WMOHeaderString = BinLightningAESKey + .getProps().getProperty( + "binlightning.gld360WMOHeaderStartString", + ""); + if (gld360WMOHeaderString.trim().equals("") == false + && wmoHdr.getWmoHeader().startsWith( + gld360WMOHeaderString)) { + // GLD360 data based on the setup + strike.setLightSource("GLD"); + } + strikes.add(strike); + } + break; + } + default: { + continueDecode = false; + } + } + } + return strikes; + } + /** * Set a trace identifier for the source data. * diff --git a/edexOsgi/com.raytheon.uf.edex.binlightning.feature/feature.xml b/edexOsgi/com.raytheon.uf.edex.binlightning.feature/feature.xml index ec27e25224..1974500c63 100644 --- a/edexOsgi/com.raytheon.uf.edex.binlightning.feature/feature.xml +++ b/edexOsgi/com.raytheon.uf.edex.binlightning.feature/feature.xml @@ -24,4 +24,11 @@ version="0.0.0" unpack="false"/> + + diff --git a/ost/gov.noaa.nws.ost.edex.plugin.binlightning/.classpath b/ost/gov.noaa.nws.ost.edex.plugin.binlightning/.classpath new file mode 100644 index 0000000000..098194ca4b --- /dev/null +++ b/ost/gov.noaa.nws.ost.edex.plugin.binlightning/.classpath @@ -0,0 +1,7 @@ + + + + + + + diff --git a/ost/gov.noaa.nws.ost.edex.plugin.binlightning/.project b/ost/gov.noaa.nws.ost.edex.plugin.binlightning/.project new file mode 100644 index 0000000000..f771e433ad --- /dev/null +++ b/ost/gov.noaa.nws.ost.edex.plugin.binlightning/.project @@ -0,0 +1,28 @@ + + + gov.noaa.nws.ost.edex.plugin.binlightning + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.pde.ManifestBuilder + + + + + org.eclipse.pde.SchemaBuilder + + + + + + org.eclipse.pde.PluginNature + org.eclipse.jdt.core.javanature + + diff --git a/ost/gov.noaa.nws.ost.edex.plugin.binlightning/.settings/org.eclipse.jdt.core.prefs b/ost/gov.noaa.nws.ost.edex.plugin.binlightning/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000000..f42de363af --- /dev/null +++ b/ost/gov.noaa.nws.ost.edex.plugin.binlightning/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,7 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 +org.eclipse.jdt.core.compiler.compliance=1.7 +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.source=1.7 diff --git a/ost/gov.noaa.nws.ost.edex.plugin.binlightning/META-INF/MANIFEST.MF b/ost/gov.noaa.nws.ost.edex.plugin.binlightning/META-INF/MANIFEST.MF new file mode 100644 index 0000000000..d0aea43e79 --- /dev/null +++ b/ost/gov.noaa.nws.ost.edex.plugin.binlightning/META-INF/MANIFEST.MF @@ -0,0 +1,13 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: OST Binlightning +Bundle-SymbolicName: gov.noaa.nws.ost.edex.plugin.binlightning +Bundle-Version: 1.14.0.qualifier +Bundle-RequiredExecutionEnvironment: JavaSE-1.7 +Import-Package: com.raytheon.uf.common.dataplugin.binlightning, + com.raytheon.uf.common.dataplugin.binlightning.dataaccess, + com.raytheon.uf.common.dataplugin.binlightning.impl, + com.raytheon.uf.common.status, + com.raytheon.uf.common.wmo, + com.raytheon.uf.edex.decodertools.core +Export-Package: gov.noaa.nws.ost.edex.plugin.binlightning diff --git a/ost/gov.noaa.nws.ost.edex.plugin.binlightning/build.properties b/ost/gov.noaa.nws.ost.edex.plugin.binlightning/build.properties new file mode 100644 index 0000000000..34d2e4d2da --- /dev/null +++ b/ost/gov.noaa.nws.ost.edex.plugin.binlightning/build.properties @@ -0,0 +1,4 @@ +source.. = src/ +output.. = bin/ +bin.includes = META-INF/,\ + . diff --git a/edexOsgi/com.raytheon.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningAESKey.java b/ost/gov.noaa.nws.ost.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningAESKey.java similarity index 95% rename from edexOsgi/com.raytheon.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningAESKey.java rename to ost/gov.noaa.nws.ost.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningAESKey.java index 5761d78944..75a70bd1ec 100755 --- a/edexOsgi/com.raytheon.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningAESKey.java +++ b/ost/gov.noaa.nws.ost.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningAESKey.java @@ -27,13 +27,14 @@ import java.util.TreeMap; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import com.raytheon.uf.common.status.IUFStatusHandler; +import com.raytheon.uf.common.status.UFStatus; /** * BinLightningAESKey - * - * Simple representation of bin lightning AES encryption key and its associated key aliases in the keystore + * + * Simple representation of bin lightning AES encryption key and its associated + * key aliases in the keystore * * 
  * 
@@ -42,11 +43,12 @@ import org.apache.commons.logging.LogFactory;
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * 20130503      DCS 112   Wufeng Zhou To handle both the new encrypted data and legacy bit-shifted data
+ * Jun 03, 2014 3226       bclement    moved from com.raytheon.edex.plugin.binlightning to gov.noaa.nws.ost.edex.plugin.binlightning
  * 
  *
* * @author Wufeng Zhou - * + * */ public class BinLightningAESKey { /** Default location to search for BinLightningAESKey.properties file, and keystore file (normally binLightningAESKeystore.jce as configured in properties file) */ @@ -63,7 +65,8 @@ public class BinLightningAESKey { private static final Pattern KEY_ALIAS_PREFIX_PATTERN = Pattern.compile(KEY_ALIAS_PREFIX); private static final SimpleDateFormat KEY_ALIAS_DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd"); - private static Log logger = LogFactory.getLog(BinLightningAESKey.class); + private static IUFStatusHandler logger = UFStatus + .getHandler(BinLightningAESKey.class); private static Properties props = new Properties(); private static KeyStore keystore; diff --git a/edexOsgi/com.raytheon.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningAESKey.properties b/ost/gov.noaa.nws.ost.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningAESKey.properties similarity index 100% rename from edexOsgi/com.raytheon.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningAESKey.properties rename to ost/gov.noaa.nws.ost.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningAESKey.properties diff --git a/edexOsgi/com.raytheon.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningDataDecryptionException.java b/ost/gov.noaa.nws.ost.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningDataDecryptionException.java similarity index 100% rename from edexOsgi/com.raytheon.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningDataDecryptionException.java rename to ost/gov.noaa.nws.ost.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningDataDecryptionException.java diff --git a/edexOsgi/com.raytheon.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLigntningDecoderUtil.java b/ost/gov.noaa.nws.ost.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningDecoderUtil.java similarity index 63% rename from edexOsgi/com.raytheon.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLigntningDecoderUtil.java rename to ost/gov.noaa.nws.ost.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningDecoderUtil.java index 2d2e648623..200b80e5bb 100644 --- a/edexOsgi/com.raytheon.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLigntningDecoderUtil.java +++ b/ost/gov.noaa.nws.ost.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningDecoderUtil.java @@ -9,24 +9,14 @@ import java.nio.ByteOrder; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; -import java.util.Date; import java.util.List; -import javax.crypto.BadPaddingException; -import javax.crypto.IllegalBlockSizeException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import com.raytheon.edex.plugin.binlightning.impl.BinLightningFactory; -import com.raytheon.edex.plugin.binlightning.impl.IBinLightningDecoder; -import com.raytheon.edex.plugin.binlightning.impl.LightningDataSource; import com.raytheon.uf.common.dataplugin.binlightning.impl.LightningStrikePoint; import com.raytheon.uf.common.dataplugin.binlightning.impl.LtgMsgType; import com.raytheon.uf.common.dataplugin.binlightning.impl.LtgStrikeType; -import com.raytheon.uf.common.wmo.WMOHeader; +import com.raytheon.uf.common.status.IUFStatusHandler; +import com.raytheon.uf.common.status.UFStatus; import com.raytheon.uf.edex.decodertools.core.BasePoint; -import com.raytheon.uf.edex.decodertools.core.IBinDataSource; /** * BinLigntningDecoderUtil @@ -44,12 +34,16 @@ import com.raytheon.uf.edex.decodertools.core.IBinDataSource; * ------------ ---------- ----------- -------------------------- * 20130503 DCS 112 Wufeng Zhou To handle both the new encrypted data and legacy bit-shifted data * 20140501 Wufeng Zhou Fix the encrypted decoding with correct offset + * Jun 03, 2014 3226 bclement renamed from BinLigntningDecoderUtil to BinLightningDecoderUtil + * moved from com.raytheon.edex.plugin.binlightning to gov.noaa.nws.ost.edex.plugin.binlightning + * moved decodeBinLightningData() and decodeBitShiftedBinLightningData() + * to BinLightningDecoder to solve circular dependency * * * @author Wufeng Zhou * */ -public class BinLigntningDecoderUtil { +public class BinLightningDecoderUtil { /** * Message type for keep alive data records. @@ -84,47 +78,10 @@ public class BinLigntningDecoderUtil { static final byte OTHER_RPT = (byte)0xD0; static final byte COMM_RPT = (byte)0xD1; - private static Log logger = LogFactory.getLog(BinLigntningDecoderUtil.class); + private static IUFStatusHandler logger = UFStatus + .getHandler(BinLightningDecoderUtil.class); - /** - * extracted from the decode() of the original - * com.raytheon.edex.plugin.binlightning.BinLightningDecoder class - * - * @param pdata - * @param wmoHdr - WMOHeader, added 12/24/2013 to help distinguish bit-shifted NLDN and GLD360 data (GLD data will have header starts like SFPA) - * @return - */ - public static List decodeBitShiftedBinLightningData(byte[] pdata, WMOHeader wmoHdr) { - List strikes = new ArrayList(); - IBinDataSource msgData = new LightningDataSource(pdata); - - boolean continueDecode = true; - while (continueDecode) { - IBinLightningDecoder decoder = BinLightningFactory.getDecoder(msgData); - - switch (decoder.getError()) { - case IBinLightningDecoder.NO_ERROR: { - for (LightningStrikePoint strike : decoder) { - // use WMO Header to distinguish NLDN or GLD360 data because no bit-shifted data spec available for GLD360. 12/24/2013, WZ - // The WMO header start string is defined in BinLightningAESKey.properties file (normally, GLD360 data will have WMO header - // starts with SFPA41, or SFPA99 for test data.) - String gld360WMOHeaderString = BinLightningAESKey.getProps().getProperty("binlightning.gld360WMOHeaderStartString", ""); - if (gld360WMOHeaderString.trim().equals("") == false && wmoHdr.getWmoHeader().startsWith(gld360WMOHeaderString)) { - // GLD360 data based on the setup - strike.setLightSource("GLD"); - } - strikes.add(strike); - } - break; - } - default: { - continueDecode = false; - } - } - } - return strikes; - } /** * decode the new bin lightning data, after the data record is decrypted, and it is not keep-alive record @@ -170,8 +127,9 @@ public class BinLigntningDecoderUtil { int strokeType = buffer.getShort() & 0xffff; // 0x0000 for cloud-to-ground, 0x00ff for cloud-to-cloud, 0xffff for total flash short strokeKiloAmps = buffer.getShort(); // valid range: -254 to 254, specifically 16 bit signed integer int strokeMultiplicity = buffer.getShort() & 0xffff; // i.e. stroke count, valid range: 0 to 15 - int strokeDuration = buffer.getShort() & 0xffff; // valid range: 0 to 65535 (i.e., looks like unsigned short) - int reserved = buffer.getShort() & 0xffff; + // int strokeDuration = buffer.getShort() & 0xffff; // valid range: + // 0 to 65535 (i.e., looks like unsigned short) + // int reserved = buffer.getShort() & 0xffff; // Create the strike record from the report info and base time information. BasePoint base = new BasePoint(lat, lon); @@ -236,134 +194,7 @@ public class BinLigntningDecoderUtil { return strikes; } - /** - * Decode bin lightning data, able to handle both legacy bit-shifted and new encryted data - * - * The modified BinLightningDecoder.decode() method will use this method to decode data, which - * will try to decrypt first, and decode the old fashioned way when decryption fails - * - * @param data - data content from file, including WMO header section - * @param pdata - data with WMO header stripped, optional, if null, will strip WMO header internally from passed in data parameter - * @param traceId - the file name of the data to be deoced - * @param wmoHdr - WMOHeader, added 12/24/2013 to help distinguish bit-shifted NLDN and GLD360 data (GLD data will have header starts like SFPA) - * @param dataDate - date of the data, optional, used as a hint to find appropriate encryption key faster - * @return null if keep-alive record, otherwise a list (could be empty) of LightningStrikePoint - */ - public static List decodeBinLightningData(byte[] data, byte[] pdata, String traceId, WMOHeader wmoHdr, Date dataDate) { - if (pdata == null) { // if data without header not passed, we'll strip the WMO header here - WMOHeader header = new WMOHeader(data); - if (header.isValid() && header.getMessageDataStart() > 0) { - pdata = new byte[data.length - header.getMessageDataStart()]; - System.arraycopy(data, header.getMessageDataStart(), pdata, 0, data.length - header.getMessageDataStart()); - } - } - - List strikes = new ArrayList(); - boolean needDecrypt = true; // set as default unless clear evidence says otherwise - boolean decodeDone = false; - EncryptedBinLightningCipher cipher = new EncryptedBinLightningCipher(); - // - // Using different WMO headers to indicate whether the data is encrypted or not would be a nice option. - // However, that idea has been discussed but not adopted. - // If in the future, WMO header can be different for legacy and encrypted data, or some other metadata can be used to decide - // whether deceyption is needed, logic can be added here. - // - // Before that happens, we'll use hints and trial & error to decode the data - // Hints: Per lightning data format spec, there are 3 bytes in the WMO header starting line that indicates the size of the encrypted block - // or the ASCII sequence # for legacy bit-shifted data - // However, the starting line is optional and AWIPS decode may not see it at all because TG will strip that starting line away - // We'll try to use this hint first, if is is not found, then trial and error way to decrypt and decode - // - // As of 11/05/2013, There is change in data spec. that the 3-bytes will not be encoded as encrypted block size anymore (it will always be transmission sequence # if present) - // So there should have some minor changes in the logic below for decoding the data. - // However, as reading into the com.raytheon.edex.plugin.binlightning.impl.BinLightningFactory.getDecoder() and follow-on code, we see the following data patterns - // for legacy bit-shifted data, which could be used to reduce guess-work in data decryption: - // The bit-shifted data will have multiple groups of the following patterns: - // 1-byte (unsigned byte): for size count - // 1-byte (unsigned byte): for flash type: - // 0x96 for FLASH_RPT (message size is 6 bytes each) - // 0x97 for RT_FLASH_RPT (message size is 8 bytes each) - // 0xd0 for OTHER_RPT (The D2D decoders declare but do not define this message, so unimplemented decoder) - // 0xd1 for COMM_RPT (The D2D decoders declare but do not define this message, so unimplemented decoder) - // 4-bytes: date time - // multiple of 6 or 8 bytes (as determined by 2nd byte flash type) with count indicated in 1st byte - // - // So this is be used to determine whether the data need to be decrypted. - - /* - // looks like previous assumption on block size bytes are not valid any more. 11/20/2013 - if (data != null) { - byte[] sizeSeqBytes = BinLigntningDecoderUtil.findSizeOrSeqBytesFromWMOHeader(data); - if (sizeSeqBytes != null) { - // if this is in the header (which may not), use that as a hint to determine which decoding route to go - if (BinLigntningDecoderUtil.isPossibleWMOHeaderSequenceNumber(sizeSeqBytes) - && BinLigntningDecoderUtil.getEncryptedBlockSizeFromWMOHeader(sizeSeqBytes) != pdata.length) { - // looks like a sequence #, and if treat as size, it does not equal to the data block size, so most likely legacy data - needDecrypt = false; - } - } - } - */ - - if (needDecrypt) { - try { - // NOTE: 11/14/2013 WZ: - // encrypted test data on TNCF (got from Melissa Porricelli) seems to have extra 4 bytes (0x0d 0x0d 0x0a 0x03) at the end, - // making the data size not a multiple of 16. However, original test data do not have this trailing bytes. while NCEP test - // data has extra 8 trailing bytes. - // Brain Rapp's email on 11/13/2013 confirms that Unidata LDM software used by AWIPS II will strips off all SBN protocol headers - // that precede the WMO header and adds its own 11 byte header like this: "soh cr cr nl 2 5 4 sp cr cr nl". It - // also adds a four byte trailer consisting of "cr cr nl etx" (0x0d 0x0d 0x0a 0x03) - // So, it seems necessary to trim trailing bytes if it is not multiple of 16, warning messages will be logged though - int dataLengthToBeDecrypted = pdata.length; - if (pdata.length % 16 != 0) { - dataLengthToBeDecrypted = pdata.length - (pdata.length % 16); - logger.warn(traceId + " - Data length from file " + traceId + " is " + pdata.length + " bytes, trailing " + - (pdata.length - dataLengthToBeDecrypted) + " bytes has been trimmed to " + dataLengthToBeDecrypted + " bytes for decryption."); - } - byte[] encryptedData = new byte[dataLengthToBeDecrypted]; - encryptedData = Arrays.copyOfRange(pdata, 0, dataLengthToBeDecrypted); - - byte[] decryptedData = cipher.decryptData(encryptedData, dataDate); - // decrypt ok, then decode, first check if keep-alive record - if (BinLigntningDecoderUtil.isKeepAliveRecord(decryptedData)) { - logger.info(traceId + " - Keep-alive record detected, ignore for now."); - decodeDone = true; - return null; - } - // not keep-alive record, then check data validity and decode into an ArrayList of strikes - if (BinLigntningDecoderUtil.isLightningDataRecords(decryptedData)) { - strikes = BinLigntningDecoderUtil.decodeDecryptedBinLightningData(decryptedData); - decodeDone = true; - } else { - logger.info(traceId + " - Failed data validity check of the decrypted data, will try decode the old-fashioned way."); - decodeDone = false; - } - } catch (IllegalBlockSizeException e) { - logger.info(traceId + " - " + e.getMessage() + ": Decryption failed, will try decode the old-fashioned way."); - decodeDone = false; - } catch (BadPaddingException e) { - logger.info(traceId + " - " + e.getMessage() + ": Decryption failed, will try decode the old-fashioned way."); - decodeDone = false; - } catch (BinLightningDataDecryptionException e) { - logger.info(traceId + " - " + e.getMessage() + ": Decryption failed, will try decode the old-fashioned way."); - decodeDone = false; - } - } - - if (decodeDone == false) { // not decoded through decrypt->decode process, try the legacy decoder - logger.info(traceId + " - decoding as bit-shifted data"); - // bit-shifting data format check call here will get us some more information on the data, also can compare the strikes with the decoder result - int estimatedStrikes = getBitShiftedDataStrikeCount(pdata); - strikes = BinLigntningDecoderUtil.decodeBitShiftedBinLightningData(pdata, wmoHdr); - if (estimatedStrikes != strikes.size()) { - logger.warn(traceId + ": bit-shifted decoder found " + strikes + " strikes, which is different from estimate from data pattern examination: " + estimatedStrikes); - } - } - - return strikes; - } /** * Determines if the bytes passed are a standard "NWS Keep Alive" message. diff --git a/edexOsgi/com.raytheon.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/EncryptedBinLightningCipher.java b/ost/gov.noaa.nws.ost.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/EncryptedBinLightningCipher.java similarity index 87% rename from edexOsgi/com.raytheon.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/EncryptedBinLightningCipher.java rename to ost/gov.noaa.nws.ost.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/EncryptedBinLightningCipher.java index 8c874c6c8f..8e482f2de9 100755 --- a/edexOsgi/com.raytheon.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/EncryptedBinLightningCipher.java +++ b/ost/gov.noaa.nws.ost.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/EncryptedBinLightningCipher.java @@ -6,6 +6,7 @@ package gov.noaa.nws.ost.edex.plugin.binlightning; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -15,13 +16,14 @@ import javax.crypto.Cipher; import javax.crypto.IllegalBlockSizeException; import javax.crypto.spec.SecretKeySpec; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import com.raytheon.uf.common.status.IUFStatusHandler; +import com.raytheon.uf.common.status.UFStatus; /** * EncryptedBinLightningCipher - * - * Use AES secret keys found in configured keystore to decrypt bin lightning data + * + * Use AES secret keys found in configured keystore to decrypt bin lightning + * data * * 
  * 
@@ -30,11 +32,13 @@ import org.apache.commons.logging.LogFactory;
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * 20130503        DCS 112 Wufeng Zhou To handle both the new encrypted data and legacy bit-shifted data
+ * Jun 03, 2014 3226       bclement    moved from com.raytheon.edex.plugin.binlightning to gov.noaa.nws.ost.edex.plugin.binlightning
+ *                                      handled null return from BinLightningAESKey.getBinLightningAESKeys()
  * 
  *
* * @author Wufeng Zhou - * + * */ public class EncryptedBinLightningCipher { private static final String BINLIGHTNING_CIPHER_TYPE = "AES"; @@ -51,6 +55,9 @@ public class EncryptedBinLightningCipher { protected HashMap initialValue() { // get AES keys from keystore and create encryption and decryption ciphers from them BinLightningAESKey[] keys = BinLightningAESKey.getBinLightningAESKeys(); + if (keys == null) { + keys = new BinLightningAESKey[0]; + } HashMap cipherMap = new HashMap(); for (BinLightningAESKey key : keys) { try { @@ -67,7 +74,8 @@ public class EncryptedBinLightningCipher { } }; - private static Log logger = LogFactory.getLog(EncryptedBinLightningCipher.class); + private static IUFStatusHandler logger = UFStatus + .getHandler(EncryptedBinLightningCipher.class); public EncryptedBinLightningCipher() { @@ -122,7 +130,7 @@ public class EncryptedBinLightningCipher { // wrong key will decrypt data into random noise/garbage, so we need to do a sanity check to make sure // we are decrypting with the right key - if ( BinLigntningDecoderUtil.isKeepAliveRecord(decryptedData) == false && BinLigntningDecoderUtil.isLightningDataRecords(decryptedData) == false) { + if ( BinLightningDecoderUtil.isKeepAliveRecord(decryptedData) == false && BinLightningDecoderUtil.isLightningDataRecords(decryptedData) == false) { //if (BinLigntningDecoderUtil.isValidMixedRecordData(decryptedData) == false) { // use this only if keep-alive record could be mixed with lightning records throw new BinLightningDataDecryptionException("Decrypted data (" + decryptedData.length + " bytes) with key " + preferredKeyList.get(i).getAlias() + " is not valid keep-alive or binLightning records.", decryptedData); @@ -163,7 +171,11 @@ public class EncryptedBinLightningCipher { * @return preferred key list order */ private List findPreferredKeyOrderForData(Date dataDate) { - List defKeyList = Arrays.asList(BinLightningAESKey.getBinLightningAESKeys()); + BinLightningAESKey[] binLightningAESKeys = BinLightningAESKey.getBinLightningAESKeys(); + if (binLightningAESKeys == null || binLightningAESKeys.length < 1) { + return Collections.emptyList(); + } + List defKeyList = Arrays.asList(binLightningAESKeys); if (dataDate == null) { return defKeyList; // use default order }