Omaha #3226 moved ost lightning code to new plugin

fixed NPE when missing encryption config

Change-Id: I8ef6004f44b686918220e7729a499e456787a834

Former-commit-id: 5421b666fc52e849e4ad5b39e2328a5c90a42408

edexOsgi/com.raytheon.edex.plugin.binlightning/.settings/org.eclipse.jdt.core.prefs

@@ -1,7 +1,11 @@
-#Thu Mar 26 10:16:39 CDT 2009
 eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
+org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.compliance=1.6
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
+org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
+org.eclipse.jdt.core.compiler.compliance=1.7
+org.eclipse.jdt.core.compiler.debug.lineNumber=generate
+org.eclipse.jdt.core.compiler.debug.localVariable=generate
+org.eclipse.jdt.core.compiler.debug.sourceFile=generate
 org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
 org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.source=1.6
+org.eclipse.jdt.core.compiler.source=1.7

edexOsgi/com.raytheon.edex.plugin.binlightning/META-INF/MANIFEST.MF

@@ -2,14 +2,16 @@ Manifest-Version: 1.0
 Bundle-ManifestVersion: 2
 Bundle-Name: Binlightning Plug-in
 Bundle-SymbolicName: com.raytheon.edex.plugin.binlightning
-Bundle-Version: 1.12.1174.qualifier
+Bundle-Version: 1.14.0.qualifier
 Bundle-Vendor: RAYTHEON
-Bundle-RequiredExecutionEnvironment: JavaSE-1.6
+Bundle-RequiredExecutionEnvironment: JavaSE-1.7
 Export-Package: com.raytheon.edex.plugin.binlightning.dao
 Import-Package: com.raytheon.edex.esb,
  com.raytheon.edex.exception,
  com.raytheon.edex.plugin,
+ com.raytheon.uf.common.status,
  com.raytheon.uf.common.wmo,
+ gov.noaa.nws.ost.edex.plugin.binlightning,
  org.apache.commons.logging
 Require-Bundle: com.raytheon.uf.common.dataplugin.binlightning;bundle-version="1.12.1174",
  com.raytheon.uf.common.dataplugin;bundle-version="1.12.1174",

edexOsgi/com.raytheon.edex.plugin.binlightning/src/com/raytheon/edex/plugin/binlightning/BinLightningDecoder.java

@@ -19,30 +19,41 @@
  **/
 package com.raytheon.edex.plugin.binlightning;
 
-import gov.noaa.nws.ost.edex.plugin.binlightning.BinLigntningDecoderUtil;
+import gov.noaa.nws.ost.edex.plugin.binlightning.BinLightningAESKey;
+import gov.noaa.nws.ost.edex.plugin.binlightning.BinLightningDataDecryptionException;
+import gov.noaa.nws.ost.edex.plugin.binlightning.BinLightningDecoderUtil;
+import gov.noaa.nws.ost.edex.plugin.binlightning.EncryptedBinLightningCipher;
 
 import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Calendar;
+import java.util.Date;
 import java.util.List;
 import java.util.TimeZone;
 
-import org.apache.commons.logging.Log;
+import javax.crypto.BadPaddingException;
-import org.apache.commons.logging.LogFactory;
+import javax.crypto.IllegalBlockSizeException;
 
 import com.raytheon.edex.esb.Headers;
 import com.raytheon.edex.exception.DecoderException;
 import com.raytheon.edex.plugin.AbstractDecoder;
+import com.raytheon.edex.plugin.binlightning.impl.BinLightningFactory;
+import com.raytheon.edex.plugin.binlightning.impl.IBinLightningDecoder;
+import com.raytheon.edex.plugin.binlightning.impl.LightningDataSource;
 import com.raytheon.uf.common.dataplugin.PluginDataObject;
-import com.raytheon.uf.common.dataplugin.PluginException;
 import com.raytheon.uf.common.dataplugin.binlightning.BinLightningRecord;
 import com.raytheon.uf.common.dataplugin.binlightning.impl.LightningStrikePoint;
 import com.raytheon.uf.common.dataplugin.binlightning.impl.LtgStrikeType;
+import com.raytheon.uf.common.status.IUFStatusHandler;
+import com.raytheon.uf.common.status.UFStatus;
 import com.raytheon.uf.common.time.DataTime;
 import com.raytheon.uf.common.time.TimeRange;
+import com.raytheon.uf.common.time.util.TimeUtil;
 import com.raytheon.uf.common.wmo.WMOHeader;
 import com.raytheon.uf.common.wmo.WMOTimeParser;
 import com.raytheon.uf.edex.decodertools.core.DecoderTools;
-import com.raytheon.uf.edex.decodertools.time.TimeTools;
+import com.raytheon.uf.edex.decodertools.core.IBinDataSource;
 
 /**
  * AWIPS decoder adapter strategy for binary lightning data.<br/>
@@ -84,6 +95,9 @@ import com.raytheon.uf.edex.decodertools.time.TimeTools;
  *                                     and to used WMO header to distinguish bit-shifted 
  *                                     GLD360 and NLDN data.
  * May 14, 2014 2536       bclement    moved WMO Header to common
+ * Jun 03, 2014 3226       bclement    removed unused WMO patterns, switched to UFStatus
+ *                                      removed TimeTools usage, removed constructDataURI() call
+ *                                      added decodeBinLightningData() and decodeBitShiftedBinLightningData() from BinLightningDecoderUtil
  * 
  * </pre>
  * 
@@ -91,16 +105,14 @@ import com.raytheon.uf.edex.decodertools.time.TimeTools;
  * @version 1.0
  */
 public class BinLightningDecoder extends AbstractDecoder {
-    private static final String SFUS_PATTERN = "SFUS41 KWBC \\d{6}[^\\r\\n]*[\\r\\n]+";
-
-    private static final String SFPA_PATTERN = "SFPA41 KWBC \\d{6}[^\\r\\n]*[\\r\\n]+";
 
     // Allow ingest up to 10 minutes into the future.
     private static final long TEN_MINUTES = 10 * 60 * 1000L;
 
     private final SimpleDateFormat SDF;
 
-    private final Log logger = LogFactory.getLog(getClass());
+    private static final IUFStatusHandler logger = UFStatus
+            .getHandler(BinLightningDecoder.class);
 
     /**
      * Default lightning strike type for FLASH messages. RT_FLASH documents
@@ -166,7 +178,8 @@ public class BinLightningDecoder extends AbstractDecoder {
                 //     both encrypted data and legacy data
                 //  
                 
-                List<LightningStrikePoint> strikes = BinLigntningDecoderUtil.decodeBinLightningData(data, pdata, traceId, wmoHdr, baseTime.getTime());
+                List<LightningStrikePoint> strikes = decodeBinLightningData(
+                        data, pdata, traceId, wmoHdr, baseTime.getTime());
 
                 if (strikes == null) { // keep-alive record, log and return
                 	logger.info(traceId + " - found keep-alive record. ignore for now.");
@@ -189,7 +202,7 @@ public class BinLightningDecoder extends AbstractDecoder {
                     return new PluginDataObject[0];
                 }
 
-                Calendar c = TimeTools.copy(baseTime);
+                Calendar c = TimeUtil.newCalendar(baseTime);
                 if (c == null) {
                     throw new DecoderException(traceId +  " - Error decoding times");
                 }
@@ -212,14 +225,7 @@ public class BinLightningDecoder extends AbstractDecoder {
 
                     if (report != null) {
                         report.setTraceId(traceId);
-                        //report.setPluginName("binlightning"); // line disappear in OB15.5.3
+                        reports = new PluginDataObject[] { report };
-                        try {
-                            report.constructDataURI();
-                            reports = new PluginDataObject[] { report };
-                        } catch (PluginException e) {
-                        	logger.error("Error constructing datauri", e);
-                            throw new DecoderException("Error constructing datauri", e);
-                        }
                     }
                 }
             }
@@ -229,6 +235,261 @@ public class BinLightningDecoder extends AbstractDecoder {
         return reports;
     }
 
+    /**
+     * Decode bin lightning data, able to handle both legacy bit-shifted and new
+     * encryted data
+     * 
+     * The BinLightningDecoder.decode() method will use this method to decode
+     * data, which will try to decrypt first, and decode the old fashioned way
+     * when decryption fails
+     * 
+     * @author Wufeng Zhou
+     * 
+     * @param data
+     *            - data content from file, including WMO header section
+     * @param pdata
+     *            - data with WMO header stripped, optional, if null, will strip
+     *            WMO header internally from passed in data parameter
+     * @param traceId
+     *            - the file name of the data to be deoced
+     * @param wmoHdr
+     *            - WMOHeader, added 12/24/2013 to help distinguish bit-shifted
+     *            NLDN and GLD360 data (GLD data will have header starts like
+     *            SFPA)
+     * @param dataDate
+     *            - date of the data, optional, used as a hint to find
+     *            appropriate encryption key faster
+     * @return null if keep-alive record, otherwise a list (could be empty) of
+     *         LightningStrikePoint
+     */
+    public static List<LightningStrikePoint> decodeBinLightningData(
+            byte[] data, byte[] pdata, String traceId, WMOHeader wmoHdr,
+            Date dataDate) {
+        if (pdata == null) { // if data without header not passed, we'll strip
+                             // the WMO header here
+            WMOHeader header = new WMOHeader(data);
+            if (header.isValid() && header.getMessageDataStart() > 0) {
+                pdata = new byte[data.length - header.getMessageDataStart()];
+                System.arraycopy(data, header.getMessageDataStart(), pdata, 0,
+                        data.length - header.getMessageDataStart());
+            }
+        }
+
+        List<LightningStrikePoint> strikes = new ArrayList<LightningStrikePoint>();
+        boolean needDecrypt = true; // set as default unless clear evidence says
+                                    // otherwise
+        boolean decodeDone = false;
+        EncryptedBinLightningCipher cipher = new EncryptedBinLightningCipher();
+
+        //
+        // Using different WMO headers to indicate whether the data is encrypted
+        // or not would be a nice option.
+        // However, that idea has been discussed but not adopted.
+        // If in the future, WMO header can be different for legacy and
+        // encrypted data, or some other metadata can be used to decide
+        // whether deceyption is needed, logic can be added here.
+        //
+        // Before that happens, we'll use hints and trial & error to decode the
+        // data
+        // Hints: Per lightning data format spec, there are 3 bytes in the WMO
+        // header starting line that indicates the size of the encrypted block
+        // or the ASCII sequence # for legacy bit-shifted data
+        // However, the starting line is optional and AWIPS decode may not see
+        // it at all because TG will strip that starting line away
+        // We'll try to use this hint first, if is is not found, then trial and
+        // error way to decrypt and decode
+        //
+        // As of 11/05/2013, There is change in data spec. that the 3-bytes will
+        // not be encoded as encrypted block size anymore (it will always be
+        // transmission sequence # if present)
+        // So there should have some minor changes in the logic below for
+        // decoding the data.
+        // However, as reading into the
+        // com.raytheon.edex.plugin.binlightning.impl.BinLightningFactory.getDecoder()
+        // and follow-on code, we see the following data patterns
+        // for legacy bit-shifted data, which could be used to reduce guess-work
+        // in data decryption:
+        // The bit-shifted data will have multiple groups of the following
+        // patterns:
+        // 1-byte (unsigned byte): for size count
+        // 1-byte (unsigned byte): for flash type:
+        // 0x96 for FLASH_RPT (message size is 6 bytes each)
+        // 0x97 for RT_FLASH_RPT (message size is 8 bytes each)
+        // 0xd0 for OTHER_RPT (The D2D decoders declare but do not define this
+        // message, so unimplemented decoder)
+        // 0xd1 for COMM_RPT (The D2D decoders declare but do not define this
+        // message, so unimplemented decoder)
+        // 4-bytes: date time
+        // multiple of 6 or 8 bytes (as determined by 2nd byte flash type) with
+        // count indicated in 1st byte
+        //
+        // So this is be used to determine whether the data need to be
+        // decrypted.
+
+        /*
+         * // looks like previous assumption on block size bytes are not valid
+         * any more. 11/20/2013 if (data != null) { byte[] sizeSeqBytes =
+         * BinLigntningDecoderUtil.findSizeOrSeqBytesFromWMOHeader(data); if
+         * (sizeSeqBytes != null) { // if this is in the header (which may not),
+         * use that as a hint to determine which decoding route to go if
+         * (BinLigntningDecoderUtil
+         * .isPossibleWMOHeaderSequenceNumber(sizeSeqBytes) &&
+         * BinLigntningDecoderUtil
+         * .getEncryptedBlockSizeFromWMOHeader(sizeSeqBytes) != pdata.length) {
+         * // looks like a sequence #, and if treat as size, it does not equal
+         * to the data block size, so most likely legacy data needDecrypt =
+         * false; } } }
+         */
+
+        if (needDecrypt) {
+            try {
+                // NOTE: 11/14/2013 WZ:
+                // encrypted test data on TNCF (got from Melissa Porricelli)
+                // seems to have extra 4 bytes (0x0d 0x0d 0x0a 0x03) at the end,
+                // making the data size not a multiple of 16. However, original
+                // test data do not have this trailing bytes. while NCEP test
+                // data has extra 8 trailing bytes.
+                // Brain Rapp's email on 11/13/2013 confirms that Unidata LDM
+                // software used by AWIPS II will strips off all SBN protocol
+                // headers
+                // that precede the WMO header and adds its own 11 byte header
+                // like this: "soh  cr  cr  nl   2   5   4  sp  cr  cr  nl". It
+                // also adds a four byte trailer consisting of "cr cr nl etx"
+                // (0x0d 0x0d 0x0a 0x03)
+                // So, it seems necessary to trim trailing bytes if it is not
+                // multiple of 16, warning messages will be logged though
+                int dataLengthToBeDecrypted = pdata.length;
+                if (pdata.length % 16 != 0) {
+                    dataLengthToBeDecrypted = pdata.length
+                            - (pdata.length % 16);
+                    logger.warn(traceId + " - Data length from file " + traceId
+                            + " is " + pdata.length + " bytes, trailing "
+                            + (pdata.length - dataLengthToBeDecrypted)
+                            + " bytes has been trimmed to "
+                            + dataLengthToBeDecrypted
+                            + " bytes for decryption.");
+                }
+                byte[] encryptedData = new byte[dataLengthToBeDecrypted];
+                encryptedData = Arrays.copyOfRange(pdata, 0,
+                        dataLengthToBeDecrypted);
+
+                byte[] decryptedData = cipher.decryptData(encryptedData,
+                        dataDate);
+                // decrypt ok, then decode, first check if keep-alive record
+                if (BinLightningDecoderUtil.isKeepAliveRecord(decryptedData)) {
+                    logger.info(traceId
+                            + " - Keep-alive record detected, ignore for now.");
+                    decodeDone = true;
+                    return null;
+                }
+                // not keep-alive record, then check data validity and decode
+                // into an ArrayList<LightningStrikePoint> of strikes
+                if (BinLightningDecoderUtil
+                        .isLightningDataRecords(decryptedData)) {
+                    strikes = BinLightningDecoderUtil
+                            .decodeDecryptedBinLightningData(decryptedData);
+                    decodeDone = true;
+                } else {
+                    logger.info(traceId
+                            + " - Failed data validity check of the decrypted data, will try decode the old-fashioned way.");
+                    decodeDone = false;
+                }
+            } catch (IllegalBlockSizeException e) {
+                logger.info(traceId
+                        + " - "
+                        + e.getMessage()
+                        + ": Decryption failed, will try decode the old-fashioned way.");
+                decodeDone = false;
+            } catch (BadPaddingException e) {
+                logger.info(traceId
+                        + " - "
+                        + e.getMessage()
+                        + ": Decryption failed, will try decode the old-fashioned way.");
+                decodeDone = false;
+            } catch (BinLightningDataDecryptionException e) {
+                logger.info(traceId
+                        + " - "
+                        + e.getMessage()
+                        + ": Decryption failed, will try decode the old-fashioned way.");
+                decodeDone = false;
+            }
+        }
+
+        if (decodeDone == false) { // not decoded through decrypt->decode
+                                   // process, try the legacy decoder
+            logger.info(traceId + " - decoding as bit-shifted data");
+            // bit-shifting data format check call here will get us some more
+            // information on the data, also can compare the strikes with the
+            // decoder result
+            int estimatedStrikes = BinLightningDecoderUtil
+                    .getBitShiftedDataStrikeCount(pdata);
+            strikes = decodeBitShiftedBinLightningData(pdata, wmoHdr);
+            if (estimatedStrikes != strikes.size()) {
+                logger.warn(traceId
+                        + ": bit-shifted decoder found "
+                        + strikes
+                        + " strikes, which is different from estimate from data pattern examination: "
+                        + estimatedStrikes);
+            }
+        }
+
+        return strikes;
+    }
+
+    /**
+     * extracted from the original {@link #decode(byte[], Headers)} method then
+     * modified by Wufeng Zhou
+     * 
+     * @param pdata
+     * @param wmoHdr
+     *            - WMOHeader, added 12/24/2013 to help distinguish bit-shifted
+     *            NLDN and GLD360 data (GLD data will have header starts like
+     *            SFPA)
+     * @return
+     */
+    public static List<LightningStrikePoint> decodeBitShiftedBinLightningData(
+            byte[] pdata, WMOHeader wmoHdr) {
+        List<LightningStrikePoint> strikes = new ArrayList<LightningStrikePoint>();
+
+        IBinDataSource msgData = new LightningDataSource(pdata);
+
+        boolean continueDecode = true;
+        while (continueDecode) {
+            IBinLightningDecoder decoder = BinLightningFactory
+                    .getDecoder(msgData);
+
+            switch (decoder.getError()) {
+            case IBinLightningDecoder.NO_ERROR: {
+                for (LightningStrikePoint strike : decoder) {
+                    // use WMO Header to distinguish NLDN or GLD360 data because
+                    // no bit-shifted data spec available for GLD360.
+                    // 12/24/2013, WZ
+                    // The WMO header start string is defined in
+                    // BinLightningAESKey.properties file (normally, GLD360 data
+                    // will have WMO header
+                    // starts with SFPA41, or SFPA99 for test data.)
+                    String gld360WMOHeaderString = BinLightningAESKey
+                            .getProps().getProperty(
+                                    "binlightning.gld360WMOHeaderStartString",
+                                    "");
+                    if (gld360WMOHeaderString.trim().equals("") == false
+                            && wmoHdr.getWmoHeader().startsWith(
+                                    gld360WMOHeaderString)) {
+                        // GLD360 data based on the setup
+                        strike.setLightSource("GLD");
+                    }
+                    strikes.add(strike);
+                }
+                break;
+            }
+            default: {
+                continueDecode = false;
+            }
+            }
+        }
+        return strikes;
+    }
+
     /**
      * Set a trace identifier for the source data.
      * 

edexOsgi/com.raytheon.uf.edex.binlightning.feature/feature.xml

@@ -24,4 +24,11 @@
          version="0.0.0"
          unpack="false"/>
 
+   <plugin
+         id="gov.noaa.nws.ost.edex.plugin.binlightning"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"
+         unpack="false"/>
+
 </feature>

ost/gov.noaa.nws.ost.edex.plugin.binlightning/.classpath

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7"/>
+	<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
+	<classpathentry kind="src" path="src"/>
+	<classpathentry kind="output" path="bin"/>
+</classpath>

ost/gov.noaa.nws.ost.edex.plugin.binlightning/.project

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>gov.noaa.nws.ost.edex.plugin.binlightning</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.pde.ManifestBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.pde.SchemaBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.pde.PluginNature</nature>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+	</natures>
+</projectDescription>

ost/gov.noaa.nws.ost.edex.plugin.binlightning/.settings/org.eclipse.jdt.core.prefs

@@ -0,0 +1,7 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
+org.eclipse.jdt.core.compiler.compliance=1.7
+org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
+org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
+org.eclipse.jdt.core.compiler.source=1.7

ost/gov.noaa.nws.ost.edex.plugin.binlightning/META-INF/MANIFEST.MF

@@ -0,0 +1,13 @@
+Manifest-Version: 1.0
+Bundle-ManifestVersion: 2
+Bundle-Name: OST Binlightning
+Bundle-SymbolicName: gov.noaa.nws.ost.edex.plugin.binlightning
+Bundle-Version: 1.14.0.qualifier
+Bundle-RequiredExecutionEnvironment: JavaSE-1.7
+Import-Package: com.raytheon.uf.common.dataplugin.binlightning,
+ com.raytheon.uf.common.dataplugin.binlightning.dataaccess,
+ com.raytheon.uf.common.dataplugin.binlightning.impl,
+ com.raytheon.uf.common.status,
+ com.raytheon.uf.common.wmo,
+ com.raytheon.uf.edex.decodertools.core
+Export-Package: gov.noaa.nws.ost.edex.plugin.binlightning

ost/gov.noaa.nws.ost.edex.plugin.binlightning/build.properties

@@ -0,0 +1,4 @@
+source.. = src/
+output.. = bin/
+bin.includes = META-INF/,\
+               .

ost/gov.noaa.nws.ost.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningAESKey.java

@@ -27,13 +27,14 @@ import java.util.TreeMap;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import org.apache.commons.logging.Log;
+import com.raytheon.uf.common.status.IUFStatusHandler;
-import org.apache.commons.logging.LogFactory;
+import com.raytheon.uf.common.status.UFStatus;
 
 /**
  * BinLightningAESKey
  * 
- * Simple representation of bin lightning AES encryption key and its associated key aliases in the keystore
+ * Simple representation of bin lightning AES encryption key and its associated
+ * key aliases in the keystore
  * 
  * <pre>
  * 
@@ -42,6 +43,7 @@ import org.apache.commons.logging.LogFactory;
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * 20130503      DCS 112   Wufeng Zhou To handle both the new encrypted data and legacy bit-shifted data
+ * Jun 03, 2014 3226       bclement    moved from com.raytheon.edex.plugin.binlightning to gov.noaa.nws.ost.edex.plugin.binlightning
  * 
  * </pre>
  * 
@@ -63,7 +65,8 @@ public class BinLightningAESKey {
     private static final Pattern KEY_ALIAS_PREFIX_PATTERN = Pattern.compile(KEY_ALIAS_PREFIX);
     private static final SimpleDateFormat KEY_ALIAS_DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd");
 	
-    private static Log logger = LogFactory.getLog(BinLightningAESKey.class);
+    private static IUFStatusHandler logger = UFStatus
+            .getHandler(BinLightningAESKey.class);
 
 	private static Properties props = new Properties();
 	private static KeyStore keystore; 

ost/gov.noaa.nws.ost.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningDecoderUtil.java

@@ -9,24 +9,14 @@ import java.nio.ByteOrder;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Calendar;
-import java.util.Date;
 import java.util.List;
 
-import javax.crypto.BadPaddingException;
-import javax.crypto.IllegalBlockSizeException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.raytheon.edex.plugin.binlightning.impl.BinLightningFactory;
-import com.raytheon.edex.plugin.binlightning.impl.IBinLightningDecoder;
-import com.raytheon.edex.plugin.binlightning.impl.LightningDataSource;
 import com.raytheon.uf.common.dataplugin.binlightning.impl.LightningStrikePoint;
 import com.raytheon.uf.common.dataplugin.binlightning.impl.LtgMsgType;
 import com.raytheon.uf.common.dataplugin.binlightning.impl.LtgStrikeType;
-import com.raytheon.uf.common.wmo.WMOHeader;
+import com.raytheon.uf.common.status.IUFStatusHandler;
+import com.raytheon.uf.common.status.UFStatus;
 import com.raytheon.uf.edex.decodertools.core.BasePoint;
-import com.raytheon.uf.edex.decodertools.core.IBinDataSource;
 
 /**
  * BinLigntningDecoderUtil
@@ -44,12 +34,16 @@ import com.raytheon.uf.edex.decodertools.core.IBinDataSource;
  * ------------ ---------- ----------- --------------------------
  * 20130503        DCS 112 Wufeng Zhou To handle both the new encrypted data and legacy bit-shifted data
  * 20140501	               Wufeng Zhou Fix the encrypted decoding with correct offset	
+ * Jun 03, 2014 3226       bclement    renamed from BinLigntningDecoderUtil to BinLightningDecoderUtil
+ *                                     moved from com.raytheon.edex.plugin.binlightning to gov.noaa.nws.ost.edex.plugin.binlightning
+ *                                     moved decodeBinLightningData() and decodeBitShiftedBinLightningData() 
+ *                                          to BinLightningDecoder to solve circular dependency
  * </pre>
  * 
  * @author Wufeng Zhou
  * 
  */
-public class BinLigntningDecoderUtil {
+public class BinLightningDecoderUtil {
 
 	/**
 	 * Message type for keep alive data records. 
@@ -84,47 +78,10 @@ public class BinLigntningDecoderUtil {
     static final byte OTHER_RPT = (byte)0xD0;
     static final byte COMM_RPT = (byte)0xD1;
 
-    private static Log logger = LogFactory.getLog(BinLigntningDecoderUtil.class);
+    private static IUFStatusHandler logger = UFStatus
+            .getHandler(BinLightningDecoderUtil.class);
 
-	/**
-	 * extracted from the decode() of the original
-	 * com.raytheon.edex.plugin.binlightning.BinLightningDecoder class
-	 * 
-	 * @param pdata
-	 * @param wmoHdr - WMOHeader, added 12/24/2013 to help distinguish bit-shifted NLDN and GLD360 data (GLD data will have header starts like SFPA)
-	 * @return
-	 */
-	public static List<LightningStrikePoint> decodeBitShiftedBinLightningData(byte[] pdata, WMOHeader wmoHdr) {
-		List<LightningStrikePoint> strikes = new ArrayList<LightningStrikePoint>();
 
-		IBinDataSource msgData = new LightningDataSource(pdata);
-
-		boolean continueDecode = true;
-		while (continueDecode) {
-			IBinLightningDecoder decoder = BinLightningFactory.getDecoder(msgData);
-
-			switch (decoder.getError()) {
-			case IBinLightningDecoder.NO_ERROR: {
-				for (LightningStrikePoint strike : decoder) {
-					// use WMO Header to distinguish NLDN or GLD360 data because no bit-shifted data spec available for GLD360. 12/24/2013, WZ
-					// The WMO header start string is defined in BinLightningAESKey.properties file (normally, GLD360 data will have WMO header 
-					//  starts with SFPA41, or SFPA99 for test data.) 
-					String gld360WMOHeaderString = BinLightningAESKey.getProps().getProperty("binlightning.gld360WMOHeaderStartString", "");
-					if (gld360WMOHeaderString.trim().equals("") == false && wmoHdr.getWmoHeader().startsWith(gld360WMOHeaderString)) { 
-						// GLD360 data based on the setup
-						strike.setLightSource("GLD");
-					}
-					strikes.add(strike);
-				}
-				break;
-			}
-			default: {
-				continueDecode = false;
-			}
-			}
-		}
-		return strikes;
-	}
 	
 	/**
 	 * decode the new bin lightning data, after the data record is decrypted, and it is not keep-alive record
@@ -170,8 +127,9 @@ public class BinLigntningDecoderUtil {
 			int strokeType = buffer.getShort() & 0xffff; // 0x0000 for cloud-to-ground, 0x00ff for cloud-to-cloud, 0xffff for total flash
 			short strokeKiloAmps = buffer.getShort(); // valid range: -254 to 254, specifically 16 bit signed integer
 			int strokeMultiplicity = buffer.getShort() & 0xffff; // i.e. stroke count, valid range: 0 to 15
-			int strokeDuration = buffer.getShort() & 0xffff; // valid range: 0 to 65535 (i.e., looks like unsigned short)
+            // int strokeDuration = buffer.getShort() & 0xffff; // valid range:
-			int reserved = buffer.getShort() & 0xffff;
+            // 0 to 65535 (i.e., looks like unsigned short)
+            // int reserved = buffer.getShort() & 0xffff;
 
             // Create the strike record from the report info and base time information. 
 			BasePoint base = new BasePoint(lat, lon);
@@ -236,134 +194,7 @@ public class BinLigntningDecoderUtil {
 		return strikes;
 	}
 	
-	/**
-	 * Decode bin lightning data, able to handle both legacy bit-shifted and new encryted data
-	 * 
-	 * The modified BinLightningDecoder.decode() method will use this method to decode data, which 
-	 *   will try to decrypt first, and decode the old fashioned way when decryption fails   
-	 * 
-	 * @param data - data content from file, including WMO header section
-	 * @param pdata - data with WMO header stripped, optional, if null, will strip WMO header internally from passed in data parameter 
-	 * @param traceId - the file name of the data to be deoced
-	 * @param wmoHdr - WMOHeader, added 12/24/2013 to help distinguish bit-shifted NLDN and GLD360 data (GLD data will have header starts like SFPA)
-	 * @param dataDate - date of the data, optional, used as a hint to find appropriate encryption key faster
-	 * @return null if keep-alive record, otherwise a list (could be empty) of LightningStrikePoint
-	 */
-	public static List<LightningStrikePoint> decodeBinLightningData(byte[] data, byte[] pdata, String traceId, WMOHeader wmoHdr, Date dataDate) {
-		if (pdata == null) { // if data without header not passed, we'll strip the WMO header here
-			WMOHeader header = new WMOHeader(data);
-			if (header.isValid() && header.getMessageDataStart() > 0) {
-				pdata = new byte[data.length - header.getMessageDataStart()];
-				System.arraycopy(data, header.getMessageDataStart(), pdata, 0, data.length - header.getMessageDataStart());
-			} 
-		}
 
-        List<LightningStrikePoint> strikes = new ArrayList<LightningStrikePoint>();
-        boolean needDecrypt = true; // set as default unless clear evidence says otherwise
-        boolean decodeDone = false;
-		EncryptedBinLightningCipher cipher = new EncryptedBinLightningCipher();
-
-		//
-		// Using different WMO headers to indicate whether the data is encrypted or not would be a nice option. 
-		// However, that idea has been discussed but not adopted.   
-        // If in the future, WMO header can be different for legacy and encrypted data, or some other metadata can be used to decide
-        //   whether deceyption is needed, logic can be added here.
-        //
-        // Before that happens, we'll use hints and trial & error to decode the data
-		//   Hints: Per lightning data format spec, there are 3 bytes in the WMO header starting line that indicates the size of the encrypted block 
-		//          or the ASCII sequence # for legacy bit-shifted data 
-		// However, the starting line is optional and AWIPS decode may not see it at all because TG will strip that starting line away
-		// We'll try to use this hint first, if is is not found, then trial and error way to decrypt and decode
-		//
-		// As of 11/05/2013, There is change in data spec. that the 3-bytes will not be encoded as encrypted block size anymore (it will always be transmission sequence # if present)
-		//     So there should have some minor changes in the logic below for decoding the data.
-		// However, as reading into the com.raytheon.edex.plugin.binlightning.impl.BinLightningFactory.getDecoder() and follow-on code, we see the following data patterns
-		//     for legacy bit-shifted data, which could be used to reduce guess-work in data decryption:
-		// The bit-shifted data will have multiple groups of the following patterns:
-		//     1-byte (unsigned byte): for size count
-		//     1-byte (unsigned byte): for flash type:  
-		//                                 0x96 for FLASH_RPT (message size is 6 bytes each)
-		//                                 0x97 for RT_FLASH_RPT (message size is 8 bytes each)
-		//                                 0xd0 for OTHER_RPT (The D2D decoders declare but do not define this message, so unimplemented decoder)
-		//                                 0xd1 for COMM_RPT  (The D2D decoders declare but do not define this message, so unimplemented decoder)
-		//     4-bytes: date time
-		//     multiple of 6 or 8 bytes (as determined by 2nd byte flash type) with count indicated in 1st byte
-		// 
-		// So this is be used to determine whether the data need to be decrypted.  
-		
-		/*
-		// looks like previous assumption on block size bytes are not valid any more.   11/20/2013
-        if (data != null) {
-			byte[] sizeSeqBytes = BinLigntningDecoderUtil.findSizeOrSeqBytesFromWMOHeader(data);
-	        if (sizeSeqBytes != null) { 
-	        	// if this is in the header (which may not), use that as a hint to determine which decoding route to go
-	        	if (BinLigntningDecoderUtil.isPossibleWMOHeaderSequenceNumber(sizeSeqBytes)  
-	        			&& BinLigntningDecoderUtil.getEncryptedBlockSizeFromWMOHeader(sizeSeqBytes) != pdata.length) {
-	        		// looks like a sequence #, and if treat as size, it does not equal to the data block size, so most likely legacy data
-	        		needDecrypt = false;
-	        	}
-	        }
-        }
-		*/
-		
-        if (needDecrypt) {
-    		try {
-    			// NOTE: 11/14/2013 WZ:
-    			//       encrypted test data on TNCF (got from Melissa Porricelli) seems to have extra 4 bytes (0x0d 0x0d 0x0a 0x03) at the end,
-    			//       making the data size not a multiple of 16.  However, original test data do not have this trailing bytes. while NCEP test 
-    			//       data has extra 8 trailing bytes.  
-    			//  Brain Rapp's email on 11/13/2013 confirms that Unidata LDM software used by AWIPS II will strips off all SBN protocol headers 
-    			//       that precede the WMO header and adds its own 11 byte header like this: "soh  cr  cr  nl   2   5   4  sp  cr  cr  nl".  It 
-    			//       also adds a four byte trailer consisting of "cr cr nl etx" (0x0d 0x0d 0x0a 0x03)
-    			//  So, it seems necessary to trim trailing bytes if it is not multiple of 16, warning messages will be logged though 
-    			int dataLengthToBeDecrypted = pdata.length;
-    			if (pdata.length % 16 != 0) {
-    				dataLengthToBeDecrypted = pdata.length - (pdata.length % 16);
-					logger.warn(traceId + " - Data length from file " + traceId + " is " + pdata.length + " bytes, trailing " + 
-							(pdata.length - dataLengthToBeDecrypted) + " bytes has been trimmed to " + dataLengthToBeDecrypted + " bytes for decryption.");
-    			}
-    			byte[] encryptedData = new byte[dataLengthToBeDecrypted];
-    			encryptedData = Arrays.copyOfRange(pdata, 0, dataLengthToBeDecrypted);
-    				
-    			byte[] decryptedData = cipher.decryptData(encryptedData, dataDate);	                    
-    			// decrypt ok, then decode, first check if keep-alive record
-    			if (BinLigntningDecoderUtil.isKeepAliveRecord(decryptedData)) {
-    				logger.info(traceId + " - Keep-alive record detected, ignore for now.");
-    				decodeDone = true;
-    				return null; 
-    			}
-    			// not keep-alive record, then check data validity and decode into an ArrayList<LightningStrikePoint> of strikes
-    			if (BinLigntningDecoderUtil.isLightningDataRecords(decryptedData)) {
-	    			strikes = BinLigntningDecoderUtil.decodeDecryptedBinLightningData(decryptedData); 
-	    			decodeDone = true;
-    			} else {
-    				logger.info(traceId + " - Failed data validity check of the decrypted data, will try decode the old-fashioned way.");
-        			decodeDone = false;
-    			}
-    		} catch (IllegalBlockSizeException e) {
-				logger.info(traceId + " - " + e.getMessage() + ": Decryption failed, will try decode the old-fashioned way.");
-    			decodeDone = false;
-    		} catch (BadPaddingException e) {
-				logger.info(traceId + " - " + e.getMessage() + ": Decryption failed, will try decode the old-fashioned way.");
-    			decodeDone = false;
-    		} catch (BinLightningDataDecryptionException e) {
-				logger.info(traceId + " - " + e.getMessage() + ": Decryption failed, will try decode the old-fashioned way.");
-    			decodeDone = false;
-			}
-        }
-
-        if (decodeDone == false) { // not decoded through decrypt->decode process, try the legacy decoder
-			logger.info(traceId + " - decoding as bit-shifted data");
-			// bit-shifting data format check call here will get us some more information on the data, also can compare the strikes with the decoder result
-			int estimatedStrikes = getBitShiftedDataStrikeCount(pdata);  
-        	strikes = BinLigntningDecoderUtil.decodeBitShiftedBinLightningData(pdata, wmoHdr);
-        	if (estimatedStrikes != strikes.size()) {
-    			logger.warn(traceId + ": bit-shifted decoder found " + strikes + " strikes, which is different from estimate from data pattern examination: " + estimatedStrikes);
-        	}
-        }
-        
-        return strikes;
-	}
 	
 	/**
 	 * Determines if the bytes passed are a standard "NWS Keep Alive" message.

ost/gov.noaa.nws.ost.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/EncryptedBinLightningCipher.java

@@ -6,6 +6,7 @@ package gov.noaa.nws.ost.edex.plugin.binlightning;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
@@ -15,13 +16,14 @@ import javax.crypto.Cipher;
 import javax.crypto.IllegalBlockSizeException;
 import javax.crypto.spec.SecretKeySpec;
 
-import org.apache.commons.logging.Log;
+import com.raytheon.uf.common.status.IUFStatusHandler;
-import org.apache.commons.logging.LogFactory;
+import com.raytheon.uf.common.status.UFStatus;
 
 /**
  * EncryptedBinLightningCipher
  * 
- * Use AES secret keys found in configured keystore to decrypt bin lightning data
+ * Use AES secret keys found in configured keystore to decrypt bin lightning
+ * data
  * 
  * <pre>
  * 
@@ -30,6 +32,8 @@ import org.apache.commons.logging.LogFactory;
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * 20130503        DCS 112 Wufeng Zhou To handle both the new encrypted data and legacy bit-shifted data
+ * Jun 03, 2014 3226       bclement    moved from com.raytheon.edex.plugin.binlightning to gov.noaa.nws.ost.edex.plugin.binlightning
+ *                                      handled null return from BinLightningAESKey.getBinLightningAESKeys()
  * 
  * </pre>
  * 
@@ -51,6 +55,9 @@ public class EncryptedBinLightningCipher {
 		protected HashMap<String, Cipher> initialValue() {
 			// get AES keys from keystore and create encryption and decryption ciphers from them
 			BinLightningAESKey[] keys = BinLightningAESKey.getBinLightningAESKeys();
+            if (keys == null) {
+                keys = new BinLightningAESKey[0];
+            }
 			HashMap<String, Cipher> cipherMap = new HashMap<String, Cipher>();
 			for (BinLightningAESKey key : keys) {
 				try {
@@ -67,7 +74,8 @@ public class EncryptedBinLightningCipher {
 		}		
 	};
 	
-    private static Log logger = LogFactory.getLog(EncryptedBinLightningCipher.class);
+    private static IUFStatusHandler logger = UFStatus
+            .getHandler(EncryptedBinLightningCipher.class);
 
     public EncryptedBinLightningCipher() {
 		
@@ -122,7 +130,7 @@ public class EncryptedBinLightningCipher {
 				
 				// wrong key will decrypt data into random noise/garbage, so we need to do a sanity check to make sure 
 				//   we are decrypting with the right key
-				if ( BinLigntningDecoderUtil.isKeepAliveRecord(decryptedData) == false && BinLigntningDecoderUtil.isLightningDataRecords(decryptedData) == false) {
+				if ( BinLightningDecoderUtil.isKeepAliveRecord(decryptedData) == false && BinLightningDecoderUtil.isLightningDataRecords(decryptedData) == false) {
 				//if (BinLigntningDecoderUtil.isValidMixedRecordData(decryptedData) == false) { // use this only if keep-alive record could be mixed with lightning records
 					throw new BinLightningDataDecryptionException("Decrypted data (" + decryptedData.length + " bytes) with key " 
 							+ preferredKeyList.get(i).getAlias() + " is not valid keep-alive or binLightning records.", decryptedData);
@@ -163,7 +171,11 @@ public class EncryptedBinLightningCipher {
 	 * @return preferred key list order
 	 */
 	private List<BinLightningAESKey> findPreferredKeyOrderForData(Date dataDate) {
-		List<BinLightningAESKey> defKeyList =  Arrays.asList(BinLightningAESKey.getBinLightningAESKeys());
+		BinLightningAESKey[] binLightningAESKeys = BinLightningAESKey.getBinLightningAESKeys();
+        if (binLightningAESKeys == null || binLightningAESKeys.length < 1) {
+            return Collections.emptyList();
+        }
+        List<BinLightningAESKey> defKeyList =  Arrays.asList(binLightningAESKeys);
 		if (dataDate == null) {
 			return defKeyList; // use default order
 		}