From d4c55244a1f6c0cefc9b3af238ba97e620cf69fa Mon Sep 17 00:00:00 2001 From: Brian Clements Date: Mon, 9 Jun 2014 10:40:34 -0500 Subject: [PATCH] Omaha #3226 encryption support for total lightning Change-Id: I87af2d986830681d258c065304e3ebfb2d2af8db Former-commit-id: fe199cb32b3fe33ac48f039888bd43d4fedd9820 --- .../binlightning/BinLightningDecoder.java | 39 +- .../total/ChecksumByteBuffer.java | 10 +- .../total/TotalLightningDecoder.java | 129 ++++++- .../binlightning/BinLightningAESKey.java | 359 ++++++++++++------ .../EncryptedBinLightningCipher.java | 220 ++++++++--- 5 files changed, 524 insertions(+), 233 deletions(-) diff --git a/edexOsgi/com.raytheon.edex.plugin.binlightning/src/com/raytheon/edex/plugin/binlightning/BinLightningDecoder.java b/edexOsgi/com.raytheon.edex.plugin.binlightning/src/com/raytheon/edex/plugin/binlightning/BinLightningDecoder.java index fe33639ffa..672eb5c438 100644 --- a/edexOsgi/com.raytheon.edex.plugin.binlightning/src/com/raytheon/edex/plugin/binlightning/BinLightningDecoder.java +++ b/edexOsgi/com.raytheon.edex.plugin.binlightning/src/com/raytheon/edex/plugin/binlightning/BinLightningDecoder.java @@ -26,7 +26,6 @@ import gov.noaa.nws.ost.edex.plugin.binlightning.EncryptedBinLightningCipher; import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Arrays; import java.util.Calendar; import java.util.Date; import java.util.List; @@ -86,6 +85,7 @@ import com.raytheon.uf.edex.decodertools.core.IBinDataSource; * removed TimeTools usage, removed constructDataURI() call * added decodeBinLightningData() and decodeBitShiftedBinLightningData() from BinLightningDecoderUtil * Jun 05, 2014 3226 bclement LightningStikePoint refactor, added extractPData() + * Jun 09, 2014 3226 bclement moved data array decrypt prep to EncryptedBinLightingCipher * * * @@ -102,6 +102,8 @@ public class BinLightningDecoder extends AbstractDecoder { private static final IUFStatusHandler logger = UFStatus .getHandler(BinLightningDecoder.class); + public static final String BINLIGHTNING_KEYSTORE_PREFIX = "binlightning"; + /** * Default lightning strike type for FLASH messages. RT_FLASH documents * indicate no default, but D2D code defaults to STRIKE_CG also. @@ -343,40 +345,11 @@ public class BinLightningDecoder extends AbstractDecoder { if (needDecrypt) { try { - /* - * NOTE: 11/14/2013 WZ: - * encrypted test data on TNCF (got from Melissa Porricelli) - * seems to have extra 4 bytes (0x0d 0x0d 0x0a 0x03) at the end, - * making the data size not a multiple of 16. However, original - * test data do not have this trailing bytes. while NCEP test - * data has extra 8 trailing bytes. - * Brain Rapp's email on 11/13/2013 confirms that Unidata LDM - * software used by AWIPS II will strips off all SBN protocol - * headers - * that precede the WMO header and adds its own 11 byte header - * like this: "soh cr cr nl 2 5 4 sp cr cr nl". It - * also adds a four byte trailer consisting of "cr cr nl etx" - * (0x0d 0x0d 0x0a 0x03) - * So, it seems necessary to trim trailing bytes if it is not - * multiple of 16, warning messages will be logged though - */ - int dataLengthToBeDecrypted = pdata.length; - if (pdata.length % 16 != 0) { - dataLengthToBeDecrypted = pdata.length - - (pdata.length % 16); - logger.warn(traceId + " - Data length from file " + traceId - + " is " + pdata.length + " bytes, trailing " - + (pdata.length - dataLengthToBeDecrypted) - + " bytes has been trimmed to " - + dataLengthToBeDecrypted - + " bytes for decryption."); - } - byte[] encryptedData = new byte[dataLengthToBeDecrypted]; - encryptedData = Arrays.copyOfRange(pdata, 0, - dataLengthToBeDecrypted); + byte[] encryptedData = EncryptedBinLightningCipher + .prepDataForDecryption(pdata, traceId); byte[] decryptedData = cipher.decryptData(encryptedData, - dataDate); + dataDate, BINLIGHTNING_KEYSTORE_PREFIX); // decrypt ok, then decode, first check if keep-alive record if (BinLightningDecoderUtil.isKeepAliveRecord(decryptedData)) { logger.info(traceId diff --git a/edexOsgi/com.raytheon.edex.plugin.binlightning/src/com/raytheon/edex/plugin/binlightning/total/ChecksumByteBuffer.java b/edexOsgi/com.raytheon.edex.plugin.binlightning/src/com/raytheon/edex/plugin/binlightning/total/ChecksumByteBuffer.java index eaf4c914f6..9e3354d200 100644 --- a/edexOsgi/com.raytheon.edex.plugin.binlightning/src/com/raytheon/edex/plugin/binlightning/total/ChecksumByteBuffer.java +++ b/edexOsgi/com.raytheon.edex.plugin.binlightning/src/com/raytheon/edex/plugin/binlightning/total/ChecksumByteBuffer.java @@ -32,7 +32,8 @@ import com.raytheon.uf.common.numeric.UnsignedNumbers; * * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- - * Jun 3, 2014 3226 bclement Initial creation + * Jun 03, 2014 3226 bclement Initial creation + * Jun 09, 2014 3226 bclement Added ByteBuffer constructor * * * @@ -62,6 +63,13 @@ public class ChecksumByteBuffer { this.buff = ByteBuffer.wrap(data); } + /** + * @param buff + */ + public ChecksumByteBuffer(ByteBuffer buff) { + this.buff = buff; + } + /** * get the sum of the next numberOfBytes worth of data * diff --git a/edexOsgi/com.raytheon.edex.plugin.binlightning/src/com/raytheon/edex/plugin/binlightning/total/TotalLightningDecoder.java b/edexOsgi/com.raytheon.edex.plugin.binlightning/src/com/raytheon/edex/plugin/binlightning/total/TotalLightningDecoder.java index a1899734b4..7f54f2a030 100644 --- a/edexOsgi/com.raytheon.edex.plugin.binlightning/src/com/raytheon/edex/plugin/binlightning/total/TotalLightningDecoder.java +++ b/edexOsgi/com.raytheon.edex.plugin.binlightning/src/com/raytheon/edex/plugin/binlightning/total/TotalLightningDecoder.java @@ -19,6 +19,9 @@ **/ package com.raytheon.edex.plugin.binlightning.total; +import gov.noaa.nws.ost.edex.plugin.binlightning.EncryptedBinLightningCipher; + +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Calendar; import java.util.Date; @@ -40,6 +43,7 @@ import com.raytheon.uf.common.status.IUFStatusHandler; import com.raytheon.uf.common.status.UFStatus; import com.raytheon.uf.common.time.util.TimeUtil; import com.raytheon.uf.common.wmo.WMOHeader; +import com.raytheon.uf.common.wmo.WMOTimeParser; /** * Decoder for Earth Networks Total Lightning data @@ -50,7 +54,8 @@ import com.raytheon.uf.common.wmo.WMOHeader; * * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- - * May 30, 2014 3226 bclement Initial creation + * May 30, 2014 3226 bclement Initial creation + * Jun 09, 2014 3226 bclement added encryption support * * * @@ -81,9 +86,19 @@ public class TotalLightningDecoder { // constant metadata public static final String DATA_SOURCE = "ENTLN"; + /* in bytes, header is total size of flash and pulses */ + private static final int COMBINATION_PACKET_HEADER_SIZE = 2; + + /* in bytes, doesn't include checksum */ + private static final int FLASH_PACKET_SIZE = 25; + private static final IUFStatusHandler log = UFStatus .getHandler(TotalLightningDecoder.class); + private static final EncryptedBinLightningCipher CIPHER = new EncryptedBinLightningCipher(); + + public static final String TOTAL_LIGHTNING_KEYSTORE_PREFIX = "total.lightning"; + /** * Parse total lightning data into BinLightningRecords * @@ -99,7 +114,7 @@ public class TotalLightningDecoder { byte[] pdata = BinLightningDecoder.extractPData(wmoHdr, data); if (pdata != null) { try { - rval = decodeInternal(fileName, pdata); + rval = decodeInternal(wmoHdr, fileName, pdata); } catch (Exception e) { error(e, headers, wmoHdr); rval = new PluginDataObject[0]; @@ -115,6 +130,27 @@ public class TotalLightningDecoder { return rval; } + /** + * @param data + * @param startIndex + * starting index of flash packet (not combination packet) + * @return true if there if a valid flash packet in data starting at index + */ + private static boolean validFlashPacket(byte[] data, int startIndex) { + /* plus one to include packet checksum */ + final int packetWithChecksum = FLASH_PACKET_SIZE + 1; + if (data.length < startIndex + packetWithChecksum) { + return false; + } + ChecksumByteBuffer buff = new ChecksumByteBuffer(ByteBuffer.wrap(data, + startIndex, packetWithChecksum)); + for (int i = 0; i < FLASH_PACKET_SIZE; ++i) { + /* build up sum in buffer */ + buff.get(); + } + return passesCheckSum(buff, false); + } + /** * Display warning message with file and header names * @@ -141,21 +177,51 @@ public class TotalLightningDecoder { /** + * @param wmoHdr * @param fileName * @param pdata * data after WMO header is removed * @return * @throws DecoderException */ - private PluginDataObject[] decodeInternal(String fileName, byte[] pdata) + private PluginDataObject[] decodeInternal(WMOHeader wmoHdr, + String fileName, byte[] pdata) throws DecoderException { - List decodeStrikes = decodeStrikes(fileName, - pdata); - BinLightningRecord record = new BinLightningRecord(decodeStrikes); - return new PluginDataObject[] { record }; + if (!validFlashPacket(pdata, COMBINATION_PACKET_HEADER_SIZE)) { + /* assume data is encrypted if we can't understand it */ + pdata = decrypt(wmoHdr, fileName, pdata); + } + List strikes = decodeStrikes(fileName, pdata); + if (!strikes.isEmpty()) { + BinLightningRecord record = new BinLightningRecord(strikes); + return new PluginDataObject[] { record }; + } else { + return new PluginDataObject[0]; + } } + /** + * @param wmoHdr + * @param fileName + * @param pdata + * @return + * @throws DecoderException + */ + private byte[] decrypt(WMOHeader wmoHdr, String fileName, byte[] pdata) + throws DecoderException { + Calendar baseTime = WMOTimeParser.findDataTime(wmoHdr.getYYGGgg(), + fileName); + pdata = EncryptedBinLightningCipher.prepDataForDecryption(pdata, + fileName); + try { + return CIPHER.decryptData(pdata, baseTime.getTime(), + TOTAL_LIGHTNING_KEYSTORE_PREFIX); + } catch (Exception e) { + throw new DecoderException("Problem decrypting total lightning", e); + } + } + /** * Extract strike data from raw binary * @@ -169,10 +235,17 @@ public class TotalLightningDecoder { List rval = new ArrayList(); ChecksumByteBuffer buff = new ChecksumByteBuffer(pdata); while (buff.position() < buff.size()) { + int startingPostion = buff.position(); int totalBytes = UnsignedNumbers.ushortToInt(buff.getShort()); - if (totalBytes > (buff.size() - buff.position())) { - log.error("Truncated total lightning packet in file: " - + fileName); + if (totalBytes > (buff.size() - startingPostion)) { + if (validFlashPacket(pdata, buff.position())) { + log.error("Truncated total lightning packet in file: " + + fileName); + } else { + int extra = buff.size() - startingPostion; + log.warn("Extra data at end of lightning packets: " + extra + + " bytes"); + } break; } /* start flash packet */ @@ -189,7 +262,7 @@ public class TotalLightningDecoder { int pulseCount = UnsignedNumbers.ubyteToShort(buff.get()); strike.setPulseCount(pulseCount); - checkSum(buff, false); + ensureCheckSum(buff, false); List pulses = new ArrayList( pulseCount); @@ -202,12 +275,14 @@ public class TotalLightningDecoder { decodeCommonFields(pulse, buff); /* discard pulse count (already set in strike) */ buff.get(); - checkSum(buff, false); + ensureCheckSum(buff, false); pulses.add(pulse); } strike.setPulses(pulses); - checkSum(buff, true); - rval.add(strike); + ensureCheckSum(buff, true); + if (strike.getType() != LtgStrikeType.KEEP_ALIVE) { + rval.add(strike); + } } return rval; } @@ -244,6 +319,19 @@ public class TotalLightningDecoder { return TimeUtil.newGmtCalendar(new Date(totalMillis)); } + /** + * @see #passesCheckSum(ChecksumByteBuffer, boolean) + * @param buff + * @param total + * @throws DecoderException + */ + private static void ensureCheckSum(ChecksumByteBuffer buff, boolean total) + throws DecoderException { + if (!passesCheckSum(buff, total)) { + throw new DecoderException("Checksum failed"); + } + } + /** * Ensure data integrity, resets appropriate sum(s) in buffer after check * @@ -251,11 +339,9 @@ public class TotalLightningDecoder { * @param total * true if total sum should be checked, otherwise checks packet * sum - * @throws DecoderException - * if check fails + * @return true if checksum passes */ - private static void checkSum(ChecksumByteBuffer buff, boolean total) - throws DecoderException { + private static boolean passesCheckSum(ChecksumByteBuffer buff, boolean total) { long rawsum = total ? buff.getTotalSum() : buff.getPacketSum(); /* convert to overflowed unsigned byte */ rawsum &= 0xFF; @@ -263,8 +349,10 @@ public class TotalLightningDecoder { long mungedSum = (256 - rawsum) & 0xFF; /* get expected after sum so it is not reflected in sum */ long expected = UnsignedNumbers.ubyteToShort(buff.get()); - if (mungedSum != expected) { - throw new DecoderException("Checksum failed: expected " + expected + + boolean rval = mungedSum == expected; + if (!rval) { + log.debug("Checksum failed: expected " + expected + " got " + mungedSum); } if (total) { @@ -272,6 +360,7 @@ public class TotalLightningDecoder { } else { buff.resetPacketSum(); } + return rval; } /** diff --git a/ost/gov.noaa.nws.ost.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningAESKey.java b/ost/gov.noaa.nws.ost.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningAESKey.java index 75a70bd1ec..7bab813cb4 100755 --- a/ost/gov.noaa.nws.ost.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningAESKey.java +++ b/ost/gov.noaa.nws.ost.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/BinLightningAESKey.java @@ -21,9 +21,11 @@ import java.util.Date; import java.util.Enumeration; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Map.Entry; import java.util.Properties; import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -44,6 +46,7 @@ import com.raytheon.uf.common.status.UFStatus; * ------------ ---------- ----------- -------------------------- * 20130503 DCS 112 Wufeng Zhou To handle both the new encrypted data and legacy bit-shifted data * Jun 03, 2014 3226 bclement moved from com.raytheon.edex.plugin.binlightning to gov.noaa.nws.ost.edex.plugin.binlightning + * Jun 09, 2014 3226 bclement refactored to support multiple stores for different data types * * * @@ -57,144 +60,258 @@ public class BinLightningAESKey { /** System property name that can used to specify configuration property file, which will overwrite the default keystore location */ public static final String SYS_PROP_FOR_CONF_FILE = "binlightning.aeskeypropfile"; - public static final String KEYSTORE_PROP = "binlightning.AESKeystore"; - public static final String KEYSTORE_PASS_PROP = "binlightning.AESKeystorePassword"; + public static final String KEYSTORE_PROP_SUFFIX = ".AESKeystore"; + + public static final String KEYSTORE_PASS_PROP_SUFFIX = ".AESKeystorePassword"; + + public static final String CIPHER_ALGORITHM_SUFFIX = ".cipherAlgorithm"; + + public static final String DEFAULT_CIPHER_ALGORITHM = "AES"; + private static final String CONF_PROPERTIES_FILE = "BinLightningAESKey.properties"; public static final String KEY_ALIAS_PREFIX = "^\\d{4}-\\d{2}-\\d{2}"; private static final Pattern KEY_ALIAS_PREFIX_PATTERN = Pattern.compile(KEY_ALIAS_PREFIX); private static final SimpleDateFormat KEY_ALIAS_DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd"); - private static IUFStatusHandler logger = UFStatus + private static final IUFStatusHandler logger = UFStatus .getHandler(BinLightningAESKey.class); - private static Properties props = new Properties(); - private static KeyStore keystore; - private static BinLightningAESKey[] keys = null; - - /** - * Helper method to selectively get all the bin lightning related AES encryption keys, ordered by key issue date in descending order. - * Keys will be ignored when its alias is not starting with yyyy-MM-dd prefix or key algorithm is not "AES" - * - * If properties file is specified through system property binlightning.aeskeypropfile, then use it to load the properties. - * Otherwise, load the default property that is at the same place as this class, and overwrite properties - * if the property is specified through system property. - * So, binlightning.aeskeypropfile has higher priority, if it is specified, other properties specified in system property will be ignored - * - * Assumption: Valid key imported/stored to the keystore will have yyyy-MM-dd prefix in its alias. - * - * @return valid bin lightning AES keys (with aliases) in descending order of key issue date - * or null when no valid keys found - */ - public static BinLightningAESKey[] getBinLightningAESKeys() { - if (keys != null) return keys; + private static final Properties props = new Properties(); + private static volatile boolean propsLoaded = false; - // if properties file is specified through system property binlightning.aeskeypropfile, then use it to load the properties - // otherwise, use default property file and overwrite with available system properties - try { - if (System.getProperty(SYS_PROP_FOR_CONF_FILE, "").equals("") == false) { - File file = new File(System.getProperty(SYS_PROP_FOR_CONF_FILE)); - if (file.exists() == false) { - logger.error("System specified property file " + file.getAbsolutePath() + " does not exist."); - } else { - FileInputStream fis = new FileInputStream(file); - props.load(fis); - fis.close(); - } - } else { - // load default properties file - Properties defProps = new Properties(); - File file = new File(DEFAULT_KEYSTORE_LOC, CONF_PROPERTIES_FILE); - if (file.exists() == false) { - logger.error("Default properties file " + file.getAbsolutePath() + " does not exist."); - } else { - FileInputStream fis = new FileInputStream(file); - defProps.load(fis); - fis.close(); - } - props.putAll(defProps); - - // now check if the properties should be overwritten, if it is specified in system properties - Iterator iter = defProps.keySet().iterator(); - while (iter.hasNext()) { - String key = (String)iter.next(); - if (System.getProperty(key, "").equals("") == false) { - props.setProperty(key, System.getProperty(key)); - } - } - } - } catch (IOException ioe) { - logger.error("Fail to load BinLightningAESCipher configuration from file or system properties.", ioe); - } - - // load keystore - try { - if (props.getProperty(KEYSTORE_PROP, "").equals("") == false) { - File ksFile = new File(props.getProperty(KEYSTORE_PROP)); - keystore = KeyStore.getInstance("JCEKS"); // type JCEKS can store AES symmetric secret key, while default JKS store can't - FileInputStream fis = null; - try { - fis = new FileInputStream(ksFile); - char[] keystorePassword = null; - if (props.getProperty(KEYSTORE_PASS_PROP) != null) { - keystorePassword = props.getProperty(KEYSTORE_PASS_PROP).toCharArray(); - } - keystore.load(fis, keystorePassword); - } finally { - if (fis != null) fis.close(); - } - - Enumeration enu = keystore.aliases(); - TreeMap treeMap = new TreeMap(); - while (enu.hasMoreElements()) { - String alias = enu.nextElement(); - Matcher matcher = KEY_ALIAS_PREFIX_PATTERN.matcher(alias); - if (matcher.lookingAt()) { // alias starts with yyyy-MM-dd pattern - Key key = keystore.getKey(alias, props.getProperty(KEYSTORE_PASS_PROP).toCharArray()); - if (key.getAlgorithm().equals("AES")) { - // valid AES key for bin lightning decryption - treeMap.put(alias, key); - } - } - } - List keyListSortedByAliasDesc = new ArrayList(); - for (Entry entry = treeMap.pollLastEntry(); entry != null; entry = treeMap.pollLastEntry()) { - Date keyDate = KEY_ALIAS_DATE_FORMAT.parse(entry.getKey().substring(0, 10)); - BinLightningAESKey blkey = new BinLightningAESKey(entry.getKey(), entry.getValue(), keyDate); - keyListSortedByAliasDesc.add(blkey); - } - keys = keyListSortedByAliasDesc.toArray(new BinLightningAESKey[] {}); - return keys; - } else { - logger.error("binlightning.AESKeystore property not set."); - } - } catch (KeyStoreException kse) { - logger.error("Fail to getInstance of JCEKS keystore.", kse); - } catch (FileNotFoundException fnfe) { - logger.error("Fail to find the keystore file configured: " + props.getProperty(KEYSTORE_PROP), fnfe); - } catch (NoSuchAlgorithmException e) { - logger.error("NoSuchAlgorithmException in loading keystore from file: " + props.getProperty(KEYSTORE_PROP), e); - } catch (CertificateException e) { - logger.error("CertificateException in loading keystore from file: " + props.getProperty(KEYSTORE_PROP), e); - } catch (IOException e) { - logger.error("IOException in loading keystore from file: " + props.getProperty(KEYSTORE_PROP), e); - } catch (UnrecoverableKeyException e) { - logger.error("UnrecoverableKeyException in loading keystore from file: " + props.getProperty(KEYSTORE_PROP), e); - } catch (ParseException e) { - logger.error("ParseException in parsing alias for key date: " + props.getProperty(KEYSTORE_PROP), e); - } - return null; + private static KeyStore keystore; + + private static final Map keyCache = new ConcurrentHashMap( + 2); + + /** + * Helper method to selectively get all the bin lightning related AES + * encryption keys for a lightning type, ordered by key issue date in + * descending order. Keys will be ignored when its alias is not starting + * with yyyy-MM-dd prefix or key algorithm is not "AES" + * + * If properties file is specified through system property + * binlightning.aeskeypropfile, then use it to load the properties. + * Otherwise, load the default property that is at the same place as this + * class, and overwrite properties if the property is specified through + * system property. So, binlightning.aeskeypropfile has higher priority, if + * it is specified, other properties specified in system property will be + * ignored + * + * Assumption: Valid key imported/stored to the keystore will have + * yyyy-MM-dd prefix in its alias. + * + * @param propertyPrefix + * prefix for properties associated with a particular lightning + * data type + * @return valid bin lightning AES keys (with aliases) in descending order + * of key issue date or null when no valid keys found + */ + public static BinLightningAESKey[] getBinLightningAESKeys( + String propertyPrefix) { + BinLightningAESKey[] rval = keyCache.get(propertyPrefix); + if (rval == null) { + if (!propsLoaded) { + synchronized (props) { + if (!propsLoaded) { + loadProperties(); + } + } + } + synchronized (keyCache) { + rval = keyCache.get(propertyPrefix); + if (rval == null) { + rval = createAESKeys(propertyPrefix); + if (rval != null) { + keyCache.put(propertyPrefix, rval); + } + } + } + } + return rval; } + /** + * load properties files. If properties file is specified through system + * property binlightning.aeskeypropfile, then use it to load the properties. + * Otherwise, load the default property that is at the same place as this + * class, and overwrite properties if the property is specified through + * system property. So, binlightning.aeskeypropfile has higher priority, if + * it is specified, other properties specified in system property will be + * ignored + */ + private static void loadProperties() { + /* + * if properties file is specified through system property + * binlightning.aeskeypropfile, then use it to load the properties + * otherwise, use default property file and overwrite with available + * system properties + */ + try { + String confFileName = System.getProperty(SYS_PROP_FOR_CONF_FILE); + if (confFileName != null && !confFileName.isEmpty()) { + File file = new File(confFileName); + if (file.exists() == false) { + logger.error("System specified property file " + + file.getAbsolutePath() + " does not exist."); + } else { + FileInputStream fis = new FileInputStream(file); + props.load(fis); + fis.close(); + } + } else { + // load default properties file + Properties defProps = new Properties(); + File file = new File(DEFAULT_KEYSTORE_LOC, CONF_PROPERTIES_FILE); + if (file.exists() == false) { + logger.error("Default properties file " + + file.getAbsolutePath() + " does not exist."); + } else { + FileInputStream fis = new FileInputStream(file); + defProps.load(fis); + fis.close(); + } + props.putAll(defProps); + + /* + * now check if the properties should be overwritten, if it is + * specified in system properties + */ + Iterator iter = defProps.keySet().iterator(); + while (iter.hasNext()) { + String key = (String) iter.next(); + if (System.getProperty(key, "").equals("") == false) { + props.setProperty(key, System.getProperty(key)); + } + } + } + propsLoaded = true; + } catch (IOException ioe) { + logger.error( + "Fail to load BinLightningAESCipher configuration from file or system properties.", + ioe); + propsLoaded = false; + } + } + + /** + * Read all keys in keystore associated with datatype, ordered by key issue + * date in descending order. Keys will be ignored when its alias is not + * starting with yyyy-MM-dd prefix or key algorithm is not "AES" + * + * @param propertyPrefix + * prefix for properties associated with a particular lightning + * data type + * @return all keys sorted by + */ + private static BinLightningAESKey[] createAESKeys(String propertyPrefix) { + // load keystore + String keystoreProp = propertyPrefix + KEYSTORE_PROP_SUFFIX; + String keystorePassProp = propertyPrefix + KEYSTORE_PASS_PROP_SUFFIX; + BinLightningAESKey[] rval = null; + try { + String ksFileName = props.getProperty(keystoreProp); + if (ksFileName != null && !ksFileName.isEmpty()) { + File ksFile = new File(ksFileName); + /* + * type JCEKS can store AES symmetric secret key, while default + * JKS store can't + */ + keystore = KeyStore.getInstance("JCEKS"); + FileInputStream fis = null; + char[] keystorePassword = null; + try { + fis = new FileInputStream(ksFile); + if (props.getProperty(keystorePassProp) != null) { + keystorePassword = props.getProperty(keystorePassProp) + .toCharArray(); + } + keystore.load(fis, keystorePassword); + } finally { + if (fis != null) + fis.close(); + } + + Enumeration enu = keystore.aliases(); + TreeMap treeMap = new TreeMap(); + while (enu.hasMoreElements()) { + String alias = enu.nextElement(); + Matcher matcher = KEY_ALIAS_PREFIX_PATTERN.matcher(alias); + /* alias starts with yyyy-MM-dd pattern */ + if (matcher.lookingAt()) { + Key key = keystore.getKey(alias, keystorePassword); + if (key.getAlgorithm().equals("AES")) { + // valid AES key for bin lightning decryption + treeMap.put(alias, key); + } + } + } + List keyListSortedByAliasDesc = new ArrayList(); + for (Entry entry = treeMap.pollLastEntry(); entry != null; entry = treeMap + .pollLastEntry()) { + Date keyDate = KEY_ALIAS_DATE_FORMAT.parse(entry.getKey() + .substring(0, 10)); + BinLightningAESKey blkey = new BinLightningAESKey( + entry.getKey(), entry.getValue(), keyDate); + keyListSortedByAliasDesc.add(blkey); + } + rval = keyListSortedByAliasDesc + .toArray(new BinLightningAESKey[] {}); + } else { + logger.error("binlightning.AESKeystore property not set."); + } + } catch (KeyStoreException kse) { + logger.error("Fail to getInstance of JCEKS keystore.", kse); + } catch (FileNotFoundException fnfe) { + logger.error( + "Fail to find the keystore file configured: " + + props.getProperty(keystoreProp), fnfe); + } catch (NoSuchAlgorithmException e) { + logger.error( + "NoSuchAlgorithmException in loading keystore from file: " + + props.getProperty(keystoreProp), e); + } catch (CertificateException e) { + logger.error("CertificateException in loading keystore from file: " + + props.getProperty(keystoreProp), e); + } catch (IOException e) { + logger.error( + "IOException in loading keystore from file: " + + props.getProperty(keystoreProp), e); + } catch (UnrecoverableKeyException e) { + logger.error( + "UnrecoverableKeyException in loading keystore from file: " + + props.getProperty(keystoreProp), e); + } catch (ParseException e) { + logger.error("ParseException in parsing alias for key date: " + + props.getProperty(keystoreProp), e); + } + return rval; + } + /** * force to reload keys, useful for testing * @return */ - public static BinLightningAESKey[] reloadBinLightningAESKeys() { - if (keys != null) keys = null; - return getBinLightningAESKeys(); + public static BinLightningAESKey[] reloadBinLightningAESKeys( + String propertyPrefix) { + keyCache.clear(); + propsLoaded = false; + return getBinLightningAESKeys(propertyPrefix); } + /** + * Get cipher algorithm name for data type + * + * @param propertyPrefix + * prefix for properties associated with a particular lightning + * data type + * @return default algorithm if no property is found for prefix + */ + public static String getCipherAlgorithm(String propertyPrefix) { + return props.getProperty(propertyPrefix + CIPHER_ALGORITHM_SUFFIX, + DEFAULT_CIPHER_ALGORITHM); + } private String alias; private Key key; diff --git a/ost/gov.noaa.nws.ost.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/EncryptedBinLightningCipher.java b/ost/gov.noaa.nws.ost.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/EncryptedBinLightningCipher.java index 8e482f2de9..04fc756759 100755 --- a/ost/gov.noaa.nws.ost.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/EncryptedBinLightningCipher.java +++ b/ost/gov.noaa.nws.ost.edex.plugin.binlightning/src/gov/noaa/nws/ost/edex/plugin/binlightning/EncryptedBinLightningCipher.java @@ -10,6 +10,8 @@ import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import javax.crypto.BadPaddingException; import javax.crypto.Cipher; @@ -34,6 +36,7 @@ import com.raytheon.uf.common.status.UFStatus; * 20130503 DCS 112 Wufeng Zhou To handle both the new encrypted data and legacy bit-shifted data * Jun 03, 2014 3226 bclement moved from com.raytheon.edex.plugin.binlightning to gov.noaa.nws.ost.edex.plugin.binlightning * handled null return from BinLightningAESKey.getBinLightningAESKeys() + * Jun 09, 2014 3226 bclement refactored to support multiple stores for different data types * * * @@ -41,39 +44,69 @@ import com.raytheon.uf.common.status.UFStatus; * */ public class EncryptedBinLightningCipher { - private static final String BINLIGHTNING_CIPHER_TYPE = "AES"; /** Maximum size of the encrypted block, determined by 3 byte length field in the header */ private static final int MAX_SIZE_ENCRYPTED_BLOCK = 0xffffff; - /** - * Cipher creation is a relatively expensive operation and would be better to reuse it in the same thread. - **/ - private static final ThreadLocal> decryptCipherMap = new ThreadLocal>() { - - @Override - protected HashMap initialValue() { - // get AES keys from keystore and create encryption and decryption ciphers from them - BinLightningAESKey[] keys = BinLightningAESKey.getBinLightningAESKeys(); - if (keys == null) { - keys = new BinLightningAESKey[0]; - } - HashMap cipherMap = new HashMap(); - for (BinLightningAESKey key : keys) { - try { - SecretKeySpec skeySpec = (SecretKeySpec)key.getKey(); - Cipher cipher = Cipher.getInstance(BINLIGHTNING_CIPHER_TYPE); - cipher.init(Cipher.DECRYPT_MODE, skeySpec); - - cipherMap.put(key.getAlias(), cipher); - } catch (Exception e) { - logger.error("Fail to create decrypt Cipher from key " + key.getAlias(), e); - } - } - return cipherMap; - } - }; + private static final Map> decryptCipherMapCache = new ConcurrentHashMap>( + 2); + /** + * Get cipher map using cache + * + * @param propertyPrefix + * datatype properties file prefix + * @return + */ + private static Map getCachedCipherMap(String propertyPrefix) { + Map rval = decryptCipherMapCache.get(propertyPrefix); + if (rval == null) { + synchronized (decryptCipherMapCache) { + if (rval == null) { + rval = createCipherMap(propertyPrefix); + decryptCipherMapCache.put(propertyPrefix, rval); + } + } + } + return rval; + } + + /** + * Create a mapping key aliases to keys for datatype + * + * @param propertyPrefix + * datatype properties file prefix + * @return + */ + private static Map createCipherMap(String propertyPrefix) { + /* + * get AES keys from keystore and create encryption and decryption + * ciphers from them + */ + BinLightningAESKey[] keys = BinLightningAESKey + .getBinLightningAESKeys(propertyPrefix); + if (keys == null) { + keys = new BinLightningAESKey[0]; + } + HashMap cipherMap = new HashMap(); + for (BinLightningAESKey key : keys) { + try { + SecretKeySpec skeySpec = (SecretKeySpec) key.getKey(); + String algorithm = BinLightningAESKey + .getCipherAlgorithm(propertyPrefix); + Cipher cipher = Cipher.getInstance(algorithm); + cipher.init(Cipher.DECRYPT_MODE, skeySpec); + + cipherMap.put(key.getAlias(), cipher); + } catch (Exception e) { + logger.error( + "Fail to create decrypt Cipher from key " + + key.getAlias(), e); + } + } + return cipherMap; + } + private static IUFStatusHandler logger = UFStatus .getHandler(EncryptedBinLightningCipher.class); @@ -85,24 +118,33 @@ public class EncryptedBinLightningCipher { * decrypt data with AES keys * * @param data + * @param propertyPrefix + * prefix for lightning type configuration * @return * @throws IllegalBlockSizeException * @throws BadPaddingException */ - public byte[] decryptData(byte[] data) throws IllegalBlockSizeException, BadPaddingException, BinLightningDataDecryptionException { - return decryptData(data, null); + public byte[] decryptData(byte[] data, String propertyPrefix) + throws IllegalBlockSizeException, BadPaddingException, + BinLightningDataDecryptionException { + return decryptData(data, null, propertyPrefix); } - /** - * decrypt data with AES keys, using data observation date as a hint to find the best suitable key to try first - * - * @param data - * @param dataDate - * @return - * @throws IllegalBlockSizeException - * @throws BadPaddingException - */ - public byte[] decryptData(byte[] data, Date dataDate) throws IllegalBlockSizeException, BadPaddingException, BinLightningDataDecryptionException { + /** + * decrypt data with AES keys, using data observation date as a hint to find + * the best suitable key to try first + * + * @param data + * @param dataDate + * @param propertyPrefix + * prefix for lightning type configuration + * @return + * @throws IllegalBlockSizeException + * @throws BadPaddingException + */ + public byte[] decryptData(byte[] data, Date dataDate, String propertyPrefix) + throws IllegalBlockSizeException, BadPaddingException, + BinLightningDataDecryptionException { if (data == null) { throw new IllegalBlockSizeException("Data is null"); } @@ -110,12 +152,14 @@ public class EncryptedBinLightningCipher { throw new IllegalBlockSizeException("Data is empty"); } if (data.length > MAX_SIZE_ENCRYPTED_BLOCK) { - throw new IllegalBlockSizeException("Block size exceeds maxinum expected."); + throw new IllegalBlockSizeException( + "Block size exceeds maximum expected."); } - HashMap cipherMap = EncryptedBinLightningCipher.decryptCipherMap.get(); + Map cipherMap = getCachedCipherMap(propertyPrefix); // find the preferred key order to try decryption based on data date - List preferredKeyList = findPreferredKeyOrderForData(dataDate); + List preferredKeyList = findPreferredKeyOrderForData( + dataDate, propertyPrefix); if (preferredKeyList == null || preferredKeyList.size() == 0) { throw new BinLightningDataDecryptionException("No AES key found to decrypt data. Please make sure keystore is properly configured with key(s)."); @@ -124,7 +168,12 @@ public class EncryptedBinLightningCipher { // try to decrypt the data using ciphers in the list until successful byte[] decryptedData = null; for (int i = 0; i < preferredKeyList.size(); i++) { - Cipher cipher = cipherMap.get(preferredKeyList.get(i).getAlias()); + String alias = preferredKeyList.get(i).getAlias(); + Cipher cipher = cipherMap.get(alias); + if (cipher == null) { + logger.warn("No cipher found for alias: " + alias); + continue; + } try { decryptedData = cipher.doFinal(data, 0, data.length); @@ -133,45 +182,63 @@ public class EncryptedBinLightningCipher { if ( BinLightningDecoderUtil.isKeepAliveRecord(decryptedData) == false && BinLightningDecoderUtil.isLightningDataRecords(decryptedData) == false) { //if (BinLigntningDecoderUtil.isValidMixedRecordData(decryptedData) == false) { // use this only if keep-alive record could be mixed with lightning records throw new BinLightningDataDecryptionException("Decrypted data (" + decryptedData.length + " bytes) with key " - + preferredKeyList.get(i).getAlias() + " is not valid keep-alive or binLightning records.", decryptedData); + + alias + + " is not valid keep-alive or binLightning records.", + decryptedData); } - logger.info("Data (" + data.length + " bytes) decrypted to " + decryptedData.length + " bytes with key: " + preferredKeyList.get(i).getAlias()); + logger.info("Data (" + data.length + " bytes) decrypted to " + + decryptedData.length + " bytes with key: " + alias); break; // decrypt ok, break out } catch (IllegalBlockSizeException e) { // ignore exception if not the last, and try next cipher - logger.info("Fail to decrypt data (" + data.length + " bytes) with key: " + preferredKeyList.get(i).getAlias() + " - " + e.getMessage() + ", will try other available key"); + logger.info("Fail to decrypt data (" + data.length + + " bytes) with key: " + alias + " - " + e.getMessage() + + ", will try other available key"); if (i == (preferredKeyList.size() - 1)) { logger.error("Fail to decrypt with all known keys, either data is not encrypted or is invalid: " + e.getMessage()); throw e; } } catch (BadPaddingException e) { // ignore exception if not the last, and try next cipher - logger.info("Fail to decrypt data (" + data.length + " bytes) with key: " + preferredKeyList.get(i).getAlias() + " - " + e.getMessage() + ", will try other available key"); + logger.info("Fail to decrypt data (" + data.length + + " bytes) with key: " + alias + " - " + e.getMessage() + + ", will try other available key"); if (i == (preferredKeyList.size() - 1)) { logger.error("Fail to decrypt with all known keys, either data is not encrypted or is invalid: " + e.getMessage()); throw e; } } catch (BinLightningDataDecryptionException e) { // ignore exception if not the last, and try next cipher - logger.info("Fail to decrypt data (" + data.length + " bytes) with key: " + preferredKeyList.get(i).getAlias() + " - " + e.getMessage() + ", will try other available key"); + logger.info("Fail to decrypt data (" + data.length + + " bytes) with key: " + alias + " - " + e.getMessage() + + ", will try other available key"); if (i == (preferredKeyList.size() - 1)) { logger.error("Fail to decrypt with all known keys, either data is not encrypted or is invalid: " + e.getMessage()); throw e; } } } + if (decryptedData == null) { + throw new BinLightningDataDecryptionException( + "No ciphers found for data type: " + propertyPrefix); + } return decryptedData; } - /** - * Assuming the best keys to decrypt data should be issued before the data observation date, so - * if there were many keys issued, this hopefully will reduce the unnecessary decryption tries - * - * @param dataDate - * @return preferred key list order - */ - private List findPreferredKeyOrderForData(Date dataDate) { - BinLightningAESKey[] binLightningAESKeys = BinLightningAESKey.getBinLightningAESKeys(); + /** + * Assuming the best keys to decrypt data should be issued before the data + * observation date, so if there were many keys issued, this hopefully will + * reduce the unnecessary decryption tries + * + * @param dataDate + * @param propertyPrefix + * prefix for lightning type configuration + * @return preferred key list order + */ + private List findPreferredKeyOrderForData( + Date dataDate, String propertyPrefix) { + BinLightningAESKey[] binLightningAESKeys = BinLightningAESKey + .getBinLightningAESKeys(propertyPrefix); if (binLightningAESKeys == null || binLightningAESKeys.length < 1) { return Collections.emptyList(); } @@ -198,4 +265,41 @@ public class EncryptedBinLightningCipher { } } + /** + * ensures that the data is the appropriate length for AES decryption. + * Copies data to new array. + * + * @param pdata + * @param traceId + * @return + */ + public static byte[] prepDataForDecryption(byte[] pdata, String traceId) { + /* + * NOTE: 11/14/2013 WZ: + * encrypted test data on TNCF (got from Melissa Porricelli) + * seems to have extra 4 bytes (0x0d 0x0d 0x0a 0x03) at the end, + * making the data size not a multiple of 16. However, original + * test data do not have this trailing bytes. while NCEP test + * data has extra 8 trailing bytes. + * Brain Rapp's email on 11/13/2013 confirms that Unidata LDM + * software used by AWIPS II will strips off all SBN protocol + * headers + * that precede the WMO header and adds its own 11 byte header + * like this: "soh cr cr nl 2 5 4 sp cr cr nl". It + * also adds a four byte trailer consisting of "cr cr nl etx" + * (0x0d 0x0d 0x0a 0x03) + * So, it seems necessary to trim trailing bytes if it is not + * multiple of 16, warning messages will be logged though + */ + int dataLengthToBeDecrypted = pdata.length; + if (pdata.length % 16 != 0) { + dataLengthToBeDecrypted = pdata.length - (pdata.length % 16); + logger.warn(traceId + " - Data length from file " + traceId + + " is " + pdata.length + " bytes, trailing " + + (pdata.length - dataLengthToBeDecrypted) + + " bytes has been trimmed to " + dataLengthToBeDecrypted + + " bytes for decryption."); + } + return Arrays.copyOfRange(pdata, 0, dataLengthToBeDecrypted); + } }