From 17f5a8b3bae7ff5213c127f3331245b63887f1f5 Mon Sep 17 00:00:00 2001 From: Brian Clements Date: Mon, 4 Aug 2014 14:53:42 -0500 Subject: [PATCH] Issue #3488 added bin range check to bin lightning decoder Change-Id: Ib445da1a33c84283bc8dd46521b6f7c3381a51f9 Former-commit-id: 5a11e28b11d5bc661b02a9b55ab00a19b010051a [formerly ba934c5411a141ce607cb49cfc0bd2545febd1bb] [formerly 8d928ab3a25c32016eb6287d407c05a4d3bc4f70] [formerly 432f43b56795b2c849339ed60d1637b5357c7237 [formerly 8d928ab3a25c32016eb6287d407c05a4d3bc4f70 [formerly 53b274aab9952d372221c686d886ac12e8eb2e6a]]] Former-commit-id: 432f43b56795b2c849339ed60d1637b5357c7237 Former-commit-id: 855189eef7bc4af2533a751f5d0dfc5daf3f4230 [formerly 61592f0e5a2d3d51228576f436f36917473b6bc9] Former-commit-id: 0455c9082035ce57f0af0b0cd5f79c10b55aab6b --- .../binlightning/BinLightningDecoder.java | 177 ++++++++++++++---- 1 file changed, 139 insertions(+), 38 deletions(-) diff --git a/edexOsgi/com.raytheon.edex.plugin.binlightning/src/com/raytheon/edex/plugin/binlightning/BinLightningDecoder.java b/edexOsgi/com.raytheon.edex.plugin.binlightning/src/com/raytheon/edex/plugin/binlightning/BinLightningDecoder.java index 7c7b0756bc..68a2b8c219 100644 --- a/edexOsgi/com.raytheon.edex.plugin.binlightning/src/com/raytheon/edex/plugin/binlightning/BinLightningDecoder.java +++ b/edexOsgi/com.raytheon.edex.plugin.binlightning/src/com/raytheon/edex/plugin/binlightning/BinLightningDecoder.java @@ -22,8 +22,14 @@ package com.raytheon.edex.plugin.binlightning; import gov.noaa.nws.ost.edex.plugin.binlightning.BinLigntningDecoderUtil; import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Calendar; -import java.util.List; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; import java.util.TimeZone; import org.apache.commons.logging.Log; @@ -33,12 +39,12 @@ import com.raytheon.edex.esb.Headers; import com.raytheon.edex.exception.DecoderException; import com.raytheon.edex.plugin.AbstractDecoder; import com.raytheon.uf.common.dataplugin.PluginDataObject; -import com.raytheon.uf.common.dataplugin.PluginException; import com.raytheon.uf.common.dataplugin.binlightning.BinLightningRecord; import com.raytheon.uf.common.dataplugin.binlightning.impl.LightningStrikePoint; import com.raytheon.uf.common.dataplugin.binlightning.impl.LtgStrikeType; import com.raytheon.uf.common.time.DataTime; import com.raytheon.uf.common.time.TimeRange; +import com.raytheon.uf.common.time.util.TimeUtil; import com.raytheon.uf.edex.decodertools.core.DecoderTools; import com.raytheon.uf.edex.decodertools.time.TimeTools; import com.raytheon.uf.edex.wmo.message.WMOHeader; @@ -82,6 +88,7 @@ import com.raytheon.uf.edex.wmo.message.WMOHeader; * Jan 24, 2014 DR 16774 Wufeng Zhou Modified for updated Bin-lightning data spec, * and to used WMO header to distinguish bit-shifted * GLD360 and NLDN data. + * Aug 04, 2014 3488 bclement added checkBinRange(), rebin() and finalizeRecords() * * * @@ -100,6 +107,9 @@ public class BinLightningDecoder extends AbstractDecoder { private final Log logger = LogFactory.getLog(getClass()); + private static final boolean REBIN_INVALID_DATA = Boolean + .getBoolean("rebin.invalid.binlightning"); + /** * Default lightning strike type for FLASH messages. RT_FLASH documents * indicate no default, but D2D code defaults to STRIKE_CG also. @@ -127,7 +137,7 @@ public class BinLightningDecoder extends AbstractDecoder { public PluginDataObject[] decode(byte[] data, Headers headers) throws DecoderException { //String traceId = null; - PluginDataObject[] reports = new PluginDataObject[0]; + PluginDataObject[] rval = new PluginDataObject[0]; if (data != null) { traceId = (String) headers.get(DecoderTools.INGEST_FILE_NAME); @@ -163,11 +173,13 @@ public class BinLightningDecoder extends AbstractDecoder { // both encrypted data and legacy data // - List strikes = BinLigntningDecoderUtil.decodeBinLightningData(data, pdata, traceId, wmoHdr, baseTime.getTime()); + Collection strikes = BinLigntningDecoderUtil + .decodeBinLightningData(data, pdata, traceId, wmoHdr, + baseTime.getTime()); if (strikes == null) { // keep-alive record, log and return logger.info(traceId + " - found keep-alive record. ignore for now."); - return reports; + return rval; } // @@ -186,44 +198,133 @@ public class BinLightningDecoder extends AbstractDecoder { return new PluginDataObject[0]; } - Calendar c = TimeTools.copy(baseTime); - if (c == null) { - throw new DecoderException(traceId + " - Error decoding times"); - } //report.setInsertTime(c); // OB13.4 source code does not have this line anymore, WZ 05/03/2013 - - Calendar cStart = report.getStartTime(); - if (cStart.getTimeInMillis() > (c.getTimeInMillis() + TEN_MINUTES)) { - synchronized (SDF) { - logger.info("Discarding future data for " + traceId - + " at " + SDF.format(cStart.getTime())); - } - } else { - Calendar cStop = report.getStopTime(); - - TimeRange range = new TimeRange(cStart.getTimeInMillis(), - cStop.getTimeInMillis()); - - DataTime dataTime = new DataTime(cStart, range); - report.setDataTime(dataTime); - - if (report != null) { - report.setTraceId(traceId); - //report.setPluginName("binlightning"); // line disappear in OB15.5.3 - try { - report.constructDataURI(); - reports = new PluginDataObject[] { report }; - } catch (PluginException e) { - logger.error("Error constructing datauri", e); - throw new DecoderException("Error constructing datauri", e); - } - } - } + Collection records = checkBinRange(report, + strikes); + rval = finalizeRecords(records, baseTime); } } else { logger.error("No WMOHeader found in data"); } - return reports; + return rval; + } + + /** + * Perform final actions on each record and populate a PDO array with them. + * Any invalid records will be omitted from the return array. + * + * @param records + * @param baseTime + * @return + * @throws DecoderException + */ + private PluginDataObject[] finalizeRecords( + Collection records, Calendar baseTime) + throws DecoderException { + Calendar c = TimeTools.copy(baseTime); + if (c == null) { + throw new DecoderException(traceId + " - Error decoding times"); + } + ArrayList rval = new ArrayList( + records.size()); + for (BinLightningRecord record : records) { + Calendar cStart = record.getStartTime(); + if (cStart.getTimeInMillis() > (c.getTimeInMillis() + TEN_MINUTES)) { + synchronized (SDF) { + logger.info("Discarding future data for " + traceId + + " at " + SDF.format(cStart.getTime())); + } + } else { + Calendar cStop = record.getStopTime(); + + TimeRange range = new TimeRange(cStart.getTimeInMillis(), + cStop.getTimeInMillis()); + + DataTime dataTime = new DataTime(cStart, range); + record.setDataTime(dataTime); + + if (record != null) { + record.setTraceId(traceId); + rval.add(record); + } + } + } + return rval.toArray(new PluginDataObject[rval.size()]); + } + + /** + * Ensure that the record has a valid bin range. If it does, it will be the + * only record in the return value. Otherwise, {@link #REBIN_INVALID_DATA} + * is used to determine if no records should be returned or the strikes + * should be split into valid bin ranges uses {@link #rebin(Collection)} + * + * @param record + * @param strikes + * @return + */ + private Collection checkBinRange( + BinLightningRecord record, Collection strikes) { + Collection rval = Collections.emptyList(); + Calendar cStart = record.getStartTime(); + Calendar cStop = record.getStopTime(); + long binRange = cStop.getTimeInMillis() - cStart.getTimeInMillis(); + if (binRange > TimeUtil.MILLIS_PER_DAY) { + if (REBIN_INVALID_DATA) { + rval = rebin(strikes); + } else { + String rangeStart; + String rangeEnd; + synchronized (SDF) { + rangeStart = SDF.format(cStart.getTime()); + rangeEnd = SDF.format(cStop.getTime()); + } + logger.error("Discarding data with invalid bin range of " + + rangeStart + " to " + rangeEnd); + } + } else { + rval = Arrays.asList(record); + } + return rval; + } + + /** + * Split the strikes into 1 day bins and create a new record for each bin + * + * @param strikes + * @return + */ + private Collection rebin( + Collection strikes) { + Map> binMap = new HashMap>( + 1); + for (LightningStrikePoint strike : strikes) { + Calendar c = TimeTools.getBaseCalendar(strike.getYear(), + strike.getMonth(), strike.getDay()); + c.set(Calendar.HOUR_OF_DAY, 0); + c.set(Calendar.MINUTE, 0); + c.set(Calendar.SECOND, 0); + c.set(Calendar.MILLISECOND, 0); + long key = c.getTimeInMillis(); + Collection bin = binMap.get(key); + if (bin == null) { + bin = new ArrayList(strikes.size()); + binMap.put(key, bin); + } + bin.add(strike); + } + Collection rval = new ArrayList( + binMap.size()); + for (Entry> e : binMap + .entrySet()) { + Collection bin = e.getValue(); + BinLightningRecord record = new BinLightningRecord(bin.size()); + for (LightningStrikePoint strike : bin) { + record.addStrike(strike); + } + rval.add(record); + } + + return rval; } /**