From 07b48207a260e4daacf5adc7cc9e3f5407d9ee1e Mon Sep 17 00:00:00 2001 From: Brian Clements Date: Thu, 11 Sep 2014 11:52:23 -0500 Subject: [PATCH] Omaha #3608 improved error handling in lightning frame retriever Change-Id: I02784a7f2d23bf37d56294e55cb9ec759d1f9080 Former-commit-id: b9dd6d798bc31b6fca2e78ff731e935ec1e2625d [formerly 0551b91b995c893fddee52cabdfbdccf83121a31] Former-commit-id: c56645dd015c0999e95aa716fc6375c5a5a80426 --- .../cache/LightningFrameRetriever.java | 145 ++++++++++++------ 1 file changed, 101 insertions(+), 44 deletions(-) diff --git a/cave/com.raytheon.viz.lightning/src/com/raytheon/viz/lightning/cache/LightningFrameRetriever.java b/cave/com.raytheon.viz.lightning/src/com/raytheon/viz/lightning/cache/LightningFrameRetriever.java index 2af8c809f3..90616f59a9 100644 --- a/cave/com.raytheon.viz.lightning/src/com/raytheon/viz/lightning/cache/LightningFrameRetriever.java +++ b/cave/com.raytheon.viz.lightning/src/com/raytheon/viz/lightning/cache/LightningFrameRetriever.java @@ -26,6 +26,7 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import com.raytheon.uf.common.dataplugin.HDF5Util; import com.raytheon.uf.common.dataplugin.annotations.DataURI; @@ -60,6 +61,7 @@ import com.raytheon.uf.viz.core.cache.CacheObject.IObjectRetrieverAndDisposer; * ------------ ---------- ----------- -------------------------- * Jul 09, 2014 3333 bclement moved from LightningResource * Jul 22, 2014 3214 bclement fixed typos in populatePulseData() and updateAndGet() + * Sep 11, 2014 3608 bclement index records by group and dataset name for better error handling * * * @@ -216,24 +218,40 @@ public class LightningFrameRetriever implements long tDS1 = System.currentTimeMillis(); dsTime += (tDS1 - tDS0); // Throw in a map for easy accessibility - Map> recordMap = createRecordMap(records); + Map> groupedRecords = createRecordMap(records); - List times = recordMap - .get(LightningConstants.TIME_DATASET); - List intensities = recordMap - .get(LightningConstants.INTENSITY_DATASET); - List lats = recordMap - .get(LightningConstants.LAT_DATASET); - List lons = recordMap - .get(LightningConstants.LON_DATASET); - List types = recordMap - .get(LightningConstants.STRIKE_TYPE_DATASET); - List pulseIndexes = recordMap - .get(LightningConstants.PULSE_INDEX_DATASET); + for (Entry> entry : groupedRecords + .entrySet()) { + Map recordMap = entry.getValue(); - int k = 0; - for (IDataRecord timeRec : times) { - if (hasPulseData(pulseIndexes, k)) { + IDataRecord timeRec = recordMap + .get(LightningConstants.TIME_DATASET); + IDataRecord intensities = recordMap + .get(LightningConstants.INTENSITY_DATASET); + IDataRecord lats = recordMap + .get(LightningConstants.LAT_DATASET); + IDataRecord lons = recordMap + .get(LightningConstants.LON_DATASET); + IDataRecord types = recordMap + .get(LightningConstants.STRIKE_TYPE_DATASET); + IDataRecord pulseIndexes = recordMap + .get(LightningConstants.PULSE_INDEX_DATASET); + + if (timeRec == null || intensities == null || lats == null + || lons == null || types == null + || pulseIndexes == null) { + List missing = getMissingDatasets(recordMap); + statusHandler.error("Group '" + entry.getKey() + + "' missing dataset(s): " + missing); + continue; + } + if (!allSameLength(timeRec, intensities, lats, lons, types, + pulseIndexes)) { + statusHandler.error("Group '" + entry.getKey() + + "' has mismatched dataset lengths"); + continue; + } + if (hasPulseData(pulseIndexes)) { populatePulseData(frame, bundle, timeRec.getGroup(), ds); } LongDataRecord time = (LongDataRecord) timeRec; @@ -243,14 +261,13 @@ public class LightningFrameRetriever implements long[] timeData = time.getLongData(); - int[] intensityData = ((IntegerDataRecord) intensities - .get(k)).getIntData(); - float[] latitudeData = ((FloatDataRecord) lats.get(k)) + int[] intensityData = ((IntegerDataRecord) intensities) + .getIntData(); + float[] latitudeData = ((FloatDataRecord) lats) .getFloatData(); - float[] longitudeData = ((FloatDataRecord) lons.get(k)) + float[] longitudeData = ((FloatDataRecord) lons) .getFloatData(); - byte[] typeData = ((ByteDataRecord) types.get(k)) - .getByteData(); + byte[] typeData = ((ByteDataRecord) types).getByteData(); for (int i = 0; i < numRecords; i++) { @@ -282,7 +299,6 @@ public class LightningFrameRetriever implements } } - k++; } } catch (StorageException e) { statusHandler.handle(Priority.PROBLEM, @@ -301,23 +317,69 @@ public class LightningFrameRetriever implements } /** - * Unpack records into map keyed by record name + * @param records + * @return true if all the records are the same length + */ + private static boolean allSameLength(IDataRecord... records) { + if (records.length == 0) { + return true; + } + boolean rval = true; + long size = records[0].getSizes()[0]; + for (IDataRecord rec : records) { + if (rec.getSizes()[0] != size) { + rval = false; + break; + } + } + return rval; + } + + /** + * @param recordMap + * @return list of lightning datasets that are not in map + */ + private static List getMissingDatasets( + Map recordMap) { + String[] datasets = { LightningConstants.TIME_DATASET, + LightningConstants.INTENSITY_DATASET, + LightningConstants.LAT_DATASET, LightningConstants.LON_DATASET, + LightningConstants.STRIKE_TYPE_DATASET, + LightningConstants.PULSE_INDEX_DATASET }; + List rval = new ArrayList<>(datasets.length); + for (String ds : datasets) { + if (!recordMap.containsKey(ds)) { + rval.add(ds); + } + } + return rval; + } + + /** + * Index records first by datastore group then by dataset name * * @param records * @return */ - public static Map> createRecordMap( + public static Map> createRecordMap( IDataRecord[] records) { - Map> recordMap = new HashMap>(); + Map> rval = new HashMap>(); for (IDataRecord rec : records) { - List recordList = recordMap.get(rec.getName()); - if (recordList == null) { - recordList = new ArrayList(); - recordMap.put(rec.getName(), recordList); + String group = rec.getGroup(); + String name = rec.getName(); + Map datasets = rval.get(group); + if (datasets == null) { + datasets = new HashMap<>(); + rval.put(group, datasets); + } + IDataRecord prevEntry = datasets.put(name, rec); + if (prevEntry != null) { + /* this should never happen */ + statusHandler.warn("Group '" + group + + "' hash multiple datasets with name " + name); } - recordList.add(rec); } - return recordMap; + return rval; } /** @@ -339,19 +401,14 @@ public class LightningFrameRetriever implements } /** - * @param pulseIndexes - * @param recordIndex - * @return true if any data record in list has a valid pulse index + * @param record + * @return true if data record has a valid pulse index */ - private static boolean hasPulseData(List pulseIndexes, - int recordIndex) { - if (pulseIndexes != null) { - IDataRecord record = pulseIndexes.get(recordIndex); - int[] indexData = ((IntegerDataRecord) record).getIntData(); - for (int i = 0; i < indexData.length; ++i) { - if (indexData[i] >= 0) { - return true; - } + private static boolean hasPulseData(IDataRecord record) { + int[] indexData = ((IntegerDataRecord) record).getIntData(); + for (int i = 0; i < indexData.length; ++i) { + if (indexData[i] >= 0) { + return true; } } return false;