From 402bef9c5b0595b20413707db1cf904647307cf1 Mon Sep 17 00:00:00 2001 From: Brian Clements Date: Wed, 22 Jan 2014 15:32:19 -0600 Subject: [PATCH] Issue #2667 added bin lightning data access factory renamed lightning record's lightSource field to source added delta script for db column change added attribute method comments for IData and DefaultGeometryData Former-commit-id: b6c50729919ea8a9b29ca963e18f68268c3a0b7f --- .../viz/lightning/LightningResource.java | 3 +- deltaScripts/14.3.1/renameLightningSource.sh | 9 + .../raytheon/uf/common/dataaccess/IData.java | 7 +- .../dataaccess/impl/DefaultGeometryData.java | 14 + .../uf/common/dataaccess/util/PDOUtil.java | 22 +- .../META-INF/MANIFEST.MF | 9 +- .../build.properties | 3 +- .../spring/binlightning-common-dataaccess.xml | 12 + .../binlightning/BinLightningRecord.java | 25 +- .../dataaccess/BinLightingAccessFactory.java | 344 ++++++++++++++++++ 10 files changed, 427 insertions(+), 21 deletions(-) create mode 100755 deltaScripts/14.3.1/renameLightningSource.sh create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/res/spring/binlightning-common-dataaccess.xml create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/src/com/raytheon/uf/common/dataplugin/binlightning/dataaccess/BinLightingAccessFactory.java diff --git a/cave/com.raytheon.viz.lightning/src/com/raytheon/viz/lightning/LightningResource.java b/cave/com.raytheon.viz.lightning/src/com/raytheon/viz/lightning/LightningResource.java index 9604088b04..56fd64050b 100644 --- a/cave/com.raytheon.viz.lightning/src/com/raytheon/viz/lightning/LightningResource.java +++ b/cave/com.raytheon.viz.lightning/src/com/raytheon/viz/lightning/LightningResource.java @@ -93,6 +93,7 @@ import com.raytheon.uf.viz.core.rsc.capabilities.MagnificationCapability; * Sep 4, 2012 15335 kshresth Will now display lightning/wind * fields when magnification set to 0 * Feb 27, 2013 DCS 152 jgerth/elau Support for WWLLN and multiple sources + * Jan 21, 2014 2667 bclement renamed record's lightSource field to source * * * @@ -487,7 +488,7 @@ public class LightningResource extends Map> recordMap = new HashMap>(); for (BinLightningRecord obj : objs) { - if (obj.getLightSource().equals(this.lightSource) || this.lightSource.isEmpty()) { + if (obj.getSource().equals(this.lightSource) || this.lightSource.isEmpty()) { DataTime time = new DataTime(obj.getStartTime()); DataTime end = new DataTime(obj.getStopTime()); time = this.getResourceData().getBinOffset() diff --git a/deltaScripts/14.3.1/renameLightningSource.sh b/deltaScripts/14.3.1/renameLightningSource.sh new file mode 100755 index 0000000000..c3003a5155 --- /dev/null +++ b/deltaScripts/14.3.1/renameLightningSource.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# DR #2667 Add binlightning support to Data Access Framework + +PSQL="/awips2/psql/bin/psql" + +SQL_COMMAND=" +ALTER TABLE binlightning RENAME COLUMN lightsource TO source; +" +${PSQL} -U awips -d metadata -c "${SQL_COMMAND}" diff --git a/edexOsgi/com.raytheon.uf.common.dataaccess/src/com/raytheon/uf/common/dataaccess/IData.java b/edexOsgi/com.raytheon.uf.common.dataaccess/src/com/raytheon/uf/common/dataaccess/IData.java index d17e4ecf7e..53a32daabe 100644 --- a/edexOsgi/com.raytheon.uf.common.dataaccess/src/com/raytheon/uf/common/dataaccess/IData.java +++ b/edexOsgi/com.raytheon.uf.common.dataaccess/src/com/raytheon/uf/common/dataaccess/IData.java @@ -36,6 +36,7 @@ import com.raytheon.uf.common.time.DataTime; * ------------ ---------- ----------- -------------------------- * Oct 10, 2012 njensen Initial creation * Jun 03, 2013 #2023 dgilling Add getAttributes(). + * Jan 21, 2014 2667 bclement attribute method comments * * * @@ -46,7 +47,8 @@ import com.raytheon.uf.common.time.DataTime; public interface IData { /** - * Gets an attribute of the data based on the key. + * Gets an attribute of the data based on the key. Attributes are metadata + * providing additional information on the dataset. * * @param key * @return the attribute @@ -54,7 +56,8 @@ public interface IData { public Object getAttribute(String key); /** - * Gets the list of attributes associated with this data. + * Gets the list of attributes associated with this data. Attributes are + * metadata providing additional information on the dataset. * * @return the attributes */ diff --git a/edexOsgi/com.raytheon.uf.common.dataaccess/src/com/raytheon/uf/common/dataaccess/impl/DefaultGeometryData.java b/edexOsgi/com.raytheon.uf.common.dataaccess/src/com/raytheon/uf/common/dataaccess/impl/DefaultGeometryData.java index 2bde0a4673..82ff38cad5 100644 --- a/edexOsgi/com.raytheon.uf.common.dataaccess/src/com/raytheon/uf/common/dataaccess/impl/DefaultGeometryData.java +++ b/edexOsgi/com.raytheon.uf.common.dataaccess/src/com/raytheon/uf/common/dataaccess/impl/DefaultGeometryData.java @@ -43,6 +43,7 @@ import com.vividsolutions.jts.geom.Geometry; * ------------ ---------- ----------- -------------------------- * Nov 09, 2012 njensen Initial creation * Jun 03, 2013 #2023 dgilling Implement getAttributes(). + * Jan 21, 2014 2667 bclement attribute method comments * * * @@ -277,6 +278,13 @@ public class DefaultGeometryData implements IGeometryData { this.dataMap.put(parameter, data); } + /** + * Add a key/value pair to the attributes map. Attributes are metadata + * providing additional information on the dataset. + * + * @param key + * @param value + */ public void addAttribute(String key, Object value) { attributes.put(key, value); } @@ -297,6 +305,12 @@ public class DefaultGeometryData implements IGeometryData { this.locationName = locationName; } + /** + * Replace the attribute map with attrs. Attributes are metadata providing + * additional information on the dataset. + * + * @param attrs + */ public void setAttributes(Map attrs) { this.attributes = attrs; } diff --git a/edexOsgi/com.raytheon.uf.common.dataaccess/src/com/raytheon/uf/common/dataaccess/util/PDOUtil.java b/edexOsgi/com.raytheon.uf.common.dataaccess/src/com/raytheon/uf/common/dataaccess/util/PDOUtil.java index 5cf0b0b14d..85fd84f46e 100644 --- a/edexOsgi/com.raytheon.uf.common.dataaccess/src/com/raytheon/uf/common/dataaccess/util/PDOUtil.java +++ b/edexOsgi/com.raytheon.uf.common.dataaccess/src/com/raytheon/uf/common/dataaccess/util/PDOUtil.java @@ -53,6 +53,7 @@ import com.raytheon.uf.common.localization.IPathManager; * ------------ ---------- ----------- -------------------------- * Jan 03, 2013 bkowal Initial creation * Jan 31, 2013 #1555 bkowal Made hdf5 variable generic + * Jan 21, 2014 2667 bclement added getHDF5File method * * * @@ -67,7 +68,13 @@ public final class PDOUtil { private PDOUtil() { } - public static IDataStore getDataStore(PluginDataObject pdo) { + /** + * Get the hdf5 file for the data object + * + * @param pdo + * @return + */ + public static File getHDF5File(PluginDataObject pdo){ final String pluginName = pdo.getPluginName(); final IPersistable persistable = (IPersistable) pdo; @@ -76,10 +83,19 @@ public final class PDOUtil { persistable); String hdf5File = pathProvider.getHDFFileName(pluginName, persistable); - File file = new File(pluginName + IPathManager.SEPARATOR + return new File(pluginName + IPathManager.SEPARATOR + hdf5Path + IPathManager.SEPARATOR + hdf5File); - return DataStoreFactory.getDataStore(file); + } + + /** + * Get the datastore for the data object + * + * @param pdo + * @return + */ + public static IDataStore getDataStore(PluginDataObject pdo) { + return DataStoreFactory.getDataStore(getHDF5File(pdo)); } /** diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/META-INF/MANIFEST.MF b/edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/META-INF/MANIFEST.MF index 584bcb27f5..261268366d 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/META-INF/MANIFEST.MF +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/META-INF/MANIFEST.MF @@ -7,12 +7,17 @@ Bundle-Vendor: RAYTHEON Eclipse-RegisterBuddy: com.raytheon.uf.common.serialization Bundle-RequiredExecutionEnvironment: JavaSE-1.6 Export-Package: com.raytheon.uf.common.dataplugin.binlightning, + com.raytheon.uf.common.dataplugin.binlightning.dataaccess, com.raytheon.uf.common.dataplugin.binlightning.impl Require-Bundle: com.raytheon.uf.common.serialization, org.geotools, javax.persistence, javax.measure, com.raytheon.uf.common.dataplugin, - com.raytheon.uf.common.datastorage -Import-Package: com.raytheon.uf.edex.decodertools.core, + com.raytheon.uf.common.datastorage, + com.raytheon.uf.common.dataaccess;bundle-version="1.0.0" +Import-Package: com.raytheon.uf.common.dataquery.requests, + com.raytheon.uf.common.dataquery.responses, + com.raytheon.uf.common.status, + com.raytheon.uf.edex.decodertools.core, com.raytheon.uf.edex.decodertools.time diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/build.properties b/edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/build.properties index 34d2e4d2da..5791d48d5f 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/build.properties +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/build.properties @@ -1,4 +1,5 @@ source.. = src/ output.. = bin/ bin.includes = META-INF/,\ - . + .,\ + res/ diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/res/spring/binlightning-common-dataaccess.xml b/edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/res/spring/binlightning-common-dataaccess.xml new file mode 100644 index 0000000000..0a196509b3 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/res/spring/binlightning-common-dataaccess.xml @@ -0,0 +1,12 @@ + + + + + + + + + + \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/src/com/raytheon/uf/common/dataplugin/binlightning/BinLightningRecord.java b/edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/src/com/raytheon/uf/common/dataplugin/binlightning/BinLightningRecord.java index 645dad3fa0..29c406ea54 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/src/com/raytheon/uf/common/dataplugin/binlightning/BinLightningRecord.java +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/src/com/raytheon/uf/common/dataplugin/binlightning/BinLightningRecord.java @@ -79,6 +79,7 @@ import com.raytheon.uf.edex.decodertools.time.TimeTools; * PluginDataObject. * Aug 30, 2013 2298 rjpeter Make getPluginName abstract * Oct 22, 2013 2361 njensen Removed XML annotations + * Jan 21, 2014 2667 bclement renamed record's lightSource field to source * * * @@ -157,7 +158,7 @@ public class BinLightningRecord extends PersistablePluginDataObject implements @Column(length = 5) @DataURI(position = 3) @DynamicSerializeElement - private String lightSource; + private String source; // Used to track @Transient @@ -231,19 +232,19 @@ public class BinLightningRecord extends PersistablePluginDataObject implements */ public void addStrike(LightningStrikePoint strike) { // jjg add - if (lightSource == null) { + if (source == null) { if (strike.getLightSource() == null) { - lightSource = "NLDN"; + source = "NLDN"; } else if (strike.getLightSource().isEmpty()) { - lightSource = "UNKN"; + source = "UNKN"; } else { - lightSource = strike.getLightSource(); + source = strike.getLightSource(); } } else { if (strike.getLightSource() == null) { - lightSource = "NLDN"; - } else if (!lightSource.equals(strike.getLightSource())) { - lightSource = "UNKN"; + source = "NLDN"; + } else if (!source.equals(strike.getLightSource())) { + source = "UNKN"; } } // end @@ -426,8 +427,8 @@ public class BinLightningRecord extends PersistablePluginDataObject implements * * @return */ - public String getLightSource() { - return lightSource; + public String getSource() { + return source; } /** @@ -435,8 +436,8 @@ public class BinLightningRecord extends PersistablePluginDataObject implements * * @param lightSource */ - public void setLightSource(String lightSource) { - this.lightSource = lightSource; + public void setSource(String lightSource) { + this.source = lightSource; } /** diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/src/com/raytheon/uf/common/dataplugin/binlightning/dataaccess/BinLightingAccessFactory.java b/edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/src/com/raytheon/uf/common/dataplugin/binlightning/dataaccess/BinLightingAccessFactory.java new file mode 100644 index 0000000000..18acffaa66 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/src/com/raytheon/uf/common/dataplugin/binlightning/dataaccess/BinLightingAccessFactory.java @@ -0,0 +1,344 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.common.dataplugin.binlightning.dataaccess; + +import java.io.File; +import java.io.FileNotFoundException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import com.raytheon.uf.common.dataaccess.IDataRequest; +import com.raytheon.uf.common.dataaccess.exception.IncompatibleRequestException; +import com.raytheon.uf.common.dataaccess.geom.IGeometryData; +import com.raytheon.uf.common.dataaccess.geom.IGeometryData.Type; +import com.raytheon.uf.common.dataaccess.impl.AbstractDataPluginFactory; +import com.raytheon.uf.common.dataaccess.impl.DefaultGeometryData; +import com.raytheon.uf.common.dataaccess.util.PDOUtil; +import com.raytheon.uf.common.dataplugin.binlightning.BinLightningRecord; +import com.raytheon.uf.common.dataquery.requests.RequestConstraint; +import com.raytheon.uf.common.dataquery.responses.DbQueryResponse; +import com.raytheon.uf.common.datastorage.DataStoreFactory; +import com.raytheon.uf.common.datastorage.IDataStore; +import com.raytheon.uf.common.datastorage.Request; +import com.raytheon.uf.common.datastorage.StorageException; +import com.raytheon.uf.common.datastorage.records.ByteDataRecord; +import com.raytheon.uf.common.datastorage.records.FloatDataRecord; +import com.raytheon.uf.common.datastorage.records.IDataRecord; +import com.raytheon.uf.common.datastorage.records.IntegerDataRecord; +import com.raytheon.uf.common.datastorage.records.LongDataRecord; +import com.raytheon.uf.common.status.IUFStatusHandler; +import com.raytheon.uf.common.status.UFStatus; +import com.raytheon.uf.common.time.DataTime; +import com.vividsolutions.jts.geom.Coordinate; +import com.vividsolutions.jts.geom.GeometryFactory; + +/** + * Data access framework factory for bin lightning + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Jan 21, 2014 2667       bclement     Initial creation
+ * 
+ *
+ * + * @author bclement + * @version 1.0 + */ +public class BinLightingAccessFactory extends AbstractDataPluginFactory { + + private static final String sourceKey = "source"; + + private static final IUFStatusHandler log = UFStatus + .getHandler(BinLightningRecord.class); + + private static final GeometryFactory geomFactory = new GeometryFactory(); + + private static final String timeKey = "obsTime"; + + private static final String latKey = "latitude"; + + private static final String lonKey = "longitude"; + + private static final String[] requiredKeys = { timeKey, latKey, lonKey }; + + /* + * (non-Javadoc) + * + * @see + * com.raytheon.uf.common.dataaccess.IDataFactory#getAvailableLocationNames + * (com.raytheon.uf.common.dataaccess.IDataRequest) + */ + @Override + public String[] getAvailableLocationNames(IDataRequest request) { + throw new IncompatibleRequestException(this.getClass() + + " does not support location names"); + } + + /* + * (non-Javadoc) + * + * @see com.raytheon.uf.common.dataaccess.impl.AbstractDataFactory# + * getRequiredIdentifiers() + */ + @Override + public String[] getRequiredIdentifiers() { + return new String[] { sourceKey }; + } + + /* + * (non-Javadoc) + * + * @see com.raytheon.uf.common.dataaccess.impl.AbstractDataFactory# + * getValidIdentifiers() + */ + @Override + public String[] getValidIdentifiers() { + return new String[] { sourceKey }; + } + + /* + * (non-Javadoc) + * + * @see com.raytheon.uf.common.dataaccess.impl.AbstractDataPluginFactory# + * buildConstraintsFromRequest + * (com.raytheon.uf.common.dataaccess.IDataRequest) + */ + @Override + protected Map buildConstraintsFromRequest( + IDataRequest request) { + Map rcMap = new HashMap(); + + Map identifiers = request.getIdentifiers(); + if (identifiers != null) { + for (Entry entry : identifiers.entrySet()) { + rcMap.put(entry.getKey(), new RequestConstraint(entry + .getValue().toString())); + } + } + return rcMap; + } + + /* + * (non-Javadoc) + * + * @see com.raytheon.uf.common.dataaccess.impl.AbstractDataPluginFactory# + * getGeometryData(com.raytheon.uf.common.dataaccess.IDataRequest, + * com.raytheon.uf.common.dataquery.responses.DbQueryResponse) + */ + @Override + protected IGeometryData[] getGeometryData(IDataRequest request, + DbQueryResponse dbQueryResponse) { + Map> results = unpackResults(dbQueryResponse); + + List rval = new ArrayList(); + for (Entry> resultEntry : results + .entrySet()) { + Map> srcDatasets = getSourceDatasets(request, + resultEntry.getValue()); + IDataStore ds = DataStoreFactory.getDataStore(resultEntry.getKey()); + for (Entry> groupEntry : srcDatasets + .entrySet()) { + addGeometryData(rval, ds, groupEntry.getKey(), + groupEntry.getValue()); + } + } + return rval.toArray(new IGeometryData[rval.size()]); + } + + + /** + * Add geometry data elements to dataList from data store + * + * @param dataList + * target result list + * @param ds + * datastore + * @param source + * lightning source value from metadata + * @param datasets + * requested datasets from datastore + */ + private void addGeometryData(List dataList, IDataStore ds, + String source, List datasets) { + // Go fetch data + try { + IDataRecord[] records = ds.retrieveDatasets( + datasets.toArray(new String[datasets.size()]), Request.ALL); + + Map> recordMap = new HashMap>(); + // Throw in a map for easy accessibility + for (IDataRecord rec : records) { + List recordList = recordMap.get(rec.getName()); + if (recordList == null) { + recordList = new ArrayList(); + recordMap.put(rec.getName(), recordList); + } + recordList.add(rec); + } + + // remove required records from map so they won't be used again when + // we look for optional records + List times = recordMap.remove(timeKey); + List lats = recordMap.remove(latKey); + List lons = recordMap.remove(lonKey); + + int k = 0; + for (IDataRecord timeRec : times) { + LongDataRecord time = (LongDataRecord) timeRec; + + long[] timeData = time.getLongData(); + float[] latitudeData = ((FloatDataRecord) lats.get(k)) + .getFloatData(); + float[] longitudeData = ((FloatDataRecord) lons.get(k)) + .getFloatData(); + + for (int i = 0; i < timeData.length; i++) { + DataTime dt = new DataTime(new Date(timeData[i])); + DefaultGeometryData data = new DefaultGeometryData(); + data.setDataTime(dt); + data.addAttribute(sourceKey, source); + data.setGeometry(geomFactory.createPoint(new Coordinate( + longitudeData[i], latitudeData[i]))); + // add the optional parameter records + addParameterData(data, recordMap, k, i); + dataList.add(data); + } + k++; + } + } catch (StorageException e) { + log.error("Storage error retrieving lightning data", e); + } catch (FileNotFoundException e) { + log.error("Unable to open lightning file", e); + } + } + + /** + * Add parameters from record map to data + * + * @param data + * target geometry data + * @param recordMap + * map of parameter names to list of data records + * @param recordIndex + * index into list of data records + * @param valueIndex + * index into the target data record's value array + */ + private void addParameterData(DefaultGeometryData data, + Map> recordMap, int recordIndex, + int valueIndex) { + for (Entry> entry : recordMap.entrySet()) { + String parameterName = entry.getKey(); + IDataRecord record = entry.getValue().get(recordIndex); + if (record instanceof IntegerDataRecord) { + int value = ((IntegerDataRecord) record).getIntData()[valueIndex]; + data.addData(parameterName, value, Type.INT); + } else if (record instanceof ByteDataRecord) { + int value = ((ByteDataRecord) record).getByteData()[valueIndex]; + data.addData(parameterName, value, Type.INT); + } else { + // lightning only uses ints and bytes, we can add support for + // more types if needed + log.warn("Unsupported parameter record type for lightning: " + + record.getClass()); + } + } + } + + /** + * Return mapping of lightning data source to list of datasets + * + * @param recList + * @return + */ + private Map> getSourceDatasets(IDataRequest request, + List recList) { + List includedDatasets = getIncludedDatasets(request); + + Map> rval = new HashMap>(); + for (BinLightningRecord record : recList) { + String src = record.getSource(); + List groups = rval.get(src); + if (groups == null) { + groups = new ArrayList(); + rval.put(src, groups); + } + for (String dataset : includedDatasets) { + groups.add(record.getDataURI() + DataStoreFactory.DEF_SEPARATOR + + dataset); + } + } + return rval; + } + + /** + * Get a list of HDF5 datasets to request + * + * @param request + * @return + */ + private List getIncludedDatasets(IDataRequest request){ + String[] parameters = request.getParameters(); + List rval = new ArrayList(parameters.length + + requiredKeys.length); + rval.addAll(Arrays.asList(requiredKeys)); + rval.addAll(Arrays.asList(parameters)); + return rval; + } + + /** + * Unpack records from response and group by HDF5 file + * + * @param dbQueryResponse + * @return + */ + private Map> unpackResults( + DbQueryResponse dbQueryResponse) { + // Bin up requests to the same hdf5 + Map> fileMap = new HashMap>(); + + for (Map result : dbQueryResponse.getResults()) { + Object object = result.get(null); + if (object == null || !(object instanceof BinLightningRecord)) { + log.warn("Unexpected result for bin lightning: " + object); + continue; + } + BinLightningRecord record = (BinLightningRecord) object; + File hdf5File = PDOUtil.getHDF5File(record); + List recList = fileMap.get(hdf5File); + if (recList == null) { + recList = new ArrayList(); + fileMap.put(hdf5File, recList); + } + recList.add(record); + } + return fileMap; + } + +}