From 05d980dfe976cfaeecd6f759c2e4eaa6d365e27c Mon Sep 17 00:00:00 2001 From: Ben Steffensmeier Date: Thu, 21 Jan 2016 15:19:35 -0600 Subject: [PATCH] Omaha #5208 Decode and store scale and offset for pointset data. Former-commit-id: 9d8ab6e63c5e85b52c9c04cb6fe762fe3a70ec39 --- .../pointset/PointSetConstants.java | 8 +- .../dataplugin/pointset/PointSetData.java | 158 ++++++++++++++++++ .../dataplugin/pointset/PointSetRecord.java | 9 +- .../netcdf/PointSetNetcdfDecoder.java | 46 +++-- .../description/ProductDescription.java | 14 +- .../META-INF/MANIFEST.MF | 4 +- .../uf/edex/plugin/pointset/PointSetDao.java | 88 +++++++++- 7 files changed, 292 insertions(+), 35 deletions(-) create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.pointset/src/com/raytheon/uf/common/dataplugin/pointset/PointSetData.java diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.pointset/src/com/raytheon/uf/common/dataplugin/pointset/PointSetConstants.java b/edexOsgi/com.raytheon.uf.common.dataplugin.pointset/src/com/raytheon/uf/common/dataplugin/pointset/PointSetConstants.java index ab03e8a788..1a1542495e 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.pointset/src/com/raytheon/uf/common/dataplugin/pointset/PointSetConstants.java +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.pointset/src/com/raytheon/uf/common/dataplugin/pointset/PointSetConstants.java @@ -29,13 +29,13 @@ package com.raytheon.uf.common.dataplugin.pointset; * SOFTWARE HISTORY * * Date Ticket# Engineer Description - * ------------- -------- --------- -------------------------- + * ------------- -------- --------- -------------------- * Aug 28, 2015 4709 bsteffen Initial creation + * Jan 21, 2016 5208 bsteffen Add scale and offset * * * * @author bsteffen - * @version 1.0 */ public class PointSetConstants { @@ -60,4 +60,8 @@ public class PointSetConstants { public static final String LEVEL_TWO = LEVEL + ".leveltwovalue"; + public static final String ADD_OFFSET = "add_offset"; + + public static final String SCALE_FACTOR = "scale_factor"; + } diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.pointset/src/com/raytheon/uf/common/dataplugin/pointset/PointSetData.java b/edexOsgi/com.raytheon.uf.common.dataplugin.pointset/src/com/raytheon/uf/common/dataplugin/pointset/PointSetData.java new file mode 100644 index 0000000000..9cd8043471 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.pointset/src/com/raytheon/uf/common/dataplugin/pointset/PointSetData.java @@ -0,0 +1,158 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.common.dataplugin.pointset; + +import java.nio.Buffer; +import java.nio.ByteBuffer; +import java.nio.FloatBuffer; +import java.nio.IntBuffer; +import java.nio.ShortBuffer; + +import javax.measure.converter.AddConverter; +import javax.measure.converter.MultiplyConverter; +import javax.measure.converter.RationalConverter; +import javax.measure.converter.UnitConverter; + +/** + * + * Store the raw data for pointset. The primary structure is a {@link Buffer} + * containing numeric data for each point. This also has support for products + * that use scale and offset values in addition to the raw numeric data(This is + * for packing the data into bytes or shorts). + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date          Ticket#  Engineer  Description
+ * ------------- -------- --------- --------------------------
+ * Jan 20, 2016  5208     bsteffen  Initial creation
+ * 
+ *
+ * + * @author bsteffen + */ +public class PointSetData { + + private Buffer data; + + private Float scale; + + private Float offset; + + public PointSetData() { + + } + + public PointSetData(Buffer data) { + this.data = data; + } + + public Buffer getData() { + return data; + } + + public void setData(Buffer data) { + this.data = data; + } + + public Float getScale() { + return scale; + } + + public void setScale(Float scale) { + this.scale = scale; + } + + public Float getOffset() { + return offset; + } + + public void setOffset(Float offset) { + this.offset = offset; + } + + /** + * Converts the data using the specified converter. If there is a scale or + * offset present and the converter is a simple add or multiply then it is + * incorporated into the scale/offset and the data is left untouched, which + * keeps packed data small. For all other cases the data is converted to + * floats and the converter is applied to the data. + * + * @param converter + */ + public void convert(UnitConverter converter) { + if (data == null || converter == null + || converter == UnitConverter.IDENTITY) { + return; + } + if (scale != null + && (converter instanceof MultiplyConverter || converter instanceof RationalConverter)) { + this.scale = (float) converter.convert(this.scale); + if (this.offset != null) { + this.offset = (float) converter.convert(this.offset); + } + } else if (offset != null && converter instanceof AddConverter) { + this.offset = (float) converter.convert(this.offset); + } else { + FloatBuffer buffer = null; + if (this.data instanceof FloatBuffer) { + buffer = (FloatBuffer) this.data; + } else if (this.data instanceof ShortBuffer) { + ShortBuffer sbuffer = (ShortBuffer) this.data; + buffer = FloatBuffer.allocate(this.data.capacity()); + for (int i = 0; i < buffer.capacity(); i += 1) { + buffer.put(i, sbuffer.get(i)); + } + } else if (this.data instanceof ByteBuffer) { + ByteBuffer bbuffer = (ByteBuffer) this.data; + buffer = FloatBuffer.allocate(this.data.capacity()); + for (int i = 0; i < buffer.capacity(); i += 1) { + buffer.put(i, bbuffer.get(i)); + } + } else if (this.data instanceof IntBuffer) { + IntBuffer ibuffer = (IntBuffer) this.data; + buffer = FloatBuffer.allocate(this.data.capacity()); + for (int i = 0; i < buffer.capacity(); i += 1) { + buffer.put(i, ibuffer.get(i)); + } + } else { + throw new IllegalStateException("Unsupported buffer of type: " + + this.data.getClass().getSimpleName()); + } + if (scale != null) { + for (int i = 0; i < buffer.capacity(); i += 1) { + buffer.put(i, buffer.get(i) * scale); + } + } + if (offset != null) { + for (int i = 0; i < buffer.capacity(); i += 1) { + buffer.put(i, buffer.get(i) + offset); + } + } + for (int i = 0; i < buffer.capacity(); i += 1) { + buffer.put(i, (float) converter.convert(buffer.get(i))); + } + this.scale = null; + this.offset = null; + this.data = buffer; + } + } +} diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.pointset/src/com/raytheon/uf/common/dataplugin/pointset/PointSetRecord.java b/edexOsgi/com.raytheon.uf.common.dataplugin.pointset/src/com/raytheon/uf/common/dataplugin/pointset/PointSetRecord.java index 9855f568c1..92808c7604 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.pointset/src/com/raytheon/uf/common/dataplugin/pointset/PointSetRecord.java +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.pointset/src/com/raytheon/uf/common/dataplugin/pointset/PointSetRecord.java @@ -19,7 +19,6 @@ **/ package com.raytheon.uf.common.dataplugin.pointset; -import java.nio.Buffer; import java.nio.file.Path; import java.nio.file.Paths; @@ -53,6 +52,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; * Date Ticket# Engineer Description * ------------- -------- --------- -------------------------- * Aug 11, 2015 4709 bsteffen Initial creation + * Jan 21, 2016 5208 bsteffen Store data in new object. * * * @@ -89,7 +89,8 @@ public class PointSetRecord extends PersistablePluginDataObject { @DynamicSerializeElement private String locationId; - private transient Buffer data; + private transient PointSetData data; + public String getDatasetId() { return datasetId; @@ -123,11 +124,11 @@ public class PointSetRecord extends PersistablePluginDataObject { this.locationId = locationId; } - public Buffer getData() { + public PointSetData getData() { return data; } - public void setData(Buffer data) { + public void setData(PointSetData data) { this.data = data; } diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.pointset.netcdf/src/com/raytheon/uf/edex/plugin/pointset/netcdf/PointSetNetcdfDecoder.java b/edexOsgi/com.raytheon.uf.edex.plugin.pointset.netcdf/src/com/raytheon/uf/edex/plugin/pointset/netcdf/PointSetNetcdfDecoder.java index 04bd442fe3..1ab0ca543a 100644 --- a/edexOsgi/com.raytheon.uf.edex.plugin.pointset.netcdf/src/com/raytheon/uf/edex/plugin/pointset/netcdf/PointSetNetcdfDecoder.java +++ b/edexOsgi/com.raytheon.uf.edex.plugin.pointset.netcdf/src/com/raytheon/uf/edex/plugin/pointset/netcdf/PointSetNetcdfDecoder.java @@ -48,6 +48,7 @@ import ucar.nc2.NetcdfFile; import ucar.nc2.Variable; import com.raytheon.uf.common.dataplugin.level.LevelFactory; +import com.raytheon.uf.common.dataplugin.pointset.PointSetData; import com.raytheon.uf.common.dataplugin.pointset.PointSetLocation; import com.raytheon.uf.common.dataplugin.pointset.PointSetRecord; import com.raytheon.uf.common.dataplugin.pointset.triangulate.DelauneyTriangulator; @@ -56,7 +57,7 @@ import com.raytheon.uf.common.datastorage.StorageException; import com.raytheon.uf.common.localization.IPathManager; import com.raytheon.uf.common.localization.LocalizationFile; import com.raytheon.uf.common.localization.exception.LocalizationException; -import com.raytheon.uf.common.parameter.lookup.ParameterLookup; +import com.raytheon.uf.edex.netcdf.decoder.util.NetcdfDecoderUtils; import com.raytheon.uf.edex.netcdf.description.VariableDescription; import com.raytheon.uf.edex.netcdf.description.exception.InvalidDescriptionException; import com.raytheon.uf.edex.plugin.pointset.netcdf.description.PointSetProductDescriptions; @@ -79,8 +80,10 @@ import com.raytheon.uf.edex.plugin.pointset.netcdf.description.TriangulationDesc * SOFTWARE HISTORY * * Date Ticket# Engineer Description - * ------------- -------- --------- -------------------------- + * ------------- -------- --------- -------------------------------------------- * Aug 11, 2015 4709 bsteffen Initial creation + * Jan 21, 2016 5208 bsteffen Decode scale, offset, units, long_name when + * they are present * * * @@ -103,15 +106,10 @@ public class PointSetNetcdfDecoder { private LevelFactory levelFactory; - private ParameterLookup parameterLookup; - public PointSetRecord[] decode(File file) { if (levelFactory == null) { levelFactory = LevelFactory.getInstance(); } - if (parameterLookup == null) { - parameterLookup = ParameterLookup.getInstance(); - } try { NetcdfFile netcdfFile = NetcdfFile.open(file.getAbsolutePath()); Map locationCache = new HashMap(); @@ -244,7 +242,7 @@ public class PointSetNetcdfDecoder { } } - PointSetRecord record = description.getRecord(file, parameterLookup, + PointSetRecord record = description.getRecord(file, levelFactory); if (record == null) { if (debug) { @@ -309,7 +307,27 @@ public class PointSetNetcdfDecoder { dataVarName, dataType); return null; } - record.setData(numericData); + + Attribute longNameAttribute = dataVariable.findAttribute("long_name"); + if (longNameAttribute != null) { + record.getParameter().setName(longNameAttribute.getStringValue()); + } + Attribute unitsAttribute = dataVariable.findAttribute("units"); + if (unitsAttribute != null) { + record.getParameter() + .setUnitString(unitsAttribute.getStringValue()); + } + PointSetData data = new PointSetData(numericData); + + Number scale = NetcdfDecoderUtils.getScaleFactor(dataVariable); + if (scale != NetcdfDecoderUtils.DEFAULT_SCALE_FACTOR) { + data.setScale(scale.floatValue()); + } + Number offset = NetcdfDecoderUtils.getAddOffset(dataVariable); + if (offset != NetcdfDecoderUtils.DEFAULT_ADD_OFFSET) { + data.setOffset(offset.floatValue()); + } + record.setData(data); StringBuilder locationKeyBuilder = new StringBuilder(); lonVariable.getNameAndDimensions(locationKeyBuilder); @@ -377,19 +395,19 @@ public class PointSetNetcdfDecoder { PointSetProductDescriptions descriptions = new PointSetProductDescriptions(); for (LocalizationFile file : files) { logger.info("Loading pointset data description from " - + file.getName()); + + file.getPath()); try (InputStream inputStream = file.openInputStream()) { PointSetProductDescriptions unmarshalled = JAXB.unmarshal( inputStream, PointSetProductDescriptions.class); for (ProductDescription description : unmarshalled .getDescriptions()) { - if (validate(file.getName(), description)) { + if (validate(file.getPath(), description)) { descriptions.addDescription(description); } } } catch (LocalizationException | IOException e) { logger.error("Unable to load product descriptions from {}", - file.getName(), e); + file.getPath(), e); } } this.descriptions = descriptions; @@ -403,10 +421,6 @@ public class PointSetNetcdfDecoder { this.levelFactory = levelFactory; } - public void setParameterLookup(ParameterLookup parameterLookup) { - this.parameterLookup = parameterLookup; - } - protected boolean validate(String fileName, ProductDescription description) { VariableDescription data = description.getData(); if (data == null) { diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.pointset.netcdf/src/com/raytheon/uf/edex/plugin/pointset/netcdf/description/ProductDescription.java b/edexOsgi/com.raytheon.uf.edex.plugin.pointset.netcdf/src/com/raytheon/uf/edex/plugin/pointset/netcdf/description/ProductDescription.java index 0e9824e60d..b604b24916 100644 --- a/edexOsgi/com.raytheon.uf.edex.plugin.pointset.netcdf/src/com/raytheon/uf/edex/plugin/pointset/netcdf/description/ProductDescription.java +++ b/edexOsgi/com.raytheon.uf.edex.plugin.pointset.netcdf/src/com/raytheon/uf/edex/plugin/pointset/netcdf/description/ProductDescription.java @@ -30,7 +30,7 @@ import ucar.nc2.NetcdfFile; import com.raytheon.uf.common.dataplugin.level.Level; import com.raytheon.uf.common.dataplugin.level.LevelFactory; import com.raytheon.uf.common.dataplugin.pointset.PointSetRecord; -import com.raytheon.uf.common.parameter.lookup.ParameterLookup; +import com.raytheon.uf.common.parameter.Parameter; import com.raytheon.uf.common.time.DataTime; import com.raytheon.uf.edex.netcdf.description.AbstractFieldDescription; import com.raytheon.uf.edex.netcdf.description.AttributeDescription; @@ -52,8 +52,9 @@ import com.raytheon.uf.edex.plugin.pointset.netcdf.PointSetNetcdfDecoder; * SOFTWARE HISTORY * * Date Ticket# Engineer Description - * ------------- -------- --------- -------------------------- + * ------------- -------- --------- ---------------------------------- * Aug 11, 2015 4709 bsteffen Initial creation + * Jan 21, 2016 5208 bsteffen Move parameter persistence to dao * * * @@ -183,15 +184,14 @@ public class ProductDescription { * Extract the the datasetId, parameter, level and datatime from the file * using the attributes contained in this description. */ - public PointSetRecord getRecord(NetcdfFile file, - ParameterLookup parameterLookup, LevelFactory levelFactory) + public PointSetRecord getRecord(NetcdfFile file, LevelFactory levelFactory) throws InvalidDescriptionException { String datasetId = this.datasetId.getString(file); if (datasetId == null) { return null; } - String parameter = this.parameter.getString(file); - if (parameter == null) { + String parameterAbbrev = this.parameter.getString(file); + if (parameterAbbrev == null) { return null; } Level level = this.level.getLevel(file, levelFactory); @@ -204,7 +204,7 @@ public class ProductDescription { } PointSetRecord record = new PointSetRecord(); record.setDatasetId(datasetId); - record.setParameter(parameterLookup.getParameter(parameter)); + record.setParameter(new Parameter(parameterAbbrev)); record.setLevel(level); record.setDataTime(dataTime); return record; diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.pointset/META-INF/MANIFEST.MF b/edexOsgi/com.raytheon.uf.edex.plugin.pointset/META-INF/MANIFEST.MF index f9682e5058..e8b1a897fe 100644 --- a/edexOsgi/com.raytheon.uf.edex.plugin.pointset/META-INF/MANIFEST.MF +++ b/edexOsgi/com.raytheon.uf.edex.plugin.pointset/META-INF/MANIFEST.MF @@ -8,4 +8,6 @@ Bundle-RequiredExecutionEnvironment: JavaSE-1.7 Require-Bundle: com.raytheon.uf.common.dataplugin;bundle-version="1.14.0", com.raytheon.uf.common.dataplugin.pointset;bundle-version="1.15.0", com.raytheon.uf.common.datastorage;bundle-version="1.15.0", - com.raytheon.uf.edex.database;bundle-version="1.15.0" + com.raytheon.uf.edex.database;bundle-version="1.15.0", + javax.measure;bundle-version="1.0.0", + com.raytheon.uf.common.status;bundle-version="1.15.0" diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.pointset/src/com/raytheon/uf/edex/plugin/pointset/PointSetDao.java b/edexOsgi/com.raytheon.uf.edex.plugin.pointset/src/com/raytheon/uf/edex/plugin/pointset/PointSetDao.java index 23e7d03e2d..b693444c37 100644 --- a/edexOsgi/com.raytheon.uf.edex.plugin.pointset/src/com/raytheon/uf/edex/plugin/pointset/PointSetDao.java +++ b/edexOsgi/com.raytheon.uf.edex.plugin.pointset/src/com/raytheon/uf/edex/plugin/pointset/PointSetDao.java @@ -20,12 +20,25 @@ package com.raytheon.uf.edex.plugin.pointset; import java.nio.Buffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.measure.converter.UnitConverter; + +import com.raytheon.uf.common.dataplugin.PluginDataObject; import com.raytheon.uf.common.dataplugin.PluginException; import com.raytheon.uf.common.dataplugin.persist.IPersistable; +import com.raytheon.uf.common.dataplugin.pointset.PointSetConstants; +import com.raytheon.uf.common.dataplugin.pointset.PointSetData; import com.raytheon.uf.common.dataplugin.pointset.PointSetRecord; import com.raytheon.uf.common.datastorage.DataStoreFactory; import com.raytheon.uf.common.datastorage.IDataStore; +import com.raytheon.uf.common.datastorage.StorageStatus; +import com.raytheon.uf.common.datastorage.records.IDataRecord; +import com.raytheon.uf.common.parameter.Parameter; +import com.raytheon.uf.common.parameter.lookup.ParameterLookup; import com.raytheon.uf.edex.database.plugin.PluginDao; /** @@ -37,15 +50,16 @@ import com.raytheon.uf.edex.database.plugin.PluginDao; * SOFTWARE HISTORY * * Date Ticket# Engineer Description - * ------------- -------- --------- -------------------------- + * ------------- -------- --------- ----------------------- * Aug 11, 2015 4709 bsteffen Initial creation + * Jan 21, 2016 5208 bsteffen Store scale and offset * * * * @author bsteffen */ public class PointSetDao extends PluginDao { - + public PointSetDao(String pluginName) throws PluginException { super(pluginName); } @@ -54,14 +68,78 @@ public class PointSetDao extends PluginDao { super("pointset"); } + @Override + public PluginDataObject[] persistToDatabase(PluginDataObject... records) { + return super.persistToDatabase(verifyRecords(records)); + } + + @Override + public StorageStatus persistToHDF5(PluginDataObject... records) + throws PluginException { + return super.persistToHDF5(verifyRecords(records)); + } + + private PluginDataObject[] verifyRecords(PluginDataObject... records) { + List toPersist = new ArrayList( + records.length); + for (PluginDataObject record : records) { + PointSetRecord rec = (PointSetRecord) record; + if (validateParameter(rec)) { + toPersist.add(rec); + } + } + return toPersist.toArray(new PointSetRecord[toPersist.size()]); + } + + protected boolean validateParameter(PointSetRecord record) { + Parameter parameter = record.getParameter(); + boolean result = true; + if (parameter == null) { + result = false; + } else if (parameter.getName() == null) { + result = false; + } else if (parameter.getName().equals("Missing")) { + result = false; + } else { + Parameter dbParameter = ParameterLookup.getInstance().getParameter( + parameter, true); + if (!parameter.equals(dbParameter)) { + UnitConverter converter = Parameter.compareUnits(parameter, + dbParameter); + record.getData().convert(converter); + } + record.setParameter(dbParameter); + } + if (!result) { + logger.info("Discarding record due to missing or unknown parameter mapping: " + + record); + } + return result; + } + @Override protected IDataStore populateDataStore(IDataStore dataStore, IPersistable obj) throws Exception { if (obj instanceof PointSetRecord) { PointSetRecord points = (PointSetRecord) obj; - Buffer data = points.getData(); - dataStore.addDataRecord(DataStoreFactory.createStorageRecord( - "Data", points.getDataURI(), data.array())); + PointSetData data = points.getData(); + Buffer buffer = data.getData(); + IDataRecord dataRecord = DataStoreFactory.createStorageRecord( + "Data", points.getDataURI(), buffer.array()); + if (data.getOffset() != null || data.getScale() != null) { + Map dataAttributes = new HashMap<>(); + if (data.getOffset() != null) { + dataAttributes.put(PointSetConstants.ADD_OFFSET, + data.getOffset()); + } + if (data.getScale() != null) { + dataAttributes.put(PointSetConstants.SCALE_FACTOR, + data.getScale()); + } + + dataRecord.setDataAttributes(dataAttributes); + } + dataStore.addDataRecord(dataRecord); } else if (obj != null) { throw new IllegalArgumentException("Cannot handle " + obj.getClass().getSimpleName());