From 60c93552a330041f1eafcad77bb8e35b68d87a80 Mon Sep 17 00:00:00 2001 From: Ben Steffensmeier Date: Tue, 14 Apr 2015 15:58:51 -0500 Subject: [PATCH] Omaha #4388 Use the record fill value in satellite resource. Former-commit-id: d13b4cfda688ecec9dfa0563787ff6370c980443 --- .../inventory/DerivedSatelliteRecord.java | 42 +++++++++++ .../inventory/SatelliteRequestableData.java | 13 +--- .../viz/satellite/rsc/SatResource.java | 72 ++++++++++++++----- .../plugin/satellite/dao/SatelliteDao.java | 28 +++++--- .../satellite/SatelliteMessageData.java | 12 ++-- .../dataplugin/satellite/SatelliteRecord.java | 6 +- .../satellite/gini/GiniSatelliteDecoder.java | 12 ++-- 7 files changed, 135 insertions(+), 50 deletions(-) diff --git a/cave/com.raytheon.viz.satellite/src/com/raytheon/viz/satellite/inventory/DerivedSatelliteRecord.java b/cave/com.raytheon.viz.satellite/src/com/raytheon/viz/satellite/inventory/DerivedSatelliteRecord.java index 713a6a4694..4db681c8ce 100644 --- a/cave/com.raytheon.viz.satellite/src/com/raytheon/viz/satellite/inventory/DerivedSatelliteRecord.java +++ b/cave/com.raytheon.viz.satellite/src/com/raytheon/viz/satellite/inventory/DerivedSatelliteRecord.java @@ -19,11 +19,13 @@ **/ package com.raytheon.viz.satellite.inventory; +import java.awt.Point; import java.awt.Rectangle; import java.util.Collections; import java.util.HashSet; import java.util.Set; +import javax.measure.unit.Unit; import javax.measure.unit.UnitFormat; import org.geotools.coverage.grid.GridEnvelope2D; @@ -33,11 +35,15 @@ import org.opengis.referencing.operation.TransformException; import com.raytheon.uf.common.dataplugin.satellite.SatMapCoverage; import com.raytheon.uf.common.dataplugin.satellite.SatelliteRecord; +import com.raytheon.uf.common.dataplugin.satellite.units.generic.GenericPixel; import com.raytheon.uf.common.datastorage.DataStoreFactory; import com.raytheon.uf.common.datastorage.Request; +import com.raytheon.uf.common.datastorage.records.ByteDataRecord; +import com.raytheon.uf.common.datastorage.records.IDataRecord; import com.raytheon.uf.common.geospatial.interpolation.GridDownscaler; import com.raytheon.uf.common.inventory.data.AbstractRequestableData; import com.raytheon.uf.common.inventory.exception.DataCubeException; +import com.raytheon.viz.satellite.tileset.SatDataRetriever; /** * Satellite record which performs derived parameter calculations to get data @@ -50,6 +56,7 @@ import com.raytheon.uf.common.inventory.exception.DataCubeException; * Date Ticket# Engineer Description * ------------- -------- ----------- -------------------------- * Apr 09, 2014 2947 bsteffen Initial creation + * Apr 15, 2014 4388 bsteffen Set Fill Value. * * * @@ -134,15 +141,50 @@ public class DerivedSatelliteRecord extends SatelliteRecord { throw new DataCubeException(e); } gridGeometry = new GridGeometry2D(range, env); + } else if (req.getType() == Request.Type.POINT + && req.getPoints().length == 1) { + Point p = req.getPoints()[0]; + GridEnvelope2D range = new GridEnvelope2D(p.x, p.y, 1, 1); + Envelope env; + try { + env = gridGeometry.gridToWorld(range); + } catch (TransformException e) { + throw new DataCubeException(e); + } + gridGeometry = new GridGeometry2D(range, env); } else if (req.getType() != Request.Type.ALL) { throw new DataCubeException( "Unsupported request type for derived satellite data: " + req.getType()); } Object dataValue = requestableData.getDataValue(gridGeometry); + setDerivedFillValue(dataValue); setMessageData(dataValue); } + /** + * Set a fill value on the dataValue if applicable. Some derived parameters + * operate on unsigned byte data and return signed byte data and need a fill + * value set to composite properly. + */ + private void setDerivedFillValue(Object dataValue) { + IDataRecord dataRecord = null; + if (dataValue instanceof IDataRecord[]) { + IDataRecord[] dataRecords = (IDataRecord[]) dataValue; + if (dataRecords.length == 1) { + dataRecord = dataRecords[0]; + } + } else if (dataValue instanceof IDataRecord) { + dataRecord = (IDataRecord) dataValue; + } + if (dataRecord instanceof ByteDataRecord) { + Unit unit = SatDataRetriever.getRecordUnit(this); + if (unit instanceof GenericPixel) { + dataRecord.setFillValue(Byte.MIN_VALUE); + } + } + } + private static Set findBaseRecords( AbstractRequestableData data) { if (data instanceof SatelliteRequestableData) { diff --git a/cave/com.raytheon.viz.satellite/src/com/raytheon/viz/satellite/inventory/SatelliteRequestableData.java b/cave/com.raytheon.viz.satellite/src/com/raytheon/viz/satellite/inventory/SatelliteRequestableData.java index bf1754d39d..8246dce879 100644 --- a/cave/com.raytheon.viz.satellite/src/com/raytheon/viz/satellite/inventory/SatelliteRequestableData.java +++ b/cave/com.raytheon.viz.satellite/src/com/raytheon/viz/satellite/inventory/SatelliteRequestableData.java @@ -21,7 +21,6 @@ package com.raytheon.viz.satellite.inventory; import java.awt.Point; import java.awt.Rectangle; -import java.util.Map; import javax.measure.unit.Unit; @@ -61,6 +60,7 @@ import com.raytheon.viz.satellite.tileset.SatDataRetriever; * Date Ticket# Engineer Description * ------------- -------- ----------- -------------------------- * Apr 09, 2014 2947 bsteffen Initial creation + * Apr 15, 2014 4388 bsteffen Set Fill Value. * * * @@ -153,20 +153,13 @@ public class SatelliteRequestableData extends AbstractRequestableData { } catch (FactoryException e) { throw new DataCubeException(e); } - Map attrs = dataRecord.getDataAttributes(); + Number fillValue = dataRecord.getFillValue(); dataRecord = DataStoreFactory.createStorageRecord( dataRecord.getName(), dataRecord.getGroup(), dest.getArray(), 2, new long[] { requestRange.width, requestRange.height }); - dataRecord.setFillValue(0); - if (attrs != null) { - Number fill = (Number) attrs - .get(SatelliteRecord.SAT_FILL_VALUE); - if (fill != null) { - dataRecord.setFillValue(fill); - } - } + dataRecord.setFillValue(fillValue); return dataRecord; } else { return CubeUtil.retrieveData(record, record.getPluginName(), diff --git a/cave/com.raytheon.viz.satellite/src/com/raytheon/viz/satellite/rsc/SatResource.java b/cave/com.raytheon.viz.satellite/src/com/raytheon/viz/satellite/rsc/SatResource.java index f3ecae6399..bb9c2b6a83 100644 --- a/cave/com.raytheon.viz.satellite/src/com/raytheon/viz/satellite/rsc/SatResource.java +++ b/cave/com.raytheon.viz.satellite/src/com/raytheon/viz/satellite/rsc/SatResource.java @@ -38,6 +38,8 @@ import org.opengis.referencing.FactoryException; import org.opengis.referencing.crs.CoordinateReferenceSystem; import org.opengis.referencing.operation.TransformException; +import com.raytheon.uf.common.colormap.ColorMapException; +import com.raytheon.uf.common.colormap.ColorMapLoader; import com.raytheon.uf.common.colormap.IColorMap; import com.raytheon.uf.common.colormap.prefs.ColorMapParameters; import com.raytheon.uf.common.colormap.prefs.ColorMapParameters.PersistedParameters; @@ -45,10 +47,15 @@ import com.raytheon.uf.common.colormap.prefs.DataMappingPreferences; import com.raytheon.uf.common.dataplugin.PluginDataObject; import com.raytheon.uf.common.dataplugin.satellite.SatMapCoverage; import com.raytheon.uf.common.dataplugin.satellite.SatelliteRecord; -import com.raytheon.uf.common.dataplugin.satellite.units.generic.GenericPixel; +import com.raytheon.uf.common.datastorage.DataStoreFactory; +import com.raytheon.uf.common.datastorage.Request; +import com.raytheon.uf.common.datastorage.records.IDataRecord; +import com.raytheon.uf.common.datastorage.records.ShortDataRecord; import com.raytheon.uf.common.geospatial.IGridGeometryProvider; import com.raytheon.uf.common.geospatial.ReferencedCoordinate; import com.raytheon.uf.common.geospatial.data.GeographicDataSource; +import com.raytheon.uf.common.inventory.exception.DataCubeException; +import com.raytheon.uf.common.status.UFStatus.Priority; import com.raytheon.uf.common.style.ParamLevelMatchCriteria; import com.raytheon.uf.common.style.StyleException; import com.raytheon.uf.common.style.StyleManager; @@ -62,7 +69,6 @@ import com.raytheon.uf.common.style.level.SingleLevel; import com.raytheon.uf.common.time.DataTime; import com.raytheon.uf.viz.core.DrawableImage; import com.raytheon.uf.viz.core.IGraphicsTarget; -import com.raytheon.uf.viz.core.drawables.ColorMapLoader; import com.raytheon.uf.viz.core.drawables.IRenderable; import com.raytheon.uf.viz.core.drawables.PaintProperties; import com.raytheon.uf.viz.core.drawables.ext.IImagingExtension.ImageProvider; @@ -79,6 +85,7 @@ import com.raytheon.uf.viz.core.rsc.interrogation.InterrogateMap; import com.raytheon.uf.viz.core.rsc.interrogation.InterrogationKey; import com.raytheon.uf.viz.core.rsc.interrogation.Interrogator; import com.raytheon.uf.viz.core.rsc.interrogation.StringInterrogationKey; +import com.raytheon.uf.viz.datacube.DataCubeContainer; import com.raytheon.viz.satellite.SatelliteConstants; import com.raytheon.viz.satellite.inventory.DerivedSatelliteRecord; import com.raytheon.viz.satellite.tileset.SatDataRetriever; @@ -127,7 +134,7 @@ import com.raytheon.viz.awipstools.IToolChangedListener; * Oct 15, 2014 3681 bsteffen create renderable in interrogate if necessary. * Feb 17, 2015 4135 bsteffen Set no data value for derived products. * Mar 3, 2015 DCS 14960 jgerth Retrieve legend from style rules if available - * + * Apr 15, 2014 4388 bsteffen Use fill value from record. * * * @@ -333,6 +340,41 @@ public class SatResource extends cmName = colorMapParameters.getColorMapName(); } } + Unit unit = SatDataRetriever.getRecordUnit(record); + + + /* + * TODO default to NaN instead of 0. 0 is the default to support older + * data(before the gini decoder set a fill value), when all decoders and + * all decoded data has a proper fill value this should be changed to + * Double.NaN. + */ + double fillValue = 0; + double defaultMin = 0; + double defaultMax = 0xff; + try { + String dataSetName = DataStoreFactory.createDataSetName(null, + SatelliteRecord.SAT_DATASET_NAME, 0); + IDataRecord dataRecord = DataCubeContainer.getDataRecord(record, + Request.buildPointRequest(new java.awt.Point(0, 0)), + dataSetName)[0]; + if (dataRecord instanceof ShortDataRecord) { + defaultMax = 0xffff; + } + Number fillObj = dataRecord.getFillValue(); + if (fillObj != null) { + fillValue = fillObj.doubleValue(); + } + Unit dataUnit = SatDataRetriever.getDataUnit(unit, dataRecord); + if (dataUnit != null && unit != null && dataUnit.isCompatible(unit)) { + UnitConverter converter = dataUnit.getConverterTo(unit); + defaultMin = converter.convert(defaultMin); + defaultMax = converter.convert(defaultMax); + } + } catch (DataCubeException e) { + statusHandler.handle(Priority.WARN, + "Unable to request sample record", e); + } SingleLevel level = new SingleLevel(Level.LevelType.SURFACE); String physicalElement = record.getPhysicalElement(); @@ -342,7 +384,6 @@ public class SatResource extends match.setParameterName(Arrays.asList(physicalElement)); match.setLevels(Arrays.asList((Level) level)); match.setCreatingEntityNames(Arrays.asList(record.getCreatingEntity())); - Unit unit = SatDataRetriever.getRecordUnit(record); String lg = null; try { StyleRule sr = StyleManager.getInstance().getStyleRule( @@ -360,8 +401,8 @@ public class SatResource extends } DataScale range = new DataScale(); range.setScaleType(DataScale.Type.LINEAR); - range.setMinValue(0.0); - range.setMaxValue(255.0); + range.setMinValue(defaultMin); + range.setMaxValue(defaultMax); preferences.setDataScale(range); } else { preferences = (ImagePreferences) sr.getPreferences(); @@ -392,7 +433,11 @@ public class SatResource extends if (cmName == null) { cmName = "Sat/VIS/ZA (Vis Default)"; } - colorMap = ColorMapLoader.loadColorMap(cmName); + try { + colorMap = ColorMapLoader.loadColorMap(cmName); + } catch (ColorMapException e) { + throw new VizException("Unable to load clormap: " + cmName, e); + } } if (colorMap != null) { @@ -402,18 +447,9 @@ public class SatResource extends if (persisted != null) { colorMapParameters.applyPersistedParameters(persisted); } - if (colorMapParameters.getDataMapping() == null) { - if (unit instanceof GenericPixel) { - /** - * Generic Pixel only comes from derived parameter which used - * signed data so 0 is valid but -128 is used as a no data - * value. - */ - colorMapParameters.setNoDataValue(Byte.MIN_VALUE); - } else { - colorMapParameters.setNoDataValue(0); - } + if (colorMapParameters.getDataMapping() == null) { + colorMapParameters.setNoDataValue(fillValue); } getCapability(ColorMapCapability.class).setColorMapParameters( diff --git a/edexOsgi/com.raytheon.edex.plugin.satellite/src/com/raytheon/edex/plugin/satellite/dao/SatelliteDao.java b/edexOsgi/com.raytheon.edex.plugin.satellite/src/com/raytheon/edex/plugin/satellite/dao/SatelliteDao.java index a1fcaf0276..2d9b90daa2 100644 --- a/edexOsgi/com.raytheon.edex.plugin.satellite/src/com/raytheon/edex/plugin/satellite/dao/SatelliteDao.java +++ b/edexOsgi/com.raytheon.edex.plugin.satellite/src/com/raytheon/edex/plugin/satellite/dao/SatelliteDao.java @@ -29,7 +29,6 @@ import com.raytheon.uf.common.dataplugin.PluginDataObject; import com.raytheon.uf.common.dataplugin.PluginException; import com.raytheon.uf.common.dataplugin.persist.IPersistable; import com.raytheon.uf.common.dataplugin.satellite.SatMapCoverage; -import com.raytheon.uf.common.dataplugin.satellite.SatelliteMessageData; import com.raytheon.uf.common.dataplugin.satellite.SatelliteRecord; import com.raytheon.uf.common.datastorage.DataStoreFactory; import com.raytheon.uf.common.datastorage.IDataStore; @@ -67,6 +66,8 @@ import com.raytheon.uf.edex.database.query.DatabaseQuery; * Mar 07, 2014 2791 bsteffen Move Data Source/Destination to numeric * plugin. * Nov 04, 2014 2714 bclement removed GINI specific DAOs + * Apr 15, 2014 4388 bsteffen Preserve fill value across interpolation levels. + * * * * @author bphillip @@ -113,8 +114,12 @@ public class SatelliteDao extends PluginDao { storageRecord.setCorrelationObject(satRecord); final Map attributes = storageRecord .getDataAttributes(); - final Float fillValue = getAttribute(attributes, - SatelliteRecord.SAT_FILL_VALUE, 0.0f); + if (storageRecord.getFillValue() == null) { + Number fillValue = getAttribute(attributes, + SatelliteRecord.SAT_FILL_VALUE, 0.0f); + storageRecord.setFillValue(fillValue); + } + final Number fillValue = storageRecord.getFillValue(); // Store the base record. dataStore.addDataRecord(storageRecord); @@ -142,13 +147,14 @@ public class SatelliteDao extends PluginDao { // data. dr.setDataAttributes(attributes); dr.setProperties(props); + dr.setFillValue(fillValue); return dr; } @Override public double getFillValue() { // always the same fill value - return fillValue; + return fillValue.doubleValue(); } @Override @@ -381,10 +387,10 @@ public class SatelliteDao extends PluginDao { */ private IDataRecord createDataRecord(SatelliteRecord satRec, Object data, int downscaleLevel, Rectangle size) { - SatelliteMessageData msgData = null; - msgData = new SatelliteMessageData(data, size.width, size.height); - IDataRecord rec = msgData.getStorageRecord(satRec, - String.valueOf(downscaleLevel)); + long[] sizes = new long[] { size.width, size.height }; + IDataRecord rec = DataStoreFactory.createStorageRecord( + String.valueOf(downscaleLevel), + satRec.getDataURI(), data, 2, sizes); rec.setCorrelationObject(satRec); rec.setGroup(DataStoreFactory.createGroupName(satRec.getDataURI(), SatelliteRecord.SAT_DATASET_NAME, true)); @@ -403,11 +409,11 @@ public class SatelliteDao extends PluginDao { * A default value. * @return */ - public static Float getAttribute(Map attrs, + public static Number getAttribute(Map attrs, String attrName, Float defValue) { - Float retValue = defValue; + Number retValue = defValue; if ((attrs != null) && (attrName != null)) { - retValue = (Float) attrs.get(attrName); + retValue = (Number) attrs.get(attrName); } return retValue; } diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.satellite/src/com/raytheon/uf/common/dataplugin/satellite/SatelliteMessageData.java b/edexOsgi/com.raytheon.uf.common.dataplugin.satellite/src/com/raytheon/uf/common/dataplugin/satellite/SatelliteMessageData.java index 0d2d55c149..9d5d222232 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.satellite/src/com/raytheon/uf/common/dataplugin/satellite/SatelliteMessageData.java +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.satellite/src/com/raytheon/uf/common/dataplugin/satellite/SatelliteMessageData.java @@ -28,11 +28,13 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; /** - * Encapsulate satellite image data as well as the dimensions of the image grid. - * Attributes about the data may also be added. As an example these attributes - * could include "scale factor" and/or "fill_value". + * @Deprecated this class is not providing anything unique enough to justify + * its existence, wherever it is used an IDataRecord of the + * appropriate type can be used instead. * - * 
+ *             TODO this class can be deleted when no decoders are using it.
+ * 
+ *             
  * 
  * SOFTWARE HISTORY
  * 
@@ -41,6 +43,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
  * Jun 27, 2012        798     jkorman Initial creation
  * Nov 14, 2013  2393       bclement   use datastore factory in
  *                                     getStorageRecord()
+ * Apr 15, 2014  4388     bsteffen    Deprecate
  * 
  * 

  * 
@@ -48,6 +51,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
  * @version 1.0
  */
 @DynamicSerialize
+@Deprecated
 public class SatelliteMessageData {
 
     private static final int DATA_DIMS = 2;
diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.satellite/src/com/raytheon/uf/common/dataplugin/satellite/SatelliteRecord.java b/edexOsgi/com.raytheon.uf.common.dataplugin.satellite/src/com/raytheon/uf/common/dataplugin/satellite/SatelliteRecord.java
index 6820f642cf..07bd4b8006 100644
--- a/edexOsgi/com.raytheon.uf.common.dataplugin.satellite/src/com/raytheon/uf/common/dataplugin/satellite/SatelliteRecord.java
+++ b/edexOsgi/com.raytheon.uf.common.dataplugin.satellite/src/com/raytheon/uf/common/dataplugin/satellite/SatelliteRecord.java
@@ -74,6 +74,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
  *                                    PluginDataObject.
  * Aug 30, 2013  2298     rjpeter     Make getPluginName abstract
  * Jun 11, 2014  2061     bsteffen    Remove IDecoderGettable
+ * Apr 15, 2014  4388     bsteffen    Deprecate SAT_FILL_VALUE
  * 
  *
* @@ -105,9 +106,10 @@ public class SatelliteRecord extends PersistablePluginDataObject implements public static final String SAT_DATASET_NAME = DataStoreFactory.DEF_DATASET_NAME; /** - * The attribute name for a value that will be used to "fill" undefined - * data. + * @deprecated {@link IDataRecord#setFillValue(Number)} should be used to + * store the fill value. */ + @Deprecated public static final String SAT_FILL_VALUE = "_FillValue"; /** diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.satellite.gini/src/com/raytheon/uf/edex/plugin/satellite/gini/GiniSatelliteDecoder.java b/edexOsgi/com.raytheon.uf.edex.plugin.satellite.gini/src/com/raytheon/uf/edex/plugin/satellite/gini/GiniSatelliteDecoder.java index 49fc601ad1..d24c83e1e8 100644 --- a/edexOsgi/com.raytheon.uf.edex.plugin.satellite.gini/src/com/raytheon/uf/edex/plugin/satellite/gini/GiniSatelliteDecoder.java +++ b/edexOsgi/com.raytheon.uf.edex.plugin.satellite.gini/src/com/raytheon/uf/edex/plugin/satellite/gini/GiniSatelliteDecoder.java @@ -37,8 +37,8 @@ import com.raytheon.edex.util.satellite.SatSpatialFactory; import com.raytheon.uf.common.dataplugin.PluginDataObject; import com.raytheon.uf.common.dataplugin.exception.UnrecognizedDataException; import com.raytheon.uf.common.dataplugin.satellite.SatMapCoverage; -import com.raytheon.uf.common.dataplugin.satellite.SatelliteMessageData; import com.raytheon.uf.common.dataplugin.satellite.SatelliteRecord; +import com.raytheon.uf.common.datastorage.DataStoreFactory; import com.raytheon.uf.common.datastorage.records.IDataRecord; import com.raytheon.uf.common.localization.IPathManager; import com.raytheon.uf.common.localization.LocalizationFile; @@ -97,6 +97,7 @@ import com.raytheon.uf.edex.plugin.satellite.gini.lookup.NumericLookupTable; * Nov 04, 2014 2714 bclement moved from satellite to satellite.gini * renamed from SatelliteDecoder to GiniDecoder * replaced database lookups with in memory tables + * Apr 15, 2014 4388 bsteffen Set Fill Value. * * * @@ -447,8 +448,6 @@ public class GiniSatelliteDecoder { default: break; } - SatelliteMessageData messageData = new SatelliteMessageData( - tempBytes, nx, ny); // get the latitude of the first point byteBuffer.position(20); @@ -537,8 +536,11 @@ public class GiniSatelliteDecoder { record.setTraceId(traceId); record.setCoverage(mapCoverage); // Create the data record. - IDataRecord dataRec = messageData.getStorageRecord(record, - SatelliteRecord.SAT_DATASET_NAME); + long[] sizes = new long[] { nx, ny }; + IDataRecord dataRec = DataStoreFactory.createStorageRecord( + SatelliteRecord.SAT_DATASET_NAME, record.getDataURI(), + tempBytes, 2, sizes); + dataRec.setFillValue(0); record.setMessageData(dataRec); } timer.stop();