From fe58826a1e1cdefd992deb6f969a0c1e97a6a84a Mon Sep 17 00:00:00 2001 From: Brian Clements Date: Wed, 13 Nov 2013 13:33:49 -0600 Subject: [PATCH] Issue #2393 removed pypies interpolation updated datastore interface updated pypies python code updated python generated from java classes moved in-java interpolation code from sat dao to utility classes changed ncep mcidas dao to use in-java interpolation changed VIIRS dao to use common interpolation code Former-commit-id: 6f9ad823de5d0b30ae7c8206b4a4fdf1e3a2b22b --- .../cave/cache/CachingDataStore.java | 19 +- .../plugin/satellite/dao/SatelliteDao.java | 164 +++++------------- .../satellite/SatelliteMessageData.java | 23 +-- .../uf/common/datastorage/IDataStore.java | 24 +-- .../common/datastorage/StorageProperties.java | 24 +-- .../data/AbstractDataWrapper.java | 26 +++ .../interpolation/data/ByteArrayWrapper.java | 3 +- .../interpolation/data/ByteBufferWrapper.java | 15 ++ .../interpolation/data/DataWrapper1D.java | 8 + .../data/DoubleBufferWrapper.java | 15 ++ .../data/FloatBufferWrapper.java | 15 ++ .../interpolation/data/IntBufferWrapper.java | 15 ++ .../data/ShortBufferWrapper.java | 15 ++ .../data/UnsignedByteBufferWrapper.java | 46 ++--- .../data/UnsignedShortBufferWrapper.java | 55 +++--- .../uf/common/pypies/PyPiesDataStore.java | 19 +- .../pypies/request/RetrieveRequest.java | 12 +- .../database/plugin/DataRecordWrapUtil.java | 121 +++++++++++++ .../database/plugin/DownscaleStoreUtil.java | 157 +++++++++++++++++ .../edex/plugin/npp/viirs/dao/VIIRSDao.java | 120 ++++++------- .../dataplugin/mcidas/McidasMapCoverage.java | 80 +++++++++ .../dataplugin/mcidas/dao/McidasDao.java | 86 +++++++-- .../rsc/AbstractSatelliteResource.java | 37 +++- .../common/datastorage/StorageProperties.py | 9 +- .../common/pypies/request/RetrieveRequest.py | 7 - .../pypies/pypies/impl/DataStoreFactory.py | 2 +- .../pypies/pypies/impl/H5pyDataStore.py | 67 +------ 27 files changed, 740 insertions(+), 444 deletions(-) create mode 100644 edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/plugin/DataRecordWrapUtil.java create mode 100644 edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/plugin/DownscaleStoreUtil.java diff --git a/cave/com.raytheon.uf.viz.thinclient.cave/src/com/raytheon/uf/viz/thinclient/cave/cache/CachingDataStore.java b/cave/com.raytheon.uf.viz.thinclient.cave/src/com/raytheon/uf/viz/thinclient/cave/cache/CachingDataStore.java index fec82757e7..24b81faaf7 100644 --- a/cave/com.raytheon.uf.viz.thinclient.cave/src/com/raytheon/uf/viz/thinclient/cave/cache/CachingDataStore.java +++ b/cave/com.raytheon.uf.viz.thinclient.cave/src/com/raytheon/uf/viz/thinclient/cave/cache/CachingDataStore.java @@ -50,6 +50,7 @@ import com.raytheon.uf.common.datastorage.records.IDataRecord; * Feb 12, 2013 1608 randerso Added explicit deletes for groups and * datasets * Sep 18, 2013 2309 bsteffen Move disk acces to DataStoreCache + * Nov 14, 2013 2393 bclement removed datastore interpolation * * * @@ -108,24 +109,6 @@ public class CachingDataStore implements IDataStore { } } - /* - * (non-Javadoc) - * - * @see - * com.raytheon.uf.common.datastorage.IDataStore#retrieve(java.lang.String, - * boolean) - */ - @Override - public IDataRecord[] retrieve(String group, boolean includeInterpolated) - throws StorageException, FileNotFoundException { - if (includeInterpolated == false) { - return retrieve(group); - } else { - /* This is deprecated and unused so caching is not implemented. */ - return delegate.retrieve(group, includeInterpolated); - } - } - /* * (non-Javadoc) * diff --git a/edexOsgi/com.raytheon.edex.plugin.satellite/src/com/raytheon/edex/plugin/satellite/dao/SatelliteDao.java b/edexOsgi/com.raytheon.edex.plugin.satellite/src/com/raytheon/edex/plugin/satellite/dao/SatelliteDao.java index f9d79202a7..d54cae295d 100644 --- a/edexOsgi/com.raytheon.edex.plugin.satellite/src/com/raytheon/edex/plugin/satellite/dao/SatelliteDao.java +++ b/edexOsgi/com.raytheon.edex.plugin.satellite/src/com/raytheon/edex/plugin/satellite/dao/SatelliteDao.java @@ -25,8 +25,6 @@ import java.util.Date; import java.util.List; import java.util.Map; -import org.opengis.referencing.operation.TransformException; - import com.raytheon.edex.plugin.satellite.gini.SatelliteCreatingEntity; import com.raytheon.edex.plugin.satellite.gini.SatellitePhysicalElement; import com.raytheon.edex.plugin.satellite.gini.SatellitePosition; @@ -43,18 +41,16 @@ import com.raytheon.uf.common.datastorage.DataStoreFactory; import com.raytheon.uf.common.datastorage.IDataStore; import com.raytheon.uf.common.datastorage.StorageException; import com.raytheon.uf.common.datastorage.StorageProperties; -import com.raytheon.uf.common.datastorage.records.ByteDataRecord; import com.raytheon.uf.common.datastorage.records.IDataRecord; -import com.raytheon.uf.common.datastorage.records.ShortDataRecord; import com.raytheon.uf.common.geospatial.interpolation.GridDownscaler; -import com.raytheon.uf.common.geospatial.interpolation.data.AbstractDataWrapper; -import com.raytheon.uf.common.geospatial.interpolation.data.ByteArrayWrapper; import com.raytheon.uf.common.geospatial.interpolation.data.DataDestination; -import com.raytheon.uf.common.geospatial.interpolation.data.ShortArrayWrapper; -import com.raytheon.uf.common.geospatial.interpolation.data.UnsignedByteArrayWrapper; +import com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D; import com.raytheon.uf.common.time.TimeRange; import com.raytheon.uf.edex.core.dataplugin.PluginRegistry; import com.raytheon.uf.edex.database.DataAccessLayerException; +import com.raytheon.uf.edex.database.plugin.DataRecordWrapUtil; +import com.raytheon.uf.edex.database.plugin.DownscaleStoreUtil; +import com.raytheon.uf.edex.database.plugin.DownscaleStoreUtil.IDataRecordCreator; import com.raytheon.uf.edex.database.plugin.PluginDao; import com.raytheon.uf.edex.database.query.DatabaseQuery; @@ -75,6 +71,7 @@ import com.raytheon.uf.edex.database.query.DatabaseQuery; * input arguments. * 06/24/2013 2044 randerso Added methods to get data by TimeRange and * getInventory with maxRecord limit + * Nov 14, 2013 2393 bclement moved interpolation code to parent class * * * @author bphillip @@ -124,79 +121,61 @@ public class SatelliteDao extends PluginDao { @Override protected IDataStore populateDataStore(IDataStore dataStore, IPersistable record) throws StorageException { - SatelliteRecord satRecord = (SatelliteRecord) record; + final SatelliteRecord satRecord = (SatelliteRecord) record; IDataRecord storageRecord = (IDataRecord) satRecord.getMessageData(); if (storageRecord != null) { - StorageProperties props = new StorageProperties(); + final StorageProperties props = new StorageProperties(); String compression = PluginRegistry.getInstance() .getRegisteredObject(pluginName).getCompression(); if (compression != null) { props.setCompression(StorageProperties.Compression .valueOf(compression)); } - props.setDownscaled(false); storageRecord.setProperties(props); storageRecord.setCorrelationObject(satRecord); + final Map attributes = storageRecord + .getDataAttributes(); + final Float fillValue = getAttribute(attributes, + SatelliteRecord.SAT_FILL_VALUE, 0.0f); + // Store the base record. dataStore.addDataRecord(storageRecord); - Map attributes = storageRecord.getDataAttributes(); - - Float fillValue = getAttribute(attributes, - SatelliteRecord.SAT_FILL_VALUE, 0.0f); - SatMapCoverage coverage = satRecord.getCoverage(); - AbstractDataWrapper dataSource = getSource(storageRecord, - coverage.getNx(), coverage.getNy()); - dataSource.setFillValue(fillValue); + GridDownscaler downScaler = new GridDownscaler( coverage.getGridGeometry()); - // How many interpolation levels do we need for this data? - int levels = downScaler.getNumberOfDownscaleLevels(); + Rectangle fullScale = downScaler.getDownscaleSize(0); + DataWrapper1D dataSource = DataRecordWrapUtil.wrap( + storageRecord, fullScale.width, fullScale.height, true); + + int levels = DownscaleStoreUtil.storeInterpolated(dataStore, + downScaler, dataSource, + new IDataRecordCreator() { + + @Override + public IDataRecord create(Object data, + int downScaleLevel, Rectangle size) + throws StorageException { + IDataRecord dr = createDataRecord(satRecord, data, + downScaleLevel, size); + // Set the attributes and properties from the parent + // data. + dr.setDataAttributes(attributes); + dr.setProperties(props); + return dr; + } + + @Override + public double getFillValue() { + // always the same fill value + return fillValue; + } + }); // set the number of levels in the 'parent' satellite data. - // Subtract one for the base level data. - satRecord.setInterpolationLevels(levels - 1); - - // How many interpolation levels do we need for this data? Includes - // the base level! - // Subtract one for the base level data. - int downScaleLevels = downScaler.getNumberOfDownscaleLevels() - 1; - // set the number of downscale levels in the satellite metadata. - satRecord.setInterpolationLevels(downScaleLevels); - if (DataStoreFactory.isInterpolated(levels)) { - for (int level = 0; level < downScaleLevels; level++) { - int downScaleLevel = level + 1; - Rectangle size = downScaler - .getDownscaleSize(downScaleLevel); - - AbstractDataWrapper dest = getDestination(storageRecord, - size); - dest.setFillValue(fillValue); - try { - // Downscale from previous level - downScaler.downscale(downScaleLevel - 1, - downScaleLevel, dataSource, dest); - - IDataRecord dr = createDataRecord(satRecord, dest, - downScaleLevel, size); - // Set the attributes and properties from the parent - // data. - dr.setDataAttributes(attributes); - dr.setProperties(props); - dataStore.addDataRecord(dr); - - // Set source to current level - dataSource = dest; - } catch (TransformException e) { - throw new StorageException( - "Error creating downscaled data", - storageRecord, e); - } - } - } - + satRecord.setInterpolationLevels(levels); } return dataStore; } @@ -518,52 +497,6 @@ public class SatelliteDao extends PluginDao { this.positionDao = positionDao; } - /** - * Create an {@link AbstractDataWrapper} destination from the supplied - * {@link IDataRecord} with given dimensions. - * - * @param rec - * The record containing data to be wrapped. - * @param size - * A {@link Rectangle} containing the size of the input data. - * @return The wrapped data. - */ - private AbstractDataWrapper getDestination(IDataRecord rec, Rectangle size) { - AbstractDataWrapper dest = null; - - if (rec instanceof ByteDataRecord) { - dest = new UnsignedByteArrayWrapper(size.width, size.height); - } else if (rec instanceof ShortDataRecord) { - dest = new ShortArrayWrapper(size.width, size.height); - } - return dest; - } - - /** - * Create an {@link AbstractDataWrapper} source from the supplied - * {@link IDataRecord} with given dimensions. - * - * @param rec - * The record containing data to be wrapped. - * @param nx - * Number of items on the x axis. - * @param ny - * Number of items on the y axis. - * @return The wrapped data. - */ - private AbstractDataWrapper getSource(IDataRecord rec, int nx, int ny) { - AbstractDataWrapper source = null; - - if (rec instanceof ByteDataRecord) { - byte[] b = ((ByteDataRecord) rec).getByteData(); - source = new UnsignedByteArrayWrapper(b, nx, ny); - } else if (rec instanceof ShortDataRecord) { - short[] s = ((ShortDataRecord) rec).getShortData(); - source = new ShortArrayWrapper(s, nx, ny); - } - return source; - } - /** * Create the {@link IDataRecord} from the {@link DataDestination} using the * original satellite data, size and @@ -577,19 +510,12 @@ public class SatelliteDao extends PluginDao { * @param size * Size of the down-scaled data. * @return The created data record to be stored. + * @throws PluginException */ - private IDataRecord createDataRecord(SatelliteRecord satRec, - DataDestination data, int downscaleLevel, Rectangle size) { + private IDataRecord createDataRecord(SatelliteRecord satRec, Object data, + int downscaleLevel, Rectangle size) throws StorageException { SatelliteMessageData msgData = null; - Object o = null; - if (data instanceof ByteArrayWrapper) { - o = ((ByteArrayWrapper) data).getArray(); - } else if (data instanceof ShortArrayWrapper) { - o = ((ShortArrayWrapper) data).getArray(); - } - if (o != null) { - msgData = new SatelliteMessageData(o, size.width, size.height); - } + msgData = new SatelliteMessageData(data, size.width, size.height); IDataRecord rec = msgData.getStorageRecord(satRec, String.valueOf(downscaleLevel)); rec.setCorrelationObject(satRec); diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.satellite/src/com/raytheon/uf/common/dataplugin/satellite/SatelliteMessageData.java b/edexOsgi/com.raytheon.uf.common.dataplugin.satellite/src/com/raytheon/uf/common/dataplugin/satellite/SatelliteMessageData.java index a3b4da3016..0d2d55c149 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.satellite/src/com/raytheon/uf/common/dataplugin/satellite/SatelliteMessageData.java +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.satellite/src/com/raytheon/uf/common/dataplugin/satellite/SatelliteMessageData.java @@ -22,17 +22,15 @@ package com.raytheon.uf.common.dataplugin.satellite; import java.util.HashMap; import java.util.Map; -import com.raytheon.uf.common.dataplugin.satellite.SatelliteRecord; -import com.raytheon.uf.common.datastorage.records.ByteDataRecord; +import com.raytheon.uf.common.datastorage.DataStoreFactory; import com.raytheon.uf.common.datastorage.records.IDataRecord; -import com.raytheon.uf.common.datastorage.records.ShortDataRecord; import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; /** - * Encapsulate satellite image data as well as the dimensions of - * the image grid. Attributes about the data may also be added. As an - * example these attributes could include "scale factor" and/or "fill_value". + * Encapsulate satellite image data as well as the dimensions of the image grid. + * Attributes about the data may also be added. As an example these attributes + * could include "scale factor" and/or "fill_value". * * 
  * 
@@ -41,6 +39,8 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * Jun 27, 2012        798     jkorman Initial creation
+ * Nov 14, 2013  2393       bclement   use datastore factory in
+ *                                     getStorageRecord()
  * 
  *
* @@ -125,15 +125,8 @@ public class SatelliteMessageData { IDataRecord storageRecord = null; if ((messageData != null) && (dataRec != null)) { long[] sizes = new long[] { nx, ny }; - if (messageData instanceof byte[]) { - storageRecord = new ByteDataRecord(dataSetName, - dataRec.getDataURI(), (byte[]) messageData, DATA_DIMS, - sizes); - } else if (messageData instanceof short[]) { - storageRecord = new ShortDataRecord(dataSetName, - dataRec.getDataURI(), (short[]) messageData, DATA_DIMS, - sizes); - } + storageRecord = DataStoreFactory.createStorageRecord(dataSetName, + dataRec.getDataURI(), messageData, DATA_DIMS, sizes); } if ((storageRecord != null) && (dataAttributes != null)) { storageRecord.setDataAttributes(dataAttributes); diff --git a/edexOsgi/com.raytheon.uf.common.datastorage/src/com/raytheon/uf/common/datastorage/IDataStore.java b/edexOsgi/com.raytheon.uf.common.datastorage/src/com/raytheon/uf/common/datastorage/IDataStore.java index 5f2e309d6c..834c86ae64 100644 --- a/edexOsgi/com.raytheon.uf.common.datastorage/src/com/raytheon/uf/common/datastorage/IDataStore.java +++ b/edexOsgi/com.raytheon.uf.common.datastorage/src/com/raytheon/uf/common/datastorage/IDataStore.java @@ -25,7 +25,6 @@ import java.util.Map; import com.raytheon.uf.common.datastorage.StorageProperties.Compression; import com.raytheon.uf.common.datastorage.records.IDataRecord; -import com.raytheon.uf.common.serialization.ISerializableObject; /** * Defines the interface for operating against a hierarchical datastore @@ -41,6 +40,7 @@ import com.raytheon.uf.common.serialization.ISerializableObject; * Feb 12, 2013 1608 randerso Added explicit methods for deleting * groups and datasets * Sep 19, 2013 2309 bsteffen Deprecate retrieve(String, boolean) + * Nov 14, 2013 2393 bclement removed interpolation * * * @@ -48,7 +48,7 @@ import com.raytheon.uf.common.serialization.ISerializableObject; * @author chammack * @version 1.0 */ -public interface IDataStore extends ISerializableObject { +public interface IDataStore { public static enum HDF5_ITEM { DATASET, GROUP @@ -143,26 +143,6 @@ public interface IDataStore extends ISerializableObject { public abstract IDataRecord[] retrieve(String group) throws StorageException, FileNotFoundException; - /** - * Convenience method for retrieve - * - * Retrieves all data at a given group, with the option to retrieve - * interpolated tilesets. - * - * @param group - * the group of data to retrieve - * @param includeInterpolated - * a flag indicating whether interpolated tilesets should be - * retrieved - * @return the data records - * @throws StorageException - * @throws FileNotFoundException - */ - @Deprecated - public abstract IDataRecord[] retrieve(String group, - boolean includeInterpolated) throws StorageException, - FileNotFoundException; - /** * Retrieve a single dataset with optional subsetting * diff --git a/edexOsgi/com.raytheon.uf.common.datastorage/src/com/raytheon/uf/common/datastorage/StorageProperties.java b/edexOsgi/com.raytheon.uf.common.datastorage/src/com/raytheon/uf/common/datastorage/StorageProperties.java index 7e21d0617f..6a5757839f 100644 --- a/edexOsgi/com.raytheon.uf.common.datastorage/src/com/raytheon/uf/common/datastorage/StorageProperties.java +++ b/edexOsgi/com.raytheon.uf.common.datastorage/src/com/raytheon/uf/common/datastorage/StorageProperties.java @@ -22,7 +22,6 @@ package com.raytheon.uf.common.datastorage; import org.apache.commons.lang.Validate; -import com.raytheon.uf.common.serialization.ISerializableObject; import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; @@ -36,6 +35,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Feb 8, 2007 chammack Initial Creation. + * Nov 14, 2013 2393 bclement removed interpolation * * * @@ -43,7 +43,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; * @version 1 */ @DynamicSerialize -public class StorageProperties implements Cloneable, ISerializableObject { +public class StorageProperties implements Cloneable { /** * Compression types: @@ -66,10 +66,6 @@ public class StorageProperties implements Cloneable, ISerializableObject { @DynamicSerializeElement private boolean chunked; - /** Is the data progressively downsampled */ - @DynamicSerializeElement - private boolean downscaled; - /** * Construct a default storage properties. Default is all features disabled. */ @@ -114,21 +110,6 @@ public class StorageProperties implements Cloneable, ISerializableObject { } } - /** - * @return the isDownscaled - */ - public boolean isDownscaled() { - return downscaled; - } - - /** - * @param isDownscaled - * the isDownscaled to set - */ - public void setDownscaled(boolean isDownscaled) { - this.downscaled = isDownscaled; - } - /* * (non-Javadoc) * @@ -138,7 +119,6 @@ public class StorageProperties implements Cloneable, ISerializableObject { public StorageProperties clone() { StorageProperties sp = new StorageProperties(); sp.chunked = chunked; - sp.downscaled = downscaled; sp.compression = compression; return sp; } diff --git a/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/AbstractDataWrapper.java b/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/AbstractDataWrapper.java index 5517e44dda..98efbd61ea 100644 --- a/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/AbstractDataWrapper.java +++ b/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/AbstractDataWrapper.java @@ -19,6 +19,9 @@ **/ package com.raytheon.uf.common.geospatial.interpolation.data; +import java.awt.Rectangle; +import java.lang.reflect.Constructor; + import org.geotools.coverage.grid.GeneralGridGeometry; import com.raytheon.uf.common.geospatial.util.GridGeometryWrapChecker; @@ -35,6 +38,7 @@ import com.raytheon.uf.common.geospatial.util.GridGeometryWrapChecker; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Jun 20, 2012 bsteffen Initial creation + * Nov 19, 2013 2393 bclement added createNew method * * * @@ -113,4 +117,26 @@ public abstract class AbstractDataWrapper implements DataSource, protected abstract void setDataValueInternal(double dataValue, int x, int y); + /** + * Create a new data wrapper of type c with the provided size + * + * @param c + * desired implementation class + * @param size + * @return + * @throws IllegalArgumentException + */ + public static T createNew( + Class c, Rectangle size) + throws IllegalArgumentException { + try { + Constructor constructor = c.getConstructor(int.class, + int.class); + return constructor.newInstance(size.width, size.height); + } catch (Exception e) { + throw new IllegalArgumentException( + "Unable to instatiate instance of class: " + c, e); + } + } + } \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/ByteArrayWrapper.java b/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/ByteArrayWrapper.java index 6fa57e1541..b6b418e95e 100644 --- a/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/ByteArrayWrapper.java +++ b/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/ByteArrayWrapper.java @@ -22,7 +22,7 @@ package com.raytheon.uf.common.geospatial.interpolation.data; import org.geotools.coverage.grid.GeneralGridGeometry; /** - * {@link AbstractDataWrapper} implementation for byte array data. + * {@link AbstractDataWrapper} implementation for byte array data. * * 
  * 
@@ -113,4 +113,5 @@ public class ByteArrayWrapper extends DataWrapper1D {
     public void setDataValueInternal(double dataValue, int index) {
         array[index] = (byte) dataValue;
     }
+
 }
diff --git a/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/ByteBufferWrapper.java b/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/ByteBufferWrapper.java
index 869dd0fbdb..17f5e16fd7 100644
--- a/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/ByteBufferWrapper.java
+++ b/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/ByteBufferWrapper.java
@@ -34,6 +34,7 @@ import org.geotools.coverage.grid.GeneralGridGeometry;
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * Jun 20, 2012            bsteffen     Initial creation
+ * Nov 19, 2013  2393      bclement    added getArray
  * 
  *
* @@ -79,4 +80,18 @@ public class ByteBufferWrapper extends DataWrapper1D { buffer.put(index, (byte) dataValue); } + /* + * (non-Javadoc) + * + * @see com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D# + * getPrimitiveArray() + */ + @Override + public byte[] getArray() { + if (buffer.hasArray()) { + return buffer.array(); + } + return null; + } + } \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/DataWrapper1D.java b/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/DataWrapper1D.java index 3ecb57799e..57ec72c57d 100644 --- a/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/DataWrapper1D.java +++ b/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/DataWrapper1D.java @@ -32,6 +32,7 @@ import org.geotools.coverage.grid.GeneralGridGeometry; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Jun 20, 2012 bsteffen Initial creation + * Nov 19, 2013 2393 bclement added abstract method for primitive array * * * @@ -62,4 +63,11 @@ public abstract class DataWrapper1D extends AbstractDataWrapper { protected abstract void setDataValueInternal(double dataValue, int index); + /** + * Get the backing 1D primitive array + * + * @return null if array isn't available + */ + public abstract Object getArray(); + } \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/DoubleBufferWrapper.java b/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/DoubleBufferWrapper.java index e9fafc901b..dcedce91ae 100644 --- a/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/DoubleBufferWrapper.java +++ b/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/DoubleBufferWrapper.java @@ -34,6 +34,7 @@ import org.geotools.coverage.grid.GeneralGridGeometry; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Jun 20, 2012 bsteffen Initial creation + * Nov 19, 2013 2393 bclement added getArray * * * @@ -79,4 +80,18 @@ public class DoubleBufferWrapper extends DataWrapper1D { buffer.put(index, dataValue); } + /* + * (non-Javadoc) + * + * @see com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D# + * getPrimitiveArray() + */ + @Override + public double[] getArray() { + if (buffer.hasArray()) { + return buffer.array(); + } + return null; + } + } \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/FloatBufferWrapper.java b/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/FloatBufferWrapper.java index 17461d42de..9a3021ffd7 100644 --- a/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/FloatBufferWrapper.java +++ b/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/FloatBufferWrapper.java @@ -34,6 +34,7 @@ import org.geotools.coverage.grid.GeneralGridGeometry; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Jun 20, 2012 bsteffen Initial creation + * Nov 19, 2013 2393 bclement added getArray * * * @@ -79,4 +80,18 @@ public class FloatBufferWrapper extends DataWrapper1D { buffer.put(index, (float) dataValue); } + /* + * (non-Javadoc) + * + * @see com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D# + * getPrimitiveArray() + */ + @Override + public float[] getArray() { + if (buffer.hasArray()) { + return buffer.array(); + } + return null; + } + } \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/IntBufferWrapper.java b/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/IntBufferWrapper.java index c4d1b6d743..f5963a058d 100644 --- a/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/IntBufferWrapper.java +++ b/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/IntBufferWrapper.java @@ -34,6 +34,7 @@ import org.geotools.coverage.grid.GeneralGridGeometry; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Jun 20, 2012 bsteffen Initial creation + * Nov 19, 2013 2393 bclement added getArray * * * @@ -79,4 +80,18 @@ public class IntBufferWrapper extends DataWrapper1D { buffer.put(index, (int) dataValue); } + /* + * (non-Javadoc) + * + * @see com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D# + * getPrimitiveArray() + */ + @Override + public int[] getArray() { + if (buffer.hasArray()) { + return buffer.array(); + } + return null; + } + } \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/ShortBufferWrapper.java b/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/ShortBufferWrapper.java index 7b1a2a900c..ab10bf1b98 100644 --- a/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/ShortBufferWrapper.java +++ b/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/ShortBufferWrapper.java @@ -34,6 +34,7 @@ import org.geotools.coverage.grid.GeneralGridGeometry; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Jun 20, 2012 bsteffen Initial creation + * Nov 19, 2013 2393 bclement added getArray * * * @@ -79,4 +80,18 @@ public class ShortBufferWrapper extends DataWrapper1D { buffer.put(index, (short) dataValue); } + /* + * (non-Javadoc) + * + * @see com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D# + * getPrimitiveArray() + */ + @Override + public short[] getArray() { + if (buffer.hasArray()) { + return buffer.array(); + } + return null; + } + } \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/UnsignedByteBufferWrapper.java b/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/UnsignedByteBufferWrapper.java index d2286888cb..05ad34aec4 100644 --- a/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/UnsignedByteBufferWrapper.java +++ b/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/UnsignedByteBufferWrapper.java @@ -25,7 +25,7 @@ import org.geotools.coverage.grid.GeneralGridGeometry; /** * - * ByteBuffer data wrapper + * Unsigned ByteBuffer data wrapper * * 
  * 
@@ -34,40 +34,47 @@ import org.geotools.coverage.grid.GeneralGridGeometry;
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * Jun 20, 2012            bsteffen     Initial creation
+ * Nov 19, 2013  2393      bclement    changed to extend ByteBufferWrapper
  * 
  *
* * @author bsteffen * @version 1.0 */ -public class UnsignedByteBufferWrapper extends DataWrapper1D { +public class UnsignedByteBufferWrapper extends ByteBufferWrapper { - protected final ByteBuffer buffer; + /** + * @param buffer + * @param geometry + */ public UnsignedByteBufferWrapper(ByteBuffer buffer, GeneralGridGeometry geometry) { - super(geometry); - this.buffer = buffer; + super(buffer, geometry); } + /** + * @param buffer + * @param nx + * @param ny + */ public UnsignedByteBufferWrapper(ByteBuffer buffer, int nx, int ny) { - super(nx, ny); - this.buffer = buffer; - } - - public UnsignedByteBufferWrapper(int nx, int ny) { - this(ByteBuffer.allocate(nx * ny), nx, ny); + super(buffer, nx, ny); } + /** + * @param geometry + */ public UnsignedByteBufferWrapper(GeneralGridGeometry geometry) { - // assume this is going to be a destination and avoid passing - // geometry to super to save time on checking for wrapping. - this(geometry.getGridRange().getSpan(0), geometry.getGridRange() - .getSpan(1)); + super(geometry); } - public ByteBuffer getBuffer() { - return buffer; + /** + * @param nx + * @param ny + */ + public UnsignedByteBufferWrapper(int nx, int ny) { + super(nx, ny); } @Override @@ -75,9 +82,4 @@ public class UnsignedByteBufferWrapper extends DataWrapper1D { return buffer.get(index) & 0xFF; } - @Override - public void setDataValueInternal(double dataValue, int index) { - buffer.put(index, (byte) dataValue); - } - } \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/UnsignedShortBufferWrapper.java b/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/UnsignedShortBufferWrapper.java index 44c3dc1bc0..1f22c686bd 100644 --- a/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/UnsignedShortBufferWrapper.java +++ b/edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/UnsignedShortBufferWrapper.java @@ -25,7 +25,7 @@ import org.geotools.coverage.grid.GeneralGridGeometry; /** * - * ShortBuffer data wrapper + * Unsigned ShortBuffer data wrapper * * 
  * 
@@ -34,40 +34,46 @@ import org.geotools.coverage.grid.GeneralGridGeometry;
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * Jun 20, 2012            bsteffen     Initial creation
+ * Nov 19, 2013  2393      bclement    changed to extend ShortBufferWrapper
  * 
  *
* * @author bsteffen * @version 1.0 */ -public class UnsignedShortBufferWrapper extends DataWrapper1D { +public class UnsignedShortBufferWrapper extends ShortBufferWrapper { - protected final ShortBuffer buffer; + /** + * @param geometry + */ + public UnsignedShortBufferWrapper(GeneralGridGeometry geometry) { + super(geometry); + } + /** + * @param nx + * @param ny + */ + public UnsignedShortBufferWrapper(int nx, int ny) { + super(nx, ny); + } + + /** + * @param buffer + * @param geometry + */ public UnsignedShortBufferWrapper(ShortBuffer buffer, GeneralGridGeometry geometry) { - super(geometry); - this.buffer = buffer; + super(buffer, geometry); } + /** + * @param buffer + * @param nx + * @param ny + */ public UnsignedShortBufferWrapper(ShortBuffer buffer, int nx, int ny) { - super(nx, ny); - this.buffer = buffer; - } - - public UnsignedShortBufferWrapper(int nx, int ny) { - this(ShortBuffer.allocate(nx * ny), nx, ny); - } - - public UnsignedShortBufferWrapper(GeneralGridGeometry geometry) { - // assume this is going to be a destination and avoid passing - // geometry to super to save time on checking for wrapping. - this(geometry.getGridRange().getSpan(0), geometry.getGridRange() - .getSpan(1)); - } - - public ShortBuffer getBuffer() { - return buffer; + super(buffer, nx, ny); } @Override @@ -75,9 +81,4 @@ public class UnsignedShortBufferWrapper extends DataWrapper1D { return buffer.get(index) & 0xFFFF; } - @Override - public void setDataValueInternal(double dataValue, int index) { - buffer.put(index, (short) dataValue); - } - } \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.common.pypies/src/com/raytheon/uf/common/pypies/PyPiesDataStore.java b/edexOsgi/com.raytheon.uf.common.pypies/src/com/raytheon/uf/common/pypies/PyPiesDataStore.java index c252b9ecad..be542e2826 100644 --- a/edexOsgi/com.raytheon.uf.common.pypies/src/com/raytheon/uf/common/pypies/PyPiesDataStore.java +++ b/edexOsgi/com.raytheon.uf.common.pypies/src/com/raytheon/uf/common/pypies/PyPiesDataStore.java @@ -73,6 +73,7 @@ import com.raytheon.uf.common.util.FileUtil; * Mon 07, 2013 DR 15294 D. Friedman Stream large requests * Feb 11, 2013 1526 njensen use HttpClient.postDynamicSerialize() for memory efficiency * Feb 12, 2013 #1608 randerso Added explicit deletes for groups and datasets + * Nov 14, 2013 2393 bclement removed interpolation * * * @@ -202,23 +203,8 @@ public class PyPiesDataStore implements IDataStore { @Override public IDataRecord[] retrieve(final String group) throws StorageException, FileNotFoundException { - return retrieve(group, false); - } - - /* - * (non-Javadoc) - * - * @see - * com.raytheon.uf.common.datastorage.IDataStore#retrieve(java.lang.String, - * boolean) - */ - @Override - public IDataRecord[] retrieve(final String group, - final boolean includeInterpolated) throws StorageException, - FileNotFoundException { RetrieveRequest req = new RetrieveRequest(); req.setGroup(group); - req.setIncludeInterpolated(includeInterpolated); RetrieveResponse resp = (RetrieveResponse) cachedRequest(req); return resp.getRecords(); } @@ -448,7 +434,8 @@ public class PyPiesDataStore implements IDataStore { protected Object deserializeResponse(final byte[] response) throws StorageException { try { - return SerializationUtil.transformFromThrift(response); + return SerializationUtil + .transformFromThrift(Object.class, response); } catch (SerializationException e) { throw new StorageException( "Error deserializing response from pypies server", null, e); diff --git a/edexOsgi/com.raytheon.uf.common.pypies/src/com/raytheon/uf/common/pypies/request/RetrieveRequest.java b/edexOsgi/com.raytheon.uf.common.pypies/src/com/raytheon/uf/common/pypies/request/RetrieveRequest.java index 7b6eaac6ab..c28f7835ea 100644 --- a/edexOsgi/com.raytheon.uf.common.pypies/src/com/raytheon/uf/common/pypies/request/RetrieveRequest.java +++ b/edexOsgi/com.raytheon.uf.common.pypies/src/com/raytheon/uf/common/pypies/request/RetrieveRequest.java @@ -32,6 +32,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Jul 8, 2010 njensen Initial creation + * Nov 14, 2013 2393 bclement removed get all interpolated * * * @@ -45,9 +46,6 @@ public class RetrieveRequest extends AbstractRequest { @DynamicSerializeElement private String group; - @DynamicSerializeElement - private boolean includeInterpolated; - @DynamicSerializeElement private String dataset; @@ -62,14 +60,6 @@ public class RetrieveRequest extends AbstractRequest { this.group = group; } - public boolean isIncludeInterpolated() { - return includeInterpolated; - } - - public void setIncludeInterpolated(boolean includeInterpolated) { - this.includeInterpolated = includeInterpolated; - } - public String getDataset() { return dataset; } diff --git a/edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/plugin/DataRecordWrapUtil.java b/edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/plugin/DataRecordWrapUtil.java new file mode 100644 index 0000000000..24f0a697b8 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/plugin/DataRecordWrapUtil.java @@ -0,0 +1,121 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.edex.database.plugin; + +import com.raytheon.uf.common.datastorage.StorageException; +import com.raytheon.uf.common.datastorage.records.ByteDataRecord; +import com.raytheon.uf.common.datastorage.records.FloatDataRecord; +import com.raytheon.uf.common.datastorage.records.IDataRecord; +import com.raytheon.uf.common.datastorage.records.IntegerDataRecord; +import com.raytheon.uf.common.datastorage.records.ShortDataRecord; +import com.raytheon.uf.common.geospatial.interpolation.data.AbstractDataWrapper; +import com.raytheon.uf.common.geospatial.interpolation.data.ByteArrayWrapper; +import com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D; +import com.raytheon.uf.common.geospatial.interpolation.data.FloatArrayWrapper; +import com.raytheon.uf.common.geospatial.interpolation.data.IntArrayWrapper; +import com.raytheon.uf.common.geospatial.interpolation.data.ShortArrayWrapper; +import com.raytheon.uf.common.geospatial.interpolation.data.UnsignedByteArrayWrapper; +import com.raytheon.uf.common.geospatial.interpolation.data.UnsignedShortArrayWrapper; + +/** + * Utility for wrapping data records to be a data source/destination + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Nov 19, 2013  2393      bclement     Initial creation
+ * 
+ *
+ * + * @author bclement + * @version 1.0 + */ + +public class DataRecordWrapUtil { + + /** + * Create an {@link AbstractDataWrapper} source from the supplied + * {@link IDataRecord} with given dimensions. + * + * @param rec + * The record containing data to be wrapped. + * @param nx + * Number of items on the x axis. + * @param ny + * Number of items on the y axis. + * @return The wrapped data. + * @throws StorageException + */ + public static DataWrapper1D wrap(IDataRecord rec, int nx, int ny) + throws StorageException { + // default to signed data + return wrap(rec, nx, ny, false); + } + + /** + * Create an {@link AbstractDataWrapper} source from the supplied + * {@link IDataRecord} with given dimensions. + * + * @param rec + * The record containing data to be wrapped. + * @param nx + * Number of items on the x axis. + * @param ny + * Number of items on the y axis. + * @param unsigned + * attempt to treat the data as unsigned if possible + * @return The wrapped data. + * @throws StorageException + */ + public static DataWrapper1D wrap(IDataRecord rec, int nx, int ny, + boolean unsigned) throws StorageException { + DataWrapper1D source = null; + + if (rec instanceof ByteDataRecord) { + byte[] b = ((ByteDataRecord) rec).getByteData(); + if (unsigned) { + source = new UnsignedByteArrayWrapper(b, nx, ny); + } else { + source = new ByteArrayWrapper(b, nx, ny); + } + } else if (rec instanceof ShortDataRecord) { + short[] s = ((ShortDataRecord) rec).getShortData(); + if (unsigned) { + source = new UnsignedShortArrayWrapper(s, nx, ny); + } else { + source = new ShortArrayWrapper(s, nx, ny); + } + } else if (rec instanceof IntegerDataRecord) { + int[] i = ((IntegerDataRecord) rec).getIntData(); + source = new IntArrayWrapper(i, nx, ny); + } else if (rec instanceof FloatDataRecord) { + float[] f = ((FloatDataRecord) rec).getFloatData(); + source = new FloatArrayWrapper(f, nx, ny); + } else { + throw new StorageException("Unsupported data record type: " + + rec.getClass(), rec); + } + return source; + } + +} diff --git a/edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/plugin/DownscaleStoreUtil.java b/edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/plugin/DownscaleStoreUtil.java new file mode 100644 index 0000000000..5675d7cab2 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/plugin/DownscaleStoreUtil.java @@ -0,0 +1,157 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.edex.database.plugin; + +import java.awt.Rectangle; + +import org.opengis.referencing.operation.TransformException; + +import com.raytheon.uf.common.dataplugin.PluginDataObject; +import com.raytheon.uf.common.datastorage.DataStoreFactory; +import com.raytheon.uf.common.datastorage.IDataStore; +import com.raytheon.uf.common.datastorage.StorageException; +import com.raytheon.uf.common.datastorage.records.IDataRecord; +import com.raytheon.uf.common.geospatial.interpolation.GridDownscaler; +import com.raytheon.uf.common.geospatial.interpolation.data.AbstractDataWrapper; +import com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D; + +/** + * Utility for storing downscaled data to datastore + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Nov 19, 2013  2393      bclement     Initial creation
+ * 
+ *
+ * + * @author bclement + * @version 1.0 + */ + +public class DownscaleStoreUtil { + + /** + * Interface for creating IDataRecords for interpolation levels + */ + public static interface IDataRecordCreator { + + /** + * Create new data record for level + * + * @param data + * @param downScaleLevel + * @param size + * @return + * @throws StorageException + */ + public IDataRecord create(Object data, int downScaleLevel, + Rectangle size) throws StorageException; + + /** + * @return fill value + */ + public double getFillValue(); + } + + /** + * Create and add interpolated levels from dataSource. + * + * @param dataStore + * @param downScaler + * @param dataSource + * @param creator + * @return number of levels not including the base level + * @throws StorageException + */ + public static int storeInterpolated( + IDataStore dataStore, GridDownscaler downScaler, + DataWrapper1D dataSource, IDataRecordCreator creator) + throws StorageException { + // default to batch storage + return storeInterpolated(dataStore, downScaler, dataSource, creator, + false); + } + + /** + * Create and add interpolated levels from dataSource. + * + * @param dataStore + * @param downScaler + * @param dataSource + * @param creator + * @param storeAfterEach + * if true, call store method on dataStore after each level is + * created + * @return number of levels not including the base level + * @throws StorageException + */ + public static int storeInterpolated( + IDataStore dataStore, GridDownscaler downScaler, + DataWrapper1D dataSource, IDataRecordCreator creator, + boolean storeAfterEach) throws StorageException { + + dataSource.setFillValue(creator.getFillValue()); + + // How many interpolation levels do we need for this data? + int levels = downScaler.getNumberOfDownscaleLevels(); + + // How many interpolation levels do we need for this data? Includes + // the base level! + // Subtract one for the base level data. + int downScaleLevels = levels - 1; + if (DataStoreFactory.isInterpolated(levels)) { + for (int level = 0; level < downScaleLevels; level++) { + int downScaleLevel = level + 1; + Rectangle size = downScaler.getDownscaleSize(downScaleLevel); + + DataWrapper1D dest = AbstractDataWrapper.createNew( + dataSource.getClass(), size); + dest.setFillValue(creator.getFillValue()); + try { + // Downscale from previous level + downScaler.downscale(downScaleLevel - 1, downScaleLevel, + dataSource, dest); + Object data = dest.getArray(); + if (data == null) { + throw new StorageException( + "Unable to get downscaled data from destination type: " + + dest.getClass(), null); + } + IDataRecord dr = creator.create(data, downScaleLevel, size); + dataStore.addDataRecord(dr); + if (storeAfterEach) { + dataStore.store(); + } + // Set source to current level + dataSource = dest; + } catch (TransformException e) { + throw new StorageException( + "Error creating downscaled data", null, e); + } + } + } + return downScaleLevels; + } + +} diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.npp.viirs/src/com/raytheon/uf/edex/plugin/npp/viirs/dao/VIIRSDao.java b/edexOsgi/com.raytheon.uf.edex.plugin.npp.viirs/src/com/raytheon/uf/edex/plugin/npp/viirs/dao/VIIRSDao.java index e1221566f1..472e58baac 100644 --- a/edexOsgi/com.raytheon.uf.edex.plugin.npp.viirs/src/com/raytheon/uf/edex/plugin/npp/viirs/dao/VIIRSDao.java +++ b/edexOsgi/com.raytheon.uf.edex.plugin.npp.viirs/src/com/raytheon/uf/edex/plugin/npp/viirs/dao/VIIRSDao.java @@ -32,21 +32,22 @@ import com.raytheon.uf.common.dataplugin.PluginException; import com.raytheon.uf.common.dataplugin.npp.viirs.VIIRSDataRecord; import com.raytheon.uf.common.dataplugin.npp.viirs.VIIRSSpatialCoverage; import com.raytheon.uf.common.dataplugin.persist.IPersistable; +import com.raytheon.uf.common.datastorage.DataStoreFactory; import com.raytheon.uf.common.datastorage.IDataStore; +import com.raytheon.uf.common.datastorage.StorageException; import com.raytheon.uf.common.datastorage.StorageProperties; -import com.raytheon.uf.common.datastorage.records.FloatDataRecord; import com.raytheon.uf.common.datastorage.records.IDataRecord; -import com.raytheon.uf.common.datastorage.records.ShortDataRecord; import com.raytheon.uf.common.geospatial.interpolation.BilinearInterpolation; import com.raytheon.uf.common.geospatial.interpolation.GridDownscaler; -import com.raytheon.uf.common.geospatial.interpolation.data.AbstractDataWrapper; import com.raytheon.uf.common.geospatial.interpolation.data.DataSource; -import com.raytheon.uf.common.geospatial.interpolation.data.FloatArrayWrapper; -import com.raytheon.uf.common.geospatial.interpolation.data.UnsignedShortArrayWrapper; +import com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D; import com.raytheon.uf.common.status.UFStatus.Priority; import com.raytheon.uf.common.time.DataTime; import com.raytheon.uf.edex.core.dataplugin.PluginRegistry; import com.raytheon.uf.edex.database.DataAccessLayerException; +import com.raytheon.uf.edex.database.plugin.DataRecordWrapUtil; +import com.raytheon.uf.edex.database.plugin.DownscaleStoreUtil; +import com.raytheon.uf.edex.database.plugin.DownscaleStoreUtil.IDataRecordCreator; import com.raytheon.uf.edex.database.plugin.PluginDao; import com.raytheon.uf.edex.database.query.DatabaseQuery; import com.raytheon.uf.edex.plugin.npp.viirs.VIIRSMessageData; @@ -129,12 +130,12 @@ public class VIIRSDao extends PluginDao { @Override protected IDataStore populateDataStore(IDataStore dataStore, IPersistable obj) throws Exception { - VIIRSDataRecord record = (VIIRSDataRecord) obj; + final VIIRSDataRecord record = (VIIRSDataRecord) obj; VIIRSSpatialCoverage spatialRecord = record.getCoverage(); int nx = spatialRecord.getNx(); int ny = spatialRecord.getNy(); - StorageProperties props = new StorageProperties(); + final StorageProperties props = new StorageProperties(); String compression = PluginRegistry.getInstance() .getRegisteredObject(pluginName).getCompression(); if (compression != null) { @@ -142,71 +143,27 @@ public class VIIRSDao extends PluginDao { .valueOf(compression)); } - VIIRSMessageData messageData = (VIIRSMessageData) record + final VIIRSMessageData messageData = (VIIRSMessageData) record .getMessageData(); float[] missingValues = messageData.getMissingValues(); - float fillValue = missingValues[0]; + final float fillValue = missingValues[0]; Object rawData = messageData.getRawData(); - AbstractDataWrapper ds = null; - // Data sources are create anonymously here to avoid having the - // fillValue/validMin/validMax even checked when getting values but - // still getting the getDataValueInternal functionality - if (rawData instanceof short[]) { - ds = new UnsignedShortArrayWrapper((short[]) rawData, nx, ny); - } else if (rawData instanceof float[]) { - ds = new FloatArrayWrapper((float[]) rawData, nx, ny); - } + + IDataRecordCreator creator = new IDataRecordCreator() { - if (ds != null) { - ds.setFillValue(fillValue); - - // Wrap the source and replace set each value which will replace - // anything in missingValues with fillValue - DataSource source = new VIIRSDataSourceWrapper(ds, missingValues); - for (int y = 0; y < ny; ++y) { - for (int x = 0; x < nx; ++x) { - ds.setDataValue(source.getDataValue(x, y), x, y); - } + @Override + public double getFillValue() { + return fillValue; } - GridDownscaler downscaler = new GridDownscaler( - spatialRecord.getGridGeometry(), - new BilinearInterpolation()); - - int levels = downscaler.getNumberOfDownscaleLevels(); - for (int i = 0; i < levels; ++i) { - Rectangle bounds = downscaler.getDownscaleSize(i); - AbstractDataWrapper dd = null; - - if (i == 0) { - // No interpolation needed for level 0 - dd = ds; - } else { - if (ds instanceof UnsignedShortArrayWrapper) { - dd = new UnsignedShortArrayWrapper(bounds.width, - bounds.height); - } else if (ds instanceof FloatArrayWrapper) { - dd = new FloatArrayWrapper(bounds.width, bounds.height); - } - - dd.setFillValue(fillValue); - downscaler.downscale(i - 1, i, ds, dd); - ds = dd; - } - - IDataRecord idr = null; - if (dd instanceof UnsignedShortArrayWrapper) { - idr = new ShortDataRecord(VIIRSDataRecord.getDataSet(i), - record.getDataURI(), - ((UnsignedShortArrayWrapper) dd).getArray(), 2, - new long[] { bounds.width, bounds.height }); - } else if (dd instanceof FloatArrayWrapper) { - idr = new FloatDataRecord(VIIRSDataRecord.getDataSet(i), - record.getDataURI(), - ((FloatArrayWrapper) dd).getArray(), 2, new long[] { - bounds.width, bounds.height }); - } - + @Override + public IDataRecord create(Object data, + int downScaleLevel, Rectangle size) + throws StorageException { + long[] sizes = new long[] { size.width, size.height }; + IDataRecord idr = DataStoreFactory.createStorageRecord( + VIIRSDataRecord.getDataSet(downScaleLevel), + record.getDataURI(), data, 2, sizes); Map attributes = new HashMap(); attributes.put(VIIRSDataRecord.MISSING_VALUE_ID, fillValue); attributes.put(VIIRSDataRecord.OFFSET_ID, @@ -220,13 +177,34 @@ public class VIIRSDao extends PluginDao { idr.setDataAttributes(attributes); idr.setProperties(props); idr.setCorrelationObject(record); - dataStore.addDataRecord(idr); - dataStore.store(); + return idr; + } + }; + + IDataRecord fullSize = creator + .create(rawData, 0, new Rectangle(nx, ny)); + dataStore.addDataRecord(fullSize); + // Data sources are create anonymously here to avoid having the + // fillValue/validMin/validMax even checked when getting values but + // still getting the getDataValueInternal functionality + DataWrapper1D ds = DataRecordWrapUtil.wrap(fullSize, nx, ny, true); + + ds.setFillValue(fillValue); + + // Wrap the source and replace set each value which will replace + // anything in missingValues with fillValue + DataSource source = new VIIRSDataSourceWrapper(ds, missingValues); + for (int y = 0; y < ny; ++y) { + for (int x = 0; x < nx; ++x) { + ds.setDataValue(source.getDataValue(x, y), x, y); } - } else { - throw new Exception("Unrecognized type for rawData: " - + (rawData != null ? rawData.getClass() : null)); } + + GridDownscaler downscaler = new GridDownscaler( + spatialRecord.getGridGeometry(), new BilinearInterpolation()); + + DownscaleStoreUtil.storeInterpolated(dataStore, downscaler, ds, + creator, true); return dataStore; } diff --git a/ncep/gov.noaa.nws.ncep.common.dataplugin.mcidas/src/gov/noaa/nws/ncep/common/dataplugin/mcidas/McidasMapCoverage.java b/ncep/gov.noaa.nws.ncep.common.dataplugin.mcidas/src/gov/noaa/nws/ncep/common/dataplugin/mcidas/McidasMapCoverage.java index 495db3e3cb..a1ca872051 100644 --- a/ncep/gov.noaa.nws.ncep.common.dataplugin.mcidas/src/gov/noaa/nws/ncep/common/dataplugin/mcidas/McidasMapCoverage.java +++ b/ncep/gov.noaa.nws.ncep.common.dataplugin.mcidas/src/gov/noaa/nws/ncep/common/dataplugin/mcidas/McidasMapCoverage.java @@ -14,6 +14,7 @@ * 12/2009 144 T. Lee Migrated to TO11D6 * 01/2010 201 M. Li Split into dataplugin project * 05/2010 144 L. Lin Migration to TO11DR11. + * Nov 14, 2013 2393 bclement added getGridGeometry() * * */ @@ -22,6 +23,7 @@ package gov.noaa.nws.ncep.common.dataplugin.mcidas; import gov.noaa.nws.ncep.common.tools.IDecoderConstantsN; +import java.awt.geom.Rectangle2D; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -36,15 +38,26 @@ import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; import org.apache.commons.lang.builder.HashCodeBuilder; +import org.geotools.coverage.grid.GridEnvelope2D; +import org.geotools.coverage.grid.GridGeometry2D; +import org.geotools.geometry.DirectPosition2D; +import org.geotools.geometry.Envelope2D; import org.geotools.referencing.CRS; import org.hibernate.annotations.Cache; import org.hibernate.annotations.CacheConcurrencyStrategy; import org.hibernate.annotations.Type; +import org.opengis.coverage.grid.GridEnvelope; +import org.opengis.geometry.Envelope; +import org.opengis.geometry.MismatchedDimensionException; +import org.opengis.referencing.FactoryException; import org.opengis.referencing.crs.CoordinateReferenceSystem; import org.opengis.referencing.crs.ProjectedCRS; +import org.opengis.referencing.operation.MathTransform; +import org.opengis.referencing.operation.TransformException; import com.raytheon.uf.common.dataplugin.persist.PersistableDataObject; import com.raytheon.uf.common.geospatial.ISpatialObject; +import com.raytheon.uf.common.geospatial.MapUtil; import com.raytheon.uf.common.serialization.adapters.GeometryAdapter; import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; @@ -185,6 +198,18 @@ public class McidasMapCoverage extends PersistableDataObject implements ISpatial @DynamicSerializeElement private Polygon location; + /** + * minimum x value in crs space + */ + @Transient + private transient double minX = Double.NaN; + + /** + * minimum y value in crs space + */ + @Transient + private transient double minY = Double.NaN; + public McidasMapCoverage() { super(); } @@ -355,6 +380,61 @@ public class McidasMapCoverage extends PersistableDataObject implements ISpatial return crsObject; } + /** + * Construct grid geometry using grid and geospatial information + * + * @return + * @throws MismatchedDimensionException + * @throws FactoryException + * @throws TransformException + */ + public GridGeometry2D getGridGeometry() + throws MismatchedDimensionException, FactoryException, + TransformException { + int nx = getNx(); + int ny = getNy(); + if (Double.isNaN(this.minX) || Double.isNaN(this.minY)) { + findMins(); + } + GridEnvelope gridRange = new GridEnvelope2D(0, 0, nx, ny); + Envelope crsRange = new Envelope2D(getCrs(), new Rectangle2D.Double( + minX, minY, nx * getDx(), ny * getDy())); + return new GridGeometry2D(gridRange, crsRange); + } + + /** + * Populate CRS min x value and min y value from longitudes and latitudes + * + * @throws FactoryException + * @throws MismatchedDimensionException + * @throws TransformException + */ + private void findMins() throws FactoryException, + MismatchedDimensionException, TransformException { + MathTransform from84 = MapUtil.getTransformFromLatLon(getCrs()); + DirectPosition2D urInCrs = transform(from84, getUrlon(), getUrlat()); + DirectPosition2D llInCrs = transform(from84, getLllon(), getLllat()); + this.minX = Math.min(urInCrs.x, llInCrs.x); + this.minY = Math.min(urInCrs.y, llInCrs.y); + } + + /** + * Transform x and y using provided math transform + * + * @param mt + * @param x + * @param y + * @return + * @throws MismatchedDimensionException + * @throws TransformException + */ + private DirectPosition2D transform(MathTransform mt, double x, double y) + throws MismatchedDimensionException, TransformException { + DirectPosition2D dest = new DirectPosition2D(); + mt.transform(new DirectPosition2D(x, y), dest); + return dest; + } + public Float getDx() { return dx; } diff --git a/ncep/gov.noaa.nws.ncep.common.dataplugin.mcidas/src/gov/noaa/nws/ncep/common/dataplugin/mcidas/dao/McidasDao.java b/ncep/gov.noaa.nws.ncep.common.dataplugin.mcidas/src/gov/noaa/nws/ncep/common/dataplugin/mcidas/dao/McidasDao.java index 50cea100bd..3940622ec0 100644 --- a/ncep/gov.noaa.nws.ncep.common.dataplugin.mcidas/src/gov/noaa/nws/ncep/common/dataplugin/mcidas/dao/McidasDao.java +++ b/ncep/gov.noaa.nws.ncep.common.dataplugin.mcidas/src/gov/noaa/nws/ncep/common/dataplugin/mcidas/dao/McidasDao.java @@ -9,6 +9,7 @@ * ------------ ---------- ----------- -------------------------- * 10/2009 144 T. Lee Created * 11/2009 144 T. Lee Implemented area name DAO + * Nov 14, 2013 2393 bclement added in-java interpolation * * * @author tlee @@ -17,21 +18,33 @@ package gov.noaa.nws.ncep.common.dataplugin.mcidas.dao; +import gov.noaa.nws.ncep.common.dataplugin.mcidas.McidasMapCoverage; +import gov.noaa.nws.ncep.common.dataplugin.mcidas.McidasRecord; +import gov.noaa.nws.ncep.common.dataplugin.mcidas.fixed.McidasAreaName; +import gov.noaa.nws.ncep.common.dataplugin.mcidas.fixed.McidasImageType; +import gov.noaa.nws.ncep.common.dataplugin.mcidas.fixed.McidasSatelliteName; + +import java.awt.Rectangle; import java.util.List; + +import org.geotools.coverage.grid.GridGeometry2D; + import com.raytheon.uf.common.dataplugin.PluginException; import com.raytheon.uf.common.dataplugin.persist.IPersistable; +import com.raytheon.uf.common.datastorage.DataStoreFactory; import com.raytheon.uf.common.datastorage.IDataStore; import com.raytheon.uf.common.datastorage.StorageException; import com.raytheon.uf.common.datastorage.StorageProperties; import com.raytheon.uf.common.datastorage.records.AbstractStorageRecord; import com.raytheon.uf.common.datastorage.records.ByteDataRecord; +import com.raytheon.uf.common.datastorage.records.IDataRecord; +import com.raytheon.uf.common.geospatial.interpolation.GridDownscaler; +import com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D; +import com.raytheon.uf.edex.database.plugin.DataRecordWrapUtil; +import com.raytheon.uf.edex.database.plugin.DownscaleStoreUtil; +import com.raytheon.uf.edex.database.plugin.DownscaleStoreUtil.IDataRecordCreator; import com.raytheon.uf.edex.database.plugin.PluginDao; -import gov.noaa.nws.ncep.common.dataplugin.mcidas.McidasRecord; -import gov.noaa.nws.ncep.common.dataplugin.mcidas.fixed.McidasAreaName; -import gov.noaa.nws.ncep.common.dataplugin.mcidas.fixed.McidasSatelliteName; -import gov.noaa.nws.ncep.common.dataplugin.mcidas.fixed.McidasImageType; - public class McidasDao extends PluginDao { private McidasSatelliteNameDao satelliteNameDao = new McidasSatelliteNameDao(); private McidasImageTypeDao imageTypeDao = new McidasImageTypeDao(); @@ -45,7 +58,7 @@ public class McidasDao extends PluginDao { @Override protected IDataStore populateDataStore(IDataStore dataStore, IPersistable record) throws StorageException { - McidasRecord satRecord = (McidasRecord) record; + final McidasRecord satRecord = (McidasRecord) record; /* * Write McIDAS Area file header block. @@ -62,21 +75,62 @@ public class McidasDao extends PluginDao { * Write McIDAS image data block to HDF5. */ if ( satRecord.getMessageData() != null ) { - long xdim = satRecord.getCoverage().getNx(); - long ydim = satRecord.getCoverage().getNy(); + McidasMapCoverage coverage = satRecord.getCoverage(); + int xdim = coverage.getNx(); + int ydim = coverage.getNy(); long[] sizes = new long[] { xdim, ydim }; - AbstractStorageRecord storageRecord = new ByteDataRecord("Data", - satRecord.getDataURI(), (byte[]) satRecord.getMessageData(), 2, sizes); + AbstractStorageRecord storageRecord = new ByteDataRecord( + DataStoreFactory.DEF_DATASET_NAME, satRecord.getDataURI(), + (byte[]) satRecord.getMessageData(), 2, sizes); - StorageProperties props = new StorageProperties(); - - props.setDownscaled(true); -// props.setChunked(true); -// props.setCompressed(true); + final StorageProperties props = new StorageProperties(); + + GridGeometry2D gridGeom; + try { + gridGeom = coverage.getGridGeometry(); + } catch (Exception e) { + throw new StorageException( + "Unable to create grid geometry for record: " + + satRecord, storageRecord, e); + } + GridDownscaler downScaler = new GridDownscaler(gridGeom); storageRecord.setProperties(props); storageRecord.setCorrelationObject(satRecord); - dataStore.addDataRecord(storageRecord); + // Store the base record. + dataStore.addDataRecord(storageRecord); + + DataWrapper1D dataSource = DataRecordWrapUtil.wrap(storageRecord, + xdim, + ydim, true); + // this way of interpolating does not create the Data-interpolated/0 + // link to the full sized data. This shouldn't be an issue since the + // retrieval code checks for level 0 and requests the full sized + // data. + DownscaleStoreUtil.storeInterpolated(dataStore, downScaler, + dataSource, new IDataRecordCreator() { + + @Override + public IDataRecord create(Object data, + int downScaleLevel, Rectangle size) + throws StorageException { + long[] sizes = new long[] { size.width, size.height }; + String group = DataStoreFactory.createGroupName( + satRecord.getDataURI(), null, true); + String name = String.valueOf(downScaleLevel); + IDataRecord rval = DataStoreFactory + .createStorageRecord(name, group, data, 2, + sizes); + rval.setProperties(props); + rval.setCorrelationObject(satRecord); + return rval; + } + + @Override + public double getFillValue() { + return Double.NaN; + } + }); } return dataStore; } diff --git a/ncep/gov.noaa.nws.ncep.viz.rsc.satellite/src/gov/noaa/nws/ncep/viz/rsc/satellite/rsc/AbstractSatelliteResource.java b/ncep/gov.noaa.nws.ncep.viz.rsc.satellite/src/gov/noaa/nws/ncep/viz/rsc/satellite/rsc/AbstractSatelliteResource.java index 6753978573..6e3f79d8ec 100644 --- a/ncep/gov.noaa.nws.ncep.viz.rsc.satellite/src/gov/noaa/nws/ncep/viz/rsc/satellite/rsc/AbstractSatelliteResource.java +++ b/ncep/gov.noaa.nws.ncep.viz.rsc.satellite/src/gov/noaa/nws/ncep/viz/rsc/satellite/rsc/AbstractSatelliteResource.java @@ -1,5 +1,7 @@ package gov.noaa.nws.ncep.viz.rsc.satellite.rsc; +import gov.noaa.nws.ncep.common.dataplugin.mcidas.McidasMapCoverage; +import gov.noaa.nws.ncep.common.dataplugin.mcidas.McidasRecord; import gov.noaa.nws.ncep.edex.common.metparameters.parameterconversion.NcUnits; import gov.noaa.nws.ncep.viz.common.ColorMapUtil; import gov.noaa.nws.ncep.viz.common.area.AreaName.AreaSource; @@ -17,6 +19,7 @@ import gov.noaa.nws.ncep.viz.rsc.satellite.units.NcSatelliteUnits; import gov.noaa.nws.ncep.viz.ui.display.ColorBarFromColormap; import gov.noaa.nws.ncep.viz.ui.display.NCMapDescriptor; +import java.awt.Rectangle; import java.io.File; import java.text.ParseException; import java.text.ParsePosition; @@ -46,14 +49,15 @@ import com.raytheon.uf.common.geospatial.ISpatialEnabled; import com.raytheon.uf.common.geospatial.ISpatialObject; import com.raytheon.uf.common.geospatial.MapUtil; import com.raytheon.uf.common.geospatial.ReferencedCoordinate; +import com.raytheon.uf.common.geospatial.interpolation.GridDownscaler; import com.raytheon.uf.common.serialization.SerializationException; import com.raytheon.uf.common.serialization.SerializationUtil; import com.raytheon.uf.common.style.AbstractStylePreferences; import com.raytheon.uf.common.style.MatchCriteria; import com.raytheon.uf.common.style.ParamLevelMatchCriteria; +import com.raytheon.uf.common.style.StyleException; import com.raytheon.uf.common.style.StyleRule; import com.raytheon.uf.common.style.StyleRuleset; -import com.raytheon.uf.common.style.StyleException; import com.raytheon.uf.common.style.image.DataScale; import com.raytheon.uf.common.style.image.ImagePreferences; import com.raytheon.uf.common.style.image.SamplePreferences; @@ -110,6 +114,7 @@ import com.vividsolutions.jts.geom.Coordinate; * 04/30/2013 *886 sgilbert Changed number of levels from 4 to 2 for native * satellite projections in the McidasFileBasedTileSet * 05/20/2013 862 ghull implement IAreaProviderCapable + * Nov 14, 2013 2393 bclement changed how numLevels is calculated for mcidas * * * @author chammack @@ -486,14 +491,30 @@ public abstract class AbstractSatelliteResource extends getCapability(ImagingCapability.class).setSuppressingMenuItems(true); getCapability(ColorMapCapability.class).setSuppressingMenuItems(true); - numLevels = 1; - int newSzX = ((ISpatialEnabled) record).getSpatialObject().getNx(); - int newSzY = ((ISpatialEnabled) record).getSpatialObject().getNy(); + if (record instanceof McidasRecord) { + // mcidas is interpolated using the GridDownscaler, get number of + // levels stored in HDF5 + McidasRecord mcidas = (McidasRecord) record; + McidasMapCoverage cov = mcidas.getCoverage(); + try { + Rectangle[] downscaleSizes = GridDownscaler + .getDownscaleSizes(cov.getGridGeometry()); + numLevels = downscaleSizes.length; + } catch (Exception e) { + throw new VizException( + "Unable to get grid geometry for record: " + record); + } - while ((newSzX > 512 && newSzY > 512)) { - newSzX /= 2; - newSzY /= 2; - numLevels++; + } else { + numLevels = 1; + int newSzX = ((ISpatialEnabled) record).getSpatialObject().getNx(); + int newSzY = ((ISpatialEnabled) record).getSpatialObject().getNy(); + + while ((newSzX > 512 && newSzY > 512)) { + newSzX /= 2; + newSzY /= 2; + numLevels++; + } } } diff --git a/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/datastorage/StorageProperties.py b/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/datastorage/StorageProperties.py index 83bb669acd..ed986b6117 100644 --- a/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/datastorage/StorageProperties.py +++ b/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/datastorage/StorageProperties.py @@ -1,3 +1,4 @@ +## # This software was developed and / or modified by Raytheon Company, # pursuant to Contract DG133W-05-CQ-1067 with the US Government. # @@ -15,6 +16,7 @@ # # See the AWIPS II Master Rights File ("Master Rights File.pdf") for # further licensing information. +## # File auto-generated against equivalent DynamicSerialize Java class @@ -23,7 +25,6 @@ class StorageProperties(object): def __init__(self): self.compression = None self.chunked = None - self.downscaled = None def getCompression(self): return self.compression @@ -37,9 +38,3 @@ class StorageProperties(object): def setChunked(self, chunked): self.chunked = chunked - def getDownscaled(self): - return self.downscaled - - def setDownscaled(self, downscaled): - self.downscaled = downscaled - diff --git a/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/pypies/request/RetrieveRequest.py b/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/pypies/request/RetrieveRequest.py index e6a9ab4b71..fc88787e4b 100644 --- a/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/pypies/request/RetrieveRequest.py +++ b/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/pypies/request/RetrieveRequest.py @@ -24,7 +24,6 @@ class RetrieveRequest(object): def __init__(self): self.group = None - self.includeInterpolated = None self.dataset = None self.request = None self.filename = None @@ -35,12 +34,6 @@ class RetrieveRequest(object): def setGroup(self, group): self.group = group - def getIncludeInterpolated(self): - return self.includeInterpolated - - def setIncludeInterpolated(self, includeInterpolated): - self.includeInterpolated = includeInterpolated - def getDataset(self): return self.dataset diff --git a/pythonPackages/pypies/pypies/impl/DataStoreFactory.py b/pythonPackages/pypies/pypies/impl/DataStoreFactory.py index ec748a195f..98fe5bf333 100644 --- a/pythonPackages/pypies/pypies/impl/DataStoreFactory.py +++ b/pythonPackages/pypies/pypies/impl/DataStoreFactory.py @@ -30,6 +30,7 @@ # 07/21/10 njensen Initial Creation. # 09/19/13 2309 bsteffen Fix group name in returned # records. +# Nov 14, 2013 2393 bclement removed interpolation # # # @@ -102,7 +103,6 @@ def createStorageRecord(rawData, ds): if compression != 'LZF' and compression != 'ZLIB': compression = 'NONE' props.setCompression(compression) - # TODO downscaled? inst.setProps(props) t1=time.time() diff --git a/pythonPackages/pypies/pypies/impl/H5pyDataStore.py b/pythonPackages/pypies/pypies/impl/H5pyDataStore.py index 2419dccceb..eda52aa4f0 100644 --- a/pythonPackages/pypies/pypies/impl/H5pyDataStore.py +++ b/pythonPackages/pypies/pypies/impl/H5pyDataStore.py @@ -32,6 +32,7 @@ # 10/09/12 rjpeter Optimized __getGroup for retrievals # 01/17/13 DR 15294 D. Friedman Clear out data in response # 02/12/13 #1608 randerso Added support for explicitly deleting groups and datasets +# Nov 14, 2013 2393 bclement removed interpolation # # @@ -64,7 +65,6 @@ dataRecordMap = { DEFAULT_CHUNK_SIZE = 256 FILESYSTEM_BLOCK_SIZE = 4096 -DOWNSCALE_THRESHOLD = 512 REQUEST_ALL = Request() REQUEST_ALL.setType('ALL') @@ -88,10 +88,7 @@ class H5pyDataStore(IDataStore.IDataStore): t0=time.time() for r in recs: try: - if r.getProps() and r.getProps().getDownscaled(): - ss = self.__storeInterpolated(f, r, op) - else: - ss = self.__writeHDF(f, r, op) + ss = self.__writeHDF(f, r, op) except: logger.warn("Exception occurred on file " + fn + ":" + IDataStore._exc()) exc.append(IDataStore._exc()) @@ -135,11 +132,6 @@ class H5pyDataStore(IDataStore.IDataStore): ss = self.__writeHDFDataset(f, data, record.getDimension(), record.getSizes(), record.getName(), group, props, self.__getHdf5Datatype(record), storeOp, record) - if props and props.getDownscaled(): - intName = record.getGroup() + '/' + record.getName() + '-interpolated' - intGroup = self.__getNode(rootNode, intName, None, create=True) - self.__link(intGroup, '0', group[record.getName()]) - f.flush() if logger.isEnabledFor(logging.DEBUG): logger.debug("Stored group " + str(record.getGroup()) + " to group " + str(group)) @@ -256,47 +248,6 @@ class H5pyDataStore(IDataStore.IDataStore): for key in attrs: dataset.attrs[key] = attrs[key] - def __storeInterpolated(self, f, rec, op): - if op != 'REPLACE' and op != 'STORE_ONLY': - raise StorageException("Only replace and store modes are supported with interpolation enabled") - - # store the base product - ss = self.__writeHDF(f, rec, op) - - sizes = rec.getSizes() - newSzX = sizes[1] - newSzY = sizes[0] - originalName = rec.getName() - originalGroup = rec.getGroup() - level = 0 - - # avoid recursive links by turning off downscaling - rec.getProps().setDownscaled(False) - from PIL import Image - import time - - while newSzX > DOWNSCALE_THRESHOLD or newSzY > DOWNSCALE_THRESHOLD: - data = rec.retrieveDataObject() - # data is potentially 1-dimensional from serialization, ensure it goes to correct 2 dimensions - data = data.reshape([newSzX, newSzY]) - newSzX = newSzX / 2 - newSzY = newSzY / 2 - level += 1 - rec.setName(str(level)) - rec.setGroup(originalGroup + '/' + originalName + '-interpolated') - # satellite data comes in as signed bytes but pil requires unsigned bytes - data = numpy.array(data + 127, dtype=numpy.uint8) - image = Image.fromarray(data) - image = image.resize((newSzY, newSzX)) - downsized = numpy.array(image) - # transform back to signed bytes - downsized = numpy.array(downsized - 127, dtype=numpy.int8) - rec.putDataObject(downsized) - rec.setSizes([newSzY, newSzX]) - self.__writeHDF(f, rec, op) - - return ss - def __writePartialHDFDataset(self, f, data, dims, szDims, ds, props, minIndex): # reverse sizes for hdf5 @@ -405,7 +356,7 @@ class H5pyDataStore(IDataStore.IDataStore): result = [self.__retrieveInternal(ds, req)] else: groupNode = self.__getNode(rootNode, group) - result = self.__retrieve(groupNode, request.getIncludeInterpolated()) + result = self.__retrieve(groupNode) resp = RetrieveResponse() resp.setRecords(result) return resp @@ -418,18 +369,12 @@ class H5pyDataStore(IDataStore.IDataStore): - def __retrieve(self, group, includeInterpolated=False): + def __retrieve(self, group): records = [] datasets = group.keys() for ds in datasets: - interpDs = ds.endswith('-interpolated') - if includeInterpolated and interpDs: - subresults = self.__retrieve(group[ds], False) - if subresults: - records += subresults - elif not interpDs: - rec = self.__retrieveInternal(group[ds], REQUEST_ALL) - records.append(rec) + rec = self.__retrieveInternal(group[ds], REQUEST_ALL) + records.append(rec) return records