xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #2393 removed pypies interpolation

Browse source 
updated datastore interface
updated pypies python code
updated python generated from java classes
moved in-java interpolation code from sat dao to utility classes
changed ncep mcidas dao to use in-java interpolation
changed VIIRS dao to use common interpolation code


Former-commit-id: 6f9ad823de5d0b30ae7c8206b4a4fdf1e3a2b22b
This commit is contained in:
Brian Clements 2013-11-13 13:33:49 -06:00
parent ff968c9884
commit fe58826a1e
27 changed files with 740 additions and 444 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
cave/com.raytheon.uf.viz.thinclient.cave/src/com/raytheon/uf/viz/thinclient/cave/cache/CachingDataStore.java vendored
View file

@ -50,6 +50,7 @@ import com.raytheon.uf.common.datastorage.records.IDataRecord;
* Feb 12, 2013 1608 randerso Added explicit deletes for groups and * Feb 12, 2013 1608 randerso Added explicit deletes for groups and
* datasets * datasets
* Sep 18, 2013 2309 bsteffen Move disk acces to DataStoreCache * Sep 18, 2013 2309 bsteffen Move disk acces to DataStoreCache
* Nov 14, 2013 2393 bclement removed datastore interpolation
* *
* </pre> * </pre>
* *
@ -108,24 +109,6 @@ public class CachingDataStore implements IDataStore {
} }
} }
/*
* (non-Javadoc)
*
* @see
* com.raytheon.uf.common.datastorage.IDataStore#retrieve(java.lang.String,
* boolean)
*/
@Override
public IDataRecord[] retrieve(String group, boolean includeInterpolated)
throws StorageException, FileNotFoundException {
if (includeInterpolated == false) {
return retrieve(group);
} else {
/* This is deprecated and unused so caching is not implemented. */
return delegate.retrieve(group, includeInterpolated);
}
}
/* /*
* (non-Javadoc) * (non-Javadoc)
* *

148
edexOsgi/com.raytheon.edex.plugin.satellite/src/com/raytheon/edex/plugin/satellite/dao/SatelliteDao.java
View file

@ -25,8 +25,6 @@ import java.util.Date;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import org.opengis.referencing.operation.TransformException;
import com.raytheon.edex.plugin.satellite.gini.SatelliteCreatingEntity; import com.raytheon.edex.plugin.satellite.gini.SatelliteCreatingEntity;
import com.raytheon.edex.plugin.satellite.gini.SatellitePhysicalElement; import com.raytheon.edex.plugin.satellite.gini.SatellitePhysicalElement;
import com.raytheon.edex.plugin.satellite.gini.SatellitePosition; import com.raytheon.edex.plugin.satellite.gini.SatellitePosition;
@ -43,18 +41,16 @@ import com.raytheon.uf.common.datastorage.DataStoreFactory;
import com.raytheon.uf.common.datastorage.IDataStore; import com.raytheon.uf.common.datastorage.IDataStore;
import com.raytheon.uf.common.datastorage.StorageException; import com.raytheon.uf.common.datastorage.StorageException;
import com.raytheon.uf.common.datastorage.StorageProperties; import com.raytheon.uf.common.datastorage.StorageProperties;
import com.raytheon.uf.common.datastorage.records.ByteDataRecord;
import com.raytheon.uf.common.datastorage.records.IDataRecord; import com.raytheon.uf.common.datastorage.records.IDataRecord;
import com.raytheon.uf.common.datastorage.records.ShortDataRecord;
import com.raytheon.uf.common.geospatial.interpolation.GridDownscaler; import com.raytheon.uf.common.geospatial.interpolation.GridDownscaler;
import com.raytheon.uf.common.geospatial.interpolation.data.AbstractDataWrapper;
import com.raytheon.uf.common.geospatial.interpolation.data.ByteArrayWrapper;
import com.raytheon.uf.common.geospatial.interpolation.data.DataDestination; import com.raytheon.uf.common.geospatial.interpolation.data.DataDestination;
import com.raytheon.uf.common.geospatial.interpolation.data.ShortArrayWrapper; import com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D;
import com.raytheon.uf.common.geospatial.interpolation.data.UnsignedByteArrayWrapper;
import com.raytheon.uf.common.time.TimeRange; import com.raytheon.uf.common.time.TimeRange;
import com.raytheon.uf.edex.core.dataplugin.PluginRegistry; import com.raytheon.uf.edex.core.dataplugin.PluginRegistry;
import com.raytheon.uf.edex.database.DataAccessLayerException; import com.raytheon.uf.edex.database.DataAccessLayerException;
import com.raytheon.uf.edex.database.plugin.DataRecordWrapUtil;
import com.raytheon.uf.edex.database.plugin.DownscaleStoreUtil;
import com.raytheon.uf.edex.database.plugin.DownscaleStoreUtil.IDataRecordCreator;
import com.raytheon.uf.edex.database.plugin.PluginDao; import com.raytheon.uf.edex.database.plugin.PluginDao;
import com.raytheon.uf.edex.database.query.DatabaseQuery; import com.raytheon.uf.edex.database.query.DatabaseQuery;
@ -75,6 +71,7 @@ import com.raytheon.uf.edex.database.query.DatabaseQuery;
* input arguments. * input arguments.
* 06/24/2013 2044 randerso Added methods to get data by TimeRange and * 06/24/2013 2044 randerso Added methods to get data by TimeRange and
* getInventory with maxRecord limit * getInventory with maxRecord limit
* Nov 14, 2013 2393 bclement moved interpolation code to parent class
* </pre> * </pre>
* *
* @author bphillip * @author bphillip
@ -124,79 +121,61 @@ public class SatelliteDao extends PluginDao {
@Override @Override
protected IDataStore populateDataStore(IDataStore dataStore, protected IDataStore populateDataStore(IDataStore dataStore,
IPersistable record) throws StorageException { IPersistable record) throws StorageException {
SatelliteRecord satRecord = (SatelliteRecord) record; final SatelliteRecord satRecord = (SatelliteRecord) record;
IDataRecord storageRecord = (IDataRecord) satRecord.getMessageData(); IDataRecord storageRecord = (IDataRecord) satRecord.getMessageData();
if (storageRecord != null) { if (storageRecord != null) {
StorageProperties props = new StorageProperties(); final StorageProperties props = new StorageProperties();
String compression = PluginRegistry.getInstance() String compression = PluginRegistry.getInstance()
.getRegisteredObject(pluginName).getCompression(); .getRegisteredObject(pluginName).getCompression();
if (compression != null) { if (compression != null) {
props.setCompression(StorageProperties.Compression props.setCompression(StorageProperties.Compression
.valueOf(compression)); .valueOf(compression));
} }
props.setDownscaled(false);
storageRecord.setProperties(props); storageRecord.setProperties(props);
storageRecord.setCorrelationObject(satRecord); storageRecord.setCorrelationObject(satRecord);
final Map<String, Object> attributes = storageRecord
.getDataAttributes();
final Float fillValue = getAttribute(attributes,
SatelliteRecord.SAT_FILL_VALUE, 0.0f);
// Store the base record. // Store the base record.
dataStore.addDataRecord(storageRecord); dataStore.addDataRecord(storageRecord);
Map<String, Object> attributes = storageRecord.getDataAttributes();
Float fillValue = getAttribute(attributes,
SatelliteRecord.SAT_FILL_VALUE, 0.0f);
SatMapCoverage coverage = satRecord.getCoverage(); SatMapCoverage coverage = satRecord.getCoverage();
AbstractDataWrapper dataSource = getSource(storageRecord,
coverage.getNx(), coverage.getNy());
dataSource.setFillValue(fillValue);
GridDownscaler downScaler = new GridDownscaler( GridDownscaler downScaler = new GridDownscaler(
coverage.getGridGeometry()); coverage.getGridGeometry());
// How many interpolation levels do we need for this data? Rectangle fullScale = downScaler.getDownscaleSize(0);
int levels = downScaler.getNumberOfDownscaleLevels(); DataWrapper1D dataSource = DataRecordWrapUtil.wrap(
// set the number of levels in the 'parent' satellite data. storageRecord, fullScale.width, fullScale.height, true);
// Subtract one for the base level data.
satRecord.setInterpolationLevels(levels - 1);
// How many interpolation levels do we need for this data? Includes int levels = DownscaleStoreUtil.storeInterpolated(dataStore,
// the base level! downScaler, dataSource,
// Subtract one for the base level data. new IDataRecordCreator() {
int downScaleLevels = downScaler.getNumberOfDownscaleLevels() - 1;
// set the number of downscale levels in the satellite metadata.
satRecord.setInterpolationLevels(downScaleLevels);
if (DataStoreFactory.isInterpolated(levels)) {
for (int level = 0; level < downScaleLevels; level++) {
int downScaleLevel = level + 1;
Rectangle size = downScaler
.getDownscaleSize(downScaleLevel);
AbstractDataWrapper dest = getDestination(storageRecord, @Override
size); public IDataRecord create(Object data,
dest.setFillValue(fillValue); int downScaleLevel, Rectangle size)
try { throws StorageException {
// Downscale from previous level IDataRecord dr = createDataRecord(satRecord, data,
downScaler.downscale(downScaleLevel - 1,
downScaleLevel, dataSource, dest);
IDataRecord dr = createDataRecord(satRecord, dest,
downScaleLevel, size); downScaleLevel, size);
// Set the attributes and properties from the parent // Set the attributes and properties from the parent
// data. // data.
dr.setDataAttributes(attributes); dr.setDataAttributes(attributes);
dr.setProperties(props); dr.setProperties(props);
dataStore.addDataRecord(dr); return dr;
// Set source to current level
dataSource = dest;
} catch (TransformException e) {
throw new StorageException(
"Error creating downscaled data",
storageRecord, e);
}
}
} }
@Override
public double getFillValue() {
// always the same fill value
return fillValue;
}
});
// set the number of levels in the 'parent' satellite data.
satRecord.setInterpolationLevels(levels);
} }
return dataStore; return dataStore;
} }
@ -518,52 +497,6 @@ public class SatelliteDao extends PluginDao {
this.positionDao = positionDao; this.positionDao = positionDao;
} }
/**
* Create an {@link AbstractDataWrapper} destination from the supplied
* {@link IDataRecord} with given dimensions.
*
* @param rec
* The record containing data to be wrapped.
* @param size
* A {@link Rectangle} containing the size of the input data.
* @return The wrapped data.
*/
private AbstractDataWrapper getDestination(IDataRecord rec, Rectangle size) {
AbstractDataWrapper dest = null;
if (rec instanceof ByteDataRecord) {
dest = new UnsignedByteArrayWrapper(size.width, size.height);
} else if (rec instanceof ShortDataRecord) {
dest = new ShortArrayWrapper(size.width, size.height);
}
return dest;
}
/**
* Create an {@link AbstractDataWrapper} source from the supplied
* {@link IDataRecord} with given dimensions.
*
* @param rec
* The record containing data to be wrapped.
* @param nx
* Number of items on the x axis.
* @param ny
* Number of items on the y axis.
* @return The wrapped data.
*/
private AbstractDataWrapper getSource(IDataRecord rec, int nx, int ny) {
AbstractDataWrapper source = null;
if (rec instanceof ByteDataRecord) {
byte[] b = ((ByteDataRecord) rec).getByteData();
source = new UnsignedByteArrayWrapper(b, nx, ny);
} else if (rec instanceof ShortDataRecord) {
short[] s = ((ShortDataRecord) rec).getShortData();
source = new ShortArrayWrapper(s, nx, ny);
}
return source;
}
/** /**
* Create the {@link IDataRecord} from the {@link DataDestination} using the * Create the {@link IDataRecord} from the {@link DataDestination} using the
* original satellite data, size and * original satellite data, size and
@ -577,19 +510,12 @@ public class SatelliteDao extends PluginDao {
* @param size * @param size
* Size of the down-scaled data. * Size of the down-scaled data.
* @return The created data record to be stored. * @return The created data record to be stored.
* @throws PluginException
*/ */
private IDataRecord createDataRecord(SatelliteRecord satRec, private IDataRecord createDataRecord(SatelliteRecord satRec, Object data,
DataDestination data, int downscaleLevel, Rectangle size) { int downscaleLevel, Rectangle size) throws StorageException {
SatelliteMessageData msgData = null; SatelliteMessageData msgData = null;
Object o = null; msgData = new SatelliteMessageData(data, size.width, size.height);
if (data instanceof ByteArrayWrapper) {
o = ((ByteArrayWrapper) data).getArray();
} else if (data instanceof ShortArrayWrapper) {
o = ((ShortArrayWrapper) data).getArray();
}
if (o != null) {
msgData = new SatelliteMessageData(o, size.width, size.height);
}
IDataRecord rec = msgData.getStorageRecord(satRec, IDataRecord rec = msgData.getStorageRecord(satRec,
String.valueOf(downscaleLevel)); String.valueOf(downscaleLevel));
rec.setCorrelationObject(satRec); rec.setCorrelationObject(satRec);

23
edexOsgi/com.raytheon.uf.common.dataplugin.satellite/src/com/raytheon/uf/common/dataplugin/satellite/SatelliteMessageData.java
View file

@ -22,17 +22,15 @@ package com.raytheon.uf.common.dataplugin.satellite;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import com.raytheon.uf.common.dataplugin.satellite.SatelliteRecord; import com.raytheon.uf.common.datastorage.DataStoreFactory;
import com.raytheon.uf.common.datastorage.records.ByteDataRecord;
import com.raytheon.uf.common.datastorage.records.IDataRecord; import com.raytheon.uf.common.datastorage.records.IDataRecord;
import com.raytheon.uf.common.datastorage.records.ShortDataRecord;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
/** /**
* Encapsulate satellite image data as well as the dimensions of * Encapsulate satellite image data as well as the dimensions of the image grid.
* the image grid. Attributes about the data may also be added. As an * Attributes about the data may also be added. As an example these attributes
* example these attributes could include "scale factor" and/or "fill_value". * could include "scale factor" and/or "fill_value".
* *
* <pre> * <pre>
* *
@ -41,6 +39,8 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* Jun 27, 2012 798 jkorman Initial creation * Jun 27, 2012 798 jkorman Initial creation
* Nov 14, 2013 2393 bclement use datastore factory in
* getStorageRecord()
* *
* </pre> * </pre>
* *
@ -125,15 +125,8 @@ public class SatelliteMessageData {
IDataRecord storageRecord = null; IDataRecord storageRecord = null;
if ((messageData != null) && (dataRec != null)) { if ((messageData != null) && (dataRec != null)) {
long[] sizes = new long[] { nx, ny }; long[] sizes = new long[] { nx, ny };
if (messageData instanceof byte[]) { storageRecord = DataStoreFactory.createStorageRecord(dataSetName,
storageRecord = new ByteDataRecord(dataSetName, dataRec.getDataURI(), messageData, DATA_DIMS, sizes);
dataRec.getDataURI(), (byte[]) messageData, DATA_DIMS,
sizes);
} else if (messageData instanceof short[]) {
storageRecord = new ShortDataRecord(dataSetName,
dataRec.getDataURI(), (short[]) messageData, DATA_DIMS,
sizes);
}
} }
if ((storageRecord != null) && (dataAttributes != null)) { if ((storageRecord != null) && (dataAttributes != null)) {
storageRecord.setDataAttributes(dataAttributes); storageRecord.setDataAttributes(dataAttributes);

24
edexOsgi/com.raytheon.uf.common.datastorage/src/com/raytheon/uf/common/datastorage/IDataStore.java
View file

@ -25,7 +25,6 @@ import java.util.Map;
import com.raytheon.uf.common.datastorage.StorageProperties.Compression; import com.raytheon.uf.common.datastorage.StorageProperties.Compression;
import com.raytheon.uf.common.datastorage.records.IDataRecord; import com.raytheon.uf.common.datastorage.records.IDataRecord;
import com.raytheon.uf.common.serialization.ISerializableObject;
/** /**
* Defines the interface for operating against a hierarchical datastore * Defines the interface for operating against a hierarchical datastore
@ -41,6 +40,7 @@ import com.raytheon.uf.common.serialization.ISerializableObject;
* Feb 12, 2013 1608 randerso Added explicit methods for deleting * Feb 12, 2013 1608 randerso Added explicit methods for deleting
* groups and datasets * groups and datasets
* Sep 19, 2013 2309 bsteffen Deprecate retrieve(String, boolean) * Sep 19, 2013 2309 bsteffen Deprecate retrieve(String, boolean)
* Nov 14, 2013 2393 bclement removed interpolation
* *
* *
* </pre> * </pre>
@ -48,7 +48,7 @@ import com.raytheon.uf.common.serialization.ISerializableObject;
* @author chammack * @author chammack
* @version 1.0 * @version 1.0
*/ */
public interface IDataStore extends ISerializableObject { public interface IDataStore {
public static enum HDF5_ITEM { public static enum HDF5_ITEM {
DATASET, GROUP DATASET, GROUP
@ -143,26 +143,6 @@ public interface IDataStore extends ISerializableObject {
public abstract IDataRecord[] retrieve(String group) public abstract IDataRecord[] retrieve(String group)
throws StorageException, FileNotFoundException; throws StorageException, FileNotFoundException;
/**
* Convenience method for retrieve
*
* Retrieves all data at a given group, with the option to retrieve
* interpolated tilesets.
*
* @param group
* the group of data to retrieve
* @param includeInterpolated
* a flag indicating whether interpolated tilesets should be
* retrieved
* @return the data records
* @throws StorageException
* @throws FileNotFoundException
*/
@Deprecated
public abstract IDataRecord[] retrieve(String group,
boolean includeInterpolated) throws StorageException,
FileNotFoundException;
/** /**
* Retrieve a single dataset with optional subsetting * Retrieve a single dataset with optional subsetting
* *

24
edexOsgi/com.raytheon.uf.common.datastorage/src/com/raytheon/uf/common/datastorage/StorageProperties.java
View file

@ -22,7 +22,6 @@ package com.raytheon.uf.common.datastorage;
import org.apache.commons.lang.Validate; import org.apache.commons.lang.Validate;
import com.raytheon.uf.common.serialization.ISerializableObject;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
@ -36,6 +35,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* Feb 8, 2007 chammack Initial Creation. * Feb 8, 2007 chammack Initial Creation.
* Nov 14, 2013 2393 bclement removed interpolation
* *
* </pre> * </pre>
* *
@ -43,7 +43,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* @version 1 * @version 1
*/ */
@DynamicSerialize @DynamicSerialize
public class StorageProperties implements Cloneable, ISerializableObject { public class StorageProperties implements Cloneable {
/** /**
* Compression types: * Compression types:
@ -66,10 +66,6 @@ public class StorageProperties implements Cloneable, ISerializableObject {
@DynamicSerializeElement @DynamicSerializeElement
private boolean chunked; private boolean chunked;
/** Is the data progressively downsampled */
@DynamicSerializeElement
private boolean downscaled;
/** /**
* Construct a default storage properties. Default is all features disabled. * Construct a default storage properties. Default is all features disabled.
*/ */
@ -114,21 +110,6 @@ public class StorageProperties implements Cloneable, ISerializableObject {
} }
} }
/**
* @return the isDownscaled
*/
public boolean isDownscaled() {
return downscaled;
}
/**
* @param isDownscaled
* the isDownscaled to set
*/
public void setDownscaled(boolean isDownscaled) {
this.downscaled = isDownscaled;
}
/* /*
* (non-Javadoc) * (non-Javadoc)
* *
@ -138,7 +119,6 @@ public class StorageProperties implements Cloneable, ISerializableObject {
public StorageProperties clone() { public StorageProperties clone() {
StorageProperties sp = new StorageProperties(); StorageProperties sp = new StorageProperties();
sp.chunked = chunked; sp.chunked = chunked;
sp.downscaled = downscaled;
sp.compression = compression; sp.compression = compression;
return sp; return sp;
} }

26
edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/AbstractDataWrapper.java
View file

@ -19,6 +19,9 @@
**/ **/
package com.raytheon.uf.common.geospatial.interpolation.data; package com.raytheon.uf.common.geospatial.interpolation.data;
import java.awt.Rectangle;
import java.lang.reflect.Constructor;
import org.geotools.coverage.grid.GeneralGridGeometry; import org.geotools.coverage.grid.GeneralGridGeometry;
import com.raytheon.uf.common.geospatial.util.GridGeometryWrapChecker; import com.raytheon.uf.common.geospatial.util.GridGeometryWrapChecker;
@ -35,6 +38,7 @@ import com.raytheon.uf.common.geospatial.util.GridGeometryWrapChecker;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* Jun 20, 2012 bsteffen Initial creation * Jun 20, 2012 bsteffen Initial creation
* Nov 19, 2013 2393 bclement added createNew method
* *
* </pre> * </pre>
* *
@ -113,4 +117,26 @@ public abstract class AbstractDataWrapper implements DataSource,
protected abstract void setDataValueInternal(double dataValue, int x, int y); protected abstract void setDataValueInternal(double dataValue, int x, int y);
/**
* Create a new data wrapper of type c with the provided size
*
* @param c
* desired implementation class
* @param size
* @return
* @throws IllegalArgumentException
*/
public static <T extends AbstractDataWrapper> T createNew(
Class<? extends T> c, Rectangle size)
throws IllegalArgumentException {
try {
Constructor<? extends T> constructor = c.getConstructor(int.class,
int.class);
return constructor.newInstance(size.width, size.height);
} catch (Exception e) {
throw new IllegalArgumentException(
"Unable to instatiate instance of class: " + c, e);
}
}
} }

1
edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/ByteArrayWrapper.java
View file

@ -113,4 +113,5 @@ public class ByteArrayWrapper extends DataWrapper1D {
public void setDataValueInternal(double dataValue, int index) { public void setDataValueInternal(double dataValue, int index) {
array[index] = (byte) dataValue; array[index] = (byte) dataValue;
} }
} }

15
edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/ByteBufferWrapper.java
View file

@ -34,6 +34,7 @@ import org.geotools.coverage.grid.GeneralGridGeometry;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* Jun 20, 2012 bsteffen Initial creation * Jun 20, 2012 bsteffen Initial creation
* Nov 19, 2013 2393 bclement added getArray
* *
* </pre> * </pre>
* *
@ -79,4 +80,18 @@ public class ByteBufferWrapper extends DataWrapper1D {
buffer.put(index, (byte) dataValue); buffer.put(index, (byte) dataValue);
} }
/*
* (non-Javadoc)
*
* @see com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D#
* getPrimitiveArray()
*/
@Override
public byte[] getArray() {
if (buffer.hasArray()) {
return buffer.array();
}
return null;
}
} }

8
edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/DataWrapper1D.java
View file

@ -32,6 +32,7 @@ import org.geotools.coverage.grid.GeneralGridGeometry;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* Jun 20, 2012 bsteffen Initial creation * Jun 20, 2012 bsteffen Initial creation
* Nov 19, 2013 2393 bclement added abstract method for primitive array
* *
* </pre> * </pre>
* *
@ -62,4 +63,11 @@ public abstract class DataWrapper1D extends AbstractDataWrapper {
protected abstract void setDataValueInternal(double dataValue, int index); protected abstract void setDataValueInternal(double dataValue, int index);
/**
* Get the backing 1D primitive array
*
* @return null if array isn't available
*/
public abstract Object getArray();
} }

15
edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/DoubleBufferWrapper.java
View file

@ -34,6 +34,7 @@ import org.geotools.coverage.grid.GeneralGridGeometry;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* Jun 20, 2012 bsteffen Initial creation * Jun 20, 2012 bsteffen Initial creation
* Nov 19, 2013 2393 bclement added getArray
* *
* </pre> * </pre>
* *
@ -79,4 +80,18 @@ public class DoubleBufferWrapper extends DataWrapper1D {
buffer.put(index, dataValue); buffer.put(index, dataValue);
} }
/*
* (non-Javadoc)
*
* @see com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D#
* getPrimitiveArray()
*/
@Override
public double[] getArray() {
if (buffer.hasArray()) {
return buffer.array();
}
return null;
}
} }

15
edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/FloatBufferWrapper.java
View file

@ -34,6 +34,7 @@ import org.geotools.coverage.grid.GeneralGridGeometry;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* Jun 20, 2012 bsteffen Initial creation * Jun 20, 2012 bsteffen Initial creation
* Nov 19, 2013 2393 bclement added getArray
* *
* </pre> * </pre>
* *
@ -79,4 +80,18 @@ public class FloatBufferWrapper extends DataWrapper1D {
buffer.put(index, (float) dataValue); buffer.put(index, (float) dataValue);
} }
/*
* (non-Javadoc)
*
* @see com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D#
* getPrimitiveArray()
*/
@Override
public float[] getArray() {
if (buffer.hasArray()) {
return buffer.array();
}
return null;
}
} }

15
edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/IntBufferWrapper.java
View file

@ -34,6 +34,7 @@ import org.geotools.coverage.grid.GeneralGridGeometry;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* Jun 20, 2012 bsteffen Initial creation * Jun 20, 2012 bsteffen Initial creation
* Nov 19, 2013 2393 bclement added getArray
* *
* </pre> * </pre>
* *
@ -79,4 +80,18 @@ public class IntBufferWrapper extends DataWrapper1D {
buffer.put(index, (int) dataValue); buffer.put(index, (int) dataValue);
} }
/*
* (non-Javadoc)
*
* @see com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D#
* getPrimitiveArray()
*/
@Override
public int[] getArray() {
if (buffer.hasArray()) {
return buffer.array();
}
return null;
}
} }

15
edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/ShortBufferWrapper.java
View file

@ -34,6 +34,7 @@ import org.geotools.coverage.grid.GeneralGridGeometry;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* Jun 20, 2012 bsteffen Initial creation * Jun 20, 2012 bsteffen Initial creation
* Nov 19, 2013 2393 bclement added getArray
* *
* </pre> * </pre>
* *
@ -79,4 +80,18 @@ public class ShortBufferWrapper extends DataWrapper1D {
buffer.put(index, (short) dataValue); buffer.put(index, (short) dataValue);
} }
/*
* (non-Javadoc)
*
* @see com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D#
* getPrimitiveArray()
*/
@Override
public short[] getArray() {
if (buffer.hasArray()) {
return buffer.array();
}
return null;
}
} }

46
edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/UnsignedByteBufferWrapper.java
View file

@ -25,7 +25,7 @@ import org.geotools.coverage.grid.GeneralGridGeometry;
/** /**
* *
* ByteBuffer data wrapper * Unsigned ByteBuffer data wrapper
* *
* <pre> * <pre>
* *
@ -34,40 +34,47 @@ import org.geotools.coverage.grid.GeneralGridGeometry;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* Jun 20, 2012 bsteffen Initial creation * Jun 20, 2012 bsteffen Initial creation
* Nov 19, 2013 2393 bclement changed to extend ByteBufferWrapper
* *
* </pre> * </pre>
* *
* @author bsteffen * @author bsteffen
* @version 1.0 * @version 1.0
*/ */
public class UnsignedByteBufferWrapper extends DataWrapper1D { public class UnsignedByteBufferWrapper extends ByteBufferWrapper {
protected final ByteBuffer buffer;
/**
* @param buffer
* @param geometry
*/
public UnsignedByteBufferWrapper(ByteBuffer buffer, public UnsignedByteBufferWrapper(ByteBuffer buffer,
GeneralGridGeometry geometry) { GeneralGridGeometry geometry) {
super(geometry); super(buffer, geometry);
this.buffer = buffer;
} }
/**
* @param buffer
* @param nx
* @param ny
*/
public UnsignedByteBufferWrapper(ByteBuffer buffer, int nx, int ny) { public UnsignedByteBufferWrapper(ByteBuffer buffer, int nx, int ny) {
super(nx, ny); super(buffer, nx, ny);
this.buffer = buffer;
}
public UnsignedByteBufferWrapper(int nx, int ny) {
this(ByteBuffer.allocate(nx * ny), nx, ny);
} }
/**
* @param geometry
*/
public UnsignedByteBufferWrapper(GeneralGridGeometry geometry) { public UnsignedByteBufferWrapper(GeneralGridGeometry geometry) {
// assume this is going to be a destination and avoid passing super(geometry);
// geometry to super to save time on checking for wrapping.
this(geometry.getGridRange().getSpan(0), geometry.getGridRange()
.getSpan(1));
} }
public ByteBuffer getBuffer() { /**
return buffer; * @param nx
* @param ny
*/
public UnsignedByteBufferWrapper(int nx, int ny) {
super(nx, ny);
} }
@Override @Override
@ -75,9 +82,4 @@ public class UnsignedByteBufferWrapper extends DataWrapper1D {
return buffer.get(index) & 0xFF; return buffer.get(index) & 0xFF;
} }
@Override
public void setDataValueInternal(double dataValue, int index) {
buffer.put(index, (byte) dataValue);
}
} }

55
edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/UnsignedShortBufferWrapper.java
View file

@ -25,7 +25,7 @@ import org.geotools.coverage.grid.GeneralGridGeometry;
/** /**
* *
* ShortBuffer data wrapper * Unsigned ShortBuffer data wrapper
* *
* <pre> * <pre>
* *
@ -34,40 +34,46 @@ import org.geotools.coverage.grid.GeneralGridGeometry;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* Jun 20, 2012 bsteffen Initial creation * Jun 20, 2012 bsteffen Initial creation
* Nov 19, 2013 2393 bclement changed to extend ShortBufferWrapper
* *
* </pre> * </pre>
* *
* @author bsteffen * @author bsteffen
* @version 1.0 * @version 1.0
*/ */
public class UnsignedShortBufferWrapper extends DataWrapper1D { public class UnsignedShortBufferWrapper extends ShortBufferWrapper {
protected final ShortBuffer buffer; /**
* @param geometry
*/
public UnsignedShortBufferWrapper(GeneralGridGeometry geometry) {
super(geometry);
}
/**
* @param nx
* @param ny
*/
public UnsignedShortBufferWrapper(int nx, int ny) {
super(nx, ny);
}
/**
* @param buffer
* @param geometry
*/
public UnsignedShortBufferWrapper(ShortBuffer buffer, public UnsignedShortBufferWrapper(ShortBuffer buffer,
GeneralGridGeometry geometry) { GeneralGridGeometry geometry) {
super(geometry); super(buffer, geometry);
this.buffer = buffer;
} }
/**
* @param buffer
* @param nx
* @param ny
*/
public UnsignedShortBufferWrapper(ShortBuffer buffer, int nx, int ny) { public UnsignedShortBufferWrapper(ShortBuffer buffer, int nx, int ny) {
super(nx, ny); super(buffer, nx, ny);
this.buffer = buffer;
}
public UnsignedShortBufferWrapper(int nx, int ny) {
this(ShortBuffer.allocate(nx * ny), nx, ny);
}
public UnsignedShortBufferWrapper(GeneralGridGeometry geometry) {
// assume this is going to be a destination and avoid passing
// geometry to super to save time on checking for wrapping.
this(geometry.getGridRange().getSpan(0), geometry.getGridRange()
.getSpan(1));
}
public ShortBuffer getBuffer() {
return buffer;
} }
@Override @Override
@ -75,9 +81,4 @@ public class UnsignedShortBufferWrapper extends DataWrapper1D {
return buffer.get(index) & 0xFFFF; return buffer.get(index) & 0xFFFF;
} }
@Override
public void setDataValueInternal(double dataValue, int index) {
buffer.put(index, (short) dataValue);
}
} }

19
edexOsgi/com.raytheon.uf.common.pypies/src/com/raytheon/uf/common/pypies/PyPiesDataStore.java
View file

@ -73,6 +73,7 @@ import com.raytheon.uf.common.util.FileUtil;
* Mon 07, 2013 DR 15294 D. Friedman Stream large requests * Mon 07, 2013 DR 15294 D. Friedman Stream large requests
* Feb 11, 2013 1526 njensen use HttpClient.postDynamicSerialize() for memory efficiency * Feb 11, 2013 1526 njensen use HttpClient.postDynamicSerialize() for memory efficiency
* Feb 12, 2013 #1608 randerso Added explicit deletes for groups and datasets * Feb 12, 2013 #1608 randerso Added explicit deletes for groups and datasets
* Nov 14, 2013 2393 bclement removed interpolation
* *
* </pre> * </pre>
* *
@ -202,23 +203,8 @@ public class PyPiesDataStore implements IDataStore {
@Override @Override
public IDataRecord[] retrieve(final String group) throws StorageException, public IDataRecord[] retrieve(final String group) throws StorageException,
FileNotFoundException { FileNotFoundException {
return retrieve(group, false);
}
/*
* (non-Javadoc)
*
* @see
* com.raytheon.uf.common.datastorage.IDataStore#retrieve(java.lang.String,
* boolean)
*/
@Override
public IDataRecord[] retrieve(final String group,
final boolean includeInterpolated) throws StorageException,
FileNotFoundException {
RetrieveRequest req = new RetrieveRequest(); RetrieveRequest req = new RetrieveRequest();
req.setGroup(group); req.setGroup(group);
req.setIncludeInterpolated(includeInterpolated);
RetrieveResponse resp = (RetrieveResponse) cachedRequest(req); RetrieveResponse resp = (RetrieveResponse) cachedRequest(req);
return resp.getRecords(); return resp.getRecords();
} }
@ -448,7 +434,8 @@ public class PyPiesDataStore implements IDataStore {
protected Object deserializeResponse(final byte[] response) protected Object deserializeResponse(final byte[] response)
throws StorageException { throws StorageException {
try { try {
return SerializationUtil.transformFromThrift(response); return SerializationUtil
.transformFromThrift(Object.class, response);
} catch (SerializationException e) { } catch (SerializationException e) {
throw new StorageException( throw new StorageException(
"Error deserializing response from pypies server", null, e); "Error deserializing response from pypies server", null, e);

12
edexOsgi/com.raytheon.uf.common.pypies/src/com/raytheon/uf/common/pypies/request/RetrieveRequest.java
View file

@ -32,6 +32,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* Jul 8, 2010 njensen Initial creation * Jul 8, 2010 njensen Initial creation
* Nov 14, 2013 2393 bclement removed get all interpolated
* *
* </pre> * </pre>
* *
@ -45,9 +46,6 @@ public class RetrieveRequest extends AbstractRequest {
@DynamicSerializeElement @DynamicSerializeElement
private String group; private String group;
@DynamicSerializeElement
private boolean includeInterpolated;
@DynamicSerializeElement @DynamicSerializeElement
private String dataset; private String dataset;
@ -62,14 +60,6 @@ public class RetrieveRequest extends AbstractRequest {
this.group = group; this.group = group;
} }
public boolean isIncludeInterpolated() {
return includeInterpolated;
}
public void setIncludeInterpolated(boolean includeInterpolated) {
this.includeInterpolated = includeInterpolated;
}
public String getDataset() { public String getDataset() {
return dataset; return dataset;
} }

121
edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/plugin/DataRecordWrapUtil.java Normal file
View file

@ -0,0 +1,121 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.database.plugin;
import com.raytheon.uf.common.datastorage.StorageException;
import com.raytheon.uf.common.datastorage.records.ByteDataRecord;
import com.raytheon.uf.common.datastorage.records.FloatDataRecord;
import com.raytheon.uf.common.datastorage.records.IDataRecord;
import com.raytheon.uf.common.datastorage.records.IntegerDataRecord;
import com.raytheon.uf.common.datastorage.records.ShortDataRecord;
import com.raytheon.uf.common.geospatial.interpolation.data.AbstractDataWrapper;
import com.raytheon.uf.common.geospatial.interpolation.data.ByteArrayWrapper;
import com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D;
import com.raytheon.uf.common.geospatial.interpolation.data.FloatArrayWrapper;
import com.raytheon.uf.common.geospatial.interpolation.data.IntArrayWrapper;
import com.raytheon.uf.common.geospatial.interpolation.data.ShortArrayWrapper;
import com.raytheon.uf.common.geospatial.interpolation.data.UnsignedByteArrayWrapper;
import com.raytheon.uf.common.geospatial.interpolation.data.UnsignedShortArrayWrapper;
/**
* Utility for wrapping data records to be a data source/destination
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Nov 19, 2013 2393 bclement Initial creation
*
* </pre>
*
* @author bclement
* @version 1.0
*/
public class DataRecordWrapUtil {
/**
* Create an {@link AbstractDataWrapper} source from the supplied
* {@link IDataRecord} with given dimensions.
*
* @param rec
* The record containing data to be wrapped.
* @param nx
* Number of items on the x axis.
* @param ny
* Number of items on the y axis.
* @return The wrapped data.
* @throws StorageException
*/
public static DataWrapper1D wrap(IDataRecord rec, int nx, int ny)
throws StorageException {
// default to signed data
return wrap(rec, nx, ny, false);
}
/**
* Create an {@link AbstractDataWrapper} source from the supplied
* {@link IDataRecord} with given dimensions.
*
* @param rec
* The record containing data to be wrapped.
* @param nx
* Number of items on the x axis.
* @param ny
* Number of items on the y axis.
* @param unsigned
* attempt to treat the data as unsigned if possible
* @return The wrapped data.
* @throws StorageException
*/
public static DataWrapper1D wrap(IDataRecord rec, int nx, int ny,
boolean unsigned) throws StorageException {
DataWrapper1D source = null;
if (rec instanceof ByteDataRecord) {
byte[] b = ((ByteDataRecord) rec).getByteData();
if (unsigned) {
source = new UnsignedByteArrayWrapper(b, nx, ny);
} else {
source = new ByteArrayWrapper(b, nx, ny);
}
} else if (rec instanceof ShortDataRecord) {
short[] s = ((ShortDataRecord) rec).getShortData();
if (unsigned) {
source = new UnsignedShortArrayWrapper(s, nx, ny);
} else {
source = new ShortArrayWrapper(s, nx, ny);
}
} else if (rec instanceof IntegerDataRecord) {
int[] i = ((IntegerDataRecord) rec).getIntData();
source = new IntArrayWrapper(i, nx, ny);
} else if (rec instanceof FloatDataRecord) {
float[] f = ((FloatDataRecord) rec).getFloatData();
source = new FloatArrayWrapper(f, nx, ny);
} else {
throw new StorageException("Unsupported data record type: "
+ rec.getClass(), rec);
}
return source;
}
}

157
edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/plugin/DownscaleStoreUtil.java Normal file
View file

@ -0,0 +1,157 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.database.plugin;
import java.awt.Rectangle;
import org.opengis.referencing.operation.TransformException;
import com.raytheon.uf.common.dataplugin.PluginDataObject;
import com.raytheon.uf.common.datastorage.DataStoreFactory;
import com.raytheon.uf.common.datastorage.IDataStore;
import com.raytheon.uf.common.datastorage.StorageException;
import com.raytheon.uf.common.datastorage.records.IDataRecord;
import com.raytheon.uf.common.geospatial.interpolation.GridDownscaler;
import com.raytheon.uf.common.geospatial.interpolation.data.AbstractDataWrapper;
import com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D;
/**
* Utility for storing downscaled data to datastore
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Nov 19, 2013 2393 bclement Initial creation
*
* </pre>
*
* @author bclement
* @version 1.0
*/
public class DownscaleStoreUtil {
/**
* Interface for creating IDataRecords for interpolation levels
*/
public static interface IDataRecordCreator {
/**
* Create new data record for level
*
* @param data
* @param downScaleLevel
* @param size
* @return
* @throws StorageException
*/
public IDataRecord create(Object data, int downScaleLevel,
Rectangle size) throws StorageException;
/**
* @return fill value
*/
public double getFillValue();
}
/**
* Create and add interpolated levels from dataSource.
*
* @param dataStore
* @param downScaler
* @param dataSource
* @param creator
* @return number of levels not including the base level
* @throws StorageException
*/
public static <T extends PluginDataObject> int storeInterpolated(
IDataStore dataStore, GridDownscaler downScaler,
DataWrapper1D dataSource, IDataRecordCreator creator)
throws StorageException {
// default to batch storage
return storeInterpolated(dataStore, downScaler, dataSource, creator,
false);
}
/**
* Create and add interpolated levels from dataSource.
*
* @param dataStore
* @param downScaler
* @param dataSource
* @param creator
* @param storeAfterEach
* if true, call store method on dataStore after each level is
* created
* @return number of levels not including the base level
* @throws StorageException
*/
public static <T extends PluginDataObject> int storeInterpolated(
IDataStore dataStore, GridDownscaler downScaler,
DataWrapper1D dataSource, IDataRecordCreator creator,
boolean storeAfterEach) throws StorageException {
dataSource.setFillValue(creator.getFillValue());
// How many interpolation levels do we need for this data?
int levels = downScaler.getNumberOfDownscaleLevels();
// How many interpolation levels do we need for this data? Includes
// the base level!
// Subtract one for the base level data.
int downScaleLevels = levels - 1;
if (DataStoreFactory.isInterpolated(levels)) {
for (int level = 0; level < downScaleLevels; level++) {
int downScaleLevel = level + 1;
Rectangle size = downScaler.getDownscaleSize(downScaleLevel);
DataWrapper1D dest = AbstractDataWrapper.createNew(
dataSource.getClass(), size);
dest.setFillValue(creator.getFillValue());
try {
// Downscale from previous level
downScaler.downscale(downScaleLevel - 1, downScaleLevel,
dataSource, dest);
Object data = dest.getArray();
if (data == null) {
throw new StorageException(
"Unable to get downscaled data from destination type: "
+ dest.getClass(), null);
}
IDataRecord dr = creator.create(data, downScaleLevel, size);
dataStore.addDataRecord(dr);
if (storeAfterEach) {
dataStore.store();
}
// Set source to current level
dataSource = dest;
} catch (TransformException e) {
throw new StorageException(
"Error creating downscaled data", null, e);
}
}
}
return downScaleLevels;
}
}

122
edexOsgi/com.raytheon.uf.edex.plugin.npp.viirs/src/com/raytheon/uf/edex/plugin/npp/viirs/dao/VIIRSDao.java
View file

@ -32,21 +32,22 @@ import com.raytheon.uf.common.dataplugin.PluginException;
import com.raytheon.uf.common.dataplugin.npp.viirs.VIIRSDataRecord; import com.raytheon.uf.common.dataplugin.npp.viirs.VIIRSDataRecord;
import com.raytheon.uf.common.dataplugin.npp.viirs.VIIRSSpatialCoverage; import com.raytheon.uf.common.dataplugin.npp.viirs.VIIRSSpatialCoverage;
import com.raytheon.uf.common.dataplugin.persist.IPersistable; import com.raytheon.uf.common.dataplugin.persist.IPersistable;
import com.raytheon.uf.common.datastorage.DataStoreFactory;
import com.raytheon.uf.common.datastorage.IDataStore; import com.raytheon.uf.common.datastorage.IDataStore;
import com.raytheon.uf.common.datastorage.StorageException;
import com.raytheon.uf.common.datastorage.StorageProperties; import com.raytheon.uf.common.datastorage.StorageProperties;
import com.raytheon.uf.common.datastorage.records.FloatDataRecord;
import com.raytheon.uf.common.datastorage.records.IDataRecord; import com.raytheon.uf.common.datastorage.records.IDataRecord;
import com.raytheon.uf.common.datastorage.records.ShortDataRecord;
import com.raytheon.uf.common.geospatial.interpolation.BilinearInterpolation; import com.raytheon.uf.common.geospatial.interpolation.BilinearInterpolation;
import com.raytheon.uf.common.geospatial.interpolation.GridDownscaler; import com.raytheon.uf.common.geospatial.interpolation.GridDownscaler;
import com.raytheon.uf.common.geospatial.interpolation.data.AbstractDataWrapper;
import com.raytheon.uf.common.geospatial.interpolation.data.DataSource; import com.raytheon.uf.common.geospatial.interpolation.data.DataSource;
import com.raytheon.uf.common.geospatial.interpolation.data.FloatArrayWrapper; import com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D;
import com.raytheon.uf.common.geospatial.interpolation.data.UnsignedShortArrayWrapper;
import com.raytheon.uf.common.status.UFStatus.Priority; import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.time.DataTime; import com.raytheon.uf.common.time.DataTime;
import com.raytheon.uf.edex.core.dataplugin.PluginRegistry; import com.raytheon.uf.edex.core.dataplugin.PluginRegistry;
import com.raytheon.uf.edex.database.DataAccessLayerException; import com.raytheon.uf.edex.database.DataAccessLayerException;
import com.raytheon.uf.edex.database.plugin.DataRecordWrapUtil;
import com.raytheon.uf.edex.database.plugin.DownscaleStoreUtil;
import com.raytheon.uf.edex.database.plugin.DownscaleStoreUtil.IDataRecordCreator;
import com.raytheon.uf.edex.database.plugin.PluginDao; import com.raytheon.uf.edex.database.plugin.PluginDao;
import com.raytheon.uf.edex.database.query.DatabaseQuery; import com.raytheon.uf.edex.database.query.DatabaseQuery;
import com.raytheon.uf.edex.plugin.npp.viirs.VIIRSMessageData; import com.raytheon.uf.edex.plugin.npp.viirs.VIIRSMessageData;
@ -129,12 +130,12 @@ public class VIIRSDao extends PluginDao {
@Override @Override
protected IDataStore populateDataStore(IDataStore dataStore, protected IDataStore populateDataStore(IDataStore dataStore,
IPersistable obj) throws Exception { IPersistable obj) throws Exception {
VIIRSDataRecord record = (VIIRSDataRecord) obj; final VIIRSDataRecord record = (VIIRSDataRecord) obj;
VIIRSSpatialCoverage spatialRecord = record.getCoverage(); VIIRSSpatialCoverage spatialRecord = record.getCoverage();
int nx = spatialRecord.getNx(); int nx = spatialRecord.getNx();
int ny = spatialRecord.getNy(); int ny = spatialRecord.getNy();
StorageProperties props = new StorageProperties(); final StorageProperties props = new StorageProperties();
String compression = PluginRegistry.getInstance() String compression = PluginRegistry.getInstance()
.getRegisteredObject(pluginName).getCompression(); .getRegisteredObject(pluginName).getCompression();
if (compression != null) { if (compression != null) {
@ -142,71 +143,27 @@ public class VIIRSDao extends PluginDao {
.valueOf(compression)); .valueOf(compression));
} }
VIIRSMessageData messageData = (VIIRSMessageData) record final VIIRSMessageData messageData = (VIIRSMessageData) record
.getMessageData(); .getMessageData();
float[] missingValues = messageData.getMissingValues(); float[] missingValues = messageData.getMissingValues();
float fillValue = missingValues[0]; final float fillValue = missingValues[0];
Object rawData = messageData.getRawData(); Object rawData = messageData.getRawData();
AbstractDataWrapper ds = null;
// Data sources are create anonymously here to avoid having the IDataRecordCreator creator = new IDataRecordCreator() {
// fillValue/validMin/validMax even checked when getting values but
// still getting the getDataValueInternal functionality @Override
if (rawData instanceof short[]) { public double getFillValue() {
ds = new UnsignedShortArrayWrapper((short[]) rawData, nx, ny); return fillValue;
} else if (rawData instanceof float[]) {
ds = new FloatArrayWrapper((float[]) rawData, nx, ny);
}
if (ds != null) {
ds.setFillValue(fillValue);
// Wrap the source and replace set each value which will replace
// anything in missingValues with fillValue
DataSource source = new VIIRSDataSourceWrapper(ds, missingValues);
for (int y = 0; y < ny; ++y) {
for (int x = 0; x < nx; ++x) {
ds.setDataValue(source.getDataValue(x, y), x, y);
}
}
GridDownscaler downscaler = new GridDownscaler(
spatialRecord.getGridGeometry(),
new BilinearInterpolation());
int levels = downscaler.getNumberOfDownscaleLevels();
for (int i = 0; i < levels; ++i) {
Rectangle bounds = downscaler.getDownscaleSize(i);
AbstractDataWrapper dd = null;
if (i == 0) {
// No interpolation needed for level 0
dd = ds;
} else {
if (ds instanceof UnsignedShortArrayWrapper) {
dd = new UnsignedShortArrayWrapper(bounds.width,
bounds.height);
} else if (ds instanceof FloatArrayWrapper) {
dd = new FloatArrayWrapper(bounds.width, bounds.height);
}
dd.setFillValue(fillValue);
downscaler.downscale(i - 1, i, ds, dd);
ds = dd;
}
IDataRecord idr = null;
if (dd instanceof UnsignedShortArrayWrapper) {
idr = new ShortDataRecord(VIIRSDataRecord.getDataSet(i),
record.getDataURI(),
((UnsignedShortArrayWrapper) dd).getArray(), 2,
new long[] { bounds.width, bounds.height });
} else if (dd instanceof FloatArrayWrapper) {
idr = new FloatDataRecord(VIIRSDataRecord.getDataSet(i),
record.getDataURI(),
((FloatArrayWrapper) dd).getArray(), 2, new long[] {
bounds.width, bounds.height });
} }
@Override
public IDataRecord create(Object data,
int downScaleLevel, Rectangle size)
throws StorageException {
long[] sizes = new long[] { size.width, size.height };
IDataRecord idr = DataStoreFactory.createStorageRecord(
VIIRSDataRecord.getDataSet(downScaleLevel),
record.getDataURI(), data, 2, sizes);
Map<String, Object> attributes = new HashMap<String, Object>(); Map<String, Object> attributes = new HashMap<String, Object>();
attributes.put(VIIRSDataRecord.MISSING_VALUE_ID, fillValue); attributes.put(VIIRSDataRecord.MISSING_VALUE_ID, fillValue);
attributes.put(VIIRSDataRecord.OFFSET_ID, attributes.put(VIIRSDataRecord.OFFSET_ID,
@ -220,13 +177,34 @@ public class VIIRSDao extends PluginDao {
idr.setDataAttributes(attributes); idr.setDataAttributes(attributes);
idr.setProperties(props); idr.setProperties(props);
idr.setCorrelationObject(record); idr.setCorrelationObject(record);
dataStore.addDataRecord(idr); return idr;
dataStore.store();
} }
} else { };
throw new Exception("Unrecognized type for rawData: "
+ (rawData != null ? rawData.getClass() : null)); IDataRecord fullSize = creator
.create(rawData, 0, new Rectangle(nx, ny));
dataStore.addDataRecord(fullSize);
// Data sources are create anonymously here to avoid having the
// fillValue/validMin/validMax even checked when getting values but
// still getting the getDataValueInternal functionality
DataWrapper1D ds = DataRecordWrapUtil.wrap(fullSize, nx, ny, true);
ds.setFillValue(fillValue);
// Wrap the source and replace set each value which will replace
// anything in missingValues with fillValue
DataSource source = new VIIRSDataSourceWrapper(ds, missingValues);
for (int y = 0; y < ny; ++y) {
for (int x = 0; x < nx; ++x) {
ds.setDataValue(source.getDataValue(x, y), x, y);
} }
}
GridDownscaler downscaler = new GridDownscaler(
spatialRecord.getGridGeometry(), new BilinearInterpolation());
DownscaleStoreUtil.storeInterpolated(dataStore, downscaler, ds,
creator, true);
return dataStore; return dataStore;
} }

80
ncep/gov.noaa.nws.ncep.common.dataplugin.mcidas/src/gov/noaa/nws/ncep/common/dataplugin/mcidas/McidasMapCoverage.java
View file

@ -14,6 +14,7 @@
* 12/2009 144 T. Lee Migrated to TO11D6 * 12/2009 144 T. Lee Migrated to TO11D6
* 01/2010 201 M. Li Split into dataplugin project * 01/2010 201 M. Li Split into dataplugin project
* 05/2010 144 L. Lin Migration to TO11DR11. * 05/2010 144 L. Lin Migration to TO11DR11.
* Nov 14, 2013 2393 bclement added getGridGeometry()
* *
* </pre> * </pre>
*/ */
@ -22,6 +23,7 @@ package gov.noaa.nws.ncep.common.dataplugin.mcidas;
import gov.noaa.nws.ncep.common.tools.IDecoderConstantsN; import gov.noaa.nws.ncep.common.tools.IDecoderConstantsN;
import java.awt.geom.Rectangle2D;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@ -36,15 +38,26 @@ import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import org.apache.commons.lang.builder.HashCodeBuilder; import org.apache.commons.lang.builder.HashCodeBuilder;
import org.geotools.coverage.grid.GridEnvelope2D;
import org.geotools.coverage.grid.GridGeometry2D;
import org.geotools.geometry.DirectPosition2D;
import org.geotools.geometry.Envelope2D;
import org.geotools.referencing.CRS; import org.geotools.referencing.CRS;
import org.hibernate.annotations.Cache; import org.hibernate.annotations.Cache;
import org.hibernate.annotations.CacheConcurrencyStrategy; import org.hibernate.annotations.CacheConcurrencyStrategy;
import org.hibernate.annotations.Type; import org.hibernate.annotations.Type;
import org.opengis.coverage.grid.GridEnvelope;
import org.opengis.geometry.Envelope;
import org.opengis.geometry.MismatchedDimensionException;
import org.opengis.referencing.FactoryException;
import org.opengis.referencing.crs.CoordinateReferenceSystem; import org.opengis.referencing.crs.CoordinateReferenceSystem;
import org.opengis.referencing.crs.ProjectedCRS; import org.opengis.referencing.crs.ProjectedCRS;
import org.opengis.referencing.operation.MathTransform;
import org.opengis.referencing.operation.TransformException;
import com.raytheon.uf.common.dataplugin.persist.PersistableDataObject; import com.raytheon.uf.common.dataplugin.persist.PersistableDataObject;
import com.raytheon.uf.common.geospatial.ISpatialObject; import com.raytheon.uf.common.geospatial.ISpatialObject;
import com.raytheon.uf.common.geospatial.MapUtil;
import com.raytheon.uf.common.serialization.adapters.GeometryAdapter; import com.raytheon.uf.common.serialization.adapters.GeometryAdapter;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
@ -185,6 +198,18 @@ public class McidasMapCoverage extends PersistableDataObject implements ISpatial
@DynamicSerializeElement @DynamicSerializeElement
private Polygon location; private Polygon location;
/**
* minimum x value in crs space
*/
@Transient
private transient double minX = Double.NaN;
/**
* minimum y value in crs space
*/
@Transient
private transient double minY = Double.NaN;
public McidasMapCoverage() { public McidasMapCoverage() {
super(); super();
} }
@ -355,6 +380,61 @@ public class McidasMapCoverage extends PersistableDataObject implements ISpatial
return crsObject; return crsObject;
} }
/**
* Construct grid geometry using grid and geospatial information
*
* @return
* @throws MismatchedDimensionException
* @throws FactoryException
* @throws TransformException
*/
public GridGeometry2D getGridGeometry()
throws MismatchedDimensionException, FactoryException,
TransformException {
int nx = getNx();
int ny = getNy();
if (Double.isNaN(this.minX) || Double.isNaN(this.minY)) {
findMins();
}
GridEnvelope gridRange = new GridEnvelope2D(0, 0, nx, ny);
Envelope crsRange = new Envelope2D(getCrs(), new Rectangle2D.Double(
minX, minY, nx * getDx(), ny * getDy()));
return new GridGeometry2D(gridRange, crsRange);
}
/**
* Populate CRS min x value and min y value from longitudes and latitudes
*
* @throws FactoryException
* @throws MismatchedDimensionException
* @throws TransformException
*/
private void findMins() throws FactoryException,
MismatchedDimensionException, TransformException {
MathTransform from84 = MapUtil.getTransformFromLatLon(getCrs());
DirectPosition2D urInCrs = transform(from84, getUrlon(), getUrlat());
DirectPosition2D llInCrs = transform(from84, getLllon(), getLllat());
this.minX = Math.min(urInCrs.x, llInCrs.x);
this.minY = Math.min(urInCrs.y, llInCrs.y);
}
/**
* Transform x and y using provided math transform
*
* @param mt
* @param x
* @param y
* @return
* @throws MismatchedDimensionException
* @throws TransformException
*/
private DirectPosition2D transform(MathTransform mt, double x, double y)
throws MismatchedDimensionException, TransformException {
DirectPosition2D dest = new DirectPosition2D();
mt.transform(new DirectPosition2D(x, y), dest);
return dest;
}
public Float getDx() { public Float getDx() {
return dx; return dx;
} }

82
ncep/gov.noaa.nws.ncep.common.dataplugin.mcidas/src/gov/noaa/nws/ncep/common/dataplugin/mcidas/dao/McidasDao.java
View file

@ -9,6 +9,7 @@
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* 10/2009 144 T. Lee Created * 10/2009 144 T. Lee Created
* 11/2009 144 T. Lee Implemented area name DAO * 11/2009 144 T. Lee Implemented area name DAO
* Nov 14, 2013 2393 bclement added in-java interpolation
* </pre> * </pre>
* *
* @author tlee * @author tlee
@ -17,21 +18,33 @@
package gov.noaa.nws.ncep.common.dataplugin.mcidas.dao; package gov.noaa.nws.ncep.common.dataplugin.mcidas.dao;
import gov.noaa.nws.ncep.common.dataplugin.mcidas.McidasMapCoverage;
import gov.noaa.nws.ncep.common.dataplugin.mcidas.McidasRecord;
import gov.noaa.nws.ncep.common.dataplugin.mcidas.fixed.McidasAreaName;
import gov.noaa.nws.ncep.common.dataplugin.mcidas.fixed.McidasImageType;
import gov.noaa.nws.ncep.common.dataplugin.mcidas.fixed.McidasSatelliteName;
import java.awt.Rectangle;
import java.util.List; import java.util.List;
import org.geotools.coverage.grid.GridGeometry2D;
import com.raytheon.uf.common.dataplugin.PluginException; import com.raytheon.uf.common.dataplugin.PluginException;
import com.raytheon.uf.common.dataplugin.persist.IPersistable; import com.raytheon.uf.common.dataplugin.persist.IPersistable;
import com.raytheon.uf.common.datastorage.DataStoreFactory;
import com.raytheon.uf.common.datastorage.IDataStore; import com.raytheon.uf.common.datastorage.IDataStore;
import com.raytheon.uf.common.datastorage.StorageException; import com.raytheon.uf.common.datastorage.StorageException;
import com.raytheon.uf.common.datastorage.StorageProperties; import com.raytheon.uf.common.datastorage.StorageProperties;
import com.raytheon.uf.common.datastorage.records.AbstractStorageRecord; import com.raytheon.uf.common.datastorage.records.AbstractStorageRecord;
import com.raytheon.uf.common.datastorage.records.ByteDataRecord; import com.raytheon.uf.common.datastorage.records.ByteDataRecord;
import com.raytheon.uf.common.datastorage.records.IDataRecord;
import com.raytheon.uf.common.geospatial.interpolation.GridDownscaler;
import com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D;
import com.raytheon.uf.edex.database.plugin.DataRecordWrapUtil;
import com.raytheon.uf.edex.database.plugin.DownscaleStoreUtil;
import com.raytheon.uf.edex.database.plugin.DownscaleStoreUtil.IDataRecordCreator;
import com.raytheon.uf.edex.database.plugin.PluginDao; import com.raytheon.uf.edex.database.plugin.PluginDao;
import gov.noaa.nws.ncep.common.dataplugin.mcidas.McidasRecord;
import gov.noaa.nws.ncep.common.dataplugin.mcidas.fixed.McidasAreaName;
import gov.noaa.nws.ncep.common.dataplugin.mcidas.fixed.McidasSatelliteName;
import gov.noaa.nws.ncep.common.dataplugin.mcidas.fixed.McidasImageType;
public class McidasDao extends PluginDao { public class McidasDao extends PluginDao {
private McidasSatelliteNameDao satelliteNameDao = new McidasSatelliteNameDao(); private McidasSatelliteNameDao satelliteNameDao = new McidasSatelliteNameDao();
private McidasImageTypeDao imageTypeDao = new McidasImageTypeDao(); private McidasImageTypeDao imageTypeDao = new McidasImageTypeDao();
@ -45,7 +58,7 @@ public class McidasDao extends PluginDao {
@Override @Override
protected IDataStore populateDataStore(IDataStore dataStore, IPersistable record) protected IDataStore populateDataStore(IDataStore dataStore, IPersistable record)
throws StorageException { throws StorageException {
McidasRecord satRecord = (McidasRecord) record; final McidasRecord satRecord = (McidasRecord) record;
/* /*
* Write McIDAS Area file header block. * Write McIDAS Area file header block.
@ -62,21 +75,62 @@ public class McidasDao extends PluginDao {
* Write McIDAS image data block to HDF5. * Write McIDAS image data block to HDF5.
*/ */
if ( satRecord.getMessageData() != null ) { if ( satRecord.getMessageData() != null ) {
long xdim = satRecord.getCoverage().getNx(); McidasMapCoverage coverage = satRecord.getCoverage();
long ydim = satRecord.getCoverage().getNy(); int xdim = coverage.getNx();
int ydim = coverage.getNy();
long[] sizes = new long[] { xdim, ydim }; long[] sizes = new long[] { xdim, ydim };
AbstractStorageRecord storageRecord = new ByteDataRecord("Data", AbstractStorageRecord storageRecord = new ByteDataRecord(
satRecord.getDataURI(), (byte[]) satRecord.getMessageData(), 2, sizes); DataStoreFactory.DEF_DATASET_NAME, satRecord.getDataURI(),
(byte[]) satRecord.getMessageData(), 2, sizes);
StorageProperties props = new StorageProperties(); final StorageProperties props = new StorageProperties();
props.setDownscaled(true); GridGeometry2D gridGeom;
// props.setChunked(true); try {
// props.setCompressed(true); gridGeom = coverage.getGridGeometry();
} catch (Exception e) {
throw new StorageException(
"Unable to create grid geometry for record: "
+ satRecord, storageRecord, e);
}
GridDownscaler downScaler = new GridDownscaler(gridGeom);
storageRecord.setProperties(props); storageRecord.setProperties(props);
storageRecord.setCorrelationObject(satRecord); storageRecord.setCorrelationObject(satRecord);
// Store the base record.
dataStore.addDataRecord(storageRecord); dataStore.addDataRecord(storageRecord);
DataWrapper1D dataSource = DataRecordWrapUtil.wrap(storageRecord,
xdim,
ydim, true);
// this way of interpolating does not create the Data-interpolated/0
// link to the full sized data. This shouldn't be an issue since the
// retrieval code checks for level 0 and requests the full sized
// data.
DownscaleStoreUtil.storeInterpolated(dataStore, downScaler,
dataSource, new IDataRecordCreator() {
@Override
public IDataRecord create(Object data,
int downScaleLevel, Rectangle size)
throws StorageException {
long[] sizes = new long[] { size.width, size.height };
String group = DataStoreFactory.createGroupName(
satRecord.getDataURI(), null, true);
String name = String.valueOf(downScaleLevel);
IDataRecord rval = DataStoreFactory
.createStorageRecord(name, group, data, 2,
sizes);
rval.setProperties(props);
rval.setCorrelationObject(satRecord);
return rval;
}
@Override
public double getFillValue() {
return Double.NaN;
}
});
} }
return dataStore; return dataStore;
} }

23
ncep/gov.noaa.nws.ncep.viz.rsc.satellite/src/gov/noaa/nws/ncep/viz/rsc/satellite/rsc/AbstractSatelliteResource.java
View file

@ -1,5 +1,7 @@
package gov.noaa.nws.ncep.viz.rsc.satellite.rsc; package gov.noaa.nws.ncep.viz.rsc.satellite.rsc;
import gov.noaa.nws.ncep.common.dataplugin.mcidas.McidasMapCoverage;
import gov.noaa.nws.ncep.common.dataplugin.mcidas.McidasRecord;
import gov.noaa.nws.ncep.edex.common.metparameters.parameterconversion.NcUnits; import gov.noaa.nws.ncep.edex.common.metparameters.parameterconversion.NcUnits;
import gov.noaa.nws.ncep.viz.common.ColorMapUtil; import gov.noaa.nws.ncep.viz.common.ColorMapUtil;
import gov.noaa.nws.ncep.viz.common.area.AreaName.AreaSource; import gov.noaa.nws.ncep.viz.common.area.AreaName.AreaSource;
@ -17,6 +19,7 @@ import gov.noaa.nws.ncep.viz.rsc.satellite.units.NcSatelliteUnits;
import gov.noaa.nws.ncep.viz.ui.display.ColorBarFromColormap; import gov.noaa.nws.ncep.viz.ui.display.ColorBarFromColormap;
import gov.noaa.nws.ncep.viz.ui.display.NCMapDescriptor; import gov.noaa.nws.ncep.viz.ui.display.NCMapDescriptor;
import java.awt.Rectangle;
import java.io.File; import java.io.File;
import java.text.ParseException; import java.text.ParseException;
import java.text.ParsePosition; import java.text.ParsePosition;
@ -46,14 +49,15 @@ import com.raytheon.uf.common.geospatial.ISpatialEnabled;
import com.raytheon.uf.common.geospatial.ISpatialObject; import com.raytheon.uf.common.geospatial.ISpatialObject;
import com.raytheon.uf.common.geospatial.MapUtil; import com.raytheon.uf.common.geospatial.MapUtil;
import com.raytheon.uf.common.geospatial.ReferencedCoordinate; import com.raytheon.uf.common.geospatial.ReferencedCoordinate;
import com.raytheon.uf.common.geospatial.interpolation.GridDownscaler;
import com.raytheon.uf.common.serialization.SerializationException; import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.common.serialization.SerializationUtil; import com.raytheon.uf.common.serialization.SerializationUtil;
import com.raytheon.uf.common.style.AbstractStylePreferences; import com.raytheon.uf.common.style.AbstractStylePreferences;
import com.raytheon.uf.common.style.MatchCriteria; import com.raytheon.uf.common.style.MatchCriteria;
import com.raytheon.uf.common.style.ParamLevelMatchCriteria; import com.raytheon.uf.common.style.ParamLevelMatchCriteria;
import com.raytheon.uf.common.style.StyleException;
import com.raytheon.uf.common.style.StyleRule; import com.raytheon.uf.common.style.StyleRule;
import com.raytheon.uf.common.style.StyleRuleset; import com.raytheon.uf.common.style.StyleRuleset;
import com.raytheon.uf.common.style.StyleException;
import com.raytheon.uf.common.style.image.DataScale; import com.raytheon.uf.common.style.image.DataScale;
import com.raytheon.uf.common.style.image.ImagePreferences; import com.raytheon.uf.common.style.image.ImagePreferences;
import com.raytheon.uf.common.style.image.SamplePreferences; import com.raytheon.uf.common.style.image.SamplePreferences;
@ -110,6 +114,7 @@ import com.vividsolutions.jts.geom.Coordinate;
* 04/30/2013 *886 sgilbert Changed number of levels from 4 to 2 for native * 04/30/2013 *886 sgilbert Changed number of levels from 4 to 2 for native
* satellite projections in the McidasFileBasedTileSet * satellite projections in the McidasFileBasedTileSet
* 05/20/2013 862 ghull implement IAreaProviderCapable * 05/20/2013 862 ghull implement IAreaProviderCapable
* Nov 14, 2013 2393 bclement changed how numLevels is calculated for mcidas
* </pre> * </pre>
* *
* @author chammack * @author chammack
@ -486,6 +491,21 @@ public abstract class AbstractSatelliteResource extends
getCapability(ImagingCapability.class).setSuppressingMenuItems(true); getCapability(ImagingCapability.class).setSuppressingMenuItems(true);
getCapability(ColorMapCapability.class).setSuppressingMenuItems(true); getCapability(ColorMapCapability.class).setSuppressingMenuItems(true);
if (record instanceof McidasRecord) {
// mcidas is interpolated using the GridDownscaler, get number of
// levels stored in HDF5
McidasRecord mcidas = (McidasRecord) record;
McidasMapCoverage cov = mcidas.getCoverage();
try {
Rectangle[] downscaleSizes = GridDownscaler
.getDownscaleSizes(cov.getGridGeometry());
numLevels = downscaleSizes.length;
} catch (Exception e) {
throw new VizException(
"Unable to get grid geometry for record: " + record);
}
} else {
numLevels = 1; numLevels = 1;
int newSzX = ((ISpatialEnabled) record).getSpatialObject().getNx(); int newSzX = ((ISpatialEnabled) record).getSpatialObject().getNx();
int newSzY = ((ISpatialEnabled) record).getSpatialObject().getNy(); int newSzY = ((ISpatialEnabled) record).getSpatialObject().getNy();
@ -495,6 +515,7 @@ public abstract class AbstractSatelliteResource extends
newSzY /= 2; newSzY /= 2;
numLevels++; numLevels++;
} }
}
} }

9
pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/datastorage/StorageProperties.py
View file

@ -1,3 +1,4 @@
##
# This software was developed and / or modified by Raytheon Company, # This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government. # pursuant to Contract DG133W-05-CQ-1067 with the US Government.
# #
@ -15,6 +16,7 @@
# #
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for # See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information. # further licensing information.
##
# File auto-generated against equivalent DynamicSerialize Java class # File auto-generated against equivalent DynamicSerialize Java class
@ -23,7 +25,6 @@ class StorageProperties(object):
def __init__(self): def __init__(self):
self.compression = None self.compression = None
self.chunked = None self.chunked = None
self.downscaled = None
def getCompression(self): def getCompression(self):
return self.compression return self.compression
@ -37,9 +38,3 @@ class StorageProperties(object):
def setChunked(self, chunked): def setChunked(self, chunked):
self.chunked = chunked self.chunked = chunked
def getDownscaled(self):
return self.downscaled
def setDownscaled(self, downscaled):
self.downscaled = downscaled

7
pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/pypies/request/RetrieveRequest.py
View file

@ -24,7 +24,6 @@ class RetrieveRequest(object):
def __init__(self): def __init__(self):
self.group = None self.group = None
self.includeInterpolated = None
self.dataset = None self.dataset = None
self.request = None self.request = None
self.filename = None self.filename = None
@ -35,12 +34,6 @@ class RetrieveRequest(object):
def setGroup(self, group): def setGroup(self, group):
self.group = group self.group = group
def getIncludeInterpolated(self):
return self.includeInterpolated
def setIncludeInterpolated(self, includeInterpolated):
self.includeInterpolated = includeInterpolated
def getDataset(self): def getDataset(self):
return self.dataset return self.dataset

2
pythonPackages/pypies/pypies/impl/DataStoreFactory.py
View file

@ -30,6 +30,7 @@
# 07/21/10 njensen Initial Creation. # 07/21/10 njensen Initial Creation.
# 09/19/13 2309 bsteffen Fix group name in returned # 09/19/13 2309 bsteffen Fix group name in returned
# records. # records.
# Nov 14, 2013 2393 bclement removed interpolation
# #
# #
# #
@ -102,7 +103,6 @@ def createStorageRecord(rawData, ds):
if compression != 'LZF' and compression != 'ZLIB': if compression != 'LZF' and compression != 'ZLIB':
compression = 'NONE' compression = 'NONE'
props.setCompression(compression) props.setCompression(compression)
# TODO downscaled?
inst.setProps(props) inst.setProps(props)
t1=time.time() t1=time.time()

61
pythonPackages/pypies/pypies/impl/H5pyDataStore.py
View file

@ -32,6 +32,7 @@
# 10/09/12 rjpeter Optimized __getGroup for retrievals # 10/09/12 rjpeter Optimized __getGroup for retrievals
# 01/17/13 DR 15294 D. Friedman Clear out data in response # 01/17/13 DR 15294 D. Friedman Clear out data in response
# 02/12/13 #1608 randerso Added support for explicitly deleting groups and datasets # 02/12/13 #1608 randerso Added support for explicitly deleting groups and datasets
# Nov 14, 2013 2393 bclement removed interpolation
# #
# #
@ -64,7 +65,6 @@ dataRecordMap = {
DEFAULT_CHUNK_SIZE = 256 DEFAULT_CHUNK_SIZE = 256
FILESYSTEM_BLOCK_SIZE = 4096 FILESYSTEM_BLOCK_SIZE = 4096
DOWNSCALE_THRESHOLD = 512
REQUEST_ALL = Request() REQUEST_ALL = Request()
REQUEST_ALL.setType('ALL') REQUEST_ALL.setType('ALL')
@ -88,9 +88,6 @@ class H5pyDataStore(IDataStore.IDataStore):
t0=time.time() t0=time.time()
for r in recs: for r in recs:
try: try:
if r.getProps() and r.getProps().getDownscaled():
ss = self.__storeInterpolated(f, r, op)
else:
ss = self.__writeHDF(f, r, op) ss = self.__writeHDF(f, r, op)
except: except:
logger.warn("Exception occurred on file " + fn + ":" + IDataStore._exc()) logger.warn("Exception occurred on file " + fn + ":" + IDataStore._exc())
@ -135,11 +132,6 @@ class H5pyDataStore(IDataStore.IDataStore):
ss = self.__writeHDFDataset(f, data, record.getDimension(), record.getSizes(), record.getName(), ss = self.__writeHDFDataset(f, data, record.getDimension(), record.getSizes(), record.getName(),
group, props, self.__getHdf5Datatype(record), storeOp, record) group, props, self.__getHdf5Datatype(record), storeOp, record)
if props and props.getDownscaled():
intName = record.getGroup() + '/' + record.getName() + '-interpolated'
intGroup = self.__getNode(rootNode, intName, None, create=True)
self.__link(intGroup, '0', group[record.getName()])
f.flush() f.flush()
if logger.isEnabledFor(logging.DEBUG): if logger.isEnabledFor(logging.DEBUG):
logger.debug("Stored group " + str(record.getGroup()) + " to group " + str(group)) logger.debug("Stored group " + str(record.getGroup()) + " to group " + str(group))
@ -256,47 +248,6 @@ class H5pyDataStore(IDataStore.IDataStore):
for key in attrs: for key in attrs:
dataset.attrs[key] = attrs[key] dataset.attrs[key] = attrs[key]
def __storeInterpolated(self, f, rec, op):
if op != 'REPLACE' and op != 'STORE_ONLY':
raise StorageException("Only replace and store modes are supported with interpolation enabled")
# store the base product
ss = self.__writeHDF(f, rec, op)
sizes = rec.getSizes()
newSzX = sizes[1]
newSzY = sizes[0]
originalName = rec.getName()
originalGroup = rec.getGroup()
level = 0
# avoid recursive links by turning off downscaling
rec.getProps().setDownscaled(False)
from PIL import Image
import time
while newSzX > DOWNSCALE_THRESHOLD or newSzY > DOWNSCALE_THRESHOLD:
data = rec.retrieveDataObject()
# data is potentially 1-dimensional from serialization, ensure it goes to correct 2 dimensions
data = data.reshape([newSzX, newSzY])
newSzX = newSzX / 2
newSzY = newSzY / 2
level += 1
rec.setName(str(level))
rec.setGroup(originalGroup + '/' + originalName + '-interpolated')
# satellite data comes in as signed bytes but pil requires unsigned bytes
data = numpy.array(data + 127, dtype=numpy.uint8)
image = Image.fromarray(data)
image = image.resize((newSzY, newSzX))
downsized = numpy.array(image)
# transform back to signed bytes
downsized = numpy.array(downsized - 127, dtype=numpy.int8)
rec.putDataObject(downsized)
rec.setSizes([newSzY, newSzX])
self.__writeHDF(f, rec, op)
return ss
def __writePartialHDFDataset(self, f, data, dims, szDims, ds, props, def __writePartialHDFDataset(self, f, data, dims, szDims, ds, props,
minIndex): minIndex):
# reverse sizes for hdf5 # reverse sizes for hdf5
@ -405,7 +356,7 @@ class H5pyDataStore(IDataStore.IDataStore):
result = [self.__retrieveInternal(ds, req)] result = [self.__retrieveInternal(ds, req)]
else: else:
groupNode = self.__getNode(rootNode, group) groupNode = self.__getNode(rootNode, group)
result = self.__retrieve(groupNode, request.getIncludeInterpolated()) result = self.__retrieve(groupNode)
resp = RetrieveResponse() resp = RetrieveResponse()
resp.setRecords(result) resp.setRecords(result)
return resp return resp
@ -418,16 +369,10 @@ class H5pyDataStore(IDataStore.IDataStore):
def __retrieve(self, group, includeInterpolated=False): def __retrieve(self, group):
records = [] records = []
datasets = group.keys() datasets = group.keys()
for ds in datasets: for ds in datasets:
interpDs = ds.endswith('-interpolated')
if includeInterpolated and interpDs:
subresults = self.__retrieve(group[ds], False)
if subresults:
records += subresults
elif not interpDs:
rec = self.__retrieveInternal(group[ds], REQUEST_ALL) rec = self.__retrieveInternal(group[ds], REQUEST_ALL)
records.append(rec) records.append(rec)