xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #2393 removed pypies interpolation

Browse source 
updated datastore interface
updated pypies python code
updated python generated from java classes
moved in-java interpolation code from sat dao to utility classes
changed ncep mcidas dao to use in-java interpolation
changed VIIRS dao to use common interpolation code


Former-commit-id: 6f9ad823de5d0b30ae7c8206b4a4fdf1e3a2b22b
This commit is contained in:
Brian Clements 2013-11-13 13:33:49 -06:00
parent ff968c9884
commit fe58826a1e
27 changed files with 740 additions and 444 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
cave/com.raytheon.uf.viz.thinclient.cave/src/com/raytheon/uf/viz/thinclient/cave/cache/CachingDataStore.java vendored
View file

@ -50,6 +50,7 @@ import com.raytheon.uf.common.datastorage.records.IDataRecord;
* Feb 12, 2013 1608 randerso Added explicit deletes for groups and
* datasets
* Sep 18, 2013 2309 bsteffen Move disk acces to DataStoreCache
* Nov 14, 2013 2393 bclement removed datastore interpolation
*
* </pre>
*
@ -108,24 +109,6 @@ public class CachingDataStore implements IDataStore {
}
}
/*
* (non-Javadoc)
*
* @see
* com.raytheon.uf.common.datastorage.IDataStore#retrieve(java.lang.String,
* boolean)
*/
@Override
public IDataRecord[] retrieve(String group, boolean includeInterpolated)
throws StorageException, FileNotFoundException {
if (includeInterpolated == false) {
return retrieve(group);
} else {
/* This is deprecated and unused so caching is not implemented. */
return delegate.retrieve(group, includeInterpolated);
}
}
/*
* (non-Javadoc)
*

164
edexOsgi/com.raytheon.edex.plugin.satellite/src/com/raytheon/edex/plugin/satellite/dao/SatelliteDao.java
View file

@ -25,8 +25,6 @@ import java.util.Date;
import java.util.List;
import java.util.Map;
import org.opengis.referencing.operation.TransformException;
import com.raytheon.edex.plugin.satellite.gini.SatelliteCreatingEntity;
import com.raytheon.edex.plugin.satellite.gini.SatellitePhysicalElement;
import com.raytheon.edex.plugin.satellite.gini.SatellitePosition;
@ -43,18 +41,16 @@ import com.raytheon.uf.common.datastorage.DataStoreFactory;
import com.raytheon.uf.common.datastorage.IDataStore;
import com.raytheon.uf.common.datastorage.StorageException;
import com.raytheon.uf.common.datastorage.StorageProperties;
import com.raytheon.uf.common.datastorage.records.ByteDataRecord;
import com.raytheon.uf.common.datastorage.records.IDataRecord;
import com.raytheon.uf.common.datastorage.records.ShortDataRecord;
import com.raytheon.uf.common.geospatial.interpolation.GridDownscaler;
import com.raytheon.uf.common.geospatial.interpolation.data.AbstractDataWrapper;
import com.raytheon.uf.common.geospatial.interpolation.data.ByteArrayWrapper;
import com.raytheon.uf.common.geospatial.interpolation.data.DataDestination;
import com.raytheon.uf.common.geospatial.interpolation.data.ShortArrayWrapper;
import com.raytheon.uf.common.geospatial.interpolation.data.UnsignedByteArrayWrapper;
import com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D;
import com.raytheon.uf.common.time.TimeRange;
import com.raytheon.uf.edex.core.dataplugin.PluginRegistry;
import com.raytheon.uf.edex.database.DataAccessLayerException;
import com.raytheon.uf.edex.database.plugin.DataRecordWrapUtil;
import com.raytheon.uf.edex.database.plugin.DownscaleStoreUtil;
import com.raytheon.uf.edex.database.plugin.DownscaleStoreUtil.IDataRecordCreator;
import com.raytheon.uf.edex.database.plugin.PluginDao;
import com.raytheon.uf.edex.database.query.DatabaseQuery;
@ -75,6 +71,7 @@ import com.raytheon.uf.edex.database.query.DatabaseQuery;
* input arguments.
* 06/24/2013 2044 randerso Added methods to get data by TimeRange and
* getInventory with maxRecord limit
* Nov 14, 2013 2393 bclement moved interpolation code to parent class
* </pre>
*
* @author bphillip
@ -124,79 +121,61 @@ public class SatelliteDao extends PluginDao {
@Override
protected IDataStore populateDataStore(IDataStore dataStore,
IPersistable record) throws StorageException {
SatelliteRecord satRecord = (SatelliteRecord) record;
final SatelliteRecord satRecord = (SatelliteRecord) record;
IDataRecord storageRecord = (IDataRecord) satRecord.getMessageData();
if (storageRecord != null) {
StorageProperties props = new StorageProperties();
final StorageProperties props = new StorageProperties();
String compression = PluginRegistry.getInstance()
.getRegisteredObject(pluginName).getCompression();
if (compression != null) {
props.setCompression(StorageProperties.Compression
.valueOf(compression));
}
props.setDownscaled(false);
storageRecord.setProperties(props);
storageRecord.setCorrelationObject(satRecord);
final Map<String, Object> attributes = storageRecord
.getDataAttributes();
final Float fillValue = getAttribute(attributes,
SatelliteRecord.SAT_FILL_VALUE, 0.0f);
// Store the base record.
dataStore.addDataRecord(storageRecord);
Map<String, Object> attributes = storageRecord.getDataAttributes();
Float fillValue = getAttribute(attributes,
SatelliteRecord.SAT_FILL_VALUE, 0.0f);
SatMapCoverage coverage = satRecord.getCoverage();
AbstractDataWrapper dataSource = getSource(storageRecord,
coverage.getNx(), coverage.getNy());
dataSource.setFillValue(fillValue);
GridDownscaler downScaler = new GridDownscaler(
coverage.getGridGeometry());
// How many interpolation levels do we need for this data?
int levels = downScaler.getNumberOfDownscaleLevels();
Rectangle fullScale = downScaler.getDownscaleSize(0);
DataWrapper1D dataSource = DataRecordWrapUtil.wrap(
storageRecord, fullScale.width, fullScale.height, true);
int levels = DownscaleStoreUtil.storeInterpolated(dataStore,
downScaler, dataSource,
new IDataRecordCreator() {
@Override
public IDataRecord create(Object data,
int downScaleLevel, Rectangle size)
throws StorageException {
IDataRecord dr = createDataRecord(satRecord, data,
downScaleLevel, size);
// Set the attributes and properties from the parent
// data.
dr.setDataAttributes(attributes);
dr.setProperties(props);
return dr;
}
@Override
public double getFillValue() {
// always the same fill value
return fillValue;
}
});
// set the number of levels in the 'parent' satellite data.
// Subtract one for the base level data.
satRecord.setInterpolationLevels(levels - 1);
// How many interpolation levels do we need for this data? Includes
// the base level!
// Subtract one for the base level data.
int downScaleLevels = downScaler.getNumberOfDownscaleLevels() - 1;
// set the number of downscale levels in the satellite metadata.
satRecord.setInterpolationLevels(downScaleLevels);
if (DataStoreFactory.isInterpolated(levels)) {
for (int level = 0; level < downScaleLevels; level++) {
int downScaleLevel = level + 1;
Rectangle size = downScaler
.getDownscaleSize(downScaleLevel);
AbstractDataWrapper dest = getDestination(storageRecord,
size);
dest.setFillValue(fillValue);
try {
// Downscale from previous level
downScaler.downscale(downScaleLevel - 1,
downScaleLevel, dataSource, dest);
IDataRecord dr = createDataRecord(satRecord, dest,
downScaleLevel, size);
// Set the attributes and properties from the parent
// data.
dr.setDataAttributes(attributes);
dr.setProperties(props);
dataStore.addDataRecord(dr);
// Set source to current level
dataSource = dest;
} catch (TransformException e) {
throw new StorageException(
"Error creating downscaled data",
storageRecord, e);
}
}
}
satRecord.setInterpolationLevels(levels);
}
return dataStore;
}
@ -518,52 +497,6 @@ public class SatelliteDao extends PluginDao {
this.positionDao = positionDao;
}
/**
* Create an {@link AbstractDataWrapper} destination from the supplied
* {@link IDataRecord} with given dimensions.
*
* @param rec
* The record containing data to be wrapped.
* @param size
* A {@link Rectangle} containing the size of the input data.
* @return The wrapped data.
*/
private AbstractDataWrapper getDestination(IDataRecord rec, Rectangle size) {
AbstractDataWrapper dest = null;
if (rec instanceof ByteDataRecord) {
dest = new UnsignedByteArrayWrapper(size.width, size.height);
} else if (rec instanceof ShortDataRecord) {
dest = new ShortArrayWrapper(size.width, size.height);
}
return dest;
}
/**
* Create an {@link AbstractDataWrapper} source from the supplied
* {@link IDataRecord} with given dimensions.
*
* @param rec
* The record containing data to be wrapped.
* @param nx
* Number of items on the x axis.
* @param ny
* Number of items on the y axis.
* @return The wrapped data.
*/
private AbstractDataWrapper getSource(IDataRecord rec, int nx, int ny) {
AbstractDataWrapper source = null;
if (rec instanceof ByteDataRecord) {
byte[] b = ((ByteDataRecord) rec).getByteData();
source = new UnsignedByteArrayWrapper(b, nx, ny);
} else if (rec instanceof ShortDataRecord) {
short[] s = ((ShortDataRecord) rec).getShortData();
source = new ShortArrayWrapper(s, nx, ny);
}
return source;
}
/**
* Create the {@link IDataRecord} from the {@link DataDestination} using the
* original satellite data, size and
@ -577,19 +510,12 @@ public class SatelliteDao extends PluginDao {
* @param size
* Size of the down-scaled data.
* @return The created data record to be stored.
* @throws PluginException
*/
private IDataRecord createDataRecord(SatelliteRecord satRec,
DataDestination data, int downscaleLevel, Rectangle size) {
private IDataRecord createDataRecord(SatelliteRecord satRec, Object data,
int downscaleLevel, Rectangle size) throws StorageException {
SatelliteMessageData msgData = null;
Object o = null;
if (data instanceof ByteArrayWrapper) {
o = ((ByteArrayWrapper) data).getArray();
} else if (data instanceof ShortArrayWrapper) {
o = ((ShortArrayWrapper) data).getArray();
}
if (o != null) {
msgData = new SatelliteMessageData(o, size.width, size.height);
}
msgData = new SatelliteMessageData(data, size.width, size.height);
IDataRecord rec = msgData.getStorageRecord(satRec,
String.valueOf(downscaleLevel));
rec.setCorrelationObject(satRec);

23
edexOsgi/com.raytheon.uf.common.dataplugin.satellite/src/com/raytheon/uf/common/dataplugin/satellite/SatelliteMessageData.java
View file

@ -22,17 +22,15 @@ package com.raytheon.uf.common.dataplugin.satellite;
import java.util.HashMap;
import java.util.Map;
import com.raytheon.uf.common.dataplugin.satellite.SatelliteRecord;
import com.raytheon.uf.common.datastorage.records.ByteDataRecord;
import com.raytheon.uf.common.datastorage.DataStoreFactory;
import com.raytheon.uf.common.datastorage.records.IDataRecord;
import com.raytheon.uf.common.datastorage.records.ShortDataRecord;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
/**
* Encapsulate satellite image data as well as the dimensions of
* the image grid. Attributes about the data may also be added. As an
* example these attributes could include "scale factor" and/or "fill_value".
* Encapsulate satellite image data as well as the dimensions of the image grid.
* Attributes about the data may also be added. As an example these attributes
* could include "scale factor" and/or "fill_value".
*
* <pre>
*
@ -41,6 +39,8 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 27, 2012 798 jkorman Initial creation
* Nov 14, 2013 2393 bclement use datastore factory in
* getStorageRecord()
*
* </pre>
*
@ -125,15 +125,8 @@ public class SatelliteMessageData {
IDataRecord storageRecord = null;
if ((messageData != null) && (dataRec != null)) {
long[] sizes = new long[] { nx, ny };
if (messageData instanceof byte[]) {
storageRecord = new ByteDataRecord(dataSetName,
dataRec.getDataURI(), (byte[]) messageData, DATA_DIMS,
sizes);
} else if (messageData instanceof short[]) {
storageRecord = new ShortDataRecord(dataSetName,
dataRec.getDataURI(), (short[]) messageData, DATA_DIMS,
sizes);
}
storageRecord = DataStoreFactory.createStorageRecord(dataSetName,
dataRec.getDataURI(), messageData, DATA_DIMS, sizes);
}
if ((storageRecord != null) && (dataAttributes != null)) {
storageRecord.setDataAttributes(dataAttributes);

24
edexOsgi/com.raytheon.uf.common.datastorage/src/com/raytheon/uf/common/datastorage/IDataStore.java
View file

@ -25,7 +25,6 @@ import java.util.Map;
import com.raytheon.uf.common.datastorage.StorageProperties.Compression;
import com.raytheon.uf.common.datastorage.records.IDataRecord;
import com.raytheon.uf.common.serialization.ISerializableObject;
/**
* Defines the interface for operating against a hierarchical datastore
@ -41,6 +40,7 @@ import com.raytheon.uf.common.serialization.ISerializableObject;
* Feb 12, 2013 1608 randerso Added explicit methods for deleting
* groups and datasets
* Sep 19, 2013 2309 bsteffen Deprecate retrieve(String, boolean)
* Nov 14, 2013 2393 bclement removed interpolation
*
*
* </pre>
@ -48,7 +48,7 @@ import com.raytheon.uf.common.serialization.ISerializableObject;
* @author chammack
* @version 1.0
*/
public interface IDataStore extends ISerializableObject {
public interface IDataStore {
public static enum HDF5_ITEM {
DATASET, GROUP
@ -143,26 +143,6 @@ public interface IDataStore extends ISerializableObject {
public abstract IDataRecord[] retrieve(String group)
throws StorageException, FileNotFoundException;
/**
* Convenience method for retrieve
*
* Retrieves all data at a given group, with the option to retrieve
* interpolated tilesets.
*
* @param group
* the group of data to retrieve
* @param includeInterpolated
* a flag indicating whether interpolated tilesets should be
* retrieved
* @return the data records
* @throws StorageException
* @throws FileNotFoundException
*/
@Deprecated
public abstract IDataRecord[] retrieve(String group,
boolean includeInterpolated) throws StorageException,
FileNotFoundException;
/**
* Retrieve a single dataset with optional subsetting
*

24
edexOsgi/com.raytheon.uf.common.datastorage/src/com/raytheon/uf/common/datastorage/StorageProperties.java
View file

@ -22,7 +22,6 @@ package com.raytheon.uf.common.datastorage;
import org.apache.commons.lang.Validate;
import com.raytheon.uf.common.serialization.ISerializableObject;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
@ -36,6 +35,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Feb 8, 2007 chammack Initial Creation.
* Nov 14, 2013 2393 bclement removed interpolation
*
* </pre>
*
@ -43,7 +43,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* @version 1
*/
@DynamicSerialize
public class StorageProperties implements Cloneable, ISerializableObject {
public class StorageProperties implements Cloneable {
/**
* Compression types:
@ -66,10 +66,6 @@ public class StorageProperties implements Cloneable, ISerializableObject {
@DynamicSerializeElement
private boolean chunked;
/** Is the data progressively downsampled */
@DynamicSerializeElement
private boolean downscaled;
/**
* Construct a default storage properties. Default is all features disabled.
*/
@ -114,21 +110,6 @@ public class StorageProperties implements Cloneable, ISerializableObject {
}
}
/**
* @return the isDownscaled
*/
public boolean isDownscaled() {
return downscaled;
}
/**
* @param isDownscaled
* the isDownscaled to set
*/
public void setDownscaled(boolean isDownscaled) {
this.downscaled = isDownscaled;
}
/*
* (non-Javadoc)
*
@ -138,7 +119,6 @@ public class StorageProperties implements Cloneable, ISerializableObject {
public StorageProperties clone() {
StorageProperties sp = new StorageProperties();
sp.chunked = chunked;
sp.downscaled = downscaled;
sp.compression = compression;
return sp;
}

26
edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/AbstractDataWrapper.java
View file

@ -19,6 +19,9 @@
**/
package com.raytheon.uf.common.geospatial.interpolation.data;
import java.awt.Rectangle;
import java.lang.reflect.Constructor;
import org.geotools.coverage.grid.GeneralGridGeometry;
import com.raytheon.uf.common.geospatial.util.GridGeometryWrapChecker;
@ -35,6 +38,7 @@ import com.raytheon.uf.common.geospatial.util.GridGeometryWrapChecker;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 20, 2012 bsteffen Initial creation
* Nov 19, 2013 2393 bclement added createNew method
*
* </pre>
*
@ -113,4 +117,26 @@ public abstract class AbstractDataWrapper implements DataSource,
protected abstract void setDataValueInternal(double dataValue, int x, int y);
/**
* Create a new data wrapper of type c with the provided size
*
* @param c
* desired implementation class
* @param size
* @return
* @throws IllegalArgumentException
*/
public static <T extends AbstractDataWrapper> T createNew(
Class<? extends T> c, Rectangle size)
throws IllegalArgumentException {
try {
Constructor<? extends T> constructor = c.getConstructor(int.class,
int.class);
return constructor.newInstance(size.width, size.height);
} catch (Exception e) {
throw new IllegalArgumentException(
"Unable to instatiate instance of class: " + c, e);
}
}
}

3
edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/ByteArrayWrapper.java
View file

@ -22,7 +22,7 @@ package com.raytheon.uf.common.geospatial.interpolation.data;
import org.geotools.coverage.grid.GeneralGridGeometry;
/**
* {@link AbstractDataWrapper} implementation for byte array data.
* {@link AbstractDataWrapper} implementation for byte array data.
*
* <pre>
*
@ -113,4 +113,5 @@ public class ByteArrayWrapper extends DataWrapper1D {
public void setDataValueInternal(double dataValue, int index) {
array[index] = (byte) dataValue;
}
}

15
edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/ByteBufferWrapper.java
View file

@ -34,6 +34,7 @@ import org.geotools.coverage.grid.GeneralGridGeometry;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 20, 2012 bsteffen Initial creation
* Nov 19, 2013 2393 bclement added getArray
*
* </pre>
*
@ -79,4 +80,18 @@ public class ByteBufferWrapper extends DataWrapper1D {
buffer.put(index, (byte) dataValue);
}
/*
* (non-Javadoc)
*
* @see com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D#
* getPrimitiveArray()
*/
@Override
public byte[] getArray() {
if (buffer.hasArray()) {
return buffer.array();
}
return null;
}
}

8
edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/DataWrapper1D.java
View file

@ -32,6 +32,7 @@ import org.geotools.coverage.grid.GeneralGridGeometry;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 20, 2012 bsteffen Initial creation
* Nov 19, 2013 2393 bclement added abstract method for primitive array
*
* </pre>
*
@ -62,4 +63,11 @@ public abstract class DataWrapper1D extends AbstractDataWrapper {
protected abstract void setDataValueInternal(double dataValue, int index);
/**
* Get the backing 1D primitive array
*
* @return null if array isn't available
*/
public abstract Object getArray();
}

15
edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/DoubleBufferWrapper.java
View file

@ -34,6 +34,7 @@ import org.geotools.coverage.grid.GeneralGridGeometry;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 20, 2012 bsteffen Initial creation
* Nov 19, 2013 2393 bclement added getArray
*
* </pre>
*
@ -79,4 +80,18 @@ public class DoubleBufferWrapper extends DataWrapper1D {
buffer.put(index, dataValue);
}
/*
* (non-Javadoc)
*
* @see com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D#
* getPrimitiveArray()
*/
@Override
public double[] getArray() {
if (buffer.hasArray()) {
return buffer.array();
}
return null;
}
}

15
edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/FloatBufferWrapper.java
View file

@ -34,6 +34,7 @@ import org.geotools.coverage.grid.GeneralGridGeometry;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 20, 2012 bsteffen Initial creation
* Nov 19, 2013 2393 bclement added getArray
*
* </pre>
*
@ -79,4 +80,18 @@ public class FloatBufferWrapper extends DataWrapper1D {
buffer.put(index, (float) dataValue);
}
/*
* (non-Javadoc)
*
* @see com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D#
* getPrimitiveArray()
*/
@Override
public float[] getArray() {
if (buffer.hasArray()) {
return buffer.array();
}
return null;
}
}

15
edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/IntBufferWrapper.java
View file

@ -34,6 +34,7 @@ import org.geotools.coverage.grid.GeneralGridGeometry;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 20, 2012 bsteffen Initial creation
* Nov 19, 2013 2393 bclement added getArray
*
* </pre>
*
@ -79,4 +80,18 @@ public class IntBufferWrapper extends DataWrapper1D {
buffer.put(index, (int) dataValue);
}
/*
* (non-Javadoc)
*
* @see com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D#
* getPrimitiveArray()
*/
@Override
public int[] getArray() {
if (buffer.hasArray()) {
return buffer.array();
}
return null;
}
}

15
edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/ShortBufferWrapper.java
View file

@ -34,6 +34,7 @@ import org.geotools.coverage.grid.GeneralGridGeometry;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 20, 2012 bsteffen Initial creation
* Nov 19, 2013 2393 bclement added getArray
*
* </pre>
*
@ -79,4 +80,18 @@ public class ShortBufferWrapper extends DataWrapper1D {
buffer.put(index, (short) dataValue);
}
/*
* (non-Javadoc)
*
* @see com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D#
* getPrimitiveArray()
*/
@Override
public short[] getArray() {
if (buffer.hasArray()) {
return buffer.array();
}
return null;
}
}

46
edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/UnsignedByteBufferWrapper.java
View file

@ -25,7 +25,7 @@ import org.geotools.coverage.grid.GeneralGridGeometry;
/**
*
* ByteBuffer data wrapper
* Unsigned ByteBuffer data wrapper
*
* <pre>
*
@ -34,40 +34,47 @@ import org.geotools.coverage.grid.GeneralGridGeometry;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 20, 2012 bsteffen Initial creation
* Nov 19, 2013 2393 bclement changed to extend ByteBufferWrapper
*
* </pre>
*
* @author bsteffen
* @version 1.0
*/
public class UnsignedByteBufferWrapper extends DataWrapper1D {
public class UnsignedByteBufferWrapper extends ByteBufferWrapper {
protected final ByteBuffer buffer;
/**
* @param buffer
* @param geometry
*/
public UnsignedByteBufferWrapper(ByteBuffer buffer,
GeneralGridGeometry geometry) {
super(geometry);
this.buffer = buffer;
super(buffer, geometry);
}
/**
* @param buffer
* @param nx
* @param ny
*/
public UnsignedByteBufferWrapper(ByteBuffer buffer, int nx, int ny) {
super(nx, ny);
this.buffer = buffer;
}
public UnsignedByteBufferWrapper(int nx, int ny) {
this(ByteBuffer.allocate(nx * ny), nx, ny);
super(buffer, nx, ny);
}
/**
* @param geometry
*/
public UnsignedByteBufferWrapper(GeneralGridGeometry geometry) {
// assume this is going to be a destination and avoid passing
// geometry to super to save time on checking for wrapping.
this(geometry.getGridRange().getSpan(0), geometry.getGridRange()
.getSpan(1));
super(geometry);
}
public ByteBuffer getBuffer() {
return buffer;
/**
* @param nx
* @param ny
*/
public UnsignedByteBufferWrapper(int nx, int ny) {
super(nx, ny);
}
@Override
@ -75,9 +82,4 @@ public class UnsignedByteBufferWrapper extends DataWrapper1D {
return buffer.get(index) & 0xFF;
}
@Override
public void setDataValueInternal(double dataValue, int index) {
buffer.put(index, (byte) dataValue);
}
}

55
edexOsgi/com.raytheon.uf.common.geospatial/src/com/raytheon/uf/common/geospatial/interpolation/data/UnsignedShortBufferWrapper.java
View file

@ -25,7 +25,7 @@ import org.geotools.coverage.grid.GeneralGridGeometry;
/**
*
* ShortBuffer data wrapper
* Unsigned ShortBuffer data wrapper
*
* <pre>
*
@ -34,40 +34,46 @@ import org.geotools.coverage.grid.GeneralGridGeometry;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 20, 2012 bsteffen Initial creation
* Nov 19, 2013 2393 bclement changed to extend ShortBufferWrapper
*
* </pre>
*
* @author bsteffen
* @version 1.0
*/
public class UnsignedShortBufferWrapper extends DataWrapper1D {
public class UnsignedShortBufferWrapper extends ShortBufferWrapper {
protected final ShortBuffer buffer;
/**
* @param geometry
*/
public UnsignedShortBufferWrapper(GeneralGridGeometry geometry) {
super(geometry);
}
/**
* @param nx
* @param ny
*/
public UnsignedShortBufferWrapper(int nx, int ny) {
super(nx, ny);
}
/**
* @param buffer
* @param geometry
*/
public UnsignedShortBufferWrapper(ShortBuffer buffer,
GeneralGridGeometry geometry) {
super(geometry);
this.buffer = buffer;
super(buffer, geometry);
}
/**
* @param buffer
* @param nx
* @param ny
*/
public UnsignedShortBufferWrapper(ShortBuffer buffer, int nx, int ny) {
super(nx, ny);
this.buffer = buffer;
}
public UnsignedShortBufferWrapper(int nx, int ny) {
this(ShortBuffer.allocate(nx * ny), nx, ny);
}
public UnsignedShortBufferWrapper(GeneralGridGeometry geometry) {
// assume this is going to be a destination and avoid passing
// geometry to super to save time on checking for wrapping.
this(geometry.getGridRange().getSpan(0), geometry.getGridRange()
.getSpan(1));
}
public ShortBuffer getBuffer() {
return buffer;
super(buffer, nx, ny);
}
@Override
@ -75,9 +81,4 @@ public class UnsignedShortBufferWrapper extends DataWrapper1D {
return buffer.get(index) & 0xFFFF;
}
@Override
public void setDataValueInternal(double dataValue, int index) {
buffer.put(index, (short) dataValue);
}
}

19
edexOsgi/com.raytheon.uf.common.pypies/src/com/raytheon/uf/common/pypies/PyPiesDataStore.java
View file

@ -73,6 +73,7 @@ import com.raytheon.uf.common.util.FileUtil;
* Mon 07, 2013 DR 15294 D. Friedman Stream large requests
* Feb 11, 2013 1526 njensen use HttpClient.postDynamicSerialize() for memory efficiency
* Feb 12, 2013 #1608 randerso Added explicit deletes for groups and datasets
* Nov 14, 2013 2393 bclement removed interpolation
*
* </pre>
*
@ -202,23 +203,8 @@ public class PyPiesDataStore implements IDataStore {
@Override
public IDataRecord[] retrieve(final String group) throws StorageException,
FileNotFoundException {
return retrieve(group, false);
}
/*
* (non-Javadoc)
*
* @see
* com.raytheon.uf.common.datastorage.IDataStore#retrieve(java.lang.String,
* boolean)
*/
@Override
public IDataRecord[] retrieve(final String group,
final boolean includeInterpolated) throws StorageException,
FileNotFoundException {
RetrieveRequest req = new RetrieveRequest();
req.setGroup(group);
req.setIncludeInterpolated(includeInterpolated);
RetrieveResponse resp = (RetrieveResponse) cachedRequest(req);
return resp.getRecords();
}
@ -448,7 +434,8 @@ public class PyPiesDataStore implements IDataStore {
protected Object deserializeResponse(final byte[] response)
throws StorageException {
try {
return SerializationUtil.transformFromThrift(response);
return SerializationUtil
.transformFromThrift(Object.class, response);
} catch (SerializationException e) {
throw new StorageException(
"Error deserializing response from pypies server", null, e);

12
edexOsgi/com.raytheon.uf.common.pypies/src/com/raytheon/uf/common/pypies/request/RetrieveRequest.java
View file

@ -32,6 +32,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jul 8, 2010 njensen Initial creation
* Nov 14, 2013 2393 bclement removed get all interpolated
*
* </pre>
*
@ -45,9 +46,6 @@ public class RetrieveRequest extends AbstractRequest {
@DynamicSerializeElement
private String group;
@DynamicSerializeElement
private boolean includeInterpolated;
@DynamicSerializeElement
private String dataset;
@ -62,14 +60,6 @@ public class RetrieveRequest extends AbstractRequest {
this.group = group;
}
public boolean isIncludeInterpolated() {
return includeInterpolated;
}
public void setIncludeInterpolated(boolean includeInterpolated) {
this.includeInterpolated = includeInterpolated;
}
public String getDataset() {
return dataset;
}

121
edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/plugin/DataRecordWrapUtil.java Normal file
View file

@ -0,0 +1,121 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.database.plugin;
import com.raytheon.uf.common.datastorage.StorageException;
import com.raytheon.uf.common.datastorage.records.ByteDataRecord;
import com.raytheon.uf.common.datastorage.records.FloatDataRecord;
import com.raytheon.uf.common.datastorage.records.IDataRecord;
import com.raytheon.uf.common.datastorage.records.IntegerDataRecord;
import com.raytheon.uf.common.datastorage.records.ShortDataRecord;
import com.raytheon.uf.common.geospatial.interpolation.data.AbstractDataWrapper;
import com.raytheon.uf.common.geospatial.interpolation.data.ByteArrayWrapper;
import com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D;
import com.raytheon.uf.common.geospatial.interpolation.data.FloatArrayWrapper;
import com.raytheon.uf.common.geospatial.interpolation.data.IntArrayWrapper;
import com.raytheon.uf.common.geospatial.interpolation.data.ShortArrayWrapper;
import com.raytheon.uf.common.geospatial.interpolation.data.UnsignedByteArrayWrapper;
import com.raytheon.uf.common.geospatial.interpolation.data.UnsignedShortArrayWrapper;
/**
* Utility for wrapping data records to be a data source/destination
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Nov 19, 2013 2393 bclement Initial creation
*
* </pre>
*
* @author bclement
* @version 1.0
*/
public class DataRecordWrapUtil {
/**
* Create an {@link AbstractDataWrapper} source from the supplied
* {@link IDataRecord} with given dimensions.
*
* @param rec
* The record containing data to be wrapped.
* @param nx
* Number of items on the x axis.
* @param ny
* Number of items on the y axis.
* @return The wrapped data.
* @throws StorageException
*/
public static DataWrapper1D wrap(IDataRecord rec, int nx, int ny)
throws StorageException {
// default to signed data
return wrap(rec, nx, ny, false);
}
/**
* Create an {@link AbstractDataWrapper} source from the supplied
* {@link IDataRecord} with given dimensions.
*
* @param rec
* The record containing data to be wrapped.
* @param nx
* Number of items on the x axis.
* @param ny
* Number of items on the y axis.
* @param unsigned
* attempt to treat the data as unsigned if possible
* @return The wrapped data.
* @throws StorageException
*/
public static DataWrapper1D wrap(IDataRecord rec, int nx, int ny,
boolean unsigned) throws StorageException {
DataWrapper1D source = null;
if (rec instanceof ByteDataRecord) {
byte[] b = ((ByteDataRecord) rec).getByteData();
if (unsigned) {
source = new UnsignedByteArrayWrapper(b, nx, ny);
} else {
source = new ByteArrayWrapper(b, nx, ny);
}
} else if (rec instanceof ShortDataRecord) {
short[] s = ((ShortDataRecord) rec).getShortData();
if (unsigned) {
source = new UnsignedShortArrayWrapper(s, nx, ny);
} else {
source = new ShortArrayWrapper(s, nx, ny);
}
} else if (rec instanceof IntegerDataRecord) {
int[] i = ((IntegerDataRecord) rec).getIntData();
source = new IntArrayWrapper(i, nx, ny);
} else if (rec instanceof FloatDataRecord) {
float[] f = ((FloatDataRecord) rec).getFloatData();
source = new FloatArrayWrapper(f, nx, ny);
} else {
throw new StorageException("Unsupported data record type: "
+ rec.getClass(), rec);
}
return source;
}
}

157
edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/plugin/DownscaleStoreUtil.java Normal file
View file

@ -0,0 +1,157 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.database.plugin;
import java.awt.Rectangle;
import org.opengis.referencing.operation.TransformException;
import com.raytheon.uf.common.dataplugin.PluginDataObject;
import com.raytheon.uf.common.datastorage.DataStoreFactory;
import com.raytheon.uf.common.datastorage.IDataStore;
import com.raytheon.uf.common.datastorage.StorageException;
import com.raytheon.uf.common.datastorage.records.IDataRecord;
import com.raytheon.uf.common.geospatial.interpolation.GridDownscaler;
import com.raytheon.uf.common.geospatial.interpolation.data.AbstractDataWrapper;
import com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D;
/**
* Utility for storing downscaled data to datastore
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Nov 19, 2013 2393 bclement Initial creation
*
* </pre>
*
* @author bclement
* @version 1.0
*/
public class DownscaleStoreUtil {
/**
* Interface for creating IDataRecords for interpolation levels
*/
public static interface IDataRecordCreator {
/**
* Create new data record for level
*
* @param data
* @param downScaleLevel
* @param size
* @return
* @throws StorageException
*/
public IDataRecord create(Object data, int downScaleLevel,
Rectangle size) throws StorageException;
/**
* @return fill value
*/
public double getFillValue();
}
/**
* Create and add interpolated levels from dataSource.
*
* @param dataStore
* @param downScaler
* @param dataSource
* @param creator
* @return number of levels not including the base level
* @throws StorageException
*/
public static <T extends PluginDataObject> int storeInterpolated(
IDataStore dataStore, GridDownscaler downScaler,
DataWrapper1D dataSource, IDataRecordCreator creator)
throws StorageException {
// default to batch storage
return storeInterpolated(dataStore, downScaler, dataSource, creator,
false);
}
/**
* Create and add interpolated levels from dataSource.
*
* @param dataStore
* @param downScaler
* @param dataSource
* @param creator
* @param storeAfterEach
* if true, call store method on dataStore after each level is
* created
* @return number of levels not including the base level
* @throws StorageException
*/
public static <T extends PluginDataObject> int storeInterpolated(
IDataStore dataStore, GridDownscaler downScaler,
DataWrapper1D dataSource, IDataRecordCreator creator,
boolean storeAfterEach) throws StorageException {
dataSource.setFillValue(creator.getFillValue());
// How many interpolation levels do we need for this data?
int levels = downScaler.getNumberOfDownscaleLevels();
// How many interpolation levels do we need for this data? Includes
// the base level!
// Subtract one for the base level data.
int downScaleLevels = levels - 1;
if (DataStoreFactory.isInterpolated(levels)) {
for (int level = 0; level < downScaleLevels; level++) {
int downScaleLevel = level + 1;
Rectangle size = downScaler.getDownscaleSize(downScaleLevel);
DataWrapper1D dest = AbstractDataWrapper.createNew(
dataSource.getClass(), size);
dest.setFillValue(creator.getFillValue());
try {
// Downscale from previous level
downScaler.downscale(downScaleLevel - 1, downScaleLevel,
dataSource, dest);
Object data = dest.getArray();
if (data == null) {
throw new StorageException(
"Unable to get downscaled data from destination type: "
+ dest.getClass(), null);
}
IDataRecord dr = creator.create(data, downScaleLevel, size);
dataStore.addDataRecord(dr);
if (storeAfterEach) {
dataStore.store();
}
// Set source to current level
dataSource = dest;
} catch (TransformException e) {
throw new StorageException(
"Error creating downscaled data", null, e);
}
}
}
return downScaleLevels;
}
}

120
edexOsgi/com.raytheon.uf.edex.plugin.npp.viirs/src/com/raytheon/uf/edex/plugin/npp/viirs/dao/VIIRSDao.java
View file

@ -32,21 +32,22 @@ import com.raytheon.uf.common.dataplugin.PluginException;
import com.raytheon.uf.common.dataplugin.npp.viirs.VIIRSDataRecord;
import com.raytheon.uf.common.dataplugin.npp.viirs.VIIRSSpatialCoverage;
import com.raytheon.uf.common.dataplugin.persist.IPersistable;
import com.raytheon.uf.common.datastorage.DataStoreFactory;
import com.raytheon.uf.common.datastorage.IDataStore;
import com.raytheon.uf.common.datastorage.StorageException;
import com.raytheon.uf.common.datastorage.StorageProperties;
import com.raytheon.uf.common.datastorage.records.FloatDataRecord;
import com.raytheon.uf.common.datastorage.records.IDataRecord;
import com.raytheon.uf.common.datastorage.records.ShortDataRecord;
import com.raytheon.uf.common.geospatial.interpolation.BilinearInterpolation;
import com.raytheon.uf.common.geospatial.interpolation.GridDownscaler;
import com.raytheon.uf.common.geospatial.interpolation.data.AbstractDataWrapper;
import com.raytheon.uf.common.geospatial.interpolation.data.DataSource;
import com.raytheon.uf.common.geospatial.interpolation.data.FloatArrayWrapper;
import com.raytheon.uf.common.geospatial.interpolation.data.UnsignedShortArrayWrapper;
import com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.time.DataTime;
import com.raytheon.uf.edex.core.dataplugin.PluginRegistry;
import com.raytheon.uf.edex.database.DataAccessLayerException;
import com.raytheon.uf.edex.database.plugin.DataRecordWrapUtil;
import com.raytheon.uf.edex.database.plugin.DownscaleStoreUtil;
import com.raytheon.uf.edex.database.plugin.DownscaleStoreUtil.IDataRecordCreator;
import com.raytheon.uf.edex.database.plugin.PluginDao;
import com.raytheon.uf.edex.database.query.DatabaseQuery;
import com.raytheon.uf.edex.plugin.npp.viirs.VIIRSMessageData;
@ -129,12 +130,12 @@ public class VIIRSDao extends PluginDao {
@Override
protected IDataStore populateDataStore(IDataStore dataStore,
IPersistable obj) throws Exception {
VIIRSDataRecord record = (VIIRSDataRecord) obj;
final VIIRSDataRecord record = (VIIRSDataRecord) obj;
VIIRSSpatialCoverage spatialRecord = record.getCoverage();
int nx = spatialRecord.getNx();
int ny = spatialRecord.getNy();
StorageProperties props = new StorageProperties();
final StorageProperties props = new StorageProperties();
String compression = PluginRegistry.getInstance()
.getRegisteredObject(pluginName).getCompression();
if (compression != null) {
@ -142,71 +143,27 @@ public class VIIRSDao extends PluginDao {
.valueOf(compression));
}
VIIRSMessageData messageData = (VIIRSMessageData) record
final VIIRSMessageData messageData = (VIIRSMessageData) record
.getMessageData();
float[] missingValues = messageData.getMissingValues();
float fillValue = missingValues[0];
final float fillValue = missingValues[0];
Object rawData = messageData.getRawData();
AbstractDataWrapper ds = null;
// Data sources are create anonymously here to avoid having the
// fillValue/validMin/validMax even checked when getting values but
// still getting the getDataValueInternal functionality
if (rawData instanceof short[]) {
ds = new UnsignedShortArrayWrapper((short[]) rawData, nx, ny);
} else if (rawData instanceof float[]) {
ds = new FloatArrayWrapper((float[]) rawData, nx, ny);
}
IDataRecordCreator creator = new IDataRecordCreator() {
if (ds != null) {
ds.setFillValue(fillValue);
// Wrap the source and replace set each value which will replace
// anything in missingValues with fillValue
DataSource source = new VIIRSDataSourceWrapper(ds, missingValues);
for (int y = 0; y < ny; ++y) {
for (int x = 0; x < nx; ++x) {
ds.setDataValue(source.getDataValue(x, y), x, y);
}
@Override
public double getFillValue() {
return fillValue;
}
GridDownscaler downscaler = new GridDownscaler(
spatialRecord.getGridGeometry(),
new BilinearInterpolation());
int levels = downscaler.getNumberOfDownscaleLevels();
for (int i = 0; i < levels; ++i) {
Rectangle bounds = downscaler.getDownscaleSize(i);
AbstractDataWrapper dd = null;
if (i == 0) {
// No interpolation needed for level 0
dd = ds;
} else {
if (ds instanceof UnsignedShortArrayWrapper) {
dd = new UnsignedShortArrayWrapper(bounds.width,
bounds.height);
} else if (ds instanceof FloatArrayWrapper) {
dd = new FloatArrayWrapper(bounds.width, bounds.height);
}
dd.setFillValue(fillValue);
downscaler.downscale(i - 1, i, ds, dd);
ds = dd;
}
IDataRecord idr = null;
if (dd instanceof UnsignedShortArrayWrapper) {
idr = new ShortDataRecord(VIIRSDataRecord.getDataSet(i),
record.getDataURI(),
((UnsignedShortArrayWrapper) dd).getArray(), 2,
new long[] { bounds.width, bounds.height });
} else if (dd instanceof FloatArrayWrapper) {
idr = new FloatDataRecord(VIIRSDataRecord.getDataSet(i),
record.getDataURI(),
((FloatArrayWrapper) dd).getArray(), 2, new long[] {
bounds.width, bounds.height });
}
@Override
public IDataRecord create(Object data,
int downScaleLevel, Rectangle size)
throws StorageException {
long[] sizes = new long[] { size.width, size.height };
IDataRecord idr = DataStoreFactory.createStorageRecord(
VIIRSDataRecord.getDataSet(downScaleLevel),
record.getDataURI(), data, 2, sizes);
Map<String, Object> attributes = new HashMap<String, Object>();
attributes.put(VIIRSDataRecord.MISSING_VALUE_ID, fillValue);
attributes.put(VIIRSDataRecord.OFFSET_ID,
@ -220,13 +177,34 @@ public class VIIRSDao extends PluginDao {
idr.setDataAttributes(attributes);
idr.setProperties(props);
idr.setCorrelationObject(record);
dataStore.addDataRecord(idr);
dataStore.store();
return idr;
}
};
IDataRecord fullSize = creator
.create(rawData, 0, new Rectangle(nx, ny));
dataStore.addDataRecord(fullSize);
// Data sources are create anonymously here to avoid having the
// fillValue/validMin/validMax even checked when getting values but
// still getting the getDataValueInternal functionality
DataWrapper1D ds = DataRecordWrapUtil.wrap(fullSize, nx, ny, true);
ds.setFillValue(fillValue);
// Wrap the source and replace set each value which will replace
// anything in missingValues with fillValue
DataSource source = new VIIRSDataSourceWrapper(ds, missingValues);
for (int y = 0; y < ny; ++y) {
for (int x = 0; x < nx; ++x) {
ds.setDataValue(source.getDataValue(x, y), x, y);
}
} else {
throw new Exception("Unrecognized type for rawData: "
+ (rawData != null ? rawData.getClass() : null));
}
GridDownscaler downscaler = new GridDownscaler(
spatialRecord.getGridGeometry(), new BilinearInterpolation());
DownscaleStoreUtil.storeInterpolated(dataStore, downscaler, ds,
creator, true);
return dataStore;
}

80
ncep/gov.noaa.nws.ncep.common.dataplugin.mcidas/src/gov/noaa/nws/ncep/common/dataplugin/mcidas/McidasMapCoverage.java
View file

@ -14,6 +14,7 @@
* 12/2009 144 T. Lee Migrated to TO11D6
* 01/2010 201 M. Li Split into dataplugin project
* 05/2010 144 L. Lin Migration to TO11DR11.
* Nov 14, 2013 2393 bclement added getGridGeometry()
*
* </pre>
*/
@ -22,6 +23,7 @@ package gov.noaa.nws.ncep.common.dataplugin.mcidas;
import gov.noaa.nws.ncep.common.tools.IDecoderConstantsN;
import java.awt.geom.Rectangle2D;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -36,15 +38,26 @@ import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.geotools.coverage.grid.GridEnvelope2D;
import org.geotools.coverage.grid.GridGeometry2D;
import org.geotools.geometry.DirectPosition2D;
import org.geotools.geometry.Envelope2D;
import org.geotools.referencing.CRS;
import org.hibernate.annotations.Cache;
import org.hibernate.annotations.CacheConcurrencyStrategy;
import org.hibernate.annotations.Type;
import org.opengis.coverage.grid.GridEnvelope;
import org.opengis.geometry.Envelope;
import org.opengis.geometry.MismatchedDimensionException;
import org.opengis.referencing.FactoryException;
import org.opengis.referencing.crs.CoordinateReferenceSystem;
import org.opengis.referencing.crs.ProjectedCRS;
import org.opengis.referencing.operation.MathTransform;
import org.opengis.referencing.operation.TransformException;
import com.raytheon.uf.common.dataplugin.persist.PersistableDataObject;
import com.raytheon.uf.common.geospatial.ISpatialObject;
import com.raytheon.uf.common.geospatial.MapUtil;
import com.raytheon.uf.common.serialization.adapters.GeometryAdapter;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
@ -185,6 +198,18 @@ public class McidasMapCoverage extends PersistableDataObject implements ISpatial
@DynamicSerializeElement
private Polygon location;
/**
* minimum x value in crs space
*/
@Transient
private transient double minX = Double.NaN;
/**
* minimum y value in crs space
*/
@Transient
private transient double minY = Double.NaN;
public McidasMapCoverage() {
super();
}
@ -355,6 +380,61 @@ public class McidasMapCoverage extends PersistableDataObject implements ISpatial
return crsObject;
}
/**
* Construct grid geometry using grid and geospatial information
*
* @return
* @throws MismatchedDimensionException
* @throws FactoryException
* @throws TransformException
*/
public GridGeometry2D getGridGeometry()
throws MismatchedDimensionException, FactoryException,
TransformException {
int nx = getNx();
int ny = getNy();
if (Double.isNaN(this.minX) || Double.isNaN(this.minY)) {
findMins();
}
GridEnvelope gridRange = new GridEnvelope2D(0, 0, nx, ny);
Envelope crsRange = new Envelope2D(getCrs(), new Rectangle2D.Double(
minX, minY, nx * getDx(), ny * getDy()));
return new GridGeometry2D(gridRange, crsRange);
}
/**
* Populate CRS min x value and min y value from longitudes and latitudes
*
* @throws FactoryException
* @throws MismatchedDimensionException
* @throws TransformException
*/
private void findMins() throws FactoryException,
MismatchedDimensionException, TransformException {
MathTransform from84 = MapUtil.getTransformFromLatLon(getCrs());
DirectPosition2D urInCrs = transform(from84, getUrlon(), getUrlat());
DirectPosition2D llInCrs = transform(from84, getLllon(), getLllat());
this.minX = Math.min(urInCrs.x, llInCrs.x);
this.minY = Math.min(urInCrs.y, llInCrs.y);
}
/**
* Transform x and y using provided math transform
*
* @param mt
* @param x
* @param y
* @return
* @throws MismatchedDimensionException
* @throws TransformException
*/
private DirectPosition2D transform(MathTransform mt, double x, double y)
throws MismatchedDimensionException, TransformException {
DirectPosition2D dest = new DirectPosition2D();
mt.transform(new DirectPosition2D(x, y), dest);
return dest;
}
public Float getDx() {
return dx;
}

86
ncep/gov.noaa.nws.ncep.common.dataplugin.mcidas/src/gov/noaa/nws/ncep/common/dataplugin/mcidas/dao/McidasDao.java
View file

@ -9,6 +9,7 @@
* ------------ ---------- ----------- --------------------------
* 10/2009 144 T. Lee Created
* 11/2009 144 T. Lee Implemented area name DAO
* Nov 14, 2013 2393 bclement added in-java interpolation
* </pre>
*
* @author tlee
@ -17,21 +18,33 @@
package gov.noaa.nws.ncep.common.dataplugin.mcidas.dao;
import gov.noaa.nws.ncep.common.dataplugin.mcidas.McidasMapCoverage;
import gov.noaa.nws.ncep.common.dataplugin.mcidas.McidasRecord;
import gov.noaa.nws.ncep.common.dataplugin.mcidas.fixed.McidasAreaName;
import gov.noaa.nws.ncep.common.dataplugin.mcidas.fixed.McidasImageType;
import gov.noaa.nws.ncep.common.dataplugin.mcidas.fixed.McidasSatelliteName;
import java.awt.Rectangle;
import java.util.List;
import org.geotools.coverage.grid.GridGeometry2D;
import com.raytheon.uf.common.dataplugin.PluginException;
import com.raytheon.uf.common.dataplugin.persist.IPersistable;
import com.raytheon.uf.common.datastorage.DataStoreFactory;
import com.raytheon.uf.common.datastorage.IDataStore;
import com.raytheon.uf.common.datastorage.StorageException;
import com.raytheon.uf.common.datastorage.StorageProperties;
import com.raytheon.uf.common.datastorage.records.AbstractStorageRecord;
import com.raytheon.uf.common.datastorage.records.ByteDataRecord;
import com.raytheon.uf.common.datastorage.records.IDataRecord;
import com.raytheon.uf.common.geospatial.interpolation.GridDownscaler;
import com.raytheon.uf.common.geospatial.interpolation.data.DataWrapper1D;
import com.raytheon.uf.edex.database.plugin.DataRecordWrapUtil;
import com.raytheon.uf.edex.database.plugin.DownscaleStoreUtil;
import com.raytheon.uf.edex.database.plugin.DownscaleStoreUtil.IDataRecordCreator;
import com.raytheon.uf.edex.database.plugin.PluginDao;
import gov.noaa.nws.ncep.common.dataplugin.mcidas.McidasRecord;
import gov.noaa.nws.ncep.common.dataplugin.mcidas.fixed.McidasAreaName;
import gov.noaa.nws.ncep.common.dataplugin.mcidas.fixed.McidasSatelliteName;
import gov.noaa.nws.ncep.common.dataplugin.mcidas.fixed.McidasImageType;
public class McidasDao extends PluginDao {
private McidasSatelliteNameDao satelliteNameDao = new McidasSatelliteNameDao();
private McidasImageTypeDao imageTypeDao = new McidasImageTypeDao();
@ -45,7 +58,7 @@ public class McidasDao extends PluginDao {
@Override
protected IDataStore populateDataStore(IDataStore dataStore, IPersistable record)
throws StorageException {
McidasRecord satRecord = (McidasRecord) record;
final McidasRecord satRecord = (McidasRecord) record;
/*
* Write McIDAS Area file header block.
@ -62,21 +75,62 @@ public class McidasDao extends PluginDao {
* Write McIDAS image data block to HDF5.
*/
if ( satRecord.getMessageData() != null ) {
long xdim = satRecord.getCoverage().getNx();
long ydim = satRecord.getCoverage().getNy();
McidasMapCoverage coverage = satRecord.getCoverage();
int xdim = coverage.getNx();
int ydim = coverage.getNy();
long[] sizes = new long[] { xdim, ydim };
AbstractStorageRecord storageRecord = new ByteDataRecord("Data",
satRecord.getDataURI(), (byte[]) satRecord.getMessageData(), 2, sizes);
AbstractStorageRecord storageRecord = new ByteDataRecord(
DataStoreFactory.DEF_DATASET_NAME, satRecord.getDataURI(),
(byte[]) satRecord.getMessageData(), 2, sizes);
StorageProperties props = new StorageProperties();
props.setDownscaled(true);
// props.setChunked(true);
// props.setCompressed(true);
final StorageProperties props = new StorageProperties();
GridGeometry2D gridGeom;
try {
gridGeom = coverage.getGridGeometry();
} catch (Exception e) {
throw new StorageException(
"Unable to create grid geometry for record: "
+ satRecord, storageRecord, e);
}
GridDownscaler downScaler = new GridDownscaler(gridGeom);
storageRecord.setProperties(props);
storageRecord.setCorrelationObject(satRecord);
dataStore.addDataRecord(storageRecord);
// Store the base record.
dataStore.addDataRecord(storageRecord);
DataWrapper1D dataSource = DataRecordWrapUtil.wrap(storageRecord,
xdim,
ydim, true);
// this way of interpolating does not create the Data-interpolated/0
// link to the full sized data. This shouldn't be an issue since the
// retrieval code checks for level 0 and requests the full sized
// data.
DownscaleStoreUtil.storeInterpolated(dataStore, downScaler,
dataSource, new IDataRecordCreator() {
@Override
public IDataRecord create(Object data,
int downScaleLevel, Rectangle size)
throws StorageException {
long[] sizes = new long[] { size.width, size.height };
String group = DataStoreFactory.createGroupName(
satRecord.getDataURI(), null, true);
String name = String.valueOf(downScaleLevel);
IDataRecord rval = DataStoreFactory
.createStorageRecord(name, group, data, 2,
sizes);
rval.setProperties(props);
rval.setCorrelationObject(satRecord);
return rval;
}
@Override
public double getFillValue() {
return Double.NaN;
}
});
}
return dataStore;
}

37
ncep/gov.noaa.nws.ncep.viz.rsc.satellite/src/gov/noaa/nws/ncep/viz/rsc/satellite/rsc/AbstractSatelliteResource.java
View file

@ -1,5 +1,7 @@
package gov.noaa.nws.ncep.viz.rsc.satellite.rsc;
import gov.noaa.nws.ncep.common.dataplugin.mcidas.McidasMapCoverage;
import gov.noaa.nws.ncep.common.dataplugin.mcidas.McidasRecord;
import gov.noaa.nws.ncep.edex.common.metparameters.parameterconversion.NcUnits;
import gov.noaa.nws.ncep.viz.common.ColorMapUtil;
import gov.noaa.nws.ncep.viz.common.area.AreaName.AreaSource;
@ -17,6 +19,7 @@ import gov.noaa.nws.ncep.viz.rsc.satellite.units.NcSatelliteUnits;
import gov.noaa.nws.ncep.viz.ui.display.ColorBarFromColormap;
import gov.noaa.nws.ncep.viz.ui.display.NCMapDescriptor;
import java.awt.Rectangle;
import java.io.File;
import java.text.ParseException;
import java.text.ParsePosition;
@ -46,14 +49,15 @@ import com.raytheon.uf.common.geospatial.ISpatialEnabled;
import com.raytheon.uf.common.geospatial.ISpatialObject;
import com.raytheon.uf.common.geospatial.MapUtil;
import com.raytheon.uf.common.geospatial.ReferencedCoordinate;
import com.raytheon.uf.common.geospatial.interpolation.GridDownscaler;
import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.common.serialization.SerializationUtil;
import com.raytheon.uf.common.style.AbstractStylePreferences;
import com.raytheon.uf.common.style.MatchCriteria;
import com.raytheon.uf.common.style.ParamLevelMatchCriteria;
import com.raytheon.uf.common.style.StyleException;
import com.raytheon.uf.common.style.StyleRule;
import com.raytheon.uf.common.style.StyleRuleset;
import com.raytheon.uf.common.style.StyleException;
import com.raytheon.uf.common.style.image.DataScale;
import com.raytheon.uf.common.style.image.ImagePreferences;
import com.raytheon.uf.common.style.image.SamplePreferences;
@ -110,6 +114,7 @@ import com.vividsolutions.jts.geom.Coordinate;
* 04/30/2013 *886 sgilbert Changed number of levels from 4 to 2 for native
* satellite projections in the McidasFileBasedTileSet
* 05/20/2013 862 ghull implement IAreaProviderCapable
* Nov 14, 2013 2393 bclement changed how numLevels is calculated for mcidas
* </pre>
*
* @author chammack
@ -486,14 +491,30 @@ public abstract class AbstractSatelliteResource extends
getCapability(ImagingCapability.class).setSuppressingMenuItems(true);
getCapability(ColorMapCapability.class).setSuppressingMenuItems(true);
numLevels = 1;
int newSzX = ((ISpatialEnabled) record).getSpatialObject().getNx();
int newSzY = ((ISpatialEnabled) record).getSpatialObject().getNy();
if (record instanceof McidasRecord) {
// mcidas is interpolated using the GridDownscaler, get number of
// levels stored in HDF5
McidasRecord mcidas = (McidasRecord) record;
McidasMapCoverage cov = mcidas.getCoverage();
try {
Rectangle[] downscaleSizes = GridDownscaler
.getDownscaleSizes(cov.getGridGeometry());
numLevels = downscaleSizes.length;
} catch (Exception e) {
throw new VizException(
"Unable to get grid geometry for record: " + record);
}
while ((newSzX > 512 && newSzY > 512)) {
newSzX /= 2;
newSzY /= 2;
numLevels++;
} else {
numLevels = 1;
int newSzX = ((ISpatialEnabled) record).getSpatialObject().getNx();
int newSzY = ((ISpatialEnabled) record).getSpatialObject().getNy();
while ((newSzX > 512 && newSzY > 512)) {
newSzX /= 2;
newSzY /= 2;
numLevels++;
}
}
}

9
pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/datastorage/StorageProperties.py
View file

@ -1,3 +1,4 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
@ -15,6 +16,7 @@
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
# File auto-generated against equivalent DynamicSerialize Java class
@ -23,7 +25,6 @@ class StorageProperties(object):
def __init__(self):
self.compression = None
self.chunked = None
self.downscaled = None
def getCompression(self):
return self.compression
@ -37,9 +38,3 @@ class StorageProperties(object):
def setChunked(self, chunked):
self.chunked = chunked
def getDownscaled(self):
return self.downscaled
def setDownscaled(self, downscaled):
self.downscaled = downscaled

7
pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/pypies/request/RetrieveRequest.py
View file

@ -24,7 +24,6 @@ class RetrieveRequest(object):
def __init__(self):
self.group = None
self.includeInterpolated = None
self.dataset = None
self.request = None
self.filename = None
@ -35,12 +34,6 @@ class RetrieveRequest(object):
def setGroup(self, group):
self.group = group
def getIncludeInterpolated(self):
return self.includeInterpolated
def setIncludeInterpolated(self, includeInterpolated):
self.includeInterpolated = includeInterpolated
def getDataset(self):
return self.dataset

2
pythonPackages/pypies/pypies/impl/DataStoreFactory.py
View file

@ -30,6 +30,7 @@
# 07/21/10 njensen Initial Creation.
# 09/19/13 2309 bsteffen Fix group name in returned
# records.
# Nov 14, 2013 2393 bclement removed interpolation
#
#
#
@ -102,7 +103,6 @@ def createStorageRecord(rawData, ds):
if compression != 'LZF' and compression != 'ZLIB':
compression = 'NONE'
props.setCompression(compression)
# TODO downscaled?
inst.setProps(props)
t1=time.time()

67
pythonPackages/pypies/pypies/impl/H5pyDataStore.py
View file

@ -32,6 +32,7 @@
# 10/09/12 rjpeter Optimized __getGroup for retrievals
# 01/17/13 DR 15294 D. Friedman Clear out data in response
# 02/12/13 #1608 randerso Added support for explicitly deleting groups and datasets
# Nov 14, 2013 2393 bclement removed interpolation
#
#
@ -64,7 +65,6 @@ dataRecordMap = {
DEFAULT_CHUNK_SIZE = 256
FILESYSTEM_BLOCK_SIZE = 4096
DOWNSCALE_THRESHOLD = 512
REQUEST_ALL = Request()
REQUEST_ALL.setType('ALL')
@ -88,10 +88,7 @@ class H5pyDataStore(IDataStore.IDataStore):
t0=time.time()
for r in recs:
try:
if r.getProps() and r.getProps().getDownscaled():
ss = self.__storeInterpolated(f, r, op)
else:
ss = self.__writeHDF(f, r, op)
ss = self.__writeHDF(f, r, op)
except:
logger.warn("Exception occurred on file " + fn + ":" + IDataStore._exc())
exc.append(IDataStore._exc())
@ -135,11 +132,6 @@ class H5pyDataStore(IDataStore.IDataStore):
ss = self.__writeHDFDataset(f, data, record.getDimension(), record.getSizes(), record.getName(),
group, props, self.__getHdf5Datatype(record), storeOp, record)
if props and props.getDownscaled():
intName = record.getGroup() + '/' + record.getName() + '-interpolated'
intGroup = self.__getNode(rootNode, intName, None, create=True)
self.__link(intGroup, '0', group[record.getName()])
f.flush()
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Stored group " + str(record.getGroup()) + " to group " + str(group))
@ -256,47 +248,6 @@ class H5pyDataStore(IDataStore.IDataStore):
for key in attrs:
dataset.attrs[key] = attrs[key]
def __storeInterpolated(self, f, rec, op):
if op != 'REPLACE' and op != 'STORE_ONLY':
raise StorageException("Only replace and store modes are supported with interpolation enabled")
# store the base product
ss = self.__writeHDF(f, rec, op)
sizes = rec.getSizes()
newSzX = sizes[1]
newSzY = sizes[0]
originalName = rec.getName()
originalGroup = rec.getGroup()
level = 0
# avoid recursive links by turning off downscaling
rec.getProps().setDownscaled(False)
from PIL import Image
import time
while newSzX > DOWNSCALE_THRESHOLD or newSzY > DOWNSCALE_THRESHOLD:
data = rec.retrieveDataObject()
# data is potentially 1-dimensional from serialization, ensure it goes to correct 2 dimensions
data = data.reshape([newSzX, newSzY])
newSzX = newSzX / 2
newSzY = newSzY / 2
level += 1
rec.setName(str(level))
rec.setGroup(originalGroup + '/' + originalName + '-interpolated')
# satellite data comes in as signed bytes but pil requires unsigned bytes
data = numpy.array(data + 127, dtype=numpy.uint8)
image = Image.fromarray(data)
image = image.resize((newSzY, newSzX))
downsized = numpy.array(image)
# transform back to signed bytes
downsized = numpy.array(downsized - 127, dtype=numpy.int8)
rec.putDataObject(downsized)
rec.setSizes([newSzY, newSzX])
self.__writeHDF(f, rec, op)
return ss
def __writePartialHDFDataset(self, f, data, dims, szDims, ds, props,
minIndex):
# reverse sizes for hdf5
@ -405,7 +356,7 @@ class H5pyDataStore(IDataStore.IDataStore):
result = [self.__retrieveInternal(ds, req)]
else:
groupNode = self.__getNode(rootNode, group)
result = self.__retrieve(groupNode, request.getIncludeInterpolated())
result = self.__retrieve(groupNode)
resp = RetrieveResponse()
resp.setRecords(result)
return resp
@ -418,18 +369,12 @@ class H5pyDataStore(IDataStore.IDataStore):
def __retrieve(self, group, includeInterpolated=False):
def __retrieve(self, group):
records = []
datasets = group.keys()
for ds in datasets:
interpDs = ds.endswith('-interpolated')
if includeInterpolated and interpDs:
subresults = self.__retrieve(group[ds], False)
if subresults:
records += subresults
elif not interpDs:
rec = self.__retrieveInternal(group[ds], REQUEST_ALL)
records.append(rec)
rec = self.__retrieveInternal(group[ds], REQUEST_ALL)
records.append(rec)
return records