xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge "Issue #1257: Enable gfe compression, add population of storage properties" into development

Browse source 
Former-commit-id: dd8d0d9215 [formerly e49f5e4545] [formerly 3f874c0f6f [formerly 2c85ad207e90f46fa0cba0090fbd5266eb11b5a8]]
Former-commit-id: 3f874c0f6f
Former-commit-id: 758dc43192
This commit is contained in:
Ron Anderson 2012-10-11 12:04:53 -05:00 • committed by Gerrit Code Review
commit 98112ae284
2 changed files with 52 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
edexOsgi/com.raytheon.edex.plugin.gfe/res/spring/gfe-common.xml
View file

@ -12,6 +12,7 @@
value="com.raytheon.uf.common.dataplugin.gfe.db.objects.GFERecord" /> value="com.raytheon.uf.common.dataplugin.gfe.db.objects.GFERecord" />
<property name="initializer" <property name="initializer"
value="com.raytheon.edex.plugin.gfe.GfePluginInitializer" /> value="com.raytheon.edex.plugin.gfe.GfePluginInitializer" />
<property name="compression" value="LZF"/>
</bean> </bean>
<bean id="gfeDbPluginProperties" class="com.raytheon.uf.edex.database.DatabasePluginProperties"> <bean id="gfeDbPluginProperties" class="com.raytheon.uf.edex.database.DatabasePluginProperties">

85
edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/database/IFPGridDatabase.java
View file

@ -74,6 +74,7 @@ import com.raytheon.uf.common.datastorage.IDataStore;
import com.raytheon.uf.common.datastorage.IDataStore.StoreOp; import com.raytheon.uf.common.datastorage.IDataStore.StoreOp;
import com.raytheon.uf.common.datastorage.Request; import com.raytheon.uf.common.datastorage.Request;
import com.raytheon.uf.common.datastorage.StorageException; import com.raytheon.uf.common.datastorage.StorageException;
import com.raytheon.uf.common.datastorage.StorageProperties;
import com.raytheon.uf.common.datastorage.StorageStatus; import com.raytheon.uf.common.datastorage.StorageStatus;
import com.raytheon.uf.common.datastorage.records.ByteDataRecord; import com.raytheon.uf.common.datastorage.records.ByteDataRecord;
import com.raytheon.uf.common.datastorage.records.FloatDataRecord; import com.raytheon.uf.common.datastorage.records.FloatDataRecord;
@ -84,6 +85,7 @@ import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus; import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority; import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.time.TimeRange; import com.raytheon.uf.common.time.TimeRange;
import com.raytheon.uf.edex.core.dataplugin.PluginRegistry;
import com.raytheon.uf.edex.database.DataAccessLayerException; import com.raytheon.uf.edex.database.DataAccessLayerException;
import com.raytheon.uf.edex.database.plugin.PluginFactory; import com.raytheon.uf.edex.database.plugin.PluginFactory;
import com.vividsolutions.jts.geom.Coordinate; import com.vividsolutions.jts.geom.Coordinate;
@ -128,9 +130,9 @@ public class IFPGridDatabase extends GridDatabase {
private static final float VECTOR_DIR_DATA_OFFSET = 0.0f; private static final float VECTOR_DIR_DATA_OFFSET = 0.0f;
private Map<String, GridParmInfo> parmInfo = new HashMap<String, GridParmInfo>(); private final Map<String, GridParmInfo> parmInfo = new HashMap<String, GridParmInfo>();
private Map<String, ParmStorageInfo> parmStorageInfo = new HashMap<String, ParmStorageInfo>(); private final Map<String, ParmStorageInfo> parmStorageInfo = new HashMap<String, ParmStorageInfo>();
/** The grid configuration for this database */ /** The grid configuration for this database */
protected GridDbConfig gridConfig; protected GridDbConfig gridConfig;
@ -315,13 +317,13 @@ public class IFPGridDatabase extends GridDatabase {
} }
// max/min changes // max/min changes
if (userGPI.getMaxValue() != dbGPI.getMaxValue() if ((userGPI.getMaxValue() != dbGPI.getMaxValue())
|| userGPI.getMinValue() != dbGPI.getMinValue() || (userGPI.getMinValue() != dbGPI.getMinValue())
|| unitsChanged || !userPSI.equals(dbPSI)) { || unitsChanged || !userPSI.equals(dbPSI)) {
// If units were changed, the values need to be clamped to the // If units were changed, the values need to be clamped to the
// min and max values // min and max values
if (userGPI.getMaxValue() != dbGPI.getMaxValue() if ((userGPI.getMaxValue() != dbGPI.getMaxValue())
|| userGPI.getMinValue() != dbGPI.getMinValue()) { || (userGPI.getMinValue() != dbGPI.getMinValue())) {
statusHandler.handle( statusHandler.handle(
Priority.INFO, Priority.INFO,
"Changing Max/Min: " + dbGPI.getParmID() "Changing Max/Min: " + dbGPI.getParmID()
@ -863,7 +865,7 @@ public class IFPGridDatabase extends GridDatabase {
} }
// Save off the individual failures (if any), and then save what we can // Save off the individual failures (if any), and then save what we can
if (failedGrids != null && failedGrids.length > 0) { if ((failedGrids != null) && (failedGrids.length > 0)) {
for (GFERecord gfeRecord : failedGrids) { for (GFERecord gfeRecord : failedGrids) {
sr.addMessage("Failed to save grid to HDF5: " + gfeRecord); sr.addMessage("Failed to save grid to HDF5: " + gfeRecord);
} }
@ -873,7 +875,7 @@ public class IFPGridDatabase extends GridDatabase {
GFERecord[] gridsToStore = records.toArray(new GFERecord[records GFERecord[] gridsToStore = records.toArray(new GFERecord[records
.size()]); .size()]);
if (failedGrids != null && failedGrids.length > 0) { if ((failedGrids != null) && (failedGrids.length > 0)) {
Set<GFERecord> workingSet = new HashSet<GFERecord>(records); Set<GFERecord> workingSet = new HashSet<GFERecord>(records);
workingSet.removeAll(Arrays.asList(failedGrids)); workingSet.removeAll(Arrays.asList(failedGrids));
gridsToStore = workingSet.toArray(new GFERecord[workingSet gridsToStore = workingSet.toArray(new GFERecord[workingSet
@ -1491,7 +1493,7 @@ public class IFPGridDatabase extends GridDatabase {
private ServerResponse<?> dbIsValid() { private ServerResponse<?> dbIsValid() {
ServerResponse<?> sr = new ServerResponse<String>(); ServerResponse<?> sr = new ServerResponse<String>();
if (dbId == null || !dbId.isValid()) { if ((dbId == null) || !dbId.isValid()) {
sr.addMessage("DBInvalid - The database is not valid."); sr.addMessage("DBInvalid - The database is not valid.");
} }
return sr; return sr;
@ -1522,7 +1524,8 @@ public class IFPGridDatabase extends GridDatabase {
gfeBaseDataDir, this.dbId)); gfeBaseDataDir, this.dbId));
try { try {
if (gridConfig != null && gridConfig.parmAndLevelList().size() > 0) { if ((gridConfig != null)
&& (gridConfig.parmAndLevelList().size() > 0)) {
ds.getDatasets(GRID_PARM_INFO_GRP); ds.getDatasets(GRID_PARM_INFO_GRP);
parmInfoInitialized = true; parmInfoInitialized = true;
} }
@ -1750,6 +1753,15 @@ public class IFPGridDatabase extends GridDatabase {
ParmStorageInfo parmStorageInfo) throws GfeException { ParmStorageInfo parmStorageInfo) throws GfeException {
List<GFERecord> failedGrids = new ArrayList<GFERecord>(); List<GFERecord> failedGrids = new ArrayList<GFERecord>();
try { try {
StorageProperties sp = null;
String compression = PluginRegistry.getInstance()
.getRegisteredObject("gfe").getCompression();
if (compression != null) {
sp = new StorageProperties();
sp.setCompression(StorageProperties.Compression
.valueOf(compression));
}
Map<File, List<GFERecord>> recordMap = new HashMap<File, List<GFERecord>>(); Map<File, List<GFERecord>> recordMap = new HashMap<File, List<GFERecord>>();
for (GFERecord rec : dataObjects) { for (GFERecord rec : dataObjects) {
@ -1780,24 +1792,24 @@ public class IFPGridDatabase extends GridDatabase {
// Get storage info (for float and vector data) // Get storage info (for float and vector data)
String storageType = parmStorageInfo.storageType(); String storageType = parmStorageInfo.storageType();
if (data instanceof FloatDataRecord if ((data instanceof FloatDataRecord)
&& !"float".equals(storageType)) { && !"float".equals(storageType)) {
storeConvertedFloatRecord((FloatDataRecord) data, storeConvertedFloatRecord((FloatDataRecord) data,
dataStore, groupName, parmStorageInfo, dataStore, sp, groupName, parmStorageInfo,
correlationMap, rec); correlationMap, rec);
} else if (data instanceof IDataRecord) { } else if (data instanceof IDataRecord) {
// store without conversion // store without conversion
((IDataRecord) data).setGroup(groupName); ((IDataRecord) data).setGroup(groupName);
dataStore.addDataRecord((IDataRecord) data); dataStore.addDataRecord((IDataRecord) data, sp);
correlationMap.put(((IDataRecord) data), rec); correlationMap.put(((IDataRecord) data), rec);
} else if (data instanceof VectorGridSlice) { } else if (data instanceof VectorGridSlice) {
storeVectorGridSlice(data, dataStore, groupName, storeVectorGridSlice(data, dataStore, sp, groupName,
parmStorageInfo, correlationMap, rec); parmStorageInfo, correlationMap, rec);
} else if (data instanceof ScalarGridSlice) { } else if (data instanceof ScalarGridSlice) {
storeScalarGridSlice(data, dataStore, groupName, storeScalarGridSlice(data, dataStore, sp, groupName,
parmStorageInfo, correlationMap, rec); parmStorageInfo, correlationMap, rec);
} else if (data instanceof DiscreteGridSlice) { } else if (data instanceof DiscreteGridSlice) {
storeDiscreteGridSlice(data, dataStore, groupName, storeDiscreteGridSlice(data, dataStore, sp, groupName,
parmStorageInfo, correlationMap, rec); parmStorageInfo, correlationMap, rec);
} else if (data instanceof WeatherGridSlice) { } else if (data instanceof WeatherGridSlice) {
WeatherGridSlice slice = (WeatherGridSlice) data; WeatherGridSlice slice = (WeatherGridSlice) data;
@ -1810,7 +1822,7 @@ public class IFPGridDatabase extends GridDatabase {
.getNx(), .getNx(),
slice.getGridInfo().getGridLoc() slice.getGridInfo().getGridLoc()
.getNy() }); .getNy() });
dataStore.addDataRecord(rawRecord); dataStore.addDataRecord(rawRecord, sp);
StringBuffer sb = new StringBuffer(); StringBuffer sb = new StringBuffer();
boolean first = true; boolean first = true;
@ -1826,7 +1838,7 @@ public class IFPGridDatabase extends GridDatabase {
ByteDataRecord keyRecord = new ByteDataRecord( ByteDataRecord keyRecord = new ByteDataRecord(
"Keys", groupName, keyBytes, 1, "Keys", groupName, keyBytes, 1,
new long[] { keyBytes.length }); new long[] { keyBytes.length });
dataStore.addDataRecord(keyRecord); dataStore.addDataRecord(keyRecord, sp);
correlationMap.put(rawRecord, rec); correlationMap.put(rawRecord, rec);
correlationMap.put(keyRecord, rec); correlationMap.put(keyRecord, rec);
} }
@ -1835,7 +1847,7 @@ public class IFPGridDatabase extends GridDatabase {
StorageStatus ss = dataStore.store(StoreOp.REPLACE); StorageStatus ss = dataStore.store(StoreOp.REPLACE);
StorageException[] exceptions = ss.getExceptions(); StorageException[] exceptions = ss.getExceptions();
if (exceptions != null && exceptions.length > 0) { if ((exceptions != null) && (exceptions.length > 0)) {
// Describe the errors, then // Describe the errors, then
// only log the first one, don't flood the log with // only log the first one, don't flood the log with
// duplicates. // duplicates.
@ -1883,7 +1895,8 @@ public class IFPGridDatabase extends GridDatabase {
* @throws StorageException * @throws StorageException
*/ */
protected void storeScalarGridSlice(Object data, IDataStore dataStore, protected void storeScalarGridSlice(Object data, IDataStore dataStore,
String groupName, ParmStorageInfo parmStorageInfo, StorageProperties sp, String groupName,
ParmStorageInfo parmStorageInfo,
Map<IDataRecord, GFERecord> correlationMap, GFERecord rec) Map<IDataRecord, GFERecord> correlationMap, GFERecord rec)
throws StorageException { throws StorageException {
ScalarGridSlice slice = (ScalarGridSlice) data; ScalarGridSlice slice = (ScalarGridSlice) data;
@ -1893,7 +1906,7 @@ public class IFPGridDatabase extends GridDatabase {
rawData, 2, new long[] { rawData, 2, new long[] {
slice.getGridInfo().getGridLoc().getNx(), slice.getGridInfo().getGridLoc().getNx(),
slice.getGridInfo().getGridLoc().getNy() }); slice.getGridInfo().getGridLoc().getNy() });
this.storeConvertedFloatRecord(rawRecord, dataStore, groupName, this.storeConvertedFloatRecord(rawRecord, dataStore, sp, groupName,
parmStorageInfo, correlationMap, rec); parmStorageInfo, correlationMap, rec);
} }
} }
@ -1917,11 +1930,12 @@ public class IFPGridDatabase extends GridDatabase {
* @throws StorageException * @throws StorageException
*/ */
protected void storeVectorGridSlice(Object data, IDataStore dataStore, protected void storeVectorGridSlice(Object data, IDataStore dataStore,
String groupName, ParmStorageInfo parmStorageInfo, StorageProperties sp, String groupName,
ParmStorageInfo parmStorageInfo,
Map<IDataRecord, GFERecord> correlationMap, GFERecord rec) Map<IDataRecord, GFERecord> correlationMap, GFERecord rec)
throws StorageException { throws StorageException {
VectorGridSlice slice = (VectorGridSlice) data; VectorGridSlice slice = (VectorGridSlice) data;
if (slice.getMagGrid() != null || slice.getDirGrid() != null) { if ((slice.getMagGrid() != null) || (slice.getDirGrid() != null)) {
float[] rawMagData = slice.getMagGrid().getFloats(); float[] rawMagData = slice.getMagGrid().getFloats();
float[] rawDirData = slice.getDirGrid().getFloats(); float[] rawDirData = slice.getDirGrid().getFloats();
FloatDataRecord rawMagRecord = new FloatDataRecord("Mag", FloatDataRecord rawMagRecord = new FloatDataRecord("Mag",
@ -1941,10 +1955,10 @@ public class IFPGridDatabase extends GridDatabase {
parmStorageInfo.parmName(), parmStorageInfo.level(), parmStorageInfo.parmName(), parmStorageInfo.level(),
VECTOR_DIR_DATA_OFFSET, VECTOR_DIR_DATA_MULTIPLIER, VECTOR_DIR_DATA_OFFSET, VECTOR_DIR_DATA_MULTIPLIER,
parmStorageInfo.storageType()); parmStorageInfo.storageType());
this.storeConvertedFloatRecord(rawMagRecord, dataStore, groupName, this.storeConvertedFloatRecord(rawMagRecord, dataStore, sp,
parmStorageInfo, correlationMap, rec); groupName, parmStorageInfo, correlationMap, rec);
this.storeConvertedFloatRecord(rawDirRecord, dataStore, groupName, this.storeConvertedFloatRecord(rawDirRecord, dataStore, sp,
dirStorageInfo, correlationMap, rec); groupName, dirStorageInfo, correlationMap, rec);
} }
} }
@ -1955,6 +1969,8 @@ public class IFPGridDatabase extends GridDatabase {
* The discrete grid slice * The discrete grid slice
* @param dataStore * @param dataStore
* The data store in which to save the slice * The data store in which to save the slice
* @param sp
* The storage properties for the slice
* @param groupName * @param groupName
* The group name under which to save the slice * The group name under which to save the slice
* @param parmStorageInfo * @param parmStorageInfo
@ -1967,7 +1983,8 @@ public class IFPGridDatabase extends GridDatabase {
* @throws StorageException * @throws StorageException
*/ */
protected void storeDiscreteGridSlice(Object data, IDataStore dataStore, protected void storeDiscreteGridSlice(Object data, IDataStore dataStore,
String groupName, ParmStorageInfo parmStorageInfo, StorageProperties sp, String groupName,
ParmStorageInfo parmStorageInfo,
Map<IDataRecord, GFERecord> correlationMap, GFERecord rec) Map<IDataRecord, GFERecord> correlationMap, GFERecord rec)
throws StorageException { throws StorageException {
DiscreteGridSlice slice = (DiscreteGridSlice) data; DiscreteGridSlice slice = (DiscreteGridSlice) data;
@ -1977,7 +1994,7 @@ public class IFPGridDatabase extends GridDatabase {
rawData, 2, new long[] { rawData, 2, new long[] {
slice.getGridInfo().getGridLoc().getNx(), slice.getGridInfo().getGridLoc().getNx(),
slice.getGridInfo().getGridLoc().getNy() }); slice.getGridInfo().getGridLoc().getNy() });
dataStore.addDataRecord(rawRecord); dataStore.addDataRecord(rawRecord, sp);
StringBuffer sb = new StringBuffer(); StringBuffer sb = new StringBuffer();
boolean first = true; boolean first = true;
@ -1992,7 +2009,7 @@ public class IFPGridDatabase extends GridDatabase {
byte[] keyBytes = sb.toString().getBytes(); byte[] keyBytes = sb.toString().getBytes();
ByteDataRecord keyRecord = new ByteDataRecord("Keys", groupName, ByteDataRecord keyRecord = new ByteDataRecord("Keys", groupName,
keyBytes, 1, new long[] { keyBytes.length }); keyBytes, 1, new long[] { keyBytes.length });
dataStore.addDataRecord(keyRecord); dataStore.addDataRecord(keyRecord, sp);
correlationMap.put(rawRecord, rec); correlationMap.put(rawRecord, rec);
correlationMap.put(keyRecord, rec); correlationMap.put(keyRecord, rec);
} }
@ -2041,7 +2058,7 @@ public class IFPGridDatabase extends GridDatabase {
* The GFE record being stored * The GFE record being stored
*/ */
protected void storeConvertedFloatRecord(FloatDataRecord data, protected void storeConvertedFloatRecord(FloatDataRecord data,
IDataStore dataStore, String groupName, IDataStore dataStore, StorageProperties sp, String groupName,
ParmStorageInfo parmStorageInfo, ParmStorageInfo parmStorageInfo,
Map<IDataRecord, GFERecord> correlationMap, GFERecord rec) Map<IDataRecord, GFERecord> correlationMap, GFERecord rec)
throws StorageException { throws StorageException {
@ -2052,7 +2069,7 @@ public class IFPGridDatabase extends GridDatabase {
float multiplier = parmStorageInfo.dataMultiplier(); float multiplier = parmStorageInfo.dataMultiplier();
float fcvt; float fcvt;
IDataRecord storeDataRec = null; IDataRecord storeDataRec = null;
if ("short".equals(storageType) && multiplier != 0.0f) { if ("short".equals(storageType) && (multiplier != 0.0f)) {
short[] converted = new short[fdata.length]; short[] converted = new short[fdata.length];
for (int i = 0; i < fdata.length; i++) { for (int i = 0; i < fdata.length; i++) {
fcvt = (fdata[i] - offset) * multiplier; fcvt = (fdata[i] - offset) * multiplier;
@ -2061,7 +2078,7 @@ public class IFPGridDatabase extends GridDatabase {
} }
storeDataRec = new ShortDataRecord(data.getName(), data.getGroup(), storeDataRec = new ShortDataRecord(data.getName(), data.getGroup(),
converted, data.getDimension(), data.getSizes().clone()); converted, data.getDimension(), data.getSizes().clone());
} else if ("byte".equals(storageType) && multiplier != 0.0f) { } else if ("byte".equals(storageType) && (multiplier != 0.0f)) {
byte[] converted = new byte[fdata.length]; byte[] converted = new byte[fdata.length];
for (int i = 0; i < fdata.length; i++) { for (int i = 0; i < fdata.length; i++) {
fcvt = (fdata[i] - offset) * multiplier; fcvt = (fdata[i] - offset) * multiplier;
@ -2074,7 +2091,7 @@ public class IFPGridDatabase extends GridDatabase {
} }
storeDataRec.setGroup(groupName); storeDataRec.setGroup(groupName);
dataStore.addDataRecord(storeDataRec); dataStore.addDataRecord(storeDataRec, sp);
correlationMap.put(storeDataRec, rec); correlationMap.put(storeDataRec, rec);
} }