xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge "Issue #1257: Enable gfe compression, add population of storage properties" into development

Browse source 
Former-commit-id: dd8d0d9215 [formerly e49f5e4545] [formerly dd8d0d9215 [formerly e49f5e4545] [formerly 3f874c0f6f [formerly 2c85ad207e90f46fa0cba0090fbd5266eb11b5a8]]]
Former-commit-id: 3f874c0f6f
Former-commit-id: 98112ae284 [formerly 758dc43192]
Former-commit-id: 69fcb899e7
This commit is contained in:
Ron Anderson 2012-10-11 12:04:53 -05:00 • committed by Gerrit Code Review
commit 765bb7b9c2
2 changed files with 52 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
edexOsgi/com.raytheon.edex.plugin.gfe/res/spring/gfe-common.xml
View file

@ -12,6 +12,7 @@
value="com.raytheon.uf.common.dataplugin.gfe.db.objects.GFERecord" />
<property name="initializer"
value="com.raytheon.edex.plugin.gfe.GfePluginInitializer" />
<property name="compression" value="LZF"/>
</bean>
<bean id="gfeDbPluginProperties" class="com.raytheon.uf.edex.database.DatabasePluginProperties">

85
edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/database/IFPGridDatabase.java
View file

@ -74,6 +74,7 @@ import com.raytheon.uf.common.datastorage.IDataStore;
import com.raytheon.uf.common.datastorage.IDataStore.StoreOp;
import com.raytheon.uf.common.datastorage.Request;
import com.raytheon.uf.common.datastorage.StorageException;
import com.raytheon.uf.common.datastorage.StorageProperties;
import com.raytheon.uf.common.datastorage.StorageStatus;
import com.raytheon.uf.common.datastorage.records.ByteDataRecord;
import com.raytheon.uf.common.datastorage.records.FloatDataRecord;
@ -84,6 +85,7 @@ import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.time.TimeRange;
import com.raytheon.uf.edex.core.dataplugin.PluginRegistry;
import com.raytheon.uf.edex.database.DataAccessLayerException;
import com.raytheon.uf.edex.database.plugin.PluginFactory;
import com.vividsolutions.jts.geom.Coordinate;
@ -128,9 +130,9 @@ public class IFPGridDatabase extends GridDatabase {
private static final float VECTOR_DIR_DATA_OFFSET = 0.0f;
private Map<String, GridParmInfo> parmInfo = new HashMap<String, GridParmInfo>();
private final Map<String, GridParmInfo> parmInfo = new HashMap<String, GridParmInfo>();
private Map<String, ParmStorageInfo> parmStorageInfo = new HashMap<String, ParmStorageInfo>();
private final Map<String, ParmStorageInfo> parmStorageInfo = new HashMap<String, ParmStorageInfo>();
/** The grid configuration for this database */
protected GridDbConfig gridConfig;
@ -315,13 +317,13 @@ public class IFPGridDatabase extends GridDatabase {
}
// max/min changes
if (userGPI.getMaxValue() != dbGPI.getMaxValue()
|| userGPI.getMinValue() != dbGPI.getMinValue()
if ((userGPI.getMaxValue() != dbGPI.getMaxValue())
|| (userGPI.getMinValue() != dbGPI.getMinValue())
|| unitsChanged || !userPSI.equals(dbPSI)) {
// If units were changed, the values need to be clamped to the
// min and max values
if (userGPI.getMaxValue() != dbGPI.getMaxValue()
|| userGPI.getMinValue() != dbGPI.getMinValue()) {
if ((userGPI.getMaxValue() != dbGPI.getMaxValue())
|| (userGPI.getMinValue() != dbGPI.getMinValue())) {
statusHandler.handle(
Priority.INFO,
"Changing Max/Min: " + dbGPI.getParmID()
@ -863,7 +865,7 @@ public class IFPGridDatabase extends GridDatabase {
}
// Save off the individual failures (if any), and then save what we can
if (failedGrids != null && failedGrids.length > 0) {
if ((failedGrids != null) && (failedGrids.length > 0)) {
for (GFERecord gfeRecord : failedGrids) {
sr.addMessage("Failed to save grid to HDF5: " + gfeRecord);
}
@ -873,7 +875,7 @@ public class IFPGridDatabase extends GridDatabase {
GFERecord[] gridsToStore = records.toArray(new GFERecord[records
.size()]);
if (failedGrids != null && failedGrids.length > 0) {
if ((failedGrids != null) && (failedGrids.length > 0)) {
Set<GFERecord> workingSet = new HashSet<GFERecord>(records);
workingSet.removeAll(Arrays.asList(failedGrids));
gridsToStore = workingSet.toArray(new GFERecord[workingSet
@ -1491,7 +1493,7 @@ public class IFPGridDatabase extends GridDatabase {
private ServerResponse<?> dbIsValid() {
ServerResponse<?> sr = new ServerResponse<String>();
if (dbId == null || !dbId.isValid()) {
if ((dbId == null) || !dbId.isValid()) {
sr.addMessage("DBInvalid - The database is not valid.");
}
return sr;
@ -1522,7 +1524,8 @@ public class IFPGridDatabase extends GridDatabase {
gfeBaseDataDir, this.dbId));
try {
if (gridConfig != null && gridConfig.parmAndLevelList().size() > 0) {
if ((gridConfig != null)
&& (gridConfig.parmAndLevelList().size() > 0)) {
ds.getDatasets(GRID_PARM_INFO_GRP);
parmInfoInitialized = true;
}
@ -1750,6 +1753,15 @@ public class IFPGridDatabase extends GridDatabase {
ParmStorageInfo parmStorageInfo) throws GfeException {
List<GFERecord> failedGrids = new ArrayList<GFERecord>();
try {
StorageProperties sp = null;
String compression = PluginRegistry.getInstance()
.getRegisteredObject("gfe").getCompression();
if (compression != null) {
sp = new StorageProperties();
sp.setCompression(StorageProperties.Compression
.valueOf(compression));
}
Map<File, List<GFERecord>> recordMap = new HashMap<File, List<GFERecord>>();
for (GFERecord rec : dataObjects) {
@ -1780,24 +1792,24 @@ public class IFPGridDatabase extends GridDatabase {
// Get storage info (for float and vector data)
String storageType = parmStorageInfo.storageType();
if (data instanceof FloatDataRecord
if ((data instanceof FloatDataRecord)
&& !"float".equals(storageType)) {
storeConvertedFloatRecord((FloatDataRecord) data,
dataStore, groupName, parmStorageInfo,
dataStore, sp, groupName, parmStorageInfo,
correlationMap, rec);
} else if (data instanceof IDataRecord) {
// store without conversion
((IDataRecord) data).setGroup(groupName);
dataStore.addDataRecord((IDataRecord) data);
dataStore.addDataRecord((IDataRecord) data, sp);
correlationMap.put(((IDataRecord) data), rec);
} else if (data instanceof VectorGridSlice) {
storeVectorGridSlice(data, dataStore, groupName,
storeVectorGridSlice(data, dataStore, sp, groupName,
parmStorageInfo, correlationMap, rec);
} else if (data instanceof ScalarGridSlice) {
storeScalarGridSlice(data, dataStore, groupName,
storeScalarGridSlice(data, dataStore, sp, groupName,
parmStorageInfo, correlationMap, rec);
} else if (data instanceof DiscreteGridSlice) {
storeDiscreteGridSlice(data, dataStore, groupName,
storeDiscreteGridSlice(data, dataStore, sp, groupName,
parmStorageInfo, correlationMap, rec);
} else if (data instanceof WeatherGridSlice) {
WeatherGridSlice slice = (WeatherGridSlice) data;
@ -1810,7 +1822,7 @@ public class IFPGridDatabase extends GridDatabase {
.getNx(),
slice.getGridInfo().getGridLoc()
.getNy() });
dataStore.addDataRecord(rawRecord);
dataStore.addDataRecord(rawRecord, sp);
StringBuffer sb = new StringBuffer();
boolean first = true;
@ -1826,7 +1838,7 @@ public class IFPGridDatabase extends GridDatabase {
ByteDataRecord keyRecord = new ByteDataRecord(
"Keys", groupName, keyBytes, 1,
new long[] { keyBytes.length });
dataStore.addDataRecord(keyRecord);
dataStore.addDataRecord(keyRecord, sp);
correlationMap.put(rawRecord, rec);
correlationMap.put(keyRecord, rec);
}
@ -1835,7 +1847,7 @@ public class IFPGridDatabase extends GridDatabase {
StorageStatus ss = dataStore.store(StoreOp.REPLACE);
StorageException[] exceptions = ss.getExceptions();
if (exceptions != null && exceptions.length > 0) {
if ((exceptions != null) && (exceptions.length > 0)) {
// Describe the errors, then
// only log the first one, don't flood the log with
// duplicates.
@ -1883,7 +1895,8 @@ public class IFPGridDatabase extends GridDatabase {
* @throws StorageException
*/
protected void storeScalarGridSlice(Object data, IDataStore dataStore,
String groupName, ParmStorageInfo parmStorageInfo,
StorageProperties sp, String groupName,
ParmStorageInfo parmStorageInfo,
Map<IDataRecord, GFERecord> correlationMap, GFERecord rec)
throws StorageException {
ScalarGridSlice slice = (ScalarGridSlice) data;
@ -1893,7 +1906,7 @@ public class IFPGridDatabase extends GridDatabase {
rawData, 2, new long[] {
slice.getGridInfo().getGridLoc().getNx(),
slice.getGridInfo().getGridLoc().getNy() });
this.storeConvertedFloatRecord(rawRecord, dataStore, groupName,
this.storeConvertedFloatRecord(rawRecord, dataStore, sp, groupName,
parmStorageInfo, correlationMap, rec);
}
}
@ -1917,11 +1930,12 @@ public class IFPGridDatabase extends GridDatabase {
* @throws StorageException
*/
protected void storeVectorGridSlice(Object data, IDataStore dataStore,
String groupName, ParmStorageInfo parmStorageInfo,
StorageProperties sp, String groupName,
ParmStorageInfo parmStorageInfo,
Map<IDataRecord, GFERecord> correlationMap, GFERecord rec)
throws StorageException {
VectorGridSlice slice = (VectorGridSlice) data;
if (slice.getMagGrid() != null || slice.getDirGrid() != null) {
if ((slice.getMagGrid() != null) || (slice.getDirGrid() != null)) {
float[] rawMagData = slice.getMagGrid().getFloats();
float[] rawDirData = slice.getDirGrid().getFloats();
FloatDataRecord rawMagRecord = new FloatDataRecord("Mag",
@ -1941,10 +1955,10 @@ public class IFPGridDatabase extends GridDatabase {
parmStorageInfo.parmName(), parmStorageInfo.level(),
VECTOR_DIR_DATA_OFFSET, VECTOR_DIR_DATA_MULTIPLIER,
parmStorageInfo.storageType());
this.storeConvertedFloatRecord(rawMagRecord, dataStore, groupName,
parmStorageInfo, correlationMap, rec);
this.storeConvertedFloatRecord(rawDirRecord, dataStore, groupName,
dirStorageInfo, correlationMap, rec);
this.storeConvertedFloatRecord(rawMagRecord, dataStore, sp,
groupName, parmStorageInfo, correlationMap, rec);
this.storeConvertedFloatRecord(rawDirRecord, dataStore, sp,
groupName, dirStorageInfo, correlationMap, rec);
}
}
@ -1955,6 +1969,8 @@ public class IFPGridDatabase extends GridDatabase {
* The discrete grid slice
* @param dataStore
* The data store in which to save the slice
* @param sp
* The storage properties for the slice
* @param groupName
* The group name under which to save the slice
* @param parmStorageInfo
@ -1967,7 +1983,8 @@ public class IFPGridDatabase extends GridDatabase {
* @throws StorageException
*/
protected void storeDiscreteGridSlice(Object data, IDataStore dataStore,
String groupName, ParmStorageInfo parmStorageInfo,
StorageProperties sp, String groupName,
ParmStorageInfo parmStorageInfo,
Map<IDataRecord, GFERecord> correlationMap, GFERecord rec)
throws StorageException {
DiscreteGridSlice slice = (DiscreteGridSlice) data;
@ -1977,7 +1994,7 @@ public class IFPGridDatabase extends GridDatabase {
rawData, 2, new long[] {
slice.getGridInfo().getGridLoc().getNx(),
slice.getGridInfo().getGridLoc().getNy() });
dataStore.addDataRecord(rawRecord);
dataStore.addDataRecord(rawRecord, sp);
StringBuffer sb = new StringBuffer();
boolean first = true;
@ -1992,7 +2009,7 @@ public class IFPGridDatabase extends GridDatabase {
byte[] keyBytes = sb.toString().getBytes();
ByteDataRecord keyRecord = new ByteDataRecord("Keys", groupName,
keyBytes, 1, new long[] { keyBytes.length });
dataStore.addDataRecord(keyRecord);
dataStore.addDataRecord(keyRecord, sp);
correlationMap.put(rawRecord, rec);
correlationMap.put(keyRecord, rec);
}
@ -2041,7 +2058,7 @@ public class IFPGridDatabase extends GridDatabase {
* The GFE record being stored
*/
protected void storeConvertedFloatRecord(FloatDataRecord data,
IDataStore dataStore, String groupName,
IDataStore dataStore, StorageProperties sp, String groupName,
ParmStorageInfo parmStorageInfo,
Map<IDataRecord, GFERecord> correlationMap, GFERecord rec)
throws StorageException {
@ -2052,7 +2069,7 @@ public class IFPGridDatabase extends GridDatabase {
float multiplier = parmStorageInfo.dataMultiplier();
float fcvt;
IDataRecord storeDataRec = null;
if ("short".equals(storageType) && multiplier != 0.0f) {
if ("short".equals(storageType) && (multiplier != 0.0f)) {
short[] converted = new short[fdata.length];
for (int i = 0; i < fdata.length; i++) {
fcvt = (fdata[i] - offset) * multiplier;
@ -2061,7 +2078,7 @@ public class IFPGridDatabase extends GridDatabase {
}
storeDataRec = new ShortDataRecord(data.getName(), data.getGroup(),
converted, data.getDimension(), data.getSizes().clone());
} else if ("byte".equals(storageType) && multiplier != 0.0f) {
} else if ("byte".equals(storageType) && (multiplier != 0.0f)) {
byte[] converted = new byte[fdata.length];
for (int i = 0; i < fdata.length; i++) {
fcvt = (fdata[i] - offset) * multiplier;
@ -2074,7 +2091,7 @@ public class IFPGridDatabase extends GridDatabase {
}
storeDataRec.setGroup(groupName);
dataStore.addDataRecord(storeDataRec);
dataStore.addDataRecord(storeDataRec, sp);
correlationMap.put(storeDataRec, rec);
}