From 5ceabc1ec2cae5965b9f875e0c1dc5145e57851a Mon Sep 17 00:00:00 2001 From: Richard Peter Date: Thu, 11 Oct 2012 10:43:43 -0500 Subject: [PATCH] Issue #1257: Enable gfe compression, add population of storage properties Change-Id: If7a60e6896fa3384b9bb6a537f3e643d6b67423f Former-commit-id: 5b02977c521d0f45376aa11fae3c90521f7c32af [formerly 129e37e44c8b6c04396444ee5646b375c92a0dff] [formerly f21c91da0030d3db3a91256aa134915116388be5 [formerly 5b3e76a6870cdbc3eb552c19b4abcb21d4edbd70]] Former-commit-id: f21c91da0030d3db3a91256aa134915116388be5 Former-commit-id: 11673062757c14cfc03d9fe71694ea11e7f04d45 --- .../res/spring/gfe-common.xml | 1 + .../gfe/server/database/IFPGridDatabase.java | 85 +++++++++++-------- 2 files changed, 52 insertions(+), 34 deletions(-) diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/res/spring/gfe-common.xml b/edexOsgi/com.raytheon.edex.plugin.gfe/res/spring/gfe-common.xml index a5ef875c3b..79301ee876 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/res/spring/gfe-common.xml +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/res/spring/gfe-common.xml @@ -12,6 +12,7 @@ value="com.raytheon.uf.common.dataplugin.gfe.db.objects.GFERecord" /> + diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/database/IFPGridDatabase.java b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/database/IFPGridDatabase.java index 78b7845bd2..e3a92e7670 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/database/IFPGridDatabase.java +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/database/IFPGridDatabase.java @@ -74,6 +74,7 @@ import com.raytheon.uf.common.datastorage.IDataStore; import com.raytheon.uf.common.datastorage.IDataStore.StoreOp; import com.raytheon.uf.common.datastorage.Request; import com.raytheon.uf.common.datastorage.StorageException; +import com.raytheon.uf.common.datastorage.StorageProperties; import com.raytheon.uf.common.datastorage.StorageStatus; import com.raytheon.uf.common.datastorage.records.ByteDataRecord; import com.raytheon.uf.common.datastorage.records.FloatDataRecord; @@ -84,6 +85,7 @@ import com.raytheon.uf.common.status.IUFStatusHandler; import com.raytheon.uf.common.status.UFStatus; import com.raytheon.uf.common.status.UFStatus.Priority; import com.raytheon.uf.common.time.TimeRange; +import com.raytheon.uf.edex.core.dataplugin.PluginRegistry; import com.raytheon.uf.edex.database.DataAccessLayerException; import com.raytheon.uf.edex.database.plugin.PluginFactory; import com.vividsolutions.jts.geom.Coordinate; @@ -128,9 +130,9 @@ public class IFPGridDatabase extends GridDatabase { private static final float VECTOR_DIR_DATA_OFFSET = 0.0f; - private Map parmInfo = new HashMap(); + private final Map parmInfo = new HashMap(); - private Map parmStorageInfo = new HashMap(); + private final Map parmStorageInfo = new HashMap(); /** The grid configuration for this database */ protected GridDbConfig gridConfig; @@ -315,13 +317,13 @@ public class IFPGridDatabase extends GridDatabase { } // max/min changes - if (userGPI.getMaxValue() != dbGPI.getMaxValue() - || userGPI.getMinValue() != dbGPI.getMinValue() + if ((userGPI.getMaxValue() != dbGPI.getMaxValue()) + || (userGPI.getMinValue() != dbGPI.getMinValue()) || unitsChanged || !userPSI.equals(dbPSI)) { // If units were changed, the values need to be clamped to the // min and max values - if (userGPI.getMaxValue() != dbGPI.getMaxValue() - || userGPI.getMinValue() != dbGPI.getMinValue()) { + if ((userGPI.getMaxValue() != dbGPI.getMaxValue()) + || (userGPI.getMinValue() != dbGPI.getMinValue())) { statusHandler.handle( Priority.INFO, "Changing Max/Min: " + dbGPI.getParmID() @@ -863,7 +865,7 @@ public class IFPGridDatabase extends GridDatabase { } // Save off the individual failures (if any), and then save what we can - if (failedGrids != null && failedGrids.length > 0) { + if ((failedGrids != null) && (failedGrids.length > 0)) { for (GFERecord gfeRecord : failedGrids) { sr.addMessage("Failed to save grid to HDF5: " + gfeRecord); } @@ -873,7 +875,7 @@ public class IFPGridDatabase extends GridDatabase { GFERecord[] gridsToStore = records.toArray(new GFERecord[records .size()]); - if (failedGrids != null && failedGrids.length > 0) { + if ((failedGrids != null) && (failedGrids.length > 0)) { Set workingSet = new HashSet(records); workingSet.removeAll(Arrays.asList(failedGrids)); gridsToStore = workingSet.toArray(new GFERecord[workingSet @@ -1491,7 +1493,7 @@ public class IFPGridDatabase extends GridDatabase { private ServerResponse dbIsValid() { ServerResponse sr = new ServerResponse(); - if (dbId == null || !dbId.isValid()) { + if ((dbId == null) || !dbId.isValid()) { sr.addMessage("DBInvalid - The database is not valid."); } return sr; @@ -1522,7 +1524,8 @@ public class IFPGridDatabase extends GridDatabase { gfeBaseDataDir, this.dbId)); try { - if (gridConfig != null && gridConfig.parmAndLevelList().size() > 0) { + if ((gridConfig != null) + && (gridConfig.parmAndLevelList().size() > 0)) { ds.getDatasets(GRID_PARM_INFO_GRP); parmInfoInitialized = true; } @@ -1750,6 +1753,15 @@ public class IFPGridDatabase extends GridDatabase { ParmStorageInfo parmStorageInfo) throws GfeException { List failedGrids = new ArrayList(); try { + StorageProperties sp = null; + String compression = PluginRegistry.getInstance() + .getRegisteredObject("gfe").getCompression(); + if (compression != null) { + sp = new StorageProperties(); + sp.setCompression(StorageProperties.Compression + .valueOf(compression)); + } + Map> recordMap = new HashMap>(); for (GFERecord rec : dataObjects) { @@ -1780,24 +1792,24 @@ public class IFPGridDatabase extends GridDatabase { // Get storage info (for float and vector data) String storageType = parmStorageInfo.storageType(); - if (data instanceof FloatDataRecord + if ((data instanceof FloatDataRecord) && !"float".equals(storageType)) { storeConvertedFloatRecord((FloatDataRecord) data, - dataStore, groupName, parmStorageInfo, + dataStore, sp, groupName, parmStorageInfo, correlationMap, rec); } else if (data instanceof IDataRecord) { // store without conversion ((IDataRecord) data).setGroup(groupName); - dataStore.addDataRecord((IDataRecord) data); + dataStore.addDataRecord((IDataRecord) data, sp); correlationMap.put(((IDataRecord) data), rec); } else if (data instanceof VectorGridSlice) { - storeVectorGridSlice(data, dataStore, groupName, + storeVectorGridSlice(data, dataStore, sp, groupName, parmStorageInfo, correlationMap, rec); } else if (data instanceof ScalarGridSlice) { - storeScalarGridSlice(data, dataStore, groupName, + storeScalarGridSlice(data, dataStore, sp, groupName, parmStorageInfo, correlationMap, rec); } else if (data instanceof DiscreteGridSlice) { - storeDiscreteGridSlice(data, dataStore, groupName, + storeDiscreteGridSlice(data, dataStore, sp, groupName, parmStorageInfo, correlationMap, rec); } else if (data instanceof WeatherGridSlice) { WeatherGridSlice slice = (WeatherGridSlice) data; @@ -1810,7 +1822,7 @@ public class IFPGridDatabase extends GridDatabase { .getNx(), slice.getGridInfo().getGridLoc() .getNy() }); - dataStore.addDataRecord(rawRecord); + dataStore.addDataRecord(rawRecord, sp); StringBuffer sb = new StringBuffer(); boolean first = true; @@ -1826,7 +1838,7 @@ public class IFPGridDatabase extends GridDatabase { ByteDataRecord keyRecord = new ByteDataRecord( "Keys", groupName, keyBytes, 1, new long[] { keyBytes.length }); - dataStore.addDataRecord(keyRecord); + dataStore.addDataRecord(keyRecord, sp); correlationMap.put(rawRecord, rec); correlationMap.put(keyRecord, rec); } @@ -1835,7 +1847,7 @@ public class IFPGridDatabase extends GridDatabase { StorageStatus ss = dataStore.store(StoreOp.REPLACE); StorageException[] exceptions = ss.getExceptions(); - if (exceptions != null && exceptions.length > 0) { + if ((exceptions != null) && (exceptions.length > 0)) { // Describe the errors, then // only log the first one, don't flood the log with // duplicates. @@ -1883,7 +1895,8 @@ public class IFPGridDatabase extends GridDatabase { * @throws StorageException */ protected void storeScalarGridSlice(Object data, IDataStore dataStore, - String groupName, ParmStorageInfo parmStorageInfo, + StorageProperties sp, String groupName, + ParmStorageInfo parmStorageInfo, Map correlationMap, GFERecord rec) throws StorageException { ScalarGridSlice slice = (ScalarGridSlice) data; @@ -1893,7 +1906,7 @@ public class IFPGridDatabase extends GridDatabase { rawData, 2, new long[] { slice.getGridInfo().getGridLoc().getNx(), slice.getGridInfo().getGridLoc().getNy() }); - this.storeConvertedFloatRecord(rawRecord, dataStore, groupName, + this.storeConvertedFloatRecord(rawRecord, dataStore, sp, groupName, parmStorageInfo, correlationMap, rec); } } @@ -1917,11 +1930,12 @@ public class IFPGridDatabase extends GridDatabase { * @throws StorageException */ protected void storeVectorGridSlice(Object data, IDataStore dataStore, - String groupName, ParmStorageInfo parmStorageInfo, + StorageProperties sp, String groupName, + ParmStorageInfo parmStorageInfo, Map correlationMap, GFERecord rec) throws StorageException { VectorGridSlice slice = (VectorGridSlice) data; - if (slice.getMagGrid() != null || slice.getDirGrid() != null) { + if ((slice.getMagGrid() != null) || (slice.getDirGrid() != null)) { float[] rawMagData = slice.getMagGrid().getFloats(); float[] rawDirData = slice.getDirGrid().getFloats(); FloatDataRecord rawMagRecord = new FloatDataRecord("Mag", @@ -1941,10 +1955,10 @@ public class IFPGridDatabase extends GridDatabase { parmStorageInfo.parmName(), parmStorageInfo.level(), VECTOR_DIR_DATA_OFFSET, VECTOR_DIR_DATA_MULTIPLIER, parmStorageInfo.storageType()); - this.storeConvertedFloatRecord(rawMagRecord, dataStore, groupName, - parmStorageInfo, correlationMap, rec); - this.storeConvertedFloatRecord(rawDirRecord, dataStore, groupName, - dirStorageInfo, correlationMap, rec); + this.storeConvertedFloatRecord(rawMagRecord, dataStore, sp, + groupName, parmStorageInfo, correlationMap, rec); + this.storeConvertedFloatRecord(rawDirRecord, dataStore, sp, + groupName, dirStorageInfo, correlationMap, rec); } } @@ -1955,6 +1969,8 @@ public class IFPGridDatabase extends GridDatabase { * The discrete grid slice * @param dataStore * The data store in which to save the slice + * @param sp + * The storage properties for the slice * @param groupName * The group name under which to save the slice * @param parmStorageInfo @@ -1967,7 +1983,8 @@ public class IFPGridDatabase extends GridDatabase { * @throws StorageException */ protected void storeDiscreteGridSlice(Object data, IDataStore dataStore, - String groupName, ParmStorageInfo parmStorageInfo, + StorageProperties sp, String groupName, + ParmStorageInfo parmStorageInfo, Map correlationMap, GFERecord rec) throws StorageException { DiscreteGridSlice slice = (DiscreteGridSlice) data; @@ -1977,7 +1994,7 @@ public class IFPGridDatabase extends GridDatabase { rawData, 2, new long[] { slice.getGridInfo().getGridLoc().getNx(), slice.getGridInfo().getGridLoc().getNy() }); - dataStore.addDataRecord(rawRecord); + dataStore.addDataRecord(rawRecord, sp); StringBuffer sb = new StringBuffer(); boolean first = true; @@ -1992,7 +2009,7 @@ public class IFPGridDatabase extends GridDatabase { byte[] keyBytes = sb.toString().getBytes(); ByteDataRecord keyRecord = new ByteDataRecord("Keys", groupName, keyBytes, 1, new long[] { keyBytes.length }); - dataStore.addDataRecord(keyRecord); + dataStore.addDataRecord(keyRecord, sp); correlationMap.put(rawRecord, rec); correlationMap.put(keyRecord, rec); } @@ -2041,7 +2058,7 @@ public class IFPGridDatabase extends GridDatabase { * The GFE record being stored */ protected void storeConvertedFloatRecord(FloatDataRecord data, - IDataStore dataStore, String groupName, + IDataStore dataStore, StorageProperties sp, String groupName, ParmStorageInfo parmStorageInfo, Map correlationMap, GFERecord rec) throws StorageException { @@ -2052,7 +2069,7 @@ public class IFPGridDatabase extends GridDatabase { float multiplier = parmStorageInfo.dataMultiplier(); float fcvt; IDataRecord storeDataRec = null; - if ("short".equals(storageType) && multiplier != 0.0f) { + if ("short".equals(storageType) && (multiplier != 0.0f)) { short[] converted = new short[fdata.length]; for (int i = 0; i < fdata.length; i++) { fcvt = (fdata[i] - offset) * multiplier; @@ -2061,7 +2078,7 @@ public class IFPGridDatabase extends GridDatabase { } storeDataRec = new ShortDataRecord(data.getName(), data.getGroup(), converted, data.getDimension(), data.getSizes().clone()); - } else if ("byte".equals(storageType) && multiplier != 0.0f) { + } else if ("byte".equals(storageType) && (multiplier != 0.0f)) { byte[] converted = new byte[fdata.length]; for (int i = 0; i < fdata.length; i++) { fcvt = (fdata[i] - offset) * multiplier; @@ -2074,7 +2091,7 @@ public class IFPGridDatabase extends GridDatabase { } storeDataRec.setGroup(groupName); - dataStore.addDataRecord(storeDataRec); + dataStore.addDataRecord(storeDataRec, sp); correlationMap.put(storeDataRec, rec); }