xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #1257: Update pypies group referencing and enable compression for radar and vil

Browse source 
Change-Id: Ib75b9fec9293f858b638f5eb61140da575909a34

Former-commit-id: f58dbfeb65 [formerly d40f52a0bc] [formerly f2ac3dc537] [formerly f58dbfeb65 [formerly d40f52a0bc] [formerly f2ac3dc537] [formerly c2a46e0b35 [formerly f2ac3dc537 [formerly bbca9f1eb7b3bc568ab3cabc9ad83510afd7317f]]]]
Former-commit-id: c2a46e0b35
Former-commit-id: 89c1a1add4 [formerly 67da3874f4] [formerly 894b23d3d92172d9e3d25a741714f67203a14c5f [formerly bb723dc404]]
Former-commit-id: c3d53c2b7bb046c65afcbdc7567fb0ef69f1a737 [formerly bc589fbdb3]
Former-commit-id: 0b0d0d073f
This commit is contained in:
Richard Peter 2012-10-09 17:21:56 -05:00
parent 9dda91e0aa
commit b7712c2d77
4 changed files with 49 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
edexOsgi/com.raytheon.edex.plugin.radar/res/spring/radar-common.xml
View file

@ -14,6 +14,7 @@
<property name="pathProvider"> <property name="pathProvider">
<ref bean="radarPathProvider" /> <ref bean="radarPathProvider" />
</property> </property>
<property name="compression" value="LZF"/>
</bean> </bean>
<bean id="radarRegistered" factory-bean="pluginRegistry" <bean id="radarRegistered" factory-bean="pluginRegistry"

35
edexOsgi/com.raytheon.uf.common.dataplugin.vil/src/com/raytheon/uf/common/dataplugin/vil/dao/VILDao.java
View file

@ -29,8 +29,10 @@ import com.raytheon.uf.common.dataplugin.persist.IPersistable;
import com.raytheon.uf.common.dataplugin.vil.VILRecord; import com.raytheon.uf.common.dataplugin.vil.VILRecord;
import com.raytheon.uf.common.dataplugin.vil.VILRecord.DATA_TYPE; import com.raytheon.uf.common.dataplugin.vil.VILRecord.DATA_TYPE;
import com.raytheon.uf.common.datastorage.IDataStore; import com.raytheon.uf.common.datastorage.IDataStore;
import com.raytheon.uf.common.datastorage.StorageProperties;
import com.raytheon.uf.common.datastorage.records.FloatDataRecord; import com.raytheon.uf.common.datastorage.records.FloatDataRecord;
import com.raytheon.uf.common.datastorage.records.IDataRecord; import com.raytheon.uf.common.datastorage.records.IDataRecord;
import com.raytheon.uf.edex.core.dataplugin.PluginRegistry;
import com.raytheon.uf.edex.database.plugin.PluginDao; import com.raytheon.uf.edex.database.plugin.PluginDao;
/** /**
@ -49,41 +51,50 @@ import com.raytheon.uf.edex.database.plugin.PluginDao;
*/ */
public class VILDao extends PluginDao { public class VILDao extends PluginDao {
public VILDao(String pluginName) throws PluginException { public VILDao(final String pluginName) throws PluginException {
super(pluginName); super(pluginName);
} }
@Override @Override
protected IDataStore populateDataStore(IDataStore dataStore, protected IDataStore populateDataStore(final IDataStore dataStore,
IPersistable obj) throws Exception { final IPersistable obj) throws Exception {
VILRecord VILRec = (VILRecord) obj; VILRecord VILRec = (VILRecord) obj;
if (VILRec.getDataArray() != null String compression = PluginRegistry.getInstance()
.getRegisteredObject(pluginName).getCompression();
StorageProperties sp = null;
if (compression != null) {
sp = new StorageProperties();
sp.setCompression(StorageProperties.Compression
.valueOf(compression));
}
if ((VILRec.getDataArray() != null)
&& VILRec.getFieldName().equals(DATA_TYPE.VILD.name())) { && VILRec.getFieldName().equals(DATA_TYPE.VILD.name())) {
IDataRecord rec = new FloatDataRecord("Data", VILRec.getDataURI(), IDataRecord rec = new FloatDataRecord("Data", VILRec.getDataURI(),
VILRec.getDataArray(), 2, new long[] { VILRec.getNx(), VILRec.getDataArray(), 2, new long[] { VILRec.getNx(),
VILRec.getNy() }); VILRec.getNy() });
rec.setCorrelationObject(VILRec); rec.setCorrelationObject(VILRec);
dataStore.addDataRecord(rec); dataStore.addDataRecord(rec, sp);
} }
if (VILRec.getDataArray() != null if ((VILRec.getDataArray() != null)
&& VILRec.getFieldName().equals(DATA_TYPE.DVILD.name())) { && VILRec.getFieldName().equals(DATA_TYPE.DVILD.name())) {
IDataRecord rec = new FloatDataRecord("Data", VILRec.getDataURI(), IDataRecord rec = new FloatDataRecord("Data", VILRec.getDataURI(),
VILRec.getDataArray(), 2, new long[] { VILRec.getNx(), VILRec.getDataArray(), 2, new long[] { VILRec.getNx(),
VILRec.getNy() }); VILRec.getNy() });
rec.setCorrelationObject(VILRec); rec.setCorrelationObject(VILRec);
dataStore.addDataRecord(rec); dataStore.addDataRecord(rec, sp);
} }
if (VILRec.getDataArray() != null if ((VILRec.getDataArray() != null)
&& VILRec.getFieldName().equals(DATA_TYPE.EDVILD.name())) { && VILRec.getFieldName().equals(DATA_TYPE.EDVILD.name())) {
IDataRecord rec = new FloatDataRecord("Data", VILRec.getDataURI(), IDataRecord rec = new FloatDataRecord("Data", VILRec.getDataURI(),
VILRec.getDataArray(), 2, new long[] { VILRec.getNx(), VILRec.getDataArray(), 2, new long[] { VILRec.getNx(),
VILRec.getNy() }); VILRec.getNy() });
rec.setCorrelationObject(VILRec); rec.setCorrelationObject(VILRec);
dataStore.addDataRecord(rec); dataStore.addDataRecord(rec, sp);
} }
logger.debug("VILDao: writing " + VILRec.toString()); logger.debug("VILDao: writing " + VILRec.toString());
@ -92,8 +103,9 @@ public class VILDao extends PluginDao {
} }
@Override @Override
public List<IDataRecord[]> getHDF5Data(List<PluginDataObject> objects, public List<IDataRecord[]> getHDF5Data(
int tileSet) throws PluginException { final List<PluginDataObject> objects, final int tileSet)
throws PluginException {
List<IDataRecord[]> retVal = new ArrayList<IDataRecord[]>(); List<IDataRecord[]> retVal = new ArrayList<IDataRecord[]>();
for (PluginDataObject obj : objects) { for (PluginDataObject obj : objects) {
@ -114,4 +126,3 @@ public class VILDao extends PluginDao {
return retVal; return retVal;
} }
} }

1
edexOsgi/com.raytheon.uf.edex.plugin.vil/res/spring/vil-common.xml
View file

@ -20,6 +20,7 @@
<value>com.raytheon.uf.common.dataplugin.radar</value> <value>com.raytheon.uf.common.dataplugin.radar</value>
</list> </list>
</property> </property>
<property name="compression" value="LZF"/>
</bean> </bean>
<bean factory-bean="pluginRegistry" factory-method="register" depends-on="radarRegistered"> <bean factory-bean="pluginRegistry" factory-method="register" depends-on="radarRegistered">

10
pythonPackages/pypies/pypies/impl/H5pyDataStore.py
View file

@ -29,6 +29,7 @@
# ------------ ---------- ----------- -------------------------- # ------------ ---------- ----------- --------------------------
# 06/16/10 njensen Initial Creation. # 06/16/10 njensen Initial Creation.
# 05/03/11 9134 njensen Optimized for pointdata # 05/03/11 9134 njensen Optimized for pointdata
# 10/09/12 rjpeter Optimized __getGroup for retrievals
# #
# #
# #
@ -528,6 +529,7 @@ class H5pyDataStore(IDataStore.IDataStore):
if chunks: if chunks:
plc.set_chunk(chunks) plc.set_chunk(chunks)
if compression == 'LZF': if compression == 'LZF':
plc.set_shuffle()
plc.set_filter(h5py.h5z.FILTER_LZF, h5py.h5z.FLAG_OPTIONAL) plc.set_filter(h5py.h5z.FILTER_LZF, h5py.h5z.FLAG_OPTIONAL)
szDims = tuple(szDims) szDims = tuple(szDims)
@ -612,6 +614,7 @@ class H5pyDataStore(IDataStore.IDataStore):
def __getGroup(self, f, name, create=False): def __getGroup(self, f, name, create=False):
t0=time.time() t0=time.time()
if create:
parts = name.split('/') parts = name.split('/')
grp = None grp = None
for s in parts: for s in parts:
@ -621,18 +624,17 @@ class H5pyDataStore(IDataStore.IDataStore):
if s == '/' or s in f.keys(): if s == '/' or s in f.keys():
grp = f[s] grp = f[s]
else: else:
if create:
grp = f.create_group(s) grp = f.create_group(s)
else:
raise StorageException("No group " + name + " found")
else: else:
if s: if s:
if s in grp.keys(): if s in grp.keys():
grp = grp[s] grp = grp[s]
else: else:
if create:
grp = grp.create_group(s) grp = grp.create_group(s)
else: else:
try:
grp = f[name]
except KeyError:
raise StorageException("No group " + name + " found") raise StorageException("No group " + name + " found")
t1=time.time() t1=time.time()