Issue #1608 Fixed group deletes in H5pyDataStore.py
Change-Id: I9cd6e186646f9db40b0e498740fb4c63bc99b79a Former-commit-id:7c0a5d74bf[formerlya2a89fb4e7[formerly dc9ac54903107224633b4e18d5caa345649d8266]] Former-commit-id:a2a89fb4e7Former-commit-id:dc7c9edb87
This commit is contained in:
Ron Anderson
parent
cb0efa78cf
commit
9156b9ffe8
67 changed files with 172 additions and 4942 deletions
|
|
@ -56,13 +56,6 @@
|
||||||
version="0.0.0"
|
version="0.0.0"
|
||||||
unpack="false"/>
|
unpack="false"/>
|
||||||
|
|
||||||
<plugin
|
|
||||||
id="com.raytheon.uf.common.datastorage.hdf5"
|
|
||||||
download-size="0"
|
|
||||||
install-size="0"
|
|
||||||
version="0.0.0"
|
|
||||||
unpack="false"/>
|
|
||||||
|
|
||||||
<plugin
|
<plugin
|
||||||
id="com.raytheon.uf.common.derivparam"
|
id="com.raytheon.uf.common.derivparam"
|
||||||
download-size="0"
|
download-size="0"
|
||||||
|
|
|
||||||
|
|
@ -120,30 +120,6 @@
|
||||||
version="0.0.0"
|
version="0.0.0"
|
||||||
fragment="true"/>
|
fragment="true"/>
|
||||||
|
|
||||||
<plugin
|
|
||||||
id="ncsa.hdf5"
|
|
||||||
download-size="0"
|
|
||||||
install-size="0"
|
|
||||||
version="0.0.0"/>
|
|
||||||
|
|
||||||
<plugin
|
|
||||||
id="ncsa.hdf5.linux32"
|
|
||||||
os="linux"
|
|
||||||
arch="x86"
|
|
||||||
download-size="0"
|
|
||||||
install-size="0"
|
|
||||||
version="0.0.0"
|
|
||||||
fragment="true"/>
|
|
||||||
|
|
||||||
<plugin
|
|
||||||
id="ncsa.hdf5.win32"
|
|
||||||
os="win32"
|
|
||||||
arch="x86"
|
|
||||||
download-size="0"
|
|
||||||
install-size="0"
|
|
||||||
version="0.0.0"
|
|
||||||
fragment="true"/>
|
|
||||||
|
|
||||||
<plugin
|
<plugin
|
||||||
id="net.sf.cglib"
|
id="net.sf.cglib"
|
||||||
download-size="0"
|
download-size="0"
|
||||||
|
|
|
||||||
|
|
@ -251,12 +251,6 @@
|
||||||
install-size="0"
|
install-size="0"
|
||||||
version="0.0.0"/>
|
version="0.0.0"/>
|
||||||
|
|
||||||
<plugin
|
|
||||||
id="ncsa.hdf5"
|
|
||||||
download-size="0"
|
|
||||||
install-size="0"
|
|
||||||
version="0.0.0"/>
|
|
||||||
|
|
||||||
<plugin
|
<plugin
|
||||||
id="com.raytheon.uf.common.message"
|
id="com.raytheon.uf.common.message"
|
||||||
download-size="0"
|
download-size="0"
|
||||||
|
|
@ -271,13 +265,6 @@
|
||||||
version="0.0.0"
|
version="0.0.0"
|
||||||
unpack="false"/>
|
unpack="false"/>
|
||||||
|
|
||||||
<plugin
|
|
||||||
id="com.raytheon.uf.common.datastorage.hdf5"
|
|
||||||
download-size="0"
|
|
||||||
install-size="0"
|
|
||||||
version="0.0.0"
|
|
||||||
unpack="false"/>
|
|
||||||
|
|
||||||
<plugin
|
<plugin
|
||||||
id="com.raytheon.uf.common.geospatial"
|
id="com.raytheon.uf.common.geospatial"
|
||||||
download-size="0"
|
download-size="0"
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,6 @@ Require-Bundle: org.eclipse.ui,
|
||||||
com.raytheon.uf.common.colormap;bundle-version="1.12.1174",
|
com.raytheon.uf.common.colormap;bundle-version="1.12.1174",
|
||||||
org.geotools;bundle-version="2.6.4",
|
org.geotools;bundle-version="2.6.4",
|
||||||
com.raytheon.uf.common.datastorage;bundle-version="1.12.1174",
|
com.raytheon.uf.common.datastorage;bundle-version="1.12.1174",
|
||||||
com.raytheon.uf.common.datastorage.hdf5;bundle-version="1.12.1174",
|
|
||||||
com.raytheon.uf.common.dataplugin;bundle-version="1.12.1174",
|
com.raytheon.uf.common.dataplugin;bundle-version="1.12.1174",
|
||||||
com.raytheon.uf.common.dataplugin.npp.viirs;bundle-version="1.0.0",
|
com.raytheon.uf.common.dataplugin.npp.viirs;bundle-version="1.0.0",
|
||||||
javax.measure;bundle-version="1.0.0",
|
javax.measure;bundle-version="1.0.0",
|
||||||
|
|
|
||||||
|
|
@ -47,7 +47,8 @@ import com.raytheon.uf.common.util.cache.LRUCacheFS;
|
||||||
*
|
*
|
||||||
* Date Ticket# Engineer Description
|
* Date Ticket# Engineer Description
|
||||||
* ------------ ---------- ----------- --------------------------
|
* ------------ ---------- ----------- --------------------------
|
||||||
* Nov 8, 2011 mschenke Initial creation
|
* Nov 8, 2011 mschenke Initial creation
|
||||||
|
* Feb 12, 2013 #1608 randerso Added explicit deletes for groups and datasets
|
||||||
*
|
*
|
||||||
* </pre>
|
* </pre>
|
||||||
*
|
*
|
||||||
|
|
@ -55,6 +56,22 @@ import com.raytheon.uf.common.util.cache.LRUCacheFS;
|
||||||
* @version 1.0
|
* @version 1.0
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* TODO Add Description
|
||||||
|
*
|
||||||
|
* <pre>
|
||||||
|
*
|
||||||
|
* SOFTWARE HISTORY
|
||||||
|
*
|
||||||
|
* Date Ticket# Engineer Description
|
||||||
|
* ------------ ---------- ----------- --------------------------
|
||||||
|
* Feb 12, 2013 randerso Initial creation
|
||||||
|
*
|
||||||
|
* </pre>
|
||||||
|
*
|
||||||
|
* @author randerso
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
public class CachingDataStore implements IDataStore {
|
public class CachingDataStore implements IDataStore {
|
||||||
|
|
||||||
// quick byte string to hex conversion
|
// quick byte string to hex conversion
|
||||||
|
|
@ -348,12 +365,26 @@ public class CachingDataStore implements IDataStore {
|
||||||
* (non-Javadoc)
|
* (non-Javadoc)
|
||||||
*
|
*
|
||||||
* @see
|
* @see
|
||||||
* com.raytheon.uf.common.datastorage.IDataStore#delete(java.lang.String[])
|
* com.raytheon.uf.common.datastorage.IDataStore#deleteDatasets(java.lang
|
||||||
|
* .String[])
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public void delete(String... location) throws StorageException,
|
public void deleteDatasets(String... datasets) throws StorageException,
|
||||||
FileNotFoundException {
|
FileNotFoundException {
|
||||||
delegate.delete(location);
|
delegate.deleteDatasets(datasets);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* (non-Javadoc)
|
||||||
|
*
|
||||||
|
* @see
|
||||||
|
* com.raytheon.uf.common.datastorage.IDataStore#deleteGroups(java.lang.
|
||||||
|
* String[])
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public void deleteGroups(String... groups) throws StorageException,
|
||||||
|
FileNotFoundException {
|
||||||
|
delegate.deleteGroups(groups);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|
|
||||||
|
|
@ -1,6 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<classpath>
|
|
||||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
|
|
||||||
<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
|
|
||||||
<classpathentry kind="output" path="bin"/>
|
|
||||||
</classpath>
|
|
||||||
|
|
@ -1,28 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<projectDescription>
|
|
||||||
<name>ncsa.hdf5.linux32</name>
|
|
||||||
<comment></comment>
|
|
||||||
<projects>
|
|
||||||
</projects>
|
|
||||||
<buildSpec>
|
|
||||||
<buildCommand>
|
|
||||||
<name>org.eclipse.jdt.core.javabuilder</name>
|
|
||||||
<arguments>
|
|
||||||
</arguments>
|
|
||||||
</buildCommand>
|
|
||||||
<buildCommand>
|
|
||||||
<name>org.eclipse.pde.ManifestBuilder</name>
|
|
||||||
<arguments>
|
|
||||||
</arguments>
|
|
||||||
</buildCommand>
|
|
||||||
<buildCommand>
|
|
||||||
<name>org.eclipse.pde.SchemaBuilder</name>
|
|
||||||
<arguments>
|
|
||||||
</arguments>
|
|
||||||
</buildCommand>
|
|
||||||
</buildSpec>
|
|
||||||
<natures>
|
|
||||||
<nature>org.eclipse.pde.PluginNature</nature>
|
|
||||||
<nature>org.eclipse.jdt.core.javanature</nature>
|
|
||||||
</natures>
|
|
||||||
</projectDescription>
|
|
||||||
|
|
@ -1,7 +0,0 @@
|
||||||
#Thu Mar 26 11:22:29 CDT 2009
|
|
||||||
eclipse.preferences.version=1
|
|
||||||
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
|
|
||||||
org.eclipse.jdt.core.compiler.compliance=1.6
|
|
||||||
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
|
|
||||||
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
|
|
||||||
org.eclipse.jdt.core.compiler.source=1.6
|
|
||||||
|
|
@ -1,9 +0,0 @@
|
||||||
Manifest-Version: 1.0
|
|
||||||
Bundle-ManifestVersion: 2
|
|
||||||
Bundle-Name: NCSA HDF5 Linux32 Fragment
|
|
||||||
Bundle-SymbolicName: ncsa.hdf5.linux32
|
|
||||||
Bundle-Version: 2.4.0
|
|
||||||
Bundle-Vendor: Raytheon-bundled OSS
|
|
||||||
Fragment-Host: ncsa.hdf5;bundle-version="2.4.0"
|
|
||||||
Bundle-RequiredExecutionEnvironment: JavaSE-1.6
|
|
||||||
Eclipse-PlatformFilter: (& (osgi.os=linux) (osgi.arch=x86))
|
|
||||||
|
|
@ -1,4 +0,0 @@
|
||||||
bin.includes = META-INF/,\
|
|
||||||
.,\
|
|
||||||
libjhdf5.so
|
|
||||||
src.includes = libjhdf5.so
|
|
||||||
Binary file not shown.
|
|
@ -1,6 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<classpath>
|
|
||||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
|
|
||||||
<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
|
|
||||||
<classpathentry kind="output" path="bin"/>
|
|
||||||
</classpath>
|
|
||||||
|
|
@ -1,28 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<projectDescription>
|
|
||||||
<name>ncsa.hdf5.win32</name>
|
|
||||||
<comment></comment>
|
|
||||||
<projects>
|
|
||||||
</projects>
|
|
||||||
<buildSpec>
|
|
||||||
<buildCommand>
|
|
||||||
<name>org.eclipse.jdt.core.javabuilder</name>
|
|
||||||
<arguments>
|
|
||||||
</arguments>
|
|
||||||
</buildCommand>
|
|
||||||
<buildCommand>
|
|
||||||
<name>org.eclipse.pde.ManifestBuilder</name>
|
|
||||||
<arguments>
|
|
||||||
</arguments>
|
|
||||||
</buildCommand>
|
|
||||||
<buildCommand>
|
|
||||||
<name>org.eclipse.pde.SchemaBuilder</name>
|
|
||||||
<arguments>
|
|
||||||
</arguments>
|
|
||||||
</buildCommand>
|
|
||||||
</buildSpec>
|
|
||||||
<natures>
|
|
||||||
<nature>org.eclipse.pde.PluginNature</nature>
|
|
||||||
<nature>org.eclipse.jdt.core.javanature</nature>
|
|
||||||
</natures>
|
|
||||||
</projectDescription>
|
|
||||||
|
|
@ -1,7 +0,0 @@
|
||||||
#Thu Mar 26 11:23:11 CDT 2009
|
|
||||||
eclipse.preferences.version=1
|
|
||||||
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
|
|
||||||
org.eclipse.jdt.core.compiler.compliance=1.6
|
|
||||||
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
|
|
||||||
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
|
|
||||||
org.eclipse.jdt.core.compiler.source=1.6
|
|
||||||
|
|
@ -1,9 +0,0 @@
|
||||||
Manifest-Version: 1.0
|
|
||||||
Bundle-ManifestVersion: 2
|
|
||||||
Bundle-Name: NCSA HDF5 Win32 Fragment
|
|
||||||
Bundle-SymbolicName: ncsa.hdf5.win32
|
|
||||||
Bundle-Version: 2.4.0
|
|
||||||
Bundle-Vendor: Raytheon-bundled OSS
|
|
||||||
Fragment-Host: ncsa.hdf5;bundle-version="2.4.0"
|
|
||||||
Bundle-RequiredExecutionEnvironment: JavaSE-1.6
|
|
||||||
Eclipse-PlatformFilter: (& (osgi.os=win32) (osgi.arch=x86))
|
|
||||||
|
|
@ -1,4 +0,0 @@
|
||||||
bin.includes = META-INF/,\
|
|
||||||
.,\
|
|
||||||
jhdf5.dll
|
|
||||||
src.includes = jhdf5.dll
|
|
||||||
Binary file not shown.
|
|
@ -1,7 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<classpath>
|
|
||||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
|
|
||||||
<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
|
|
||||||
<classpathentry exported="true" kind="lib" path="jhdf5.jar" sourcepath="/home/randerso/hdf-java.zip"/>
|
|
||||||
<classpathentry kind="output" path="bin"/>
|
|
||||||
</classpath>
|
|
||||||
|
|
@ -1,28 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<projectDescription>
|
|
||||||
<name>ncsa.hdf5</name>
|
|
||||||
<comment></comment>
|
|
||||||
<projects>
|
|
||||||
</projects>
|
|
||||||
<buildSpec>
|
|
||||||
<buildCommand>
|
|
||||||
<name>org.eclipse.jdt.core.javabuilder</name>
|
|
||||||
<arguments>
|
|
||||||
</arguments>
|
|
||||||
</buildCommand>
|
|
||||||
<buildCommand>
|
|
||||||
<name>org.eclipse.pde.ManifestBuilder</name>
|
|
||||||
<arguments>
|
|
||||||
</arguments>
|
|
||||||
</buildCommand>
|
|
||||||
<buildCommand>
|
|
||||||
<name>org.eclipse.pde.SchemaBuilder</name>
|
|
||||||
<arguments>
|
|
||||||
</arguments>
|
|
||||||
</buildCommand>
|
|
||||||
</buildSpec>
|
|
||||||
<natures>
|
|
||||||
<nature>org.eclipse.pde.PluginNature</nature>
|
|
||||||
<nature>org.eclipse.jdt.core.javanature</nature>
|
|
||||||
</natures>
|
|
||||||
</projectDescription>
|
|
||||||
|
|
@ -1,7 +0,0 @@
|
||||||
#Thu Mar 26 11:34:03 CDT 2009
|
|
||||||
eclipse.preferences.version=1
|
|
||||||
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
|
|
||||||
org.eclipse.jdt.core.compiler.compliance=1.6
|
|
||||||
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
|
|
||||||
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
|
|
||||||
org.eclipse.jdt.core.compiler.source=1.6
|
|
||||||
|
|
@ -1,11 +0,0 @@
|
||||||
Manifest-Version: 1.0
|
|
||||||
Bundle-ManifestVersion: 2
|
|
||||||
Bundle-Name: NCSA HDF5 Plug-in
|
|
||||||
Bundle-SymbolicName: ncsa.hdf5
|
|
||||||
Bundle-Version: 2.4.0
|
|
||||||
Bundle-ClassPath: jhdf5.jar
|
|
||||||
Export-Package: ncsa.hdf.hdf5lib,
|
|
||||||
ncsa.hdf.hdf5lib.exceptions
|
|
||||||
Bundle-Vendor: Raytheon-bundled OSS
|
|
||||||
Edex-Deploy: jhdf5.jar
|
|
||||||
Bundle-RequiredExecutionEnvironment: JavaSE-1.6
|
|
||||||
|
|
@ -1,2 +0,0 @@
|
||||||
bin.includes = META-INF/,\
|
|
||||||
jhdf5.jar
|
|
||||||
Binary file not shown.
Binary file not shown.
|
|
@ -27,8 +27,7 @@ Export-Package: com.raytheon.edex.colormap,
|
||||||
com.raytheon.edex.util,
|
com.raytheon.edex.util,
|
||||||
com.raytheon.edex.utility,
|
com.raytheon.edex.utility,
|
||||||
com.raytheon.edex.common.shapefiles
|
com.raytheon.edex.common.shapefiles
|
||||||
Require-Bundle: ncsa.hdf5,
|
Require-Bundle: net.sf.ehcache,
|
||||||
net.sf.ehcache,
|
|
||||||
org.apache.commons.configuration,
|
org.apache.commons.configuration,
|
||||||
org.apache.velocity;bundle-version="1.6.0",
|
org.apache.velocity;bundle-version="1.6.0",
|
||||||
org.geotools,
|
org.geotools,
|
||||||
|
|
@ -57,7 +56,6 @@ Require-Bundle: ncsa.hdf5,
|
||||||
com.raytheon.uf.common.geospatial;bundle-version="1.0.0";visibility:=reexport,
|
com.raytheon.uf.common.geospatial;bundle-version="1.0.0";visibility:=reexport,
|
||||||
com.raytheon.uf.common.message;bundle-version="1.11.11",
|
com.raytheon.uf.common.message;bundle-version="1.11.11",
|
||||||
com.raytheon.uf.common.serialization.comm;bundle-version="1.11.17",
|
com.raytheon.uf.common.serialization.comm;bundle-version="1.11.17",
|
||||||
com.raytheon.uf.common.datastorage.hdf5;bundle-version="1.11.17",
|
|
||||||
org.hibernate;visibility:=reexport,
|
org.hibernate;visibility:=reexport,
|
||||||
com.raytheon.uf.common.pointdata,
|
com.raytheon.uf.common.pointdata,
|
||||||
com.raytheon.uf.common.util;visibility:=reexport,
|
com.raytheon.uf.common.util;visibility:=reexport,
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,6 @@ Require-Bundle: com.raytheon.uf.common.dataplugin.gfe;bundle-version="1.12.1174"
|
||||||
com.raytheon.uf.common.serialization,
|
com.raytheon.uf.common.serialization,
|
||||||
org.junit;bundle-version="1.0.0",
|
org.junit;bundle-version="1.0.0",
|
||||||
com.raytheon.uf.edex.plugin.grid,
|
com.raytheon.uf.edex.plugin.grid,
|
||||||
com.raytheon.uf.common.datastorage.hdf5;bundle-version="1.11.24",
|
|
||||||
com.raytheon.uf.edex.topo,
|
com.raytheon.uf.edex.topo,
|
||||||
com.raytheon.edex.common,
|
com.raytheon.edex.common,
|
||||||
com.raytheon.uf.common.dataplugin.satellite;bundle-version="1.0.0",
|
com.raytheon.uf.common.dataplugin.satellite;bundle-version="1.0.0",
|
||||||
|
|
|
||||||
|
|
@ -109,6 +109,8 @@ import com.raytheon.uf.edex.database.query.DatabaseQuery;
|
||||||
* adding it to the inventory
|
* adding it to the inventory
|
||||||
* 12/06/12 #1394 rjpeter Optimized D2D grid access.
|
* 12/06/12 #1394 rjpeter Optimized D2D grid access.
|
||||||
* 01/21/12 #1504 randerso Back ported change to use ParameterMapper into 13.1.2
|
* 01/21/12 #1504 randerso Back ported change to use ParameterMapper into 13.1.2
|
||||||
|
* 02/10/13 #1603 randerso Eliminated unnecessary conversion from lists to arrays
|
||||||
|
* 02/12/13 #1608 randerso Changed to use explicit deletes for groups and datasets
|
||||||
*
|
*
|
||||||
* </pre>
|
* </pre>
|
||||||
*
|
*
|
||||||
|
|
@ -459,7 +461,7 @@ public class GFEDao extends DefaultPluginDao {
|
||||||
String[] groupsToDelete = entry.getValue().getSecond();
|
String[] groupsToDelete = entry.getValue().getSecond();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
dataStore.delete(groupsToDelete);
|
dataStore.deleteGroups(groupsToDelete);
|
||||||
|
|
||||||
if (statusHandler.isPriorityEnabled(Priority.DEBUG)) {
|
if (statusHandler.isPriorityEnabled(Priority.DEBUG)) {
|
||||||
statusHandler.handle(Priority.DEBUG,
|
statusHandler.handle(Priority.DEBUG,
|
||||||
|
|
@ -1074,7 +1076,8 @@ public class GFEDao extends DefaultPluginDao {
|
||||||
try {
|
try {
|
||||||
IDataStore ds = DataStoreFactory.getDataStore(GfeUtil
|
IDataStore ds = DataStoreFactory.getDataStore(GfeUtil
|
||||||
.getGridParmHdf5File(GridDatabase.gfeBaseDataDir, dbId));
|
.getGridParmHdf5File(GridDatabase.gfeBaseDataDir, dbId));
|
||||||
ds.delete("/GridParmInfo/" + parmAndLevel);
|
ds.deleteDatasets("/GridParmInfo/" + parmAndLevel,
|
||||||
|
"/GridParmStorageInfo/" + parmAndLevel);
|
||||||
} catch (Exception e1) {
|
} catch (Exception e1) {
|
||||||
throw new DataAccessLayerException("Error deleting data from HDF5",
|
throw new DataAccessLayerException("Error deleting data from HDF5",
|
||||||
e1);
|
e1);
|
||||||
|
|
|
||||||
|
|
@ -103,6 +103,11 @@ import com.vividsolutions.jts.io.WKTReader;
|
||||||
* 06/18/08 njensen Added discrete/wx to getGridData()
|
* 06/18/08 njensen Added discrete/wx to getGridData()
|
||||||
* 05/04/12 #574 dgilling Restructure class to better match AWIPS1.
|
* 05/04/12 #574 dgilling Restructure class to better match AWIPS1.
|
||||||
* 07/11/12 15162 ryu No raising exception in c'tor
|
* 07/11/12 15162 ryu No raising exception in c'tor
|
||||||
|
* 02/10/12 #1603 randerso Implemented deleteDb, moved methods down from
|
||||||
|
* GridDatabase that belonged here.
|
||||||
|
* Removed unncecssary conversion from Lists to/from arrays
|
||||||
|
* Added performance logging
|
||||||
|
* 02/12/13 #1608 randerso Changed to explicitly call deleteGroups
|
||||||
*
|
*
|
||||||
* </pre>
|
* </pre>
|
||||||
*
|
*
|
||||||
|
|
@ -2449,7 +2454,7 @@ public class IFPGridDatabase extends GridDatabase {
|
||||||
IDataStore dataStore = DataStoreFactory.getDataStore(hdf5File);
|
IDataStore dataStore = DataStoreFactory.getDataStore(hdf5File);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
dataStore.delete(groupName);
|
dataStore.deleteGroups(groupName);
|
||||||
statusHandler.handle(Priority.DEBUG, "Deleted: " + groupName
|
statusHandler.handle(Priority.DEBUG, "Deleted: " + groupName
|
||||||
+ " from " + hdf5File.getName());
|
+ " from " + hdf5File.getName());
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -72,6 +72,7 @@ import com.raytheon.uf.edex.topo.TopoQuery;
|
||||||
* ------------ ---------- ----------- --------------------------
|
* ------------ ---------- ----------- --------------------------
|
||||||
* Jul 10, 2009 njensen Initial creation
|
* Jul 10, 2009 njensen Initial creation
|
||||||
* May 04, 2012 #574 dgilling Re-port to better match AWIPS1.
|
* May 04, 2012 #574 dgilling Re-port to better match AWIPS1.
|
||||||
|
* Feb 12, 2013 #1608 randerso Changed to use explicit deleteGroups
|
||||||
*
|
*
|
||||||
* </pre>
|
* </pre>
|
||||||
*
|
*
|
||||||
|
|
@ -310,7 +311,7 @@ public class TopoDatabaseManager {
|
||||||
public void revertTopoData(final GridLocation gloc) {
|
public void revertTopoData(final GridLocation gloc) {
|
||||||
String name = calcGroupName(gloc);
|
String name = calcGroupName(gloc);
|
||||||
try {
|
try {
|
||||||
dataStore.delete(name);
|
dataStore.deleteGroups(name);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
statusHandler.error("Error attempting to remove: " + name, e);
|
statusHandler.error("Error attempting to remove: " + name, e);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,6 @@ Require-Bundle: com.raytheon.edex.common;bundle-version="1.11.13",
|
||||||
org.apache.commons.logging;bundle-version="1.0.4",
|
org.apache.commons.logging;bundle-version="1.0.4",
|
||||||
com.raytheon.uf.common.localization;bundle-version="1.11.13",
|
com.raytheon.uf.common.localization;bundle-version="1.11.13",
|
||||||
com.raytheon.edex.plugin.ldad;bundle-version="1.0.0",
|
com.raytheon.edex.plugin.ldad;bundle-version="1.0.0",
|
||||||
ncsa.hdf5;bundle-version="2.4.0",
|
|
||||||
com.raytheon.uf.common.pointdata;bundle-version="1.11.17",
|
com.raytheon.uf.common.pointdata;bundle-version="1.11.17",
|
||||||
com.raytheon.uf.edex.pointdata
|
com.raytheon.uf.edex.pointdata
|
||||||
Bundle-RequiredExecutionEnvironment: JavaSE-1.6
|
Bundle-RequiredExecutionEnvironment: JavaSE-1.6
|
||||||
|
|
|
||||||
|
|
@ -1,8 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<classpath>
|
|
||||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
|
|
||||||
<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
|
|
||||||
<classpathentry kind="src" path="src"/>
|
|
||||||
<classpathentry kind="src" output="bin" path="unit-test"/>
|
|
||||||
<classpathentry kind="output" path="bin"/>
|
|
||||||
</classpath>
|
|
||||||
|
|
@ -1,28 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<projectDescription>
|
|
||||||
<name>com.raytheon.uf.common.datastorage.hdf5</name>
|
|
||||||
<comment></comment>
|
|
||||||
<projects>
|
|
||||||
</projects>
|
|
||||||
<buildSpec>
|
|
||||||
<buildCommand>
|
|
||||||
<name>org.eclipse.jdt.core.javabuilder</name>
|
|
||||||
<arguments>
|
|
||||||
</arguments>
|
|
||||||
</buildCommand>
|
|
||||||
<buildCommand>
|
|
||||||
<name>org.eclipse.pde.ManifestBuilder</name>
|
|
||||||
<arguments>
|
|
||||||
</arguments>
|
|
||||||
</buildCommand>
|
|
||||||
<buildCommand>
|
|
||||||
<name>org.eclipse.pde.SchemaBuilder</name>
|
|
||||||
<arguments>
|
|
||||||
</arguments>
|
|
||||||
</buildCommand>
|
|
||||||
</buildSpec>
|
|
||||||
<natures>
|
|
||||||
<nature>org.eclipse.pde.PluginNature</nature>
|
|
||||||
<nature>org.eclipse.jdt.core.javanature</nature>
|
|
||||||
</natures>
|
|
||||||
</projectDescription>
|
|
||||||
|
|
@ -1,12 +0,0 @@
|
||||||
#Wed Jun 17 09:51:15 CDT 2009
|
|
||||||
eclipse.preferences.version=1
|
|
||||||
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
|
|
||||||
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
|
|
||||||
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
|
|
||||||
org.eclipse.jdt.core.compiler.compliance=1.6
|
|
||||||
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
|
|
||||||
org.eclipse.jdt.core.compiler.debug.localVariable=generate
|
|
||||||
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
|
|
||||||
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
|
|
||||||
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
|
|
||||||
org.eclipse.jdt.core.compiler.source=1.6
|
|
||||||
|
|
@ -1,18 +0,0 @@
|
||||||
Manifest-Version: 1.0
|
|
||||||
Bundle-ManifestVersion: 2
|
|
||||||
Bundle-Name: Hdf5 Plug-in
|
|
||||||
Bundle-SymbolicName: com.raytheon.uf.common.datastorage.hdf5
|
|
||||||
Bundle-Version: 1.12.1174.qualifier
|
|
||||||
Bundle-Vendor: RAYTHEON
|
|
||||||
Eclipse-BuddyPolicy: registered, ext, global
|
|
||||||
Eclipse-RegisterBuddy: com.raytheon.uf.common.datastorage
|
|
||||||
Bundle-RequiredExecutionEnvironment: JavaSE-1.6
|
|
||||||
Export-Package: com.raytheon.uf.common.datastorage.hdf5
|
|
||||||
Import-Package: com.raytheon.uf.common.datastorage,
|
|
||||||
com.raytheon.uf.common.datastorage.locking,
|
|
||||||
com.raytheon.uf.common.datastorage.records,
|
|
||||||
com.raytheon.uf.common.util,
|
|
||||||
junit.framework,
|
|
||||||
org.junit
|
|
||||||
Require-Bundle: ncsa.hdf5;bundle-version="2.4.0",
|
|
||||||
com.raytheon.uf.common.serialization
|
|
||||||
|
|
@ -1,4 +0,0 @@
|
||||||
source.. = src/
|
|
||||||
output.. = bin/
|
|
||||||
bin.includes = META-INF/,\
|
|
||||||
.
|
|
||||||
|
|
@ -1,8 +0,0 @@
|
||||||
<project basedir="." default="deploy" name="com.raytheon.uf.common.datastorage.hdf5">
|
|
||||||
|
|
||||||
<available file="../build.edex" property="build.dir.location" value="../build.edex"/>
|
|
||||||
<available file="../../../../../build.edex" property="build.dir.location" value="../../../../../build.edex"/>
|
|
||||||
|
|
||||||
<import file="${build.dir.location}/basebuilds/component_deploy_base.xml" />
|
|
||||||
|
|
||||||
</project>
|
|
||||||
|
|
@ -1,446 +0,0 @@
|
||||||
/**
|
|
||||||
* This software was developed and / or modified by Raytheon Company,
|
|
||||||
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
|
|
||||||
*
|
|
||||||
* U.S. EXPORT CONTROLLED TECHNICAL DATA
|
|
||||||
* This software product contains export-restricted data whose
|
|
||||||
* export/transfer/disclosure is restricted by U.S. law. Dissemination
|
|
||||||
* to non-U.S. persons whether in the United States or abroad requires
|
|
||||||
* an export license or other authorization.
|
|
||||||
*
|
|
||||||
* Contractor Name: Raytheon Company
|
|
||||||
* Contractor Address: 6825 Pine Street, Suite 340
|
|
||||||
* Mail Stop B8
|
|
||||||
* Omaha, NE 68106
|
|
||||||
* 402.291.0100
|
|
||||||
*
|
|
||||||
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
|
|
||||||
* further licensing information.
|
|
||||||
**/
|
|
||||||
package com.raytheon.uf.common.datastorage.hdf5;
|
|
||||||
|
|
||||||
import java.awt.Point;
|
|
||||||
import java.lang.reflect.Method;
|
|
||||||
import java.util.LinkedHashMap;
|
|
||||||
import java.util.LinkedHashSet;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Set;
|
|
||||||
|
|
||||||
import ncsa.hdf.hdf5lib.H5;
|
|
||||||
import ncsa.hdf.hdf5lib.HDF5Constants;
|
|
||||||
import ncsa.hdf.hdf5lib.HDFNativeData;
|
|
||||||
import ncsa.hdf.hdf5lib.exceptions.HDF5Exception;
|
|
||||||
import ncsa.hdf.hdf5lib.exceptions.HDF5LibraryException;
|
|
||||||
|
|
||||||
import com.raytheon.uf.common.datastorage.DataStoreFactory;
|
|
||||||
import com.raytheon.uf.common.datastorage.Request;
|
|
||||||
import com.raytheon.uf.common.datastorage.StorageException;
|
|
||||||
import com.raytheon.uf.common.datastorage.Request.Type;
|
|
||||||
import com.raytheon.uf.common.datastorage.records.AbstractStorageRecord;
|
|
||||||
import com.raytheon.uf.common.datastorage.records.IDataRecord;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Defines the base implementation of an HDF5 Read
|
|
||||||
*
|
|
||||||
* <pre>
|
|
||||||
*
|
|
||||||
* SOFTWARE HISTORY
|
|
||||||
* Date Ticket# Engineer Description
|
|
||||||
* ------------ ---------- ----------- --------------------------
|
|
||||||
* Jul 27, 2009 chammack Initial creation
|
|
||||||
*
|
|
||||||
* </pre>
|
|
||||||
*
|
|
||||||
* @author chammack
|
|
||||||
* @version 1.0
|
|
||||||
*/
|
|
||||||
|
|
||||||
public abstract class AbstractHDFRead {
|
|
||||||
|
|
||||||
private static Method select;
|
|
||||||
|
|
||||||
static {
|
|
||||||
try {
|
|
||||||
Method[] methods = H5.class.getDeclaredMethods();
|
|
||||||
for (Method method : methods) {
|
|
||||||
if (method.getName().equals("H5Sselect_elements")) {
|
|
||||||
Class<?>[] clazz = method.getParameterTypes();
|
|
||||||
if (clazz.length != 4) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (clazz[0] == Integer.TYPE && clazz[1] == Integer.TYPE
|
|
||||||
&& clazz[2] == Integer.TYPE
|
|
||||||
&& clazz[3] == byte[].class) {
|
|
||||||
select = method;
|
|
||||||
select.setAccessible(true);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
} catch (Exception e) {
|
|
||||||
System.out
|
|
||||||
.println("Method signature for H5Sselect_elements not found.");
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public IDataRecord read(Request request, int file_id, String group,
|
|
||||||
String dataset, float scaleFactor) throws StorageException {
|
|
||||||
int dataset_id = -1;
|
|
||||||
int typeid = -1;
|
|
||||||
int memspace = -1;
|
|
||||||
int dataspace = -1;
|
|
||||||
int plist = -1;
|
|
||||||
Number fillValueResult = null;
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Open an existing dataset.
|
|
||||||
|
|
||||||
dataset_id = H5.H5Dopen(file_id, group + "/" + dataset);
|
|
||||||
|
|
||||||
plist = H5.H5Dget_create_plist(dataset_id);
|
|
||||||
|
|
||||||
typeid = H5.H5Dget_type(dataset_id);
|
|
||||||
|
|
||||||
int classid = H5.H5Tget_class(typeid);
|
|
||||||
|
|
||||||
int size = H5.H5Tget_size(typeid);
|
|
||||||
boolean isVL = H5.H5Tis_variable_str(typeid);
|
|
||||||
|
|
||||||
dataspace = H5.H5Dget_space(dataset_id);
|
|
||||||
|
|
||||||
int sz = H5.H5Sget_simple_extent_ndims(dataspace);
|
|
||||||
|
|
||||||
long[] dims = new long[sz];
|
|
||||||
long[] originalDims = new long[sz];
|
|
||||||
|
|
||||||
int totalSize = 1;
|
|
||||||
H5.H5Sget_simple_extent_dims(dataspace, dims, (long[]) null);
|
|
||||||
// perform a deep copy
|
|
||||||
for (int i = 0; i < originalDims.length; i++) {
|
|
||||||
originalDims[i] = dims[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
// optimization:
|
|
||||||
// if a line query was used, but it's only 1d, this is really
|
|
||||||
// a point query. We _could_ use hyperslabs to do this, but
|
|
||||||
// it would be a lot slower than a regular point query, so
|
|
||||||
// just rewrite the request as a point query.
|
|
||||||
AbstractHDFRead readObj = this;
|
|
||||||
if (dims.length == 1
|
|
||||||
&& (request.getType() == Type.XLINE || request.getType() == Type.YLINE)) {
|
|
||||||
int[] idx = request.getIndices();
|
|
||||||
int[] pts = new int[idx.length];
|
|
||||||
Set<Point> p = new LinkedHashSet<Point>(idx.length);
|
|
||||||
int k = 0;
|
|
||||||
for (int i = 0; i < idx.length; i++) {
|
|
||||||
if (request.getType() == Type.XLINE) {
|
|
||||||
if (idx[i] >= originalDims[1])
|
|
||||||
continue;
|
|
||||||
} else if (request.getType() == Type.YLINE) {
|
|
||||||
if (idx[i] >= originalDims[0])
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
p.add(new Point(idx[i], 0));
|
|
||||||
pts[k] = idx[i];
|
|
||||||
k++;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (k != idx.length) {
|
|
||||||
// Prune the caller's copy (optimization)
|
|
||||||
idx = new int[k];
|
|
||||||
System.arraycopy(pts, 0, idx, 0, k);
|
|
||||||
request.setIndices(idx);
|
|
||||||
}
|
|
||||||
|
|
||||||
request = (Request.buildPointRequest(p.toArray(new Point[p
|
|
||||||
.size()])));
|
|
||||||
readObj = HDF5OpManager.readTypes.get(request.getType());
|
|
||||||
}
|
|
||||||
|
|
||||||
totalSize = readObj.calculateSize(dataspace, sz, request, dims,
|
|
||||||
originalDims, totalSize);
|
|
||||||
|
|
||||||
LocalHDFDataType dataType = null;
|
|
||||||
Object data = null;
|
|
||||||
Object fillValue = null;
|
|
||||||
int readType = 0;
|
|
||||||
if (classid == HDF5Constants.H5T_FLOAT) {
|
|
||||||
dataType = LocalHDFDataType.FLOAT;
|
|
||||||
data = new float[totalSize];
|
|
||||||
fillValue = new float[1];
|
|
||||||
readType = HDF5Constants.H5T_NATIVE_FLOAT;
|
|
||||||
} else if (classid == HDF5Constants.H5T_INTEGER) {
|
|
||||||
if (size == 1) {
|
|
||||||
dataType = LocalHDFDataType.BYTE;
|
|
||||||
data = new byte[totalSize];
|
|
||||||
fillValue = new byte[1];
|
|
||||||
readType = HDF5Constants.H5T_NATIVE_INT8;
|
|
||||||
} else if (size == 2) {
|
|
||||||
dataType = LocalHDFDataType.SHORT;
|
|
||||||
data = new short[totalSize];
|
|
||||||
fillValue = new short[1];
|
|
||||||
readType = HDF5Constants.H5T_NATIVE_SHORT;
|
|
||||||
} else if (size == 4) {
|
|
||||||
dataType = LocalHDFDataType.INT;
|
|
||||||
data = new int[totalSize];
|
|
||||||
fillValue = new int[1];
|
|
||||||
readType = HDF5Constants.H5T_NATIVE_INT;
|
|
||||||
} else if (size == 8) {
|
|
||||||
dataType = LocalHDFDataType.LONG;
|
|
||||||
data = new long[totalSize];
|
|
||||||
fillValue = new long[1];
|
|
||||||
readType = HDF5Constants.H5T_NATIVE_INT64;
|
|
||||||
}
|
|
||||||
} else if (classid == HDF5Constants.H5T_STRING) {
|
|
||||||
data = new String[totalSize];
|
|
||||||
dataType = LocalHDFDataType.STRING;
|
|
||||||
readType = typeid;
|
|
||||||
}
|
|
||||||
|
|
||||||
memspace = H5.H5Screate_simple(sz, dims, null);
|
|
||||||
|
|
||||||
readObj.selectSet(memspace, dataspace, request, dims, originalDims);
|
|
||||||
|
|
||||||
if (dataType == LocalHDFDataType.STRING && isVL) {
|
|
||||||
H5.H5DreadVL(dataset_id, readType, memspace, dataspace,
|
|
||||||
HDF5Constants.H5P_DEFAULT, (Object[]) data);
|
|
||||||
} else {
|
|
||||||
H5.H5Dread(dataset_id, readType, memspace, dataspace,
|
|
||||||
HDF5Constants.H5P_DATASET_XFER_DEFAULT, data);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (fillValue != null) {
|
|
||||||
H5.H5Pget_fill_value(plist, readType, fillValue);
|
|
||||||
if (fillValue instanceof double[]) {
|
|
||||||
fillValueResult = new Double(((double[]) fillValue)[0]);
|
|
||||||
} else if (fillValue instanceof int[]) {
|
|
||||||
fillValueResult = new Integer(((int[]) fillValue)[0]);
|
|
||||||
} else if (fillValue instanceof byte[]) {
|
|
||||||
fillValueResult = new Byte(((byte[]) fillValue)[0]);
|
|
||||||
} else if (fillValue instanceof long[]) {
|
|
||||||
fillValueResult = new Long(((long[]) fillValue)[0]);
|
|
||||||
} else if (fillValue instanceof short[]) {
|
|
||||||
fillValueResult = new Short(((short[]) fillValue)[0]);
|
|
||||||
} else if (fillValue instanceof float[]) {
|
|
||||||
fillValueResult = new Float(((float[]) fillValue)[0]);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
// Swizzle and scale the dims to match java nomenclature
|
|
||||||
long[] dims2 = new long[dims.length];
|
|
||||||
int k = 0;
|
|
||||||
for (int i = dims2.length - 1; i >= 0; i--) {
|
|
||||||
dims2[k] = (int) (dims[i] * scaleFactor);
|
|
||||||
k++;
|
|
||||||
}
|
|
||||||
|
|
||||||
AbstractStorageRecord rec = DataStoreFactory.createStorageRecord(
|
|
||||||
dataset, group, data, sz, dims2);
|
|
||||||
if (fillValueResult != null) {
|
|
||||||
rec.setFillValue(fillValueResult);
|
|
||||||
}
|
|
||||||
readProperties(rec, dataset_id);
|
|
||||||
|
|
||||||
return rec;
|
|
||||||
} catch (Exception e) {
|
|
||||||
try {
|
|
||||||
H5.H5Eclear();
|
|
||||||
} catch (HDF5LibraryException e1) {
|
|
||||||
// ignore
|
|
||||||
}
|
|
||||||
throw new StorageException("Error occurred during retrieve ", null,
|
|
||||||
e);
|
|
||||||
} finally {
|
|
||||||
|
|
||||||
try {
|
|
||||||
if (memspace >= 0) {
|
|
||||||
H5.H5Sclose(memspace);
|
|
||||||
}
|
|
||||||
} catch (HDF5LibraryException e) {
|
|
||||||
// ignore
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
if (dataspace >= 0) {
|
|
||||||
H5.H5Sclose(dataspace);
|
|
||||||
}
|
|
||||||
} catch (HDF5LibraryException e) {
|
|
||||||
// ignore
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
if (plist >= 0) {
|
|
||||||
H5.H5Pclose(plist);
|
|
||||||
}
|
|
||||||
} catch (HDF5LibraryException e) {
|
|
||||||
// ignore
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
if (dataset_id >= 0) {
|
|
||||||
H5.H5Dclose(dataset_id);
|
|
||||||
}
|
|
||||||
} catch (HDF5LibraryException e) {
|
|
||||||
// ignore
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
if (typeid >= 0) {
|
|
||||||
H5.H5Tclose(typeid);
|
|
||||||
}
|
|
||||||
} catch (HDF5LibraryException e) {
|
|
||||||
// ignore
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void readProperties(IDataRecord dr, int dataSet) {
|
|
||||||
try {
|
|
||||||
|
|
||||||
Map<String, Object> attribMap = new LinkedHashMap<String, Object>();
|
|
||||||
int attribs = H5.H5Aget_num_attrs(dataSet);
|
|
||||||
if (attribs > 0) {
|
|
||||||
dr.setDataAttributes(attribMap);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int i = 0; i < attribs; i++) {
|
|
||||||
|
|
||||||
int attribId = 0;
|
|
||||||
int type = 0;
|
|
||||||
int nativeType = 0;
|
|
||||||
try {
|
|
||||||
attribId = H5.H5Aopen_idx(dataSet, i);
|
|
||||||
String[] str = new String[1];
|
|
||||||
H5.H5Aget_name(attribId, 256, str);
|
|
||||||
type = H5.H5Aget_type(attribId);
|
|
||||||
int spc = H5.H5Tget_size(type);
|
|
||||||
nativeType = H5.H5Tget_native_type(type);
|
|
||||||
int cls = H5.H5Tget_class(type);
|
|
||||||
|
|
||||||
if (cls == HDF5Constants.H5T_INTEGER) {
|
|
||||||
Object d = null;
|
|
||||||
switch (spc) {
|
|
||||||
case 1:
|
|
||||||
// byte
|
|
||||||
d = new byte[1];
|
|
||||||
H5.H5Aread(attribId, HDF5Constants.H5T_NATIVE_INT,
|
|
||||||
d);
|
|
||||||
attribMap.put(str[0], ((byte[]) d)[0]);
|
|
||||||
break;
|
|
||||||
case 2:
|
|
||||||
// short
|
|
||||||
d = new short[1];
|
|
||||||
H5.H5Aread(attribId, HDF5Constants.H5T_NATIVE_INT,
|
|
||||||
d);
|
|
||||||
attribMap.put(str[0], ((short[]) d)[0]);
|
|
||||||
break;
|
|
||||||
case 4:
|
|
||||||
// regular int
|
|
||||||
d = new int[1];
|
|
||||||
H5.H5Aread(attribId, HDF5Constants.H5T_NATIVE_INT,
|
|
||||||
d);
|
|
||||||
attribMap.put(str[0], ((int[]) d)[0]);
|
|
||||||
break;
|
|
||||||
case 8:
|
|
||||||
// long
|
|
||||||
d = new long[1];
|
|
||||||
H5.H5Aread(attribId, HDF5Constants.H5T_NATIVE_INT,
|
|
||||||
d);
|
|
||||||
attribMap.put(str[0], ((long[]) d)[0]);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
} else if (cls == HDF5Constants.H5T_FLOAT) {
|
|
||||||
if (spc == 4) {
|
|
||||||
float[] d = new float[1];
|
|
||||||
H5.H5Aread(attribId,
|
|
||||||
HDF5Constants.H5T_NATIVE_FLOAT, d);
|
|
||||||
attribMap.put(str[0], d[0]);
|
|
||||||
} else if (spc == 8) {
|
|
||||||
double[] d = new double[1];
|
|
||||||
H5.H5Aread(attribId,
|
|
||||||
HDF5Constants.H5T_NATIVE_DOUBLE, d);
|
|
||||||
attribMap.put(str[0], d[0]);
|
|
||||||
}
|
|
||||||
} else if (cls == HDF5Constants.H5T_STRING) {
|
|
||||||
byte[] b = new byte[spc];
|
|
||||||
H5.H5Aread(attribId, nativeType, b);
|
|
||||||
String outStr = new String(b, 0, spc - 1);
|
|
||||||
attribMap.put(str[0], outStr);
|
|
||||||
} else {
|
|
||||||
throw new IllegalArgumentException(
|
|
||||||
"Unable to handle type" + cls);
|
|
||||||
}
|
|
||||||
} catch (HDF5Exception e) {
|
|
||||||
H5.H5Eclear();
|
|
||||||
e.printStackTrace();
|
|
||||||
} finally {
|
|
||||||
if (type > 0) {
|
|
||||||
try {
|
|
||||||
H5.H5Tclose(type);
|
|
||||||
} catch (Exception e) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (attribId > 0) {
|
|
||||||
try {
|
|
||||||
H5.H5Aclose(attribId);
|
|
||||||
} catch (Exception e) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (HDF5LibraryException e) {
|
|
||||||
try {
|
|
||||||
H5.H5Eclear();
|
|
||||||
} catch (HDF5LibraryException e1) {
|
|
||||||
}
|
|
||||||
e.printStackTrace();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
protected abstract void selectSet(int memspace, int dataspace,
|
|
||||||
Request request, long[] dims, long[] originalDatasetDims)
|
|
||||||
throws HDF5Exception, HDF5LibraryException, StorageException;
|
|
||||||
|
|
||||||
protected abstract int calculateSize(int dataspace, int sz,
|
|
||||||
Request request, long[] dims, long[] originalDims, int totalSize)
|
|
||||||
throws HDF5LibraryException;
|
|
||||||
|
|
||||||
// For some reason, the hdf5 folks implemented this using reflection which
|
|
||||||
// makes it very slow. This implementation is at least 5x faster
|
|
||||||
protected static final byte[] longToBytes(long[][] indices) {
|
|
||||||
long[] inn = new long[indices.length * indices[0].length];
|
|
||||||
for (int j = 0; j < indices.length; j++) {
|
|
||||||
System.arraycopy(indices[j], 0, inn, j * indices[0].length,
|
|
||||||
indices[0].length);
|
|
||||||
}
|
|
||||||
|
|
||||||
return HDFNativeData.longToByte(0, inn.length, inn);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected static int invokeH5Sselect_elements(int space, int op,
|
|
||||||
int num_elements, byte[] coords) throws StorageException {
|
|
||||||
// synchronization not needed since caller methods are synchronized
|
|
||||||
if (select == null) {
|
|
||||||
throw new StorageException(
|
|
||||||
"Method did not initialize properly? Incompatible version of hdf5?",
|
|
||||||
null);
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
Integer i = (Integer) select.invoke(null, space, op, num_elements,
|
|
||||||
coords);
|
|
||||||
return i;
|
|
||||||
} catch (Exception e) {
|
|
||||||
throw new StorageException("Error selecting elements", null, e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,69 +0,0 @@
|
||||||
/**
|
|
||||||
* This software was developed and / or modified by Raytheon Company,
|
|
||||||
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
|
|
||||||
*
|
|
||||||
* U.S. EXPORT CONTROLLED TECHNICAL DATA
|
|
||||||
* This software product contains export-restricted data whose
|
|
||||||
* export/transfer/disclosure is restricted by U.S. law. Dissemination
|
|
||||||
* to non-U.S. persons whether in the United States or abroad requires
|
|
||||||
* an export license or other authorization.
|
|
||||||
*
|
|
||||||
* Contractor Name: Raytheon Company
|
|
||||||
* Contractor Address: 6825 Pine Street, Suite 340
|
|
||||||
* Mail Stop B8
|
|
||||||
* Omaha, NE 68106
|
|
||||||
* 402.291.0100
|
|
||||||
*
|
|
||||||
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
|
|
||||||
* further licensing information.
|
|
||||||
**/
|
|
||||||
package com.raytheon.uf.common.datastorage.hdf5;
|
|
||||||
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
import com.raytheon.uf.common.datastorage.Request;
|
|
||||||
import com.raytheon.uf.common.datastorage.StorageException;
|
|
||||||
import com.raytheon.uf.common.datastorage.Request.Type;
|
|
||||||
import com.raytheon.uf.common.datastorage.records.IDataRecord;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Provides a proxy for HDF5 to be extended for various types of requests
|
|
||||||
*
|
|
||||||
* <pre>
|
|
||||||
*
|
|
||||||
* SOFTWARE HISTORY
|
|
||||||
* Date Ticket# Engineer Description
|
|
||||||
* ------------ ---------- ----------- --------------------------
|
|
||||||
* Jul 27, 2009 chammack Initial creation
|
|
||||||
*
|
|
||||||
* </pre>
|
|
||||||
*
|
|
||||||
* @author chammack
|
|
||||||
* @version 1.0
|
|
||||||
*/
|
|
||||||
|
|
||||||
public class HDF5OpManager {
|
|
||||||
|
|
||||||
protected static final Map<Type, AbstractHDFRead> readTypes;
|
|
||||||
static {
|
|
||||||
readTypes = new HashMap<Type, AbstractHDFRead>();
|
|
||||||
readTypes.put(Type.POINT, new PointSelectionRead());
|
|
||||||
readTypes.put(Type.ALL, new WholeDatasetSelectionRead());
|
|
||||||
readTypes.put(Type.SLAB, new SlabSelectionRead());
|
|
||||||
readTypes.put(Type.XLINE, new LineSelectionRead.XLineSelectionRead());
|
|
||||||
readTypes.put(Type.YLINE, new LineSelectionRead.YLineSelectionRead());
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IDataRecord read(Request request, int file_id, String group,
|
|
||||||
String dataset, float scaleFactor) throws StorageException {
|
|
||||||
AbstractHDFRead readImpl = readTypes.get(request.getType());
|
|
||||||
if (readImpl == null) {
|
|
||||||
throw new IllegalArgumentException("Unsupported request type: "
|
|
||||||
+ request.getType());
|
|
||||||
}
|
|
||||||
|
|
||||||
return readImpl.read(request, file_id, group, dataset, scaleFactor);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
@ -1,231 +0,0 @@
|
||||||
/**
|
|
||||||
* This software was developed and / or modified by Raytheon Company,
|
|
||||||
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
|
|
||||||
*
|
|
||||||
* U.S. EXPORT CONTROLLED TECHNICAL DATA
|
|
||||||
* This software product contains export-restricted data whose
|
|
||||||
* export/transfer/disclosure is restricted by U.S. law. Dissemination
|
|
||||||
* to non-U.S. persons whether in the United States or abroad requires
|
|
||||||
* an export license or other authorization.
|
|
||||||
*
|
|
||||||
* Contractor Name: Raytheon Company
|
|
||||||
* Contractor Address: 6825 Pine Street, Suite 340
|
|
||||||
* Mail Stop B8
|
|
||||||
* Omaha, NE 68106
|
|
||||||
* 402.291.0100
|
|
||||||
*
|
|
||||||
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
|
|
||||||
* further licensing information.
|
|
||||||
**/
|
|
||||||
package com.raytheon.uf.common.datastorage.hdf5;
|
|
||||||
|
|
||||||
import ncsa.hdf.hdf5lib.H5;
|
|
||||||
import ncsa.hdf.hdf5lib.HDF5Constants;
|
|
||||||
import ncsa.hdf.hdf5lib.exceptions.HDF5Exception;
|
|
||||||
import ncsa.hdf.hdf5lib.exceptions.HDF5LibraryException;
|
|
||||||
|
|
||||||
import com.raytheon.uf.common.datastorage.Request;
|
|
||||||
import com.raytheon.uf.common.datastorage.StorageException;
|
|
||||||
import com.raytheon.uf.common.datastorage.records.IDataRecord;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Implements a set of line (one dimensional) reads
|
|
||||||
*
|
|
||||||
* Initially, this supports reading along either the X or Y axis in a two
|
|
||||||
* dimensional dataset. It could be extended to accomodate selection from
|
|
||||||
* n-dimensional datasets or selection along a diagonal.
|
|
||||||
*
|
|
||||||
* <pre>
|
|
||||||
*
|
|
||||||
* SOFTWARE HISTORY
|
|
||||||
* Date Ticket# Engineer Description
|
|
||||||
* ------------ ---------- ----------- --------------------------
|
|
||||||
* Jul 28, 2009 chammack Initial creation
|
|
||||||
*
|
|
||||||
* </pre>
|
|
||||||
*
|
|
||||||
* @author chammack
|
|
||||||
* @version 1.0
|
|
||||||
*/
|
|
||||||
|
|
||||||
public abstract class LineSelectionRead extends AbstractHDFRead {
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected int calculateSize(int dataspace, int sz, Request request,
|
|
||||||
long[] dims, long[] originalDims, int totalSize)
|
|
||||||
throws HDF5LibraryException {
|
|
||||||
|
|
||||||
int[] points = request.getIndices();
|
|
||||||
|
|
||||||
return points.length;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected abstract void pruneIndices(Request request, long[] originalDims);
|
|
||||||
|
|
||||||
/*
|
|
||||||
* (non-Javadoc)
|
|
||||||
*
|
|
||||||
* @see
|
|
||||||
* com.raytheon.uf.common.datastorage.hdf5.AbstractHDFRead#read(com.raytheon
|
|
||||||
* .uf.common.datastorage.Request, int, java.lang.String, java.lang.String,
|
|
||||||
* float)
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
public IDataRecord read(Request request, int file_id, String group,
|
|
||||||
String dataset, float scaleFactor) throws StorageException {
|
|
||||||
return super.read(request, file_id, group, dataset, scaleFactor);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static class YLineSelectionRead extends LineSelectionRead {
|
|
||||||
|
|
||||||
/*
|
|
||||||
* (non-Javadoc)
|
|
||||||
*
|
|
||||||
* @see
|
|
||||||
* com.raytheon.uf.common.datastorage.hdf5.LineSelectionRead#pruneIndices
|
|
||||||
* (com.raytheon.uf.common.datastorage.Request, long[])
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
protected void pruneIndices(Request request, long[] originalDims) {
|
|
||||||
int[] indices = request.getIndices();
|
|
||||||
int[] prunedIndices = new int[indices.length];
|
|
||||||
int k = 0;
|
|
||||||
for (int i = 0; i < indices.length; i++) {
|
|
||||||
if (indices[i] < originalDims[0] && indices[i] >= 0) {
|
|
||||||
prunedIndices[k] = indices[i];
|
|
||||||
k++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
indices = new int[k];
|
|
||||||
System.arraycopy(prunedIndices, 0, indices, 0, k);
|
|
||||||
request.setIndices(indices);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected void selectSet(int memspace, int dataspace, Request request,
|
|
||||||
long[] dims, long[] originalDatasetDims) throws HDF5Exception,
|
|
||||||
HDF5LibraryException, StorageException {
|
|
||||||
int[] points = request.getIndices();
|
|
||||||
|
|
||||||
long[] memOffset = new long[] { 0, 0 };
|
|
||||||
for (int i = 0; i < points.length; i++) {
|
|
||||||
long[] start = new long[] { points[i], 0 };
|
|
||||||
long[] count = new long[] { 1, originalDatasetDims[1] };
|
|
||||||
|
|
||||||
if (i == 0) {
|
|
||||||
H5.H5Sselect_hyperslab(dataspace,
|
|
||||||
HDF5Constants.H5S_SELECT_SET, start, null, count,
|
|
||||||
null);
|
|
||||||
H5.H5Sselect_hyperslab(memspace,
|
|
||||||
HDF5Constants.H5S_SELECT_SET, memOffset, null,
|
|
||||||
count, null);
|
|
||||||
} else {
|
|
||||||
memOffset[0] += 1;
|
|
||||||
H5.H5Sselect_hyperslab(dataspace,
|
|
||||||
HDF5Constants.H5S_SELECT_OR, start, null, count,
|
|
||||||
null);
|
|
||||||
H5.H5Sselect_hyperslab(memspace,
|
|
||||||
HDF5Constants.H5S_SELECT_OR, memOffset, null,
|
|
||||||
count, null);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* (non-Javadoc)
|
|
||||||
*
|
|
||||||
* @see
|
|
||||||
* com.raytheon.uf.common.datastorage.hdf5.LineSelectionRead#calculateSize
|
|
||||||
* (int, int, com.raytheon.uf.common.datastorage.Request, long[], int)
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
protected int calculateSize(int dataspace, int sz, Request request,
|
|
||||||
long[] dims, long[] originalDims, int totalSize)
|
|
||||||
throws HDF5LibraryException {
|
|
||||||
pruneIndices(request, originalDims);
|
|
||||||
dims[0] = request.getIndices().length;
|
|
||||||
return (int) (super.calculateSize(dataspace, sz, request, dims,
|
|
||||||
originalDims, totalSize) * dims[1]);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
public static class XLineSelectionRead extends LineSelectionRead {
|
|
||||||
|
|
||||||
/*
|
|
||||||
* (non-Javadoc)
|
|
||||||
*
|
|
||||||
* @see
|
|
||||||
* com.raytheon.uf.common.datastorage.hdf5.LineSelectionRead#pruneIndices
|
|
||||||
* (com.raytheon.uf.common.datastorage.Request, long[])
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
protected void pruneIndices(Request request, long[] originalDims) {
|
|
||||||
int[] indices = request.getIndices();
|
|
||||||
int[] prunedIndices = new int[indices.length];
|
|
||||||
int k = 0;
|
|
||||||
for (int i = 0; i < indices.length; i++) {
|
|
||||||
if (indices[i] < originalDims[1] && indices[i] >= 0) {
|
|
||||||
prunedIndices[k] = indices[i];
|
|
||||||
k++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
indices = new int[k];
|
|
||||||
System.arraycopy(prunedIndices, 0, indices, 0, k);
|
|
||||||
request.setIndices(indices);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected void selectSet(int memspace, int dataspace, Request request,
|
|
||||||
long[] dims, long[] originalDatasetDims) throws HDF5Exception,
|
|
||||||
HDF5LibraryException, StorageException {
|
|
||||||
int[] points = request.getIndices();
|
|
||||||
|
|
||||||
long[] memOffset = new long[] { 0, 0 };
|
|
||||||
for (int i = 0; i < points.length; i++) {
|
|
||||||
long[] start = new long[] { 0, points[i] };
|
|
||||||
long[] count = new long[] { originalDatasetDims[0], 1 };
|
|
||||||
|
|
||||||
if (i == 0) {
|
|
||||||
H5.H5Sselect_hyperslab(dataspace,
|
|
||||||
HDF5Constants.H5S_SELECT_SET, start, null, count,
|
|
||||||
null);
|
|
||||||
H5.H5Sselect_hyperslab(memspace,
|
|
||||||
HDF5Constants.H5S_SELECT_SET, memOffset, null,
|
|
||||||
count, null);
|
|
||||||
} else {
|
|
||||||
memOffset[1] += 1;
|
|
||||||
H5.H5Sselect_hyperslab(dataspace,
|
|
||||||
HDF5Constants.H5S_SELECT_OR, start, null, count,
|
|
||||||
null);
|
|
||||||
H5.H5Sselect_hyperslab(memspace,
|
|
||||||
HDF5Constants.H5S_SELECT_OR, memOffset, null,
|
|
||||||
count, null);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* (non-Javadoc)
|
|
||||||
*
|
|
||||||
* @see
|
|
||||||
* com.raytheon.uf.common.datastorage.hdf5.LineSelectionRead#calculateSize
|
|
||||||
* (int, int, com.raytheon.uf.common.datastorage.Request, long[], int)
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
protected int calculateSize(int dataspace, int sz, Request request,
|
|
||||||
long[] dims, long[] originalDims, int totalSize)
|
|
||||||
throws HDF5LibraryException {
|
|
||||||
pruneIndices(request, originalDims);
|
|
||||||
dims[1] = request.getIndices().length;
|
|
||||||
return (int) (super.calculateSize(dataspace, sz, request, dims,
|
|
||||||
originalDims, totalSize) * dims[0]);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
@ -1,158 +0,0 @@
|
||||||
/**
|
|
||||||
* This software was developed and / or modified by Raytheon Company,
|
|
||||||
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
|
|
||||||
*
|
|
||||||
* U.S. EXPORT CONTROLLED TECHNICAL DATA
|
|
||||||
* This software product contains export-restricted data whose
|
|
||||||
* export/transfer/disclosure is restricted by U.S. law. Dissemination
|
|
||||||
* to non-U.S. persons whether in the United States or abroad requires
|
|
||||||
* an export license or other authorization.
|
|
||||||
*
|
|
||||||
* Contractor Name: Raytheon Company
|
|
||||||
* Contractor Address: 6825 Pine Street, Suite 340
|
|
||||||
* Mail Stop B8
|
|
||||||
* Omaha, NE 68106
|
|
||||||
* 402.291.0100
|
|
||||||
*
|
|
||||||
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
|
|
||||||
* further licensing information.
|
|
||||||
**/
|
|
||||||
package com.raytheon.uf.common.datastorage.hdf5;
|
|
||||||
|
|
||||||
import java.util.Collection;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
import ncsa.hdf.hdf5lib.HDF5Constants;
|
|
||||||
|
|
||||||
import com.raytheon.uf.common.datastorage.records.ByteDataRecord;
|
|
||||||
import com.raytheon.uf.common.datastorage.records.FloatDataRecord;
|
|
||||||
import com.raytheon.uf.common.datastorage.records.IDataRecord;
|
|
||||||
import com.raytheon.uf.common.datastorage.records.IntegerDataRecord;
|
|
||||||
import com.raytheon.uf.common.datastorage.records.LongDataRecord;
|
|
||||||
import com.raytheon.uf.common.datastorage.records.ShortDataRecord;
|
|
||||||
import com.raytheon.uf.common.datastorage.records.StringDataRecord;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Enum to manage HDF5 data types.
|
|
||||||
*
|
|
||||||
* <pre>
|
|
||||||
*
|
|
||||||
* SOFTWARE HISTORY
|
|
||||||
*
|
|
||||||
* Date Ticket# Engineer Description
|
|
||||||
* ------------ ---------- ----------- --------------------------
|
|
||||||
* Feb 12, 2007 chammack Initial Creation. (HDF5DataStore)
|
|
||||||
* 20070914 379 jkorman Refactored from HDFDataStore.
|
|
||||||
* </pre>
|
|
||||||
*
|
|
||||||
* @author chammack
|
|
||||||
* @version 1
|
|
||||||
*/
|
|
||||||
public enum LocalHDFDataType {
|
|
||||||
|
|
||||||
BYTE(HDF5Constants.H5T_NATIVE_INT8, "BYTE"), SHORT(
|
|
||||||
HDF5Constants.H5T_NATIVE_SHORT, "SHORT"), INT(
|
|
||||||
HDF5Constants.H5T_NATIVE_INT, "INT"), LONG(
|
|
||||||
HDF5Constants.H5T_NATIVE_INT64, "LONG"), FLOAT(
|
|
||||||
HDF5Constants.H5T_NATIVE_FLOAT, "FLOAT"), DOUBLE(
|
|
||||||
HDF5Constants.H5T_NATIVE_DOUBLE, "DOUBLE"), STRING(
|
|
||||||
HDF5Constants.H5T_STR_NULLTERM, "STRING");
|
|
||||||
|
|
||||||
private final int hdfNativeDataType;
|
|
||||||
|
|
||||||
private final String localName;
|
|
||||||
|
|
||||||
private static final Map<Integer, LocalHDFDataType> typeMap = new HashMap<Integer, LocalHDFDataType>();
|
|
||||||
static {
|
|
||||||
typeMap.put(BYTE.getHDFNativeType(), BYTE);
|
|
||||||
typeMap.put(SHORT.getHDFNativeType(), SHORT);
|
|
||||||
typeMap.put(INT.getHDFNativeType(), INT);
|
|
||||||
typeMap.put(LONG.getHDFNativeType(), LONG);
|
|
||||||
typeMap.put(FLOAT.getHDFNativeType(), FLOAT);
|
|
||||||
typeMap.put(DOUBLE.getHDFNativeType(), DOUBLE);
|
|
||||||
typeMap.put(STRING.getHDFNativeType(), STRING);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static final Map<Class<? extends IDataRecord>, LocalHDFDataType> dataRecordMap = new HashMap<Class<? extends IDataRecord>, LocalHDFDataType>();
|
|
||||||
static {
|
|
||||||
dataRecordMap.put(FloatDataRecord.class, FLOAT);
|
|
||||||
dataRecordMap.put(IntegerDataRecord.class, INT);
|
|
||||||
dataRecordMap.put(ShortDataRecord.class, SHORT);
|
|
||||||
dataRecordMap.put(LongDataRecord.class, LONG);
|
|
||||||
dataRecordMap.put(StringDataRecord.class, STRING);
|
|
||||||
dataRecordMap.put(ByteDataRecord.class, BYTE);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Construct an instance of this class.
|
|
||||||
*
|
|
||||||
* @param hdfDataType
|
|
||||||
* The HDF5 data type.
|
|
||||||
* @param name
|
|
||||||
* The name of this instance.
|
|
||||||
*/
|
|
||||||
private LocalHDFDataType(int hdfDataType, String name) {
|
|
||||||
this.hdfNativeDataType = hdfDataType;
|
|
||||||
localName = name;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the HDF5 data type associated with this instance.
|
|
||||||
*
|
|
||||||
* @return The HDF5 data type.
|
|
||||||
*/
|
|
||||||
public int getHDFNativeType() {
|
|
||||||
return hdfNativeDataType;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the string representation of this instance.
|
|
||||||
*
|
|
||||||
* @return The string representation.
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
public String toString() {
|
|
||||||
return localName;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the LocalHDFDataType enum corresponding to a specified HDF5 data
|
|
||||||
* type. If the data type is not defined, a null is returned.
|
|
||||||
*
|
|
||||||
* @param hdfDataType
|
|
||||||
* HDF5 data type to find.
|
|
||||||
* @return The LocalHDFDataType if defined, null otherwise.
|
|
||||||
*/
|
|
||||||
public static LocalHDFDataType getCorrespondingEnum(int hdfDataType) {
|
|
||||||
return typeMap.get(hdfDataType);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the LocalHDFDataType enum corresponding to a specified data type
|
|
||||||
* name. If the data type is not defined, a null is returned.
|
|
||||||
*
|
|
||||||
* @param typeName
|
|
||||||
* hdfDataType HDF5 data type name to find.
|
|
||||||
* @return The LocalHDFDataType if defined, null otherwise.
|
|
||||||
*/
|
|
||||||
public static LocalHDFDataType getCorrespondingEnum(String typeName) {
|
|
||||||
LocalHDFDataType dataType = null;
|
|
||||||
Collection<LocalHDFDataType> dataTypes = typeMap.values();
|
|
||||||
for (LocalHDFDataType type : dataTypes) {
|
|
||||||
if (type.localName.equals(typeName)) {
|
|
||||||
dataType = type;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return dataType;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static LocalHDFDataType getCorrespondingEnum(IDataRecord rec) {
|
|
||||||
if (rec == null) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return dataRecordMap.get(rec.getClass());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,99 +0,0 @@
|
||||||
/**
|
|
||||||
* This software was developed and / or modified by Raytheon Company,
|
|
||||||
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
|
|
||||||
*
|
|
||||||
* U.S. EXPORT CONTROLLED TECHNICAL DATA
|
|
||||||
* This software product contains export-restricted data whose
|
|
||||||
* export/transfer/disclosure is restricted by U.S. law. Dissemination
|
|
||||||
* to non-U.S. persons whether in the United States or abroad requires
|
|
||||||
* an export license or other authorization.
|
|
||||||
*
|
|
||||||
* Contractor Name: Raytheon Company
|
|
||||||
* Contractor Address: 6825 Pine Street, Suite 340
|
|
||||||
* Mail Stop B8
|
|
||||||
* Omaha, NE 68106
|
|
||||||
* 402.291.0100
|
|
||||||
*
|
|
||||||
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
|
|
||||||
* further licensing information.
|
|
||||||
**/
|
|
||||||
package com.raytheon.uf.common.datastorage.hdf5;
|
|
||||||
|
|
||||||
import java.awt.Point;
|
|
||||||
|
|
||||||
import ncsa.hdf.hdf5lib.HDF5Constants;
|
|
||||||
import ncsa.hdf.hdf5lib.exceptions.HDF5Exception;
|
|
||||||
import ncsa.hdf.hdf5lib.exceptions.HDF5LibraryException;
|
|
||||||
|
|
||||||
import com.raytheon.uf.common.datastorage.Request;
|
|
||||||
import com.raytheon.uf.common.datastorage.StorageException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Selects data from an hdf dataset along a discrete set of points
|
|
||||||
*
|
|
||||||
* Currently this supports either 1D or 2D selection (the java.awt.Point class
|
|
||||||
* is used, which is only at max two dimensional). In one dimensional cases, the
|
|
||||||
* y value is ignored.
|
|
||||||
*
|
|
||||||
* <pre>
|
|
||||||
*
|
|
||||||
* SOFTWARE HISTORY
|
|
||||||
* Date Ticket# Engineer Description
|
|
||||||
* ------------ ---------- ----------- --------------------------
|
|
||||||
* Jul 27, 2009 chammack Initial creation
|
|
||||||
*
|
|
||||||
* </pre>
|
|
||||||
*
|
|
||||||
* @author chammack
|
|
||||||
* @version 1.0
|
|
||||||
*/
|
|
||||||
|
|
||||||
public class PointSelectionRead extends AbstractHDFRead {
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected int calculateSize(int dataspace, int sz, Request request,
|
|
||||||
long[] dims, long[] originalDims, int totalSize)
|
|
||||||
throws HDF5LibraryException {
|
|
||||||
Point[] points = request.getPoints();
|
|
||||||
|
|
||||||
dims[0] = points.length;
|
|
||||||
if (sz > 1) {
|
|
||||||
dims[1] = 1;
|
|
||||||
}
|
|
||||||
for (int i = 0; i < dims.length; i++) {
|
|
||||||
totalSize *= dims[i];
|
|
||||||
}
|
|
||||||
return totalSize;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected void selectSet(int memspace, int dataspace, Request request,
|
|
||||||
long[] dims, long[] originalDatasetDims) throws HDF5Exception,
|
|
||||||
HDF5LibraryException, StorageException {
|
|
||||||
long[][] coords;
|
|
||||||
Point[] points = request.getPoints();
|
|
||||||
|
|
||||||
if (dims.length == 2) {
|
|
||||||
coords = new long[points.length * (int) dims[1]][dims.length];
|
|
||||||
} else {
|
|
||||||
coords = new long[points.length][dims.length];
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int i = 0; i < points.length; i++) {
|
|
||||||
|
|
||||||
if (dims.length == 2) {
|
|
||||||
coords[i][0] = points[i].y;
|
|
||||||
coords[i][1] = points[i].x;
|
|
||||||
} else {
|
|
||||||
coords[i][0] = points[i].x;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Below is a performance optimization due to
|
|
||||||
// limitations in hdf5
|
|
||||||
byte[] coordsAsBytes = longToBytes(coords);
|
|
||||||
invokeH5Sselect_elements(dataspace, HDF5Constants.H5S_SELECT_SET,
|
|
||||||
coords.length, coordsAsBytes);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
@ -1,92 +0,0 @@
|
||||||
/**
|
|
||||||
* This software was developed and / or modified by Raytheon Company,
|
|
||||||
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
|
|
||||||
*
|
|
||||||
* U.S. EXPORT CONTROLLED TECHNICAL DATA
|
|
||||||
* This software product contains export-restricted data whose
|
|
||||||
* export/transfer/disclosure is restricted by U.S. law. Dissemination
|
|
||||||
* to non-U.S. persons whether in the United States or abroad requires
|
|
||||||
* an export license or other authorization.
|
|
||||||
*
|
|
||||||
* Contractor Name: Raytheon Company
|
|
||||||
* Contractor Address: 6825 Pine Street, Suite 340
|
|
||||||
* Mail Stop B8
|
|
||||||
* Omaha, NE 68106
|
|
||||||
* 402.291.0100
|
|
||||||
*
|
|
||||||
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
|
|
||||||
* further licensing information.
|
|
||||||
**/
|
|
||||||
package com.raytheon.uf.common.datastorage.hdf5;
|
|
||||||
|
|
||||||
import ncsa.hdf.hdf5lib.H5;
|
|
||||||
import ncsa.hdf.hdf5lib.HDF5Constants;
|
|
||||||
import ncsa.hdf.hdf5lib.exceptions.HDF5Exception;
|
|
||||||
import ncsa.hdf.hdf5lib.exceptions.HDF5LibraryException;
|
|
||||||
|
|
||||||
import com.raytheon.uf.common.datastorage.Request;
|
|
||||||
import com.raytheon.uf.common.datastorage.StorageException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Read a "stripe" of data. This is much more efficient than reading by points
|
|
||||||
* as it tells the HDF5 library to do a continuous read.
|
|
||||||
*
|
|
||||||
* <pre>
|
|
||||||
*
|
|
||||||
* SOFTWARE HISTORY
|
|
||||||
* Date Ticket# Engineer Description
|
|
||||||
* ------------ ---------- ----------- --------------------------
|
|
||||||
* Jul 27, 2009 chammack Initial creation
|
|
||||||
*
|
|
||||||
* </pre>
|
|
||||||
*
|
|
||||||
* @author chammack
|
|
||||||
* @version 1.0
|
|
||||||
*/
|
|
||||||
|
|
||||||
public class SlabSelectionRead extends AbstractHDFRead {
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected int calculateSize(int dataspace, int sz, Request request,
|
|
||||||
long[] dims, long[] originalDims, int totalSize)
|
|
||||||
throws HDF5LibraryException {
|
|
||||||
int[] minIndex = request.getMinIndexForSlab();
|
|
||||||
int[] maxIndex = request.getMaxIndexForSlab();
|
|
||||||
|
|
||||||
if (minIndex != null && maxIndex != null) {
|
|
||||||
dims[0] = maxIndex[1] - minIndex[1];
|
|
||||||
dims[1] = maxIndex[0] - minIndex[0];
|
|
||||||
totalSize = (int) ((dims[0]) * dims[1]);
|
|
||||||
} else {
|
|
||||||
H5.H5Sget_simple_extent_dims(dataspace, dims, (long[]) null);
|
|
||||||
for (long dim : dims) {
|
|
||||||
totalSize *= (int) dim;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return totalSize;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected void selectSet(int memspace, int dataspace, Request request,
|
|
||||||
long[] dims, long[] originalDatasetDims) throws HDF5Exception,
|
|
||||||
HDF5LibraryException, StorageException {
|
|
||||||
long[] offset = new long[2];
|
|
||||||
long[] count = new long[2];
|
|
||||||
|
|
||||||
int[] minIndex = request.getMinIndexForSlab();
|
|
||||||
int[] maxIndex = request.getMaxIndexForSlab();
|
|
||||||
|
|
||||||
if ((minIndex[0] > maxIndex[0]) || (minIndex[1] > maxIndex[1])) {
|
|
||||||
throw new StorageException("Minimum slab dimension exceeds Maximum dimension, request failed", null);
|
|
||||||
}
|
|
||||||
|
|
||||||
offset[0] = minIndex[1];
|
|
||||||
offset[1] = minIndex[0];
|
|
||||||
count[0] = maxIndex[1] - minIndex[1];
|
|
||||||
count[1] = maxIndex[0] - minIndex[0];
|
|
||||||
|
|
||||||
H5.H5Sselect_hyperslab(dataspace, HDF5Constants.H5S_SELECT_SET, offset,
|
|
||||||
null, count, null);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
@ -1,65 +0,0 @@
|
||||||
/**
|
|
||||||
* This software was developed and / or modified by Raytheon Company,
|
|
||||||
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
|
|
||||||
*
|
|
||||||
* U.S. EXPORT CONTROLLED TECHNICAL DATA
|
|
||||||
* This software product contains export-restricted data whose
|
|
||||||
* export/transfer/disclosure is restricted by U.S. law. Dissemination
|
|
||||||
* to non-U.S. persons whether in the United States or abroad requires
|
|
||||||
* an export license or other authorization.
|
|
||||||
*
|
|
||||||
* Contractor Name: Raytheon Company
|
|
||||||
* Contractor Address: 6825 Pine Street, Suite 340
|
|
||||||
* Mail Stop B8
|
|
||||||
* Omaha, NE 68106
|
|
||||||
* 402.291.0100
|
|
||||||
*
|
|
||||||
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
|
|
||||||
* further licensing information.
|
|
||||||
**/
|
|
||||||
package com.raytheon.uf.common.datastorage.hdf5;
|
|
||||||
|
|
||||||
import ncsa.hdf.hdf5lib.H5;
|
|
||||||
import ncsa.hdf.hdf5lib.exceptions.HDF5Exception;
|
|
||||||
import ncsa.hdf.hdf5lib.exceptions.HDF5LibraryException;
|
|
||||||
|
|
||||||
import com.raytheon.uf.common.datastorage.Request;
|
|
||||||
import com.raytheon.uf.common.datastorage.StorageException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Implements the whole dataset read.
|
|
||||||
*
|
|
||||||
* <pre>
|
|
||||||
*
|
|
||||||
* SOFTWARE HISTORY
|
|
||||||
* Date Ticket# Engineer Description
|
|
||||||
* ------------ ---------- ----------- --------------------------
|
|
||||||
* Jul 27, 2009 chammack Initial creation
|
|
||||||
*
|
|
||||||
* </pre>
|
|
||||||
*
|
|
||||||
* @author chammack
|
|
||||||
* @version 1.0
|
|
||||||
*/
|
|
||||||
|
|
||||||
public class WholeDatasetSelectionRead extends AbstractHDFRead {
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected int calculateSize(int dataspace, int sz, Request request,
|
|
||||||
long[] dims, long[] originalDims, int totalSize)
|
|
||||||
throws HDF5LibraryException {
|
|
||||||
H5.H5Sget_simple_extent_dims(dataspace, dims, (long[]) null);
|
|
||||||
for (long dim : dims) {
|
|
||||||
totalSize *= (int) dim;
|
|
||||||
}
|
|
||||||
return totalSize;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected void selectSet(int memspace, int dataspace, Request request,
|
|
||||||
long[] dims, long[] originalDatasetDims) throws HDF5Exception,
|
|
||||||
HDF5LibraryException, StorageException {
|
|
||||||
// no op
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,121 +0,0 @@
|
||||||
/**
|
|
||||||
* This software was developed and / or modified by Raytheon Company,
|
|
||||||
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
|
|
||||||
*
|
|
||||||
* U.S. EXPORT CONTROLLED TECHNICAL DATA
|
|
||||||
* This software product contains export-restricted data whose
|
|
||||||
* export/transfer/disclosure is restricted by U.S. law. Dissemination
|
|
||||||
* to non-U.S. persons whether in the United States or abroad requires
|
|
||||||
* an export license or other authorization.
|
|
||||||
*
|
|
||||||
* Contractor Name: Raytheon Company
|
|
||||||
* Contractor Address: 6825 Pine Street, Suite 340
|
|
||||||
* Mail Stop B8
|
|
||||||
* Omaha, NE 68106
|
|
||||||
* 402.291.0100
|
|
||||||
*
|
|
||||||
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
|
|
||||||
* further licensing information.
|
|
||||||
**/
|
|
||||||
|
|
||||||
package com.raytheon.uf.common.datastorage.hdf5;
|
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.lang.Thread.State;
|
|
||||||
import java.util.Random;
|
|
||||||
|
|
||||||
import com.raytheon.uf.common.datastorage.DataStoreFactory;
|
|
||||||
import com.raytheon.uf.common.datastorage.IDataStore;
|
|
||||||
import com.raytheon.uf.common.datastorage.records.FloatDataRecord;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Test locking mechanism on HDF5
|
|
||||||
*
|
|
||||||
* Start two instance of this program and check for errors
|
|
||||||
*
|
|
||||||
* <pre>
|
|
||||||
*
|
|
||||||
* SOFTWARE HISTORY
|
|
||||||
*
|
|
||||||
* Date Ticket# Engineer Description
|
|
||||||
* ------------ ---------- ----------- --------------------------
|
|
||||||
* Feb 20, 2007 chammack Initial Creation.
|
|
||||||
*
|
|
||||||
* </pre>
|
|
||||||
*
|
|
||||||
* @author chammack
|
|
||||||
* @version 1
|
|
||||||
*/
|
|
||||||
public class TestHDF5Locking {
|
|
||||||
|
|
||||||
private static File TEST_FILE = new File("/tmp/test.hdf");
|
|
||||||
|
|
||||||
public static void main(String[] args) {
|
|
||||||
Random rand = new Random(System.currentTimeMillis());
|
|
||||||
int uniqueId = rand.nextInt();
|
|
||||||
|
|
||||||
Thread thread1 = new Thread(new HDF5Writer(uniqueId, 1));
|
|
||||||
Thread thread2 = new Thread(new HDF5Writer(uniqueId, 2));
|
|
||||||
|
|
||||||
try {
|
|
||||||
Thread.sleep(3000);
|
|
||||||
} catch (InterruptedException e1) {
|
|
||||||
// TODO Auto-generated catch block
|
|
||||||
e1.printStackTrace();
|
|
||||||
}
|
|
||||||
|
|
||||||
thread1.start();
|
|
||||||
thread2.start();
|
|
||||||
while (thread1.getState() != State.TERMINATED
|
|
||||||
|| thread2.getState() != State.TERMINATED) {
|
|
||||||
try {
|
|
||||||
Thread.sleep(100);
|
|
||||||
} catch (InterruptedException e) {
|
|
||||||
// TODO Auto-generated catch block
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
System.out.println("Complete");
|
|
||||||
System.exit(0);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
public static class HDF5Writer implements Runnable {
|
|
||||||
|
|
||||||
private int progId;
|
|
||||||
|
|
||||||
private int instanceId;
|
|
||||||
|
|
||||||
public HDF5Writer(int progId, int instanceId) {
|
|
||||||
this.progId = progId;
|
|
||||||
this.instanceId = instanceId;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* (non-Javadoc)
|
|
||||||
*
|
|
||||||
* @see java.lang.Runnable#run()
|
|
||||||
*/
|
|
||||||
public void run() {
|
|
||||||
IDataStore dataStore = DataStoreFactory.getDataStore(TEST_FILE);
|
|
||||||
|
|
||||||
float[] dummyFloatData = new float[1024];
|
|
||||||
|
|
||||||
for (int i = 0; i < 5000; i++) {
|
|
||||||
FloatDataRecord fdr = new FloatDataRecord("" + i, progId + "/"
|
|
||||||
+ instanceId, dummyFloatData);
|
|
||||||
try {
|
|
||||||
dataStore.addDataRecord(fdr);
|
|
||||||
dataStore.store();
|
|
||||||
// dataStore.store("/");
|
|
||||||
} catch (Exception e) {
|
|
||||||
// TODO Auto-generated catch block
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
@ -1,142 +0,0 @@
|
||||||
/**
|
|
||||||
* This software was developed and / or modified by Raytheon Company,
|
|
||||||
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
|
|
||||||
*
|
|
||||||
* U.S. EXPORT CONTROLLED TECHNICAL DATA
|
|
||||||
* This software product contains export-restricted data whose
|
|
||||||
* export/transfer/disclosure is restricted by U.S. law. Dissemination
|
|
||||||
* to non-U.S. persons whether in the United States or abroad requires
|
|
||||||
* an export license or other authorization.
|
|
||||||
*
|
|
||||||
* Contractor Name: Raytheon Company
|
|
||||||
* Contractor Address: 6825 Pine Street, Suite 340
|
|
||||||
* Mail Stop B8
|
|
||||||
* Omaha, NE 68106
|
|
||||||
* 402.291.0100
|
|
||||||
*
|
|
||||||
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
|
|
||||||
* further licensing information.
|
|
||||||
**/
|
|
||||||
|
|
||||||
package com.raytheon.uf.common.datastorage.hdf5;
|
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.lang.Thread.State;
|
|
||||||
|
|
||||||
import junit.framework.Assert;
|
|
||||||
|
|
||||||
import com.raytheon.uf.common.datastorage.locking.ClusteredLockManager;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* A Lock Manager test:
|
|
||||||
*
|
|
||||||
* Tests lock manager running inside the same JVM
|
|
||||||
*
|
|
||||||
* <pre>
|
|
||||||
*
|
|
||||||
* SOFTWARE HISTORY
|
|
||||||
* Date Ticket# Engineer Description
|
|
||||||
* ------------ ---------- ----------- --------------------------
|
|
||||||
* Feb 20, 2007 chammack Initial Creation.
|
|
||||||
*
|
|
||||||
* </pre>
|
|
||||||
*
|
|
||||||
* @author chammack
|
|
||||||
* @version 1
|
|
||||||
*/
|
|
||||||
public class TestLockManager {
|
|
||||||
|
|
||||||
public static void main(String[] args) {
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Test #1: Single Thread
|
|
||||||
|
|
||||||
ClusteredLockManager lockMgr = ClusteredLockManager.getInstance();
|
|
||||||
|
|
||||||
boolean gotLock = lockMgr.getLock(new File("/tmp/foo"), true);
|
|
||||||
if (gotLock == false) {
|
|
||||||
System.out.println("ERROR: Lock was not granted initially");
|
|
||||||
}
|
|
||||||
|
|
||||||
gotLock = lockMgr.getLock(new File("/tmp/foo"), true);
|
|
||||||
if (gotLock == true) {
|
|
||||||
System.out.println("ERROR: Lock should not have been granted");
|
|
||||||
}
|
|
||||||
|
|
||||||
lockMgr.releaseLock(new File("/tmp/foo"));
|
|
||||||
|
|
||||||
// Release and retry
|
|
||||||
gotLock = lockMgr.getLock(new File("/tmp/foo"), true);
|
|
||||||
if (gotLock == false) {
|
|
||||||
System.out.println("ERROR: Lock was not granted after unlock");
|
|
||||||
}
|
|
||||||
|
|
||||||
lockMgr.releaseLock(new File("/tmp/foo"));
|
|
||||||
} catch (Exception e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
Assert.fail(e.getMessage());
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test #2 Multi-thread
|
|
||||||
System.out
|
|
||||||
.println("Threaded test: Threads should both return approximately 0.5 load");
|
|
||||||
|
|
||||||
LockTester lt1 = new LockTester(1);
|
|
||||||
LockTester lt2 = new LockTester(2);
|
|
||||||
|
|
||||||
Thread thread1 = new Thread(lt1);
|
|
||||||
Thread thread2 = new Thread(lt2);
|
|
||||||
|
|
||||||
thread1.start();
|
|
||||||
thread2.start();
|
|
||||||
|
|
||||||
System.out.println("Running test: This will take a few seconds...");
|
|
||||||
while (thread1.getState() != State.TERMINATED
|
|
||||||
|| thread2.getState() != State.TERMINATED) {
|
|
||||||
try {
|
|
||||||
Thread.sleep(100);
|
|
||||||
} catch (InterruptedException e) {
|
|
||||||
// TODO Auto-generated catch block
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
System.exit(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static class LockTester implements Runnable {
|
|
||||||
|
|
||||||
private int id;
|
|
||||||
|
|
||||||
public LockTester(int id) {
|
|
||||||
this.id = id;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void run() {
|
|
||||||
try {
|
|
||||||
|
|
||||||
ClusteredLockManager lockMgr = ClusteredLockManager
|
|
||||||
.getInstance();
|
|
||||||
|
|
||||||
int hits = 0;
|
|
||||||
for (int i = 0; i < 1000; i++) {
|
|
||||||
boolean gotLock = lockMgr.getLock(new File("/tmp/foo"),
|
|
||||||
true);
|
|
||||||
Thread.sleep(10);
|
|
||||||
if (gotLock) {
|
|
||||||
lockMgr.releaseLock(new File("/tmp/foo"));
|
|
||||||
hits++;
|
|
||||||
}
|
|
||||||
Thread.sleep(10);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
System.out.println("Thread #" + id + ":: " + (hits) / 1000.0);
|
|
||||||
} catch (Exception e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
@ -1,123 +0,0 @@
|
||||||
/**
|
|
||||||
* This software was developed and / or modified by Raytheon Company,
|
|
||||||
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
|
|
||||||
*
|
|
||||||
* U.S. EXPORT CONTROLLED TECHNICAL DATA
|
|
||||||
* This software product contains export-restricted data whose
|
|
||||||
* export/transfer/disclosure is restricted by U.S. law. Dissemination
|
|
||||||
* to non-U.S. persons whether in the United States or abroad requires
|
|
||||||
* an export license or other authorization.
|
|
||||||
*
|
|
||||||
* Contractor Name: Raytheon Company
|
|
||||||
* Contractor Address: 6825 Pine Street, Suite 340
|
|
||||||
* Mail Stop B8
|
|
||||||
* Omaha, NE 68106
|
|
||||||
* 402.291.0100
|
|
||||||
*
|
|
||||||
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
|
|
||||||
* further licensing information.
|
|
||||||
**/
|
|
||||||
|
|
||||||
package com.raytheon.uf.common.datastorage.hdf5;
|
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.lang.Thread.State;
|
|
||||||
|
|
||||||
import com.raytheon.uf.common.datastorage.locking.ClusteredLockManager;
|
|
||||||
import com.raytheon.uf.common.datastorage.locking.LockException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clustered version of TestLockManager. This requires multicast.
|
|
||||||
*
|
|
||||||
* Run two copies of this program at the same time.
|
|
||||||
*
|
|
||||||
* Should yield approximately 0.5 on each program.
|
|
||||||
*
|
|
||||||
*
|
|
||||||
* <pre>
|
|
||||||
* SOFTWARE HISTORY
|
|
||||||
* Date Ticket# Engineer Description
|
|
||||||
* ------------ ---------- ----------- --------------------------
|
|
||||||
* Feb 20, 2007 chammack Initial Creation.
|
|
||||||
*
|
|
||||||
* </pre>
|
|
||||||
*
|
|
||||||
* @author chammack
|
|
||||||
* @version 1
|
|
||||||
*/
|
|
||||||
public class TestLockManagerClustered {
|
|
||||||
|
|
||||||
public static void main(String[] args) {
|
|
||||||
|
|
||||||
try {
|
|
||||||
// ClusteredLockManager lockMgr =
|
|
||||||
// ClusteredLockManager.getInstance();
|
|
||||||
ClusteredLockManager.getInstance();
|
|
||||||
Thread.sleep(2000);
|
|
||||||
} catch (LockException e1) {
|
|
||||||
// TODO Auto-generated catch block
|
|
||||||
e1.printStackTrace();
|
|
||||||
} catch (InterruptedException e1) {
|
|
||||||
// TODO Auto-generated catch block
|
|
||||||
e1.printStackTrace();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test #1 Clustered
|
|
||||||
System.out
|
|
||||||
.println("Clustered test: Programs should both return approximately 0.5 load");
|
|
||||||
|
|
||||||
LockTester lt1 = new LockTester(1);
|
|
||||||
|
|
||||||
Thread thread1 = new Thread(lt1);
|
|
||||||
|
|
||||||
thread1.start();
|
|
||||||
|
|
||||||
System.out.println("Running test: This will take a few seconds...");
|
|
||||||
while (thread1.getState() != State.TERMINATED) {
|
|
||||||
try {
|
|
||||||
Thread.sleep(100);
|
|
||||||
} catch (InterruptedException e) {
|
|
||||||
// TODO Auto-generated catch block
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
System.exit(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static class LockTester implements Runnable {
|
|
||||||
|
|
||||||
private int id;
|
|
||||||
|
|
||||||
public LockTester(int id) {
|
|
||||||
this.id = id;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void run() {
|
|
||||||
try {
|
|
||||||
|
|
||||||
ClusteredLockManager lockMgr = ClusteredLockManager
|
|
||||||
.getInstance();
|
|
||||||
|
|
||||||
int hits = 0;
|
|
||||||
for (int i = 0; i < 1000; i++) {
|
|
||||||
boolean gotLock = lockMgr.getLock(new File("/tmp/foo"),
|
|
||||||
true);
|
|
||||||
Thread.sleep(10);
|
|
||||||
if (gotLock) {
|
|
||||||
lockMgr.releaseLock(new File("/tmp/foo"));
|
|
||||||
hits++;
|
|
||||||
}
|
|
||||||
Thread.sleep(10);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
System.out.println("Thread #" + id + ":: " + (hits) / 1000.0);
|
|
||||||
} catch (Exception e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
@ -37,7 +37,8 @@ import com.raytheon.uf.common.serialization.ISerializableObject;
|
||||||
* Feb 9, 2007 chammack Initial Creation.
|
* Feb 9, 2007 chammack Initial Creation.
|
||||||
* Apr 1, 2008 chammack Added delete API
|
* Apr 1, 2008 chammack Added delete API
|
||||||
* Aug 3, 2009 chammack Modified to support Request
|
* Aug 3, 2009 chammack Modified to support Request
|
||||||
* Sep 27, 2010 5091 njensen Added deleteFiles(String)
|
* Sep 27, 2010 5091 njensen Added deleteFiles(String)
|
||||||
|
* Feb 12, 2013 #1608 randerso Added explicit methods for deleting groups and datasets
|
||||||
*
|
*
|
||||||
* </pre>
|
* </pre>
|
||||||
*
|
*
|
||||||
|
|
@ -88,18 +89,30 @@ public interface IDataStore extends ISerializableObject {
|
||||||
public abstract StorageStatus store() throws StorageException;
|
public abstract StorageStatus store() throws StorageException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Delete a (set of) location(s), where a location is either a group or a
|
* Delete one or more datasets. If all datasets have been deleted from a
|
||||||
* dataset. If all datasets have been deleted from a file, the file will be
|
* file, the file will be deleted also.
|
||||||
* deleted also.
|
|
||||||
*
|
*
|
||||||
* @param location
|
* @param datasets
|
||||||
* the full path to the group or dataset
|
* the full path to the dataset(s) to be deleted
|
||||||
* @throws StorageException
|
* @throws StorageException
|
||||||
* if deletion fails
|
* if deletion fails
|
||||||
* @throws FileNotFoundException
|
* @throws FileNotFoundException
|
||||||
*/
|
*/
|
||||||
public abstract void delete(String... location) throws StorageException,
|
public abstract void deleteDatasets(String... datasets)
|
||||||
FileNotFoundException;
|
throws StorageException, FileNotFoundException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete one or more groups and all subgroups/datasets they contain. If all
|
||||||
|
* datasets have been deleted from a file, the file will be deleted also.
|
||||||
|
*
|
||||||
|
* @param groups
|
||||||
|
* the full path to the group(s) to be deleted
|
||||||
|
* @throws StorageException
|
||||||
|
* if deletion fails
|
||||||
|
* @throws FileNotFoundException
|
||||||
|
*/
|
||||||
|
public abstract void deleteGroups(String... groups)
|
||||||
|
throws StorageException, FileNotFoundException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Store all data records to a given data group, or replace it the group
|
* Store all data records to a given data group, or replace it the group
|
||||||
|
|
|
||||||
|
|
@ -72,6 +72,8 @@ import com.raytheon.uf.common.util.FileUtil;
|
||||||
* Oct 01, 2010 rjpeter Added logging of requests over 300ms
|
* Oct 01, 2010 rjpeter Added logging of requests over 300ms
|
||||||
* Mon 07, 2013 DR 15294 D. Friedman Stream large requests
|
* Mon 07, 2013 DR 15294 D. Friedman Stream large requests
|
||||||
* Feb 11, 2013 1526 njensen use HttpClient.postDynamicSerialize() for memory efficiency
|
* Feb 11, 2013 1526 njensen use HttpClient.postDynamicSerialize() for memory efficiency
|
||||||
|
* Feb 12, 2013 #1608 randerso Added explicit deletes for groups and datasets
|
||||||
|
*
|
||||||
* </pre>
|
* </pre>
|
||||||
*
|
*
|
||||||
* @author njensen
|
* @author njensen
|
||||||
|
|
@ -149,13 +151,29 @@ public class PyPiesDataStore implements IDataStore {
|
||||||
* (non-Javadoc)
|
* (non-Javadoc)
|
||||||
*
|
*
|
||||||
* @see
|
* @see
|
||||||
* com.raytheon.uf.common.datastorage.IDataStore#delete(java.lang.String[])
|
* com.raytheon.uf.common.datastorage.IDataStore#deleteDatasets(java.lang
|
||||||
|
* .String[])
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public void delete(final String... location) throws StorageException,
|
public void deleteDatasets(final String... datasets)
|
||||||
|
throws StorageException, FileNotFoundException {
|
||||||
|
DeleteRequest delete = new DeleteRequest();
|
||||||
|
delete.setDatasets(datasets);
|
||||||
|
sendRequest(delete);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* (non-Javadoc)
|
||||||
|
*
|
||||||
|
* @see
|
||||||
|
* com.raytheon.uf.common.datastorage.IDataStore#deleteGroups(java.lang.
|
||||||
|
* String[])
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public void deleteGroups(final String... groups) throws StorageException,
|
||||||
FileNotFoundException {
|
FileNotFoundException {
|
||||||
DeleteRequest delete = new DeleteRequest();
|
DeleteRequest delete = new DeleteRequest();
|
||||||
delete.setLocations(location);
|
delete.setGroups(groups);
|
||||||
sendRequest(delete);
|
sendRequest(delete);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -31,6 +31,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
|
||||||
* Date Ticket# Engineer Description
|
* Date Ticket# Engineer Description
|
||||||
* ------------ ---------- ----------- --------------------------
|
* ------------ ---------- ----------- --------------------------
|
||||||
* Aug 10, 2010 njensen Initial creation
|
* Aug 10, 2010 njensen Initial creation
|
||||||
|
* Feb 12, 2013 #1608 randerso Added support for explicitly deleting grous and datasets
|
||||||
*
|
*
|
||||||
* </pre>
|
* </pre>
|
||||||
*
|
*
|
||||||
|
|
@ -42,14 +43,25 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
|
||||||
public class DeleteRequest extends AbstractRequest {
|
public class DeleteRequest extends AbstractRequest {
|
||||||
|
|
||||||
@DynamicSerializeElement
|
@DynamicSerializeElement
|
||||||
private String[] locations;
|
private String[] datasets;
|
||||||
|
|
||||||
public String[] getLocations() {
|
@DynamicSerializeElement
|
||||||
return locations;
|
private String[] groups;
|
||||||
|
|
||||||
|
public String[] getDatasets() {
|
||||||
|
return datasets;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setLocations(String[] locations) {
|
public void setDatasets(String[] datasets) {
|
||||||
this.locations = locations;
|
this.datasets = datasets;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String[] getGroups() {
|
||||||
|
return groups;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setGroups(String[] groups) {
|
||||||
|
this.groups = groups;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,6 @@ Bundle-Vendor: RAYTHEON
|
||||||
Eclipse-BuddyPolicy: registered, ext, global
|
Eclipse-BuddyPolicy: registered, ext, global
|
||||||
Bundle-RequiredExecutionEnvironment: JavaSE-1.6
|
Bundle-RequiredExecutionEnvironment: JavaSE-1.6
|
||||||
Require-Bundle: com.raytheon.uf.common.datastorage;bundle-version="1.11.17",
|
Require-Bundle: com.raytheon.uf.common.datastorage;bundle-version="1.11.17",
|
||||||
com.raytheon.uf.common.datastorage.hdf5;bundle-version="1.11.17",
|
|
||||||
com.raytheon.uf.common.serialization;bundle-version="1.11.17",
|
com.raytheon.uf.common.serialization;bundle-version="1.11.17",
|
||||||
com.raytheon.uf.common.geospatial;bundle-version="1.11.17",
|
com.raytheon.uf.common.geospatial;bundle-version="1.11.17",
|
||||||
org.geotools;bundle-version="2.5.8"
|
org.geotools;bundle-version="2.5.8"
|
||||||
|
|
|
||||||
|
|
@ -35,9 +35,9 @@ import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
import com.raytheon.uf.common.datastorage.DataStoreFactory;
|
import com.raytheon.uf.common.datastorage.DataStoreFactory;
|
||||||
|
import com.raytheon.uf.common.datastorage.IDataStore;
|
||||||
import com.raytheon.uf.common.datastorage.StorageProperties;
|
import com.raytheon.uf.common.datastorage.StorageProperties;
|
||||||
import com.raytheon.uf.common.datastorage.StorageProperties.Compression;
|
import com.raytheon.uf.common.datastorage.StorageProperties.Compression;
|
||||||
import com.raytheon.uf.common.datastorage.hdf5.HDF5DataStore;
|
|
||||||
import com.raytheon.uf.common.datastorage.records.ByteDataRecord;
|
import com.raytheon.uf.common.datastorage.records.ByteDataRecord;
|
||||||
import com.raytheon.uf.common.datastorage.records.IDataRecord;
|
import com.raytheon.uf.common.datastorage.records.IDataRecord;
|
||||||
import com.raytheon.uf.common.datastorage.records.IntegerDataRecord;
|
import com.raytheon.uf.common.datastorage.records.IntegerDataRecord;
|
||||||
|
|
@ -253,8 +253,7 @@ public class TopoImporter {
|
||||||
System.exit(-1);
|
System.exit(-1);
|
||||||
}
|
}
|
||||||
|
|
||||||
HDF5DataStore store = (HDF5DataStore) DataStoreFactory
|
IDataStore store = DataStoreFactory.getDataStore(hdf);
|
||||||
.getDataStore(hdf);
|
|
||||||
|
|
||||||
String dataset = "full";
|
String dataset = "full";
|
||||||
long[] sizes = new long[] { maxCols, maxRows };
|
long[] sizes = new long[] { maxCols, maxRows };
|
||||||
|
|
|
||||||
|
|
@ -41,10 +41,10 @@ import javax.media.jai.RasterFactory;
|
||||||
import javax.media.jai.TiledImage;
|
import javax.media.jai.TiledImage;
|
||||||
|
|
||||||
import com.raytheon.uf.common.datastorage.DataStoreFactory;
|
import com.raytheon.uf.common.datastorage.DataStoreFactory;
|
||||||
|
import com.raytheon.uf.common.datastorage.IDataStore;
|
||||||
import com.raytheon.uf.common.datastorage.Request;
|
import com.raytheon.uf.common.datastorage.Request;
|
||||||
import com.raytheon.uf.common.datastorage.StorageProperties;
|
import com.raytheon.uf.common.datastorage.StorageProperties;
|
||||||
import com.raytheon.uf.common.datastorage.StorageProperties.Compression;
|
import com.raytheon.uf.common.datastorage.StorageProperties.Compression;
|
||||||
import com.raytheon.uf.common.datastorage.hdf5.HDF5DataStore;
|
|
||||||
import com.raytheon.uf.common.datastorage.records.ByteDataRecord;
|
import com.raytheon.uf.common.datastorage.records.ByteDataRecord;
|
||||||
import com.raytheon.uf.common.datastorage.records.FloatDataRecord;
|
import com.raytheon.uf.common.datastorage.records.FloatDataRecord;
|
||||||
import com.raytheon.uf.common.datastorage.records.IDataRecord;
|
import com.raytheon.uf.common.datastorage.records.IDataRecord;
|
||||||
|
|
@ -59,7 +59,9 @@ import com.raytheon.uf.common.datastorage.records.ShortDataRecord;
|
||||||
* SOFTWARE HISTORY
|
* SOFTWARE HISTORY
|
||||||
* Date Ticket# Engineer Description
|
* Date Ticket# Engineer Description
|
||||||
* ------------ ---------- ----------- --------------------------
|
* ------------ ---------- ----------- --------------------------
|
||||||
* Oct 26, 2009 randerso Initial creation
|
* Oct 26, 2009 randerso Initial creation
|
||||||
|
* Feb 12, 2013 #1608 randerso Remove exlicit references to HDF5DataStore
|
||||||
|
* Added explicit calls to deleteGroups
|
||||||
*
|
*
|
||||||
* </pre>
|
* </pre>
|
||||||
*
|
*
|
||||||
|
|
@ -72,7 +74,7 @@ public class TopoInterpolator {
|
||||||
|
|
||||||
private static final String DEFAULT_TOPO_FILE = "/topo/srtm30.hdf";
|
private static final String DEFAULT_TOPO_FILE = "/topo/srtm30.hdf";
|
||||||
|
|
||||||
private HDF5DataStore dataStore;
|
private IDataStore dataStore;
|
||||||
|
|
||||||
public TopoInterpolator() {
|
public TopoInterpolator() {
|
||||||
this(new File(DEFAULT_TOPO_FILE));
|
this(new File(DEFAULT_TOPO_FILE));
|
||||||
|
|
@ -84,7 +86,7 @@ public class TopoInterpolator {
|
||||||
* @param file
|
* @param file
|
||||||
*/
|
*/
|
||||||
public TopoInterpolator(File hdf) {
|
public TopoInterpolator(File hdf) {
|
||||||
dataStore = (HDF5DataStore) DataStoreFactory.getDataStore(hdf);
|
dataStore = DataStoreFactory.getDataStore(hdf);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -107,7 +109,7 @@ public class TopoInterpolator {
|
||||||
// remove existing interpolated datasets
|
// remove existing interpolated datasets
|
||||||
if (Arrays.asList(dataStore.getDatasets(srcGroup)).contains(
|
if (Arrays.asList(dataStore.getDatasets(srcGroup)).contains(
|
||||||
"interpolated")) {
|
"interpolated")) {
|
||||||
dataStore.delete(dstGroup);
|
dataStore.deleteGroups(dstGroup);
|
||||||
}
|
}
|
||||||
|
|
||||||
StorageProperties properties = new StorageProperties();
|
StorageProperties properties = new StorageProperties();
|
||||||
|
|
|
||||||
|
|
@ -59,13 +59,6 @@
|
||||||
version="0.0.0"
|
version="0.0.0"
|
||||||
unpack="false"/>
|
unpack="false"/>
|
||||||
|
|
||||||
<plugin
|
|
||||||
id="com.raytheon.uf.common.datastorage.hdf5"
|
|
||||||
download-size="0"
|
|
||||||
install-size="0"
|
|
||||||
version="0.0.0"
|
|
||||||
unpack="false"/>
|
|
||||||
|
|
||||||
<plugin
|
<plugin
|
||||||
id="com.raytheon.uf.common.datastorage"
|
id="com.raytheon.uf.common.datastorage"
|
||||||
download-size="0"
|
download-size="0"
|
||||||
|
|
|
||||||
|
|
@ -65,12 +65,6 @@
|
||||||
install-size="0"
|
install-size="0"
|
||||||
version="0.0.0"/>
|
version="0.0.0"/>
|
||||||
|
|
||||||
<plugin
|
|
||||||
id="ncsa.hdf5"
|
|
||||||
download-size="0"
|
|
||||||
install-size="0"
|
|
||||||
version="0.0.0"/>
|
|
||||||
|
|
||||||
<plugin
|
<plugin
|
||||||
id="net.sf.ehcache"
|
id="net.sf.ehcache"
|
||||||
download-size="0"
|
download-size="0"
|
||||||
|
|
|
||||||
|
|
@ -75,7 +75,6 @@ import com.raytheon.uf.common.spatial.reprojection.ReferencedDataRecord;
|
||||||
import com.raytheon.uf.common.time.util.TimeUtil;
|
import com.raytheon.uf.common.time.util.TimeUtil;
|
||||||
import com.raytheon.uf.common.util.FileUtil;
|
import com.raytheon.uf.common.util.FileUtil;
|
||||||
import com.raytheon.uf.edex.core.EdexException;
|
import com.raytheon.uf.edex.core.EdexException;
|
||||||
import com.raytheon.uf.edex.core.props.PropertiesFactory;
|
|
||||||
import com.raytheon.uf.edex.database.DataAccessLayerException;
|
import com.raytheon.uf.edex.database.DataAccessLayerException;
|
||||||
import com.raytheon.uf.edex.database.dao.CoreDao;
|
import com.raytheon.uf.edex.database.dao.CoreDao;
|
||||||
import com.raytheon.uf.edex.database.dao.DaoConfig;
|
import com.raytheon.uf.edex.database.dao.DaoConfig;
|
||||||
|
|
@ -105,6 +104,8 @@ import com.vividsolutions.jts.geom.Polygon;
|
||||||
* Oct 10, 2012 1261 djohnson Add some generics wildcarding.
|
* Oct 10, 2012 1261 djohnson Add some generics wildcarding.
|
||||||
* Jan 14, 2013 1469 bkowal No longer retrieves the hdf5 data directory
|
* Jan 14, 2013 1469 bkowal No longer retrieves the hdf5 data directory
|
||||||
* from the environment.
|
* from the environment.
|
||||||
|
* Feb 12, 2013 #1608 randerso Changed to call deleteDatasets
|
||||||
|
*
|
||||||
* </pre>
|
* </pre>
|
||||||
*
|
*
|
||||||
* @author bphillip
|
* @author bphillip
|
||||||
|
|
@ -884,7 +885,8 @@ public abstract class PluginDao extends CoreDao {
|
||||||
if (uris == null) {
|
if (uris == null) {
|
||||||
ds.deleteFiles(null);
|
ds.deleteFiles(null);
|
||||||
} else {
|
} else {
|
||||||
ds.delete(uris.toArray(new String[uris.size()]));
|
ds.deleteDatasets(uris.toArray(new String[uris
|
||||||
|
.size()]));
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
PurgeLogger.logError("Error occurred purging file: "
|
PurgeLogger.logError("Error occurred purging file: "
|
||||||
|
|
@ -906,7 +908,7 @@ public abstract class PluginDao extends CoreDao {
|
||||||
if (uris == null) {
|
if (uris == null) {
|
||||||
ds.deleteFiles(null);
|
ds.deleteFiles(null);
|
||||||
} else {
|
} else {
|
||||||
ds.delete(uris.toArray(new String[uris.size()]));
|
ds.deleteDatasets(uris.toArray(new String[uris.size()]));
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
PurgeLogger.logError("Error occurred purging file: "
|
PurgeLogger.logError("Error occurred purging file: "
|
||||||
|
|
|
||||||
|
|
@ -84,6 +84,7 @@ import com.vividsolutions.jts.geom.Coordinate;
|
||||||
* 04/18/2012 DR 14694 D. Friedman Fixes for static topography generation
|
* 04/18/2012 DR 14694 D. Friedman Fixes for static topography generation
|
||||||
* 05/09/2012 DR 14939 D. Friedman Fix errors in DR 14694
|
* 05/09/2012 DR 14939 D. Friedman Fix errors in DR 14694
|
||||||
* 01/14/2013 1469 bkowal Removed the hdf5 data directory
|
* 01/14/2013 1469 bkowal Removed the hdf5 data directory
|
||||||
|
* 02/12/2013 #1608 randerso Changed to call deleteDatasets
|
||||||
*
|
*
|
||||||
* </pre>
|
* </pre>
|
||||||
*
|
*
|
||||||
|
|
@ -177,8 +178,10 @@ public class StaticTopoData {
|
||||||
try {
|
try {
|
||||||
if (!topoFileExists()) {
|
if (!topoFileExists()) {
|
||||||
// TODO: This will fail in a clustered server environment
|
// TODO: This will fail in a clustered server environment
|
||||||
// since
|
// since static topo isn't installed to dx3/4
|
||||||
// static topo isn't installed to dx3/4
|
// UPDATE: this doesn't even work in a standalone
|
||||||
|
// environment now because it can't find the gzipped source
|
||||||
|
// files since FILE_PREFIX was changed
|
||||||
statusHandler.handle(Priority.INFO,
|
statusHandler.handle(Priority.INFO,
|
||||||
"Static Topo file not found. Creating it...");
|
"Static Topo file not found. Creating it...");
|
||||||
|
|
||||||
|
|
@ -344,9 +347,7 @@ public class StaticTopoData {
|
||||||
sTopoDataStore.addDataRecord(attributeSet, sp);
|
sTopoDataStore.addDataRecord(attributeSet, sp);
|
||||||
sTopoDataStore.addDataRecord(westRecord, sp);
|
sTopoDataStore.addDataRecord(westRecord, sp);
|
||||||
sTopoDataStore.store();
|
sTopoDataStore.store();
|
||||||
sTopoDataStore.delete("pac");
|
sTopoDataStore.deleteDatasets("pac", "attrpac");
|
||||||
sTopoDataStore.delete("attrpac");
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -573,12 +574,13 @@ public class StaticTopoData {
|
||||||
|
|
||||||
for (int i = 0; i < finalData.length; i++) {
|
for (int i = 0; i < finalData.length; i++) {
|
||||||
float v = finalData[i];
|
float v = finalData[i];
|
||||||
if (Float.isNaN(v))
|
if (Float.isNaN(v)) {
|
||||||
finalData[i] = TOPO_FILL;
|
finalData[i] = TOPO_FILL;
|
||||||
else if (v == DATA_FILL || (v > -0.5 && v < 0.5))
|
} else if (v == DATA_FILL || (v > -0.5 && v < 0.5)) {
|
||||||
finalData[i] = 0.0f;
|
finalData[i] = 0.0f;
|
||||||
else
|
} else {
|
||||||
finalData[i] = v;
|
finalData[i] = v;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return finalData;
|
return finalData;
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,6 @@ Require-Bundle: com.raytheon.edex.uengine,
|
||||||
com.raytheon.uf.edex.plugin.grid,
|
com.raytheon.uf.edex.plugin.grid,
|
||||||
com.raytheon.uf.common.dataplugin.grid,
|
com.raytheon.uf.common.dataplugin.grid,
|
||||||
com.raytheon.uf.common.parameter,
|
com.raytheon.uf.common.parameter,
|
||||||
com.raytheon.uf.common.datastorage.hdf5;bundle-version="1.11.22",
|
|
||||||
gov.noaa.nws.ncep.edex.common;bundle-version="1.0.0",
|
gov.noaa.nws.ncep.edex.common;bundle-version="1.0.0",
|
||||||
gov.noaa.nws.ncep.common.dataplugin.mcidas;bundle-version="1.0.0",
|
gov.noaa.nws.ncep.common.dataplugin.mcidas;bundle-version="1.0.0",
|
||||||
gov.noaa.nws.ncep.common.dataplugin.ncgrib;bundle-version="1.0.0",
|
gov.noaa.nws.ncep.common.dataplugin.ncgrib;bundle-version="1.0.0",
|
||||||
|
|
|
||||||
|
|
@ -51,8 +51,6 @@ Import-Package: com.raytheon.edex.meteoLib,
|
||||||
gov.noaa.nws.ncep.viz.ui.display,
|
gov.noaa.nws.ncep.viz.ui.display,
|
||||||
javax.measure.converter,
|
javax.measure.converter,
|
||||||
javax.measure.unit,
|
javax.measure.unit,
|
||||||
ncsa.hdf.hdf5lib,
|
|
||||||
ncsa.hdf.hdf5lib.exceptions,
|
|
||||||
org.eclipse.swt.events,
|
org.eclipse.swt.events,
|
||||||
org.eclipse.swt.graphics,
|
org.eclipse.swt.graphics,
|
||||||
org.eclipse.swt.widgets,
|
org.eclipse.swt.widgets,
|
||||||
|
|
|
||||||
|
|
@ -21,8 +21,7 @@ Require-Bundle: org.eclipse.ui,
|
||||||
gov.noaa.nws.ncep.edex.common;bundle-version="1.0.0"
|
gov.noaa.nws.ncep.edex.common;bundle-version="1.0.0"
|
||||||
Bundle-ActivationPolicy: lazy
|
Bundle-ActivationPolicy: lazy
|
||||||
Bundle-RequiredExecutionEnvironment: JavaSE-1.6
|
Bundle-RequiredExecutionEnvironment: JavaSE-1.6
|
||||||
Import-Package: com.raytheon.uf.common.datastorage.hdf5,
|
Import-Package: com.raytheon.uf.common.pypies,
|
||||||
com.raytheon.uf.common.pypies,
|
|
||||||
com.raytheon.viz.core.gl.dataformat,
|
com.raytheon.viz.core.gl.dataformat,
|
||||||
com.raytheon.viz.satellite.rsc,
|
com.raytheon.viz.satellite.rsc,
|
||||||
gov.noaa.nws.ncep.common.dataplugin.mcidas,
|
gov.noaa.nws.ncep.common.dataplugin.mcidas,
|
||||||
|
|
|
||||||
|
|
@ -23,14 +23,21 @@
|
||||||
class DeleteRequest(object):
|
class DeleteRequest(object):
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.locations = None
|
self.datasets = None
|
||||||
|
self.groups = None
|
||||||
self.filename = None
|
self.filename = None
|
||||||
|
|
||||||
def getLocations(self):
|
def getDatasets(self):
|
||||||
return self.locations
|
return self.datasets
|
||||||
|
|
||||||
def setLocations(self, locations):
|
def setDatasets(self, datasets):
|
||||||
self.locations = locations
|
self.datasets = datasets
|
||||||
|
|
||||||
|
def getGroups(self):
|
||||||
|
return self.groups
|
||||||
|
|
||||||
|
def setGroups(self, groups):
|
||||||
|
self.groups = groups
|
||||||
|
|
||||||
def getFilename(self):
|
def getFilename(self):
|
||||||
return self.filename
|
return self.filename
|
||||||
|
|
|
||||||
|
|
@ -31,7 +31,7 @@
|
||||||
# 05/03/11 9134 njensen Optimized for pointdata
|
# 05/03/11 9134 njensen Optimized for pointdata
|
||||||
# 10/09/12 rjpeter Optimized __getGroup for retrievals
|
# 10/09/12 rjpeter Optimized __getGroup for retrievals
|
||||||
# 01/17/13 DR 15294 D. Friedman Clear out data in response
|
# 01/17/13 DR 15294 D. Friedman Clear out data in response
|
||||||
#
|
# 02/12/13 #1608 randerso Added support for explicitly deleting groups and datasets
|
||||||
#
|
#
|
||||||
#
|
#
|
||||||
|
|
||||||
|
|
@ -324,18 +324,32 @@ class H5pyDataStore(IDataStore.IDataStore):
|
||||||
deleteFile = False
|
deleteFile = False
|
||||||
|
|
||||||
try:
|
try:
|
||||||
locs = request.getLocations()
|
rootNode=f['/']
|
||||||
for dataset in locs:
|
datasets = request.getDatasets()
|
||||||
ds = None
|
if datasets is not None:
|
||||||
|
for dataset in datasets:
|
||||||
|
ds = None
|
||||||
|
try :
|
||||||
|
ds = self.__getNode(f, None, dataset)
|
||||||
|
except Exception, e:
|
||||||
|
logger.warn('Unable to find uri [' + str(dataset) + '] in file [' + str(fn) + '] to delete: ' + IDataStore._exc())
|
||||||
|
|
||||||
|
if ds:
|
||||||
|
parent = ds.parent
|
||||||
|
parent.id.unlink(ds.name)
|
||||||
|
|
||||||
try :
|
groups = request.getGroups()
|
||||||
ds = self.__getNode(f, None, dataset)
|
if groups is not None:
|
||||||
except Exception, e:
|
for group in groups:
|
||||||
logger.warn('Unable to find uri [' + str(dataset) + '] in file [' + str(fn) + '] to delete: ' + IDataStore._exc())
|
gp = None
|
||||||
|
try :
|
||||||
if ds:
|
gp = self.__getNode(f, group)
|
||||||
grp = ds.parent
|
except Exception, e:
|
||||||
grp.id.unlink(ds.name)
|
logger.warn('Unable to find uri [' + str(group) + '] in file [' + str(fn) + '] to delete: ' + IDataStore._exc())
|
||||||
|
|
||||||
|
if gp:
|
||||||
|
parent = gp.parent
|
||||||
|
parent.id.unlink(gp.name)
|
||||||
|
|
||||||
finally:
|
finally:
|
||||||
# check if file has any remaining data sets
|
# check if file has any remaining data sets
|
||||||
|
|
@ -353,6 +367,7 @@ class H5pyDataStore(IDataStore.IDataStore):
|
||||||
timeMap['closeFile']=t1-t0
|
timeMap['closeFile']=t1-t0
|
||||||
|
|
||||||
if deleteFile:
|
if deleteFile:
|
||||||
|
logger.info('Removing empty file ['+ str(fn) + ']')
|
||||||
try:
|
try:
|
||||||
os.remove(fn)
|
os.remove(fn)
|
||||||
except Exception, e:
|
except Exception, e:
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue