xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge "Omaha #4167 Propagrate hdf5 store errors StaticDataGenerator." into omaha_16.1.1

Browse source 
Former-commit-id: 8a5d42baec39079d3f5e350370db65c55030d9e0
This commit is contained in:
Nate Jensen 2015-04-30 09:19:03 -05:00 • committed by Gerrit Code Review
commit b546751b9d
3 changed files with 60 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
edexOsgi/com.raytheon.uf.edex.grid.staticdata/META-INF/MANIFEST.MF
View file

@ -2,7 +2,7 @@ Manifest-Version: 1.0
Bundle-ManifestVersion: 2 Bundle-ManifestVersion: 2
Bundle-Name: Grid Static Data Bundle-Name: Grid Static Data
Bundle-SymbolicName: com.raytheon.uf.edex.grid.staticdata Bundle-SymbolicName: com.raytheon.uf.edex.grid.staticdata
Bundle-Version: 1.14.1.qualifier Bundle-Version: 1.15.0.qualifier
Bundle-Vendor: RAYTHEON Bundle-Vendor: RAYTHEON
Bundle-RequiredExecutionEnvironment: JavaSE-1.7 Bundle-RequiredExecutionEnvironment: JavaSE-1.7
Require-Bundle: javax.measure, Require-Bundle: javax.measure,

23
edexOsgi/com.raytheon.uf.edex.grid.staticdata/src/com/raytheon/uf/edex/grid/staticdata/StaticDataGenerator.java
View file

@ -1,19 +1,19 @@
/** /**
* This software was developed and / or modified by Raytheon Company, * This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government. * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
* *
* U.S. EXPORT CONTROLLED TECHNICAL DATA * U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose * This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination * export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires * to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization. * an export license or other authorization.
* *
* Contractor Name: Raytheon Company * Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340 * Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8 * Mail Stop B8
* Omaha, NE 68106 * Omaha, NE 68106
* 402.291.0100 * 402.291.0100
* *
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information. * further licensing information.
**/ **/
@ -72,12 +72,13 @@ import com.raytheon.uf.edex.plugin.grid.dao.GridDao;
* Feb 15, 2013 1638 mschenke Moved DataURINotificationMessage to uf.common.dataplugin * Feb 15, 2013 1638 mschenke Moved DataURINotificationMessage to uf.common.dataplugin
* Mar 07, 2013 1587 bsteffen rewrite static data generation. * Mar 07, 2013 1587 bsteffen rewrite static data generation.
* Mar 14, 2013 1587 bsteffen Fix persisting to datastore. * Mar 14, 2013 1587 bsteffen Fix persisting to datastore.
* Apr 14, 2014 16752 MPorricelli Add ensembleid to hash key to allow * Apr 14, 2014 16752 MPorricelli Add ensembleid to hash key to allow
* creation of static data for all perturbations * creation of static data for all perturbations
* of Ensemble models * of Ensemble models
* Apr 21, 2014 2060 njensen Remove dependency on grid dataURI column * Apr 21, 2014 2060 njensen Remove dependency on grid dataURI column
* Sep 19, 2014 3627 mapeters Updated deprecated TimeTools usage. * Sep 19, 2014 3627 mapeters Updated deprecated TimeTools usage.
* Aug 06, 2013 3805 bsteffen Increase time of cluster locks. * Aug 06, 2013 3805 bsteffen Increase time of cluster locks.
* Apr 29, 2015 4167 nabowle Propagate exceptions to #processNotification()
* *
* </pre> * </pre>
* *
@ -153,7 +154,7 @@ public class StaticDataGenerator {
staticRecords.addAll(processRecord(record)); staticRecords.addAll(processRecord(record));
} catch (Exception e) { } catch (Exception e) {
statusHandler.handle(Priority.ERROR, statusHandler.handle(Priority.ERROR,
"Error creating/saving staticTopo data!", e); "Error creating/saving staticTopo data.", e);
} }
} }
} }
@ -347,8 +348,12 @@ public class StaticDataGenerator {
/** /**
* Populate the message data for a static record, loads the correct type of * Populate the message data for a static record, loads the correct type of
* data based of the parameter in the record. * data based of the parameter in the record.
*
* @throws Exception
* If the static topo data cannot retrieved for a GridCoverage.
*/ */
private void populateMessageData(GridRecord staticRecord) { private void populateMessageData(GridRecord staticRecord)
throws Exception {
if (staticRecord.getMessageData() != null) { if (staticRecord.getMessageData() != null) {
// already populated, hits for static topo copied from the record. // already populated, hits for static topo copied from the record.
return; return;
@ -399,7 +404,7 @@ public class StaticDataGenerator {
/** /**
* Return a set with only records which are not already in the database * Return a set with only records which are not already in the database
* *
* @throws PluginException * @throws PluginException
*/ */
private Set<GridRecord> checkDatabase(Set<GridRecord> staticRecords) private Set<GridRecord> checkDatabase(Set<GridRecord> staticRecords)
@ -461,7 +466,7 @@ public class StaticDataGenerator {
private final int forecastTime; private final int forecastTime;
private final int coverageid; private final int coverageid;
private final String ensembleid; private final String ensembleid;
public CacheKey(GridRecord record) { public CacheKey(GridRecord record) {
@ -525,7 +530,7 @@ public class StaticDataGenerator {
} }
} else if (!ensembleid.equals(other.ensembleid)) { } else if (!ensembleid.equals(other.ensembleid)) {
return false; return false;
} }
return true; return true;
} }

74
edexOsgi/com.raytheon.uf.edex.grid.staticdata/src/com/raytheon/uf/edex/grid/staticdata/topo/StaticTopoData.java
View file

@ -1,19 +1,19 @@
/** /**
* This software was developed and / or modified by Raytheon Company, * This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government. * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
* *
* U.S. EXPORT CONTROLLED TECHNICAL DATA * U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose * This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination * export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires * to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization. * an export license or other authorization.
* *
* Contractor Name: Raytheon Company * Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340 * Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8 * Mail Stop B8
* Omaha, NE 68106 * Omaha, NE 68106
* 402.291.0100 * 402.291.0100
* *
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information. * further licensing information.
**/ **/
@ -53,6 +53,7 @@ import com.raytheon.uf.common.datastorage.Request;
import com.raytheon.uf.common.datastorage.StorageException; import com.raytheon.uf.common.datastorage.StorageException;
import com.raytheon.uf.common.datastorage.StorageProperties; import com.raytheon.uf.common.datastorage.StorageProperties;
import com.raytheon.uf.common.datastorage.StorageProperties.Compression; import com.raytheon.uf.common.datastorage.StorageProperties.Compression;
import com.raytheon.uf.common.datastorage.StorageStatus;
import com.raytheon.uf.common.datastorage.records.FloatDataRecord; import com.raytheon.uf.common.datastorage.records.FloatDataRecord;
import com.raytheon.uf.common.geospatial.CRSCache; import com.raytheon.uf.common.geospatial.CRSCache;
import com.raytheon.uf.common.geospatial.MapUtil; import com.raytheon.uf.common.geospatial.MapUtil;
@ -74,7 +75,7 @@ import com.vividsolutions.jts.geom.Coordinate;
/** /**
* Class used for accessing static topography information for GFE smart inits * Class used for accessing static topography information for GFE smart inits
* *
* <pre> * <pre>
* SOFTWARE HISTORY * SOFTWARE HISTORY
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
@ -89,9 +90,11 @@ import com.vividsolutions.jts.geom.Coordinate;
* Feb 12, 2013 1608 randerso Changed to call deleteDatasets * Feb 12, 2013 1608 randerso Changed to call deleteDatasets
* Aug 06, 2013 2235 bsteffen Added Caching version of TopoQuery. * Aug 06, 2013 2235 bsteffen Added Caching version of TopoQuery.
* Aug 06, 2013 3805 bsteffen Add timing to logging. * Aug 06, 2013 3805 bsteffen Add timing to logging.
* * Apr 29, 2015 4167 nabowle Propagate exceptions from
* #initStopoData(GridCoverage)
*
* </pre> * </pre>
* *
* @author bphillip * @author bphillip
* @version 1.0 * @version 1.0
*/ */
@ -147,7 +150,7 @@ public class StaticTopoData {
/** /**
* Gets the singleton instance of StaticTopoData * Gets the singleton instance of StaticTopoData
* *
* @return The singleton instance * @return The singleton instance
*/ */
public static synchronized StaticTopoData getInstance() { public static synchronized StaticTopoData getInstance() {
@ -268,7 +271,7 @@ public class StaticTopoData {
* Prepare DataSource for interpolation. Since TiledTopoSource loads all * Prepare DataSource for interpolation. Since TiledTopoSource loads all
* data on demand this will inititialize the source but will not request any * data on demand this will inititialize the source but will not request any
* data. * data.
* *
* @throws FactoryException * @throws FactoryException
* @throws MismatchedDimensionException * @throws MismatchedDimensionException
* @throws TransformException * @throws TransformException
@ -313,7 +316,7 @@ public class StaticTopoData {
/** /**
* Splits the Pacific grid into east and west hemisphere chunks to reduce * Splits the Pacific grid into east and west hemisphere chunks to reduce
* memory usage * memory usage
* *
* @throws Exception * @throws Exception
*/ */
private void splitPacific() throws Exception { private void splitPacific() throws Exception {
@ -354,7 +357,7 @@ public class StaticTopoData {
/** /**
* Checks to see if the topo file exists * Checks to see if the topo file exists
* *
* @return True if the file exists * @return True if the file exists
*/ */
private boolean topoFileExists() { private boolean topoFileExists() {
@ -371,7 +374,7 @@ public class StaticTopoData {
/** /**
* Initializes the attributes so they can be easily retrieved * Initializes the attributes so they can be easily retrieved
* *
* @throws Exception * @throws Exception
*/ */
private void initAttributes() throws Exception { private void initAttributes() throws Exception {
@ -394,7 +397,7 @@ public class StaticTopoData {
* Initializes the topography data for the given site. The data is extracted * Initializes the topography data for the given site. The data is extracted
* from the static topo file and resampled according to the site location * from the static topo file and resampled according to the site location
* information * information
* *
* @param modelName * @param modelName
* The site for which to initalize the topo data * The site for which to initalize the topo data
* @param config * @param config
@ -425,18 +428,30 @@ public class StaticTopoData {
inGeom.getGridRange().getHigh(0) + 1, inGeom.getGridRange().getHigh(0) + 1,
inGeom.getGridRange().getHigh(1) + 1 }); inGeom.getGridRange().getHigh(1) + 1 });
siteDataStore.addDataRecord(outRecord, sp); siteDataStore.addDataRecord(outRecord, sp);
siteDataStore.store(StoreOp.REPLACE); StorageStatus storestatus = siteDataStore.store(StoreOp.REPLACE);
long endTimeMillis = System.currentTimeMillis(); long endTimeMillis = System.currentTimeMillis();
statusHandler.handle(Priority.INFO, if (storestatus.hasExceptions()) {
"Stopo data successfully initialized for " + coverage.getName() throw storestatus.getExceptions()[0];
+ " in " + (endTimeMillis - startTimeMillis) + "ms"); } else {
statusHandler.handle(
Priority.INFO,
"Stopo data successfully initialized for "
+ coverage.getName() + " in "
+ (endTimeMillis - startTimeMillis) + "ms");
}
} }
/** /**
* Private method used by the initialization code to see if the static topo * Private method used by the initialization code to see if the static topo
* data for a coverage has been initialized * data for a coverage has been initialized
*
* @throws StorageException
* @throws SerializationException
* If the topography data cannot be initialized for the given
* site
*/ */
public boolean checkModelTopo(GridCoverage coverage) { public boolean checkModelTopo(GridCoverage coverage)
throws SerializationException, StorageException {
ClusterTask ct = ClusterLockUtils.lock(TASK_NAME, ClusterTask ct = ClusterLockUtils.lock(TASK_NAME,
coverage.spatialKey(), 120000, false); coverage.spatialKey(), 120000, false);
@ -451,11 +466,6 @@ public class StaticTopoData {
+ coverage.getName()); + coverage.getName());
initStopoData(coverage); initStopoData(coverage);
} }
} catch (Exception e) {
statusHandler.handle(
Priority.INFO,
"Error storing static topo data for "
+ coverage.getName(), e);
} finally { } finally {
ClusterLockUtils.unlock(ct, false); ClusterLockUtils.unlock(ct, false);
} }
@ -466,7 +476,7 @@ public class StaticTopoData {
/** /**
* Chekcs if the static topo data for a model exists in the file * Chekcs if the static topo data for a model exists in the file
* *
* @param modelName * @param modelName
* The model to check * The model to check
* @return True if the data exists, else false * @return True if the data exists, else false
@ -491,12 +501,18 @@ public class StaticTopoData {
/** /**
* Retrieves the static topo data for the given site * Retrieves the static topo data for the given site
* *
* @param modelName * @param modelName
* The site for which to get the static topo data * The site for which to get the static topo data
* @return The static topo data * @return The static topo data
* @throws StorageException
* @throws SerializationException
* If the topography data cannot be initialized for the given
* site
*
*/ */
public FloatDataRecord getStopoData(GridCoverage coverage) { public FloatDataRecord getStopoData(GridCoverage coverage)
throws SerializationException, StorageException {
if (!topoExists(coverage)) { if (!topoExists(coverage)) {
while (!checkModelTopo(coverage)) { while (!checkModelTopo(coverage)) {
} }
@ -518,7 +534,7 @@ public class StaticTopoData {
/** /**
* Extracts the the topo data from each of the topo data sets. * Extracts the the topo data from each of the topo data sets.
* *
* @param inGeom * @param inGeom
* The geometry of the data to be requested * The geometry of the data to be requested
* @return A float array containing the extracted topo data * @return A float array containing the extracted topo data
@ -582,7 +598,7 @@ public class StaticTopoData {
/** /**
* Reads the raw data from the topo files * Reads the raw data from the topo files
* *
* @param name * @param name
* The name of the topo data set * The name of the topo data set
* @param llLat * @param llLat
@ -667,7 +683,7 @@ public class StaticTopoData {
/** /**
* Creates a grid geometry object from the provided information * Creates a grid geometry object from the provided information
* *
* @param crs * @param crs
* The coordinate reference system * The coordinate reference system
* @param llCoord * @param llCoord
@ -718,7 +734,7 @@ public class StaticTopoData {
/** /**
* Retrieves a slab of data from a data set and returns a float array * Retrieves a slab of data from a data set and returns a float array
* containing the data * containing the data
* *
* @param name * @param name
* The name of the data set to get the data slab from * The name of the data set to get the data slab from
* @param minX * @param minX