xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Omaha #3805 Increase time of cluster lock, cleanup statictopo plugin.

Browse source 
Former-commit-id: 5c65750e82 [formerly 9bbd3f9f8dea46b14d77b796d0ea00285628ea75]
Former-commit-id: 11966fc0de
This commit is contained in:
Ben Steffensmeier 2014-11-12 12:46:23 -06:00
parent 1d6bec894e
commit 0a0cad6ed5
4 changed files with 61 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
edexOsgi/com.raytheon.uf.edex.grid.staticdata/.settings/org.eclipse.jdt.core.prefs
View file

@ -1,8 +0,0 @@
#Wed Dec 12 16:13:09 CST 2012
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
org.eclipse.jdt.core.compiler.compliance=1.6
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.source=1.6

21
edexOsgi/com.raytheon.uf.edex.grid.staticdata/META-INF/MANIFEST.MF
View file

@ -2,21 +2,24 @@ Manifest-Version: 1.0
Bundle-ManifestVersion: 2
Bundle-Name: Grid Static Data
Bundle-SymbolicName: com.raytheon.uf.edex.grid.staticdata
Bundle-Version: 1.14.0.qualifier
Bundle-Version: 1.14.1.qualifier
Bundle-Vendor: RAYTHEON
Bundle-RequiredExecutionEnvironment: JavaSE-1.6
Bundle-RequiredExecutionEnvironment: JavaSE-1.7
Require-Bundle: javax.measure,
com.raytheon.uf.common.dataplugin.grid;bundle-version="1.13.0",
com.raytheon.uf.edex.plugin.grid,
com.raytheon.uf.common.dataplugin.level;bundle-version="1.14.0",
com.raytheon.uf.common.parameter,
com.raytheon.uf.common.status,
com.raytheon.uf.common.parameter;bundle-version="1.14.0",
com.raytheon.uf.common.status;bundle-version="1.14.1",
com.raytheon.uf.common.gridcoverage;bundle-version="1.13.0",
com.raytheon.uf.edex.decodertools,
com.raytheon.edex.common,
com.raytheon.edex.plugin.gfe,
com.raytheon.uf.common.comm,
com.raytheon.edex.plugin.gfe;bundle-version="1.14.0",
com.raytheon.uf.common.topo;bundle-version="1.14.0",
com.raytheon.uf.common.numeric;bundle-version="1.14.0"
com.raytheon.uf.common.numeric;bundle-version="1.14.0",
com.raytheon.uf.common.geospatial;bundle-version="1.14.2",
com.raytheon.uf.common.datastorage;bundle-version="1.14.0",
com.raytheon.uf.edex.database;bundle-version="1.14.1",
com.raytheon.uf.common.util;bundle-version="1.14.1",
com.raytheon.uf.common.dataplugin;bundle-version="1.14.0",
com.raytheon.uf.edex.core;bundle-version="1.14.1"
Export-Package: com.raytheon.uf.edex.grid.staticdata,
com.raytheon.uf.edex.grid.staticdata.topo

42
edexOsgi/com.raytheon.uf.edex.grid.staticdata/src/com/raytheon/uf/edex/grid/staticdata/StaticDataGenerator.java
View file

@ -55,7 +55,6 @@ import com.raytheon.uf.edex.database.cluster.ClusterLockUtils;
import com.raytheon.uf.edex.database.cluster.ClusterLockUtils.LockState;
import com.raytheon.uf.edex.database.cluster.ClusterTask;
import com.raytheon.uf.edex.database.plugin.DataURIDatabaseUtil;
import com.raytheon.uf.edex.decodertools.time.TimeTools;
import com.raytheon.uf.edex.grid.staticdata.topo.StaticTopoData;
import com.raytheon.uf.edex.plugin.grid.dao.GridDao;
@ -67,17 +66,18 @@ import com.raytheon.uf.edex.plugin.grid.dao.GridDao;
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Dec 3, 2010 rjpeter Initial creation
* Feb 15, 2013 1638 mschenke Moved DataURINotificationMessage to uf.common.dataplugin
* Mar 07, 2013 1587 bsteffen rewrite static data generation.
* Mar 14, 2013 1587 bsteffen Fix persisting to datastore.
* Apr 14, 2014 DR 16752 MPorricelli Add ensembleid to hash key to allow
* creation of static data for all perturbations
* of Ensemble models
* Apr 21, 2014 2060 njensen Remove dependency on grid dataURI column
* Sep 19, 2014 3627 mapeters Updated deprecated {@link TimeTools} usage.
* Date Ticket# Engineer Description
* ------------- -------- ----------- --------------------------
* Dec 03, 2010 rjpeter Initial creation
* Feb 15, 2013 1638 mschenke Moved DataURINotificationMessage to uf.common.dataplugin
* Mar 07, 2013 1587 bsteffen rewrite static data generation.
* Mar 14, 2013 1587 bsteffen Fix persisting to datastore.
* Apr 14, 2014 16752 MPorricelli Add ensembleid to hash key to allow
* creation of static data for all perturbations
* of Ensemble models
* Apr 21, 2014 2060 njensen Remove dependency on grid dataURI column
* Sep 19, 2014 3627 mapeters Updated deprecated TimeTools usage.
* Aug 06, 2013 3805 bsteffen Increase time of cluster locks.
*
* </pre>
*
@ -270,7 +270,7 @@ public class StaticDataGenerator {
staticRecords.add(staticRecord);
}
databaseRecords.addAll(checkDatabase(dao, staticRecords));
databaseRecords.addAll(checkDatabase(staticRecords));
datastoreRecords.addAll(checkDatastore(dao, databaseRecords));
if (!datastoreRecords.isEmpty()) {
for (GridRecord staticRecord : datastoreRecords) {
@ -299,7 +299,17 @@ public class StaticDataGenerator {
+ record.getDataTime().getRefTime();
ClusterTask rval = null;
do {
rval = ClusterLockUtils.lock(STATIC_TOPO, taskDetails, 1500, true);
/*
* 4 minutes is needed because when static topo has not been
* pregenerated it must go through a lengthy topo initialization
* process. This is rare but we do not want the cluster lock to time
* out in these cases. The worst case scenerio that has been
* observed is generating topo for grid 233 which took 107 seconds.
* 4 minutes leaves extra time in case other grids have worse
* performance.
*/
rval = ClusterLockUtils.lock(STATIC_TOPO, taskDetails,
4 * TimeUtil.MILLIS_PER_MINUTE, true);
} while (!rval.getLockState().equals(LockState.SUCCESSFUL));
return rval;
}
@ -392,8 +402,8 @@ public class StaticDataGenerator {
*
* @throws PluginException
*/
private Set<GridRecord> checkDatabase(GridDao dao,
Set<GridRecord> staticRecords) throws PluginException {
private Set<GridRecord> checkDatabase(Set<GridRecord> staticRecords)
throws PluginException {
if (staticRecords.isEmpty()) {
return staticRecords;
}

58
edexOsgi/com.raytheon.uf.edex.grid.staticdata/src/com/raytheon/uf/edex/grid/staticdata/topo/StaticTopoData.java
View file

@ -37,6 +37,7 @@ import org.geotools.coverage.grid.GridGeometry2D;
import org.geotools.geometry.DirectPosition2D;
import org.geotools.geometry.GeneralEnvelope;
import org.geotools.referencing.operation.builder.GridToEnvelopeMapper;
import org.hibernate.type.SerializationException;
import org.opengis.geometry.DirectPosition;
import org.opengis.geometry.MismatchedDimensionException;
import org.opengis.referencing.FactoryException;
@ -45,7 +46,6 @@ import org.opengis.referencing.datum.PixelInCell;
import org.opengis.referencing.operation.MathTransform;
import org.opengis.referencing.operation.TransformException;
import com.raytheon.edex.site.SiteUtil;
import com.raytheon.uf.common.datastorage.DataStoreFactory;
import com.raytheon.uf.common.datastorage.IDataStore;
import com.raytheon.uf.common.datastorage.IDataStore.StoreOp;
@ -61,12 +61,12 @@ import com.raytheon.uf.common.geospatial.interpolation.GridReprojection;
import com.raytheon.uf.common.geospatial.interpolation.GridSampler;
import com.raytheon.uf.common.gridcoverage.GridCoverage;
import com.raytheon.uf.common.numeric.buffer.FloatBufferWrapper;
import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.topo.TiledTopoSource;
import com.raytheon.uf.common.util.RunProcess;
import com.raytheon.uf.edex.core.EDEXUtil;
import com.raytheon.uf.edex.database.cluster.ClusterLockUtils;
import com.raytheon.uf.edex.database.cluster.ClusterLockUtils.LockState;
import com.raytheon.uf.edex.database.cluster.ClusterTask;
@ -77,17 +77,18 @@ import com.vividsolutions.jts.geom.Coordinate;
*
* <pre>
* SOFTWARE HISTORY
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Sep 27, 2010 6394 bphillip Initial creation
* Oct 08, 2010 6394 bphillip Rewrote sections for optimal reading and
* writing performance
* Sep 19, 2011 10955 rferrel Use RunProcess
* Apr 18, 2012 14694 D. Friedman Fixes for static topography generation
* May 09, 2012 14939 D. Friedman Fix errors in DR 14694
* Jan 14, 2013 1469 bkowal Removed the hdf5 data directory
* Feb 12, 2013 1608 randerso Changed to call deleteDatasets
* Aug 06, 2013 2235 bsteffen Added Caching version of TopoQuery.
* Date Ticket# Engineer Description
* ------------- -------- ----------- --------------------------
* Sep 27, 2010 6394 bphillip Initial creation
* Oct 08, 2010 6394 bphillip Rewrote sections for optimal reading and
* writing performance
* Sep 19, 2011 10955 rferrel Use RunProcess
* Apr 18, 2012 14694 D. Friedman Fixes for static topography generation
* May 09, 2012 14939 D. Friedman Fix errors in DR 14694
* Jan 14, 2013 1469 bkowal Removed the hdf5 data directory
* Feb 12, 2013 1608 randerso Changed to call deleteDatasets
* Aug 06, 2013 2235 bsteffen Added Caching version of TopoQuery.
* Aug 06, 2013 3805 bsteffen Add timing to logging.
*
* </pre>
*
@ -250,9 +251,7 @@ public class StaticTopoData {
splitPacific();
}
} finally {
if (ct != null) {
ClusterLockUtils.unlock(ct, false);
}
ClusterLockUtils.unlock(ct, false);
}
initAttributes();
@ -409,16 +408,16 @@ public class StaticTopoData {
SerializationException {
if (coverage.getNx() < 0 || coverage.getNy() < 0) {
statusHandler.handle(Priority.PROBLEM, coverage.getName()
+ " is not applicable to " + SiteUtil.getSite()
+ " is not applicable to " + EDEXUtil.getEdexSite()
+ ". Skipping.");
return;
}
long startTimeMillis = System.currentTimeMillis();
GridGeometry2D inGeom = MapUtil.getGridGeometry(coverage);
// Gets the location data and extracts it from the static topo file
float[] finalData = null;
finalData = getTopoData(inGeom, coverage.getCrs(), coverage.getNx(),
coverage.getNy());
finalData = getTopoData(inGeom);
// Create an HDF5 data record and store it
FloatDataRecord outRecord = new FloatDataRecord(STOPO_DATASET,
@ -427,11 +426,10 @@ public class StaticTopoData {
inGeom.getGridRange().getHigh(1) + 1 });
siteDataStore.addDataRecord(outRecord, sp);
siteDataStore.store(StoreOp.REPLACE);
statusHandler
.handle(Priority.INFO,
"Stopo data successfully initialized for "
+ coverage.getName());
long endTimeMillis = System.currentTimeMillis();
statusHandler.handle(Priority.INFO,
"Stopo data successfully initialized for " + coverage.getName()
+ " in " + (endTimeMillis - startTimeMillis) + "ms");
}
/**
@ -472,7 +470,6 @@ public class StaticTopoData {
* @param modelName
* The model to check
* @return True if the data exists, else false
* @throws GribException
*/
private boolean topoExists(GridCoverage coverage) {
@ -498,8 +495,6 @@ public class StaticTopoData {
* @param modelName
* The site for which to get the static topo data
* @return The static topo data
* @throws GribException
* If an error occurs while retrieving the topo data
*/
public FloatDataRecord getStopoData(GridCoverage coverage) {
if (!topoExists(coverage)) {
@ -526,14 +521,9 @@ public class StaticTopoData {
*
* @param inGeom
* The geometry of the data to be requested
* @param inCrs
* The coordinate reference system of the desired data
* @return A float array containing the extracted topo data
* @throws GribException
* If the data cannot be extracted
*/
private float[] getTopoData(GridGeometry2D inGeom,
CoordinateReferenceSystem inCrs, int coverageNx, int coverageNy) {
private float[] getTopoData(GridGeometry2D inGeom) {
FloatBufferWrapper finalDataWrapper = null;
BilinearInterpolation interp = new BilinearInterpolation();
interp.setMissingThreshold(0.0001f);
@ -692,8 +682,6 @@ public class StaticTopoData {
* @throws TransformException
* @throws MismatchedDimensionException
* @throws FactoryException
* @throws GribException
* If the geometry cannot be created
*/
private GridGeometry2D createGridGeometry(CoordinateReferenceSystem crs,
Coordinate llCoord, Coordinate urCoord, int nx, int ny)