From 0a0cad6ed5f7df734f9db9dbbc19c40b0f5d0431 Mon Sep 17 00:00:00 2001 From: Ben Steffensmeier Date: Wed, 12 Nov 2014 12:46:23 -0600 Subject: [PATCH] Omaha #3805 Increase time of cluster lock, cleanup statictopo plugin. Former-commit-id: 5c65750e82eb9c59879f19ecb85bfad4cc677b77 [formerly 9bbd3f9f8dea46b14d77b796d0ea00285628ea75] Former-commit-id: 11966fc0dee4f7073f0d081cd1ac728240e2b867 --- .../.settings/org.eclipse.jdt.core.prefs | 8 --- .../META-INF/MANIFEST.MF | 21 ++++--- .../grid/staticdata/StaticDataGenerator.java | 42 +++++++++----- .../grid/staticdata/topo/StaticTopoData.java | 58 ++++++++----------- 4 files changed, 61 insertions(+), 68 deletions(-) delete mode 100644 edexOsgi/com.raytheon.uf.edex.grid.staticdata/.settings/org.eclipse.jdt.core.prefs diff --git a/edexOsgi/com.raytheon.uf.edex.grid.staticdata/.settings/org.eclipse.jdt.core.prefs b/edexOsgi/com.raytheon.uf.edex.grid.staticdata/.settings/org.eclipse.jdt.core.prefs deleted file mode 100644 index 64129be9da..0000000000 --- a/edexOsgi/com.raytheon.uf.edex.grid.staticdata/.settings/org.eclipse.jdt.core.prefs +++ /dev/null @@ -1,8 +0,0 @@ -#Wed Dec 12 16:13:09 CST 2012 -eclipse.preferences.version=1 -org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 -org.eclipse.jdt.core.compiler.compliance=1.6 -org.eclipse.jdt.core.compiler.problem.assertIdentifier=error -org.eclipse.jdt.core.compiler.problem.enumIdentifier=error -org.eclipse.jdt.core.compiler.source=1.6 diff --git a/edexOsgi/com.raytheon.uf.edex.grid.staticdata/META-INF/MANIFEST.MF b/edexOsgi/com.raytheon.uf.edex.grid.staticdata/META-INF/MANIFEST.MF index c7e3c0d11c..47bfc5bb60 100644 --- a/edexOsgi/com.raytheon.uf.edex.grid.staticdata/META-INF/MANIFEST.MF +++ b/edexOsgi/com.raytheon.uf.edex.grid.staticdata/META-INF/MANIFEST.MF @@ -2,21 +2,24 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: Grid Static Data Bundle-SymbolicName: com.raytheon.uf.edex.grid.staticdata -Bundle-Version: 1.14.0.qualifier +Bundle-Version: 1.14.1.qualifier Bundle-Vendor: RAYTHEON -Bundle-RequiredExecutionEnvironment: JavaSE-1.6 +Bundle-RequiredExecutionEnvironment: JavaSE-1.7 Require-Bundle: javax.measure, com.raytheon.uf.common.dataplugin.grid;bundle-version="1.13.0", com.raytheon.uf.edex.plugin.grid, com.raytheon.uf.common.dataplugin.level;bundle-version="1.14.0", - com.raytheon.uf.common.parameter, - com.raytheon.uf.common.status, + com.raytheon.uf.common.parameter;bundle-version="1.14.0", + com.raytheon.uf.common.status;bundle-version="1.14.1", com.raytheon.uf.common.gridcoverage;bundle-version="1.13.0", - com.raytheon.uf.edex.decodertools, - com.raytheon.edex.common, - com.raytheon.edex.plugin.gfe, - com.raytheon.uf.common.comm, + com.raytheon.edex.plugin.gfe;bundle-version="1.14.0", com.raytheon.uf.common.topo;bundle-version="1.14.0", - com.raytheon.uf.common.numeric;bundle-version="1.14.0" + com.raytheon.uf.common.numeric;bundle-version="1.14.0", + com.raytheon.uf.common.geospatial;bundle-version="1.14.2", + com.raytheon.uf.common.datastorage;bundle-version="1.14.0", + com.raytheon.uf.edex.database;bundle-version="1.14.1", + com.raytheon.uf.common.util;bundle-version="1.14.1", + com.raytheon.uf.common.dataplugin;bundle-version="1.14.0", + com.raytheon.uf.edex.core;bundle-version="1.14.1" Export-Package: com.raytheon.uf.edex.grid.staticdata, com.raytheon.uf.edex.grid.staticdata.topo diff --git a/edexOsgi/com.raytheon.uf.edex.grid.staticdata/src/com/raytheon/uf/edex/grid/staticdata/StaticDataGenerator.java b/edexOsgi/com.raytheon.uf.edex.grid.staticdata/src/com/raytheon/uf/edex/grid/staticdata/StaticDataGenerator.java index e6712829ad..50cd0e80a0 100644 --- a/edexOsgi/com.raytheon.uf.edex.grid.staticdata/src/com/raytheon/uf/edex/grid/staticdata/StaticDataGenerator.java +++ b/edexOsgi/com.raytheon.uf.edex.grid.staticdata/src/com/raytheon/uf/edex/grid/staticdata/StaticDataGenerator.java @@ -55,7 +55,6 @@ import com.raytheon.uf.edex.database.cluster.ClusterLockUtils; import com.raytheon.uf.edex.database.cluster.ClusterLockUtils.LockState; import com.raytheon.uf.edex.database.cluster.ClusterTask; import com.raytheon.uf.edex.database.plugin.DataURIDatabaseUtil; -import com.raytheon.uf.edex.decodertools.time.TimeTools; import com.raytheon.uf.edex.grid.staticdata.topo.StaticTopoData; import com.raytheon.uf.edex.plugin.grid.dao.GridDao; @@ -67,17 +66,18 @@ import com.raytheon.uf.edex.plugin.grid.dao.GridDao; * * SOFTWARE HISTORY * - * Date Ticket# Engineer Description - * ------------ ---------- ----------- -------------------------- - * Dec 3, 2010 rjpeter Initial creation - * Feb 15, 2013 1638 mschenke Moved DataURINotificationMessage to uf.common.dataplugin - * Mar 07, 2013 1587 bsteffen rewrite static data generation. - * Mar 14, 2013 1587 bsteffen Fix persisting to datastore. - * Apr 14, 2014 DR 16752 MPorricelli Add ensembleid to hash key to allow - * creation of static data for all perturbations - * of Ensemble models - * Apr 21, 2014 2060 njensen Remove dependency on grid dataURI column - * Sep 19, 2014 3627 mapeters Updated deprecated {@link TimeTools} usage. + * Date Ticket# Engineer Description + * ------------- -------- ----------- -------------------------- + * Dec 03, 2010 rjpeter Initial creation + * Feb 15, 2013 1638 mschenke Moved DataURINotificationMessage to uf.common.dataplugin + * Mar 07, 2013 1587 bsteffen rewrite static data generation. + * Mar 14, 2013 1587 bsteffen Fix persisting to datastore. + * Apr 14, 2014 16752 MPorricelli Add ensembleid to hash key to allow + * creation of static data for all perturbations + * of Ensemble models + * Apr 21, 2014 2060 njensen Remove dependency on grid dataURI column + * Sep 19, 2014 3627 mapeters Updated deprecated TimeTools usage. + * Aug 06, 2013 3805 bsteffen Increase time of cluster locks. * * * @@ -270,7 +270,7 @@ public class StaticDataGenerator { staticRecords.add(staticRecord); } - databaseRecords.addAll(checkDatabase(dao, staticRecords)); + databaseRecords.addAll(checkDatabase(staticRecords)); datastoreRecords.addAll(checkDatastore(dao, databaseRecords)); if (!datastoreRecords.isEmpty()) { for (GridRecord staticRecord : datastoreRecords) { @@ -299,7 +299,17 @@ public class StaticDataGenerator { + record.getDataTime().getRefTime(); ClusterTask rval = null; do { - rval = ClusterLockUtils.lock(STATIC_TOPO, taskDetails, 1500, true); + /* + * 4 minutes is needed because when static topo has not been + * pregenerated it must go through a lengthy topo initialization + * process. This is rare but we do not want the cluster lock to time + * out in these cases. The worst case scenerio that has been + * observed is generating topo for grid 233 which took 107 seconds. + * 4 minutes leaves extra time in case other grids have worse + * performance. + */ + rval = ClusterLockUtils.lock(STATIC_TOPO, taskDetails, + 4 * TimeUtil.MILLIS_PER_MINUTE, true); } while (!rval.getLockState().equals(LockState.SUCCESSFUL)); return rval; } @@ -392,8 +402,8 @@ public class StaticDataGenerator { * * @throws PluginException */ - private Set checkDatabase(GridDao dao, - Set staticRecords) throws PluginException { + private Set checkDatabase(Set staticRecords) + throws PluginException { if (staticRecords.isEmpty()) { return staticRecords; } diff --git a/edexOsgi/com.raytheon.uf.edex.grid.staticdata/src/com/raytheon/uf/edex/grid/staticdata/topo/StaticTopoData.java b/edexOsgi/com.raytheon.uf.edex.grid.staticdata/src/com/raytheon/uf/edex/grid/staticdata/topo/StaticTopoData.java index 85902388fc..dec5d236c0 100644 --- a/edexOsgi/com.raytheon.uf.edex.grid.staticdata/src/com/raytheon/uf/edex/grid/staticdata/topo/StaticTopoData.java +++ b/edexOsgi/com.raytheon.uf.edex.grid.staticdata/src/com/raytheon/uf/edex/grid/staticdata/topo/StaticTopoData.java @@ -37,6 +37,7 @@ import org.geotools.coverage.grid.GridGeometry2D; import org.geotools.geometry.DirectPosition2D; import org.geotools.geometry.GeneralEnvelope; import org.geotools.referencing.operation.builder.GridToEnvelopeMapper; +import org.hibernate.type.SerializationException; import org.opengis.geometry.DirectPosition; import org.opengis.geometry.MismatchedDimensionException; import org.opengis.referencing.FactoryException; @@ -45,7 +46,6 @@ import org.opengis.referencing.datum.PixelInCell; import org.opengis.referencing.operation.MathTransform; import org.opengis.referencing.operation.TransformException; -import com.raytheon.edex.site.SiteUtil; import com.raytheon.uf.common.datastorage.DataStoreFactory; import com.raytheon.uf.common.datastorage.IDataStore; import com.raytheon.uf.common.datastorage.IDataStore.StoreOp; @@ -61,12 +61,12 @@ import com.raytheon.uf.common.geospatial.interpolation.GridReprojection; import com.raytheon.uf.common.geospatial.interpolation.GridSampler; import com.raytheon.uf.common.gridcoverage.GridCoverage; import com.raytheon.uf.common.numeric.buffer.FloatBufferWrapper; -import com.raytheon.uf.common.serialization.SerializationException; import com.raytheon.uf.common.status.IUFStatusHandler; import com.raytheon.uf.common.status.UFStatus; import com.raytheon.uf.common.status.UFStatus.Priority; import com.raytheon.uf.common.topo.TiledTopoSource; import com.raytheon.uf.common.util.RunProcess; +import com.raytheon.uf.edex.core.EDEXUtil; import com.raytheon.uf.edex.database.cluster.ClusterLockUtils; import com.raytheon.uf.edex.database.cluster.ClusterLockUtils.LockState; import com.raytheon.uf.edex.database.cluster.ClusterTask; @@ -77,17 +77,18 @@ import com.vividsolutions.jts.geom.Coordinate; * * 
  * SOFTWARE HISTORY
- * Date         Ticket#     Engineer    Description
- * ------------ ----------  ----------- --------------------------
- * Sep 27, 2010 6394        bphillip    Initial creation
- * Oct 08, 2010 6394        bphillip    Rewrote sections for optimal reading and
- *                                      writing performance
- * Sep 19, 2011 10955       rferrel     Use RunProcess
- * Apr 18, 2012 14694       D. Friedman Fixes for static topography generation
- * May 09, 2012 14939       D. Friedman Fix errors in DR 14694
- * Jan 14, 2013 1469        bkowal      Removed the hdf5 data directory
- * Feb 12, 2013 1608        randerso    Changed to call deleteDatasets
- * Aug 06, 2013 2235        bsteffen    Added Caching version of TopoQuery.
+ * Date          Ticket#  Engineer    Description
+ * ------------- -------- ----------- --------------------------
+ * Sep 27, 2010  6394     bphillip    Initial creation
+ * Oct 08, 2010  6394     bphillip    Rewrote sections for optimal reading and
+ *                                    writing performance
+ * Sep 19, 2011  10955    rferrel     Use RunProcess
+ * Apr 18, 2012  14694    D. Friedman Fixes for static topography generation
+ * May 09, 2012  14939    D. Friedman Fix errors in DR 14694
+ * Jan 14, 2013  1469     bkowal      Removed the hdf5 data directory
+ * Feb 12, 2013  1608     randerso    Changed to call deleteDatasets
+ * Aug 06, 2013  2235     bsteffen    Added Caching version of TopoQuery.
+ * Aug 06, 2013  3805     bsteffen    Add timing to logging.
  * 
  *
* @@ -250,9 +251,7 @@ public class StaticTopoData { splitPacific(); } } finally { - if (ct != null) { - ClusterLockUtils.unlock(ct, false); - } + ClusterLockUtils.unlock(ct, false); } initAttributes(); @@ -409,16 +408,16 @@ public class StaticTopoData { SerializationException { if (coverage.getNx() < 0 || coverage.getNy() < 0) { statusHandler.handle(Priority.PROBLEM, coverage.getName() - + " is not applicable to " + SiteUtil.getSite() + + " is not applicable to " + EDEXUtil.getEdexSite() + ". Skipping."); return; } + long startTimeMillis = System.currentTimeMillis(); GridGeometry2D inGeom = MapUtil.getGridGeometry(coverage); // Gets the location data and extracts it from the static topo file float[] finalData = null; - finalData = getTopoData(inGeom, coverage.getCrs(), coverage.getNx(), - coverage.getNy()); + finalData = getTopoData(inGeom); // Create an HDF5 data record and store it FloatDataRecord outRecord = new FloatDataRecord(STOPO_DATASET, @@ -427,11 +426,10 @@ public class StaticTopoData { inGeom.getGridRange().getHigh(1) + 1 }); siteDataStore.addDataRecord(outRecord, sp); siteDataStore.store(StoreOp.REPLACE); - - statusHandler - .handle(Priority.INFO, - "Stopo data successfully initialized for " - + coverage.getName()); + long endTimeMillis = System.currentTimeMillis(); + statusHandler.handle(Priority.INFO, + "Stopo data successfully initialized for " + coverage.getName() + + " in " + (endTimeMillis - startTimeMillis) + "ms"); } /** @@ -472,7 +470,6 @@ public class StaticTopoData { * @param modelName * The model to check * @return True if the data exists, else false - * @throws GribException */ private boolean topoExists(GridCoverage coverage) { @@ -498,8 +495,6 @@ public class StaticTopoData { * @param modelName * The site for which to get the static topo data * @return The static topo data - * @throws GribException - * If an error occurs while retrieving the topo data */ public FloatDataRecord getStopoData(GridCoverage coverage) { if (!topoExists(coverage)) { @@ -526,14 +521,9 @@ public class StaticTopoData { * * @param inGeom * The geometry of the data to be requested - * @param inCrs - * The coordinate reference system of the desired data * @return A float array containing the extracted topo data - * @throws GribException - * If the data cannot be extracted */ - private float[] getTopoData(GridGeometry2D inGeom, - CoordinateReferenceSystem inCrs, int coverageNx, int coverageNy) { + private float[] getTopoData(GridGeometry2D inGeom) { FloatBufferWrapper finalDataWrapper = null; BilinearInterpolation interp = new BilinearInterpolation(); interp.setMissingThreshold(0.0001f); @@ -692,8 +682,6 @@ public class StaticTopoData { * @throws TransformException * @throws MismatchedDimensionException * @throws FactoryException - * @throws GribException - * If the geometry cannot be created */ private GridGeometry2D createGridGeometry(CoordinateReferenceSystem crs, Coordinate llCoord, Coordinate urCoord, int nx, int ny)