From e2c62ab096eead271a6d7f2f1a468cce0ca5374c Mon Sep 17 00:00:00 2001 From: Ben Steffensmeier Date: Fri, 27 Jan 2012 14:48:40 -0600 Subject: [PATCH] Issue #183 change grib data catalog to use bulk requests to retrieve all coverages at once. Change-Id: I99f3dfc7900b57b074062155a4e7a65851f70170 Former-commit-id: e76323dc970a481ca4d5d133f973742f7203ff67 [formerly d90356d2661123d2ffecc4fc9edda3a925110b65] [formerly 113722a596425ea65f103af78109973962b83bfe] [formerly e76323dc970a481ca4d5d133f973742f7203ff67 [formerly d90356d2661123d2ffecc4fc9edda3a925110b65] [formerly 113722a596425ea65f103af78109973962b83bfe] [formerly dfd869c1c1fb4358f9111190f7b3da8d897061f2 [formerly 113722a596425ea65f103af78109973962b83bfe [formerly 3cdfc8f73cce96c00b7ac37a1e34e9d051b628b6]]]] Former-commit-id: dfd869c1c1fb4358f9111190f7b3da8d897061f2 Former-commit-id: 573108f2c5ddcd6af6877b09772e33f399b090d8 [formerly f2ff1331173aa1f73e241c68c695e88d42930d5b] [formerly 09449968f9cc4d3fac177bbbf9f58f412fe658c0 [formerly f0827271122f04c078593f0ccaae3b8567866df0]] Former-commit-id: 9649bd78778fa82525127c827ee65f451beab1a7 [formerly 7335f90bfb28277e8cb9558ecd12e073cf0c0121] Former-commit-id: 0d9cbcd09eda07e8df7847f70febc82c224e7fb2 --- .../raytheon/viz/grid/inv/GridInventory.java | 60 ++++++++++++----- .../raytheon/viz/grid/util/CoverageUtils.java | 64 ++++++++++++++----- .../AbstractInventoryDataCatalog.java | 11 +++- .../datacatalog/GribDataCatalog.java | 44 +++++++++---- .../res/spring/grib-request.xml | 6 ++ .../grib/handler/GetCoveragesHandler.java | 58 +++++++++++++++++ .../grib/request/GetCoveragesRequest.java | 58 +++++++++++++++++ 7 files changed, 254 insertions(+), 47 deletions(-) create mode 100644 edexOsgi/com.raytheon.edex.plugin.grib/src/com/raytheon/edex/plugin/grib/handler/GetCoveragesHandler.java create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.grib/src/com/raytheon/uf/common/dataplugin/grib/request/GetCoveragesRequest.java diff --git a/cave/com.raytheon.viz.grid/src/com/raytheon/viz/grid/inv/GridInventory.java b/cave/com.raytheon.viz.grid/src/com/raytheon/viz/grid/inv/GridInventory.java index 7f91e14869..a8514e8694 100644 --- a/cave/com.raytheon.viz.grid/src/com/raytheon/viz/grid/inv/GridInventory.java +++ b/cave/com.raytheon.viz.grid/src/com/raytheon/viz/grid/inv/GridInventory.java @@ -152,6 +152,7 @@ public class GridInventory extends AbstractInventory implements for (Map constraints : constraintsToTry) { evaluateRequestConstraints(constraints); } + } private DataTree getTreeFromEdex() { @@ -349,6 +350,7 @@ public class GridInventory extends AbstractInventory implements * @param newGridTree */ private void initAliasModels(DataTree newGridTree) { + Set allAliasModels = new HashSet(); sourceAliases.clear(); GribModelLookup lookup = GribModelLookup.getInstance(); for (String modelName : newGridTree.getSources()) { @@ -357,6 +359,8 @@ public class GridInventory extends AbstractInventory implements SourceNode source = newGridTree.getSourceNode(modelName); SourceNode dest = newGridTree.getSourceNode(model.getAlias()); if (source != null && dest != null) { + allAliasModels.add(source.getValue()); + allAliasModels.add(dest.getValue()); List aliases = sourceAliases.get(dest.getValue()); if (aliases == null) { aliases = new ArrayList(); @@ -367,28 +371,29 @@ public class GridInventory extends AbstractInventory implements } } } - for (Entry> aliases : sourceAliases.entrySet()) { - Collections.sort(aliases.getValue(), new Comparator() { + // Requesting coverages all at once is more efficient + try { + final Map coverages = CoverageUtils + .getInstance().getCoverages(allAliasModels); - @Override - public int compare(String model1, String model2) { - try { - GridCoverage coverage1 = CoverageUtils.getInstance() - .getCoverage(model1); + for (Entry> aliases : sourceAliases.entrySet()) { + Collections.sort(aliases.getValue(), new Comparator() { + + @Override + public int compare(String model1, String model2) { + GridCoverage coverage1 = coverages.get(model1); Integer res1 = coverage1.getNx() * coverage1.getNy(); - GridCoverage coverage2 = CoverageUtils.getInstance() - .getCoverage(model2); + GridCoverage coverage2 = coverages.get(model2); Integer res2 = coverage2.getNx() * coverage2.getNy(); return res2.compareTo(res1); - } catch (VizException e) { - statusHandler.handle(Priority.PROBLEM, - "Unable to create model aliases, problems with " - + model1 + " and " + model2, e); - return 0; } - } - }); + }); + } + } catch (VizException e) { + statusHandler.handle(Priority.PROBLEM, + "Unable to create model aliases", e); + return; } } @@ -449,6 +454,28 @@ public class GridInventory extends AbstractInventory implements public List getPerts(Map query) throws VizException { + RequestConstraint nameRC = query.get(MODEL_NAME_QUERY); + if (nameRC == null) { + // Only bother grabbing nodes with perts + nameRC = new RequestConstraint(null, ConstraintType.IN); + nameRC.setConstraintValueList(modelsWithPerts + .toArray(new String[0])); + query = new HashMap(query); + query.put(MODEL_NAME_QUERY, nameRC); + } else { + boolean hasPerts = false; + for (String modelName : modelsWithPerts) { + if (nameRC.evaluate(modelName)) { + hasPerts = true; + break; + } + } + // If this query is not valid for any models with perts then it has + // no perts, don't bother with a query. + if (!hasPerts) { + return Collections.emptyList(); + } + } Set perts = new HashSet(); for (AbstractRequestableLevelNode node : evaluateRequestConstraints(query)) { perts.addAll(getPerts(node)); @@ -459,7 +486,6 @@ public class GridInventory extends AbstractInventory implements protected static List getPerts(AbstractRequestableLevelNode node) throws VizException { - if (node instanceof GribRequestableLevelNode) { GribRequestableLevelNode gNode = (GribRequestableLevelNode) node; if (gNode.getPerts() != null) { diff --git a/cave/com.raytheon.viz.grid/src/com/raytheon/viz/grid/util/CoverageUtils.java b/cave/com.raytheon.viz.grid/src/com/raytheon/viz/grid/util/CoverageUtils.java index 9c1c11864f..adab327886 100644 --- a/cave/com.raytheon.viz.grid/src/com/raytheon/viz/grid/util/CoverageUtils.java +++ b/cave/com.raytheon.viz.grid/src/com/raytheon/viz/grid/util/CoverageUtils.java @@ -22,8 +22,11 @@ package com.raytheon.viz.grid.util; import java.awt.RenderingHints; import java.awt.image.Raster; import java.awt.image.RenderedImage; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; import javax.media.jai.BorderExtender; @@ -40,6 +43,7 @@ import org.geotools.coverage.processing.Operations; import org.opengis.geometry.Envelope; import com.raytheon.uf.common.dataplugin.grib.request.GetCoverageRequest; +import com.raytheon.uf.common.dataplugin.grib.request.GetCoveragesRequest; import com.raytheon.uf.common.dataplugin.grib.spatial.projections.GridCoverage; import com.raytheon.uf.common.datastorage.records.FloatDataRecord; import com.raytheon.uf.common.geospatial.MapUtil; @@ -79,28 +83,56 @@ public class CoverageUtils { } public GridCoverage getCoverage(String modelName) throws VizException { - GridCoverage rval = coverageCache.get(modelName); + synchronized (coverageCache) { + GridCoverage rval = coverageCache.get(modelName); - if (rval == null) { - GetCoverageRequest request = new GetCoverageRequest(); - request.setModelName(modelName); - Object obj = ThriftClient.sendRequest(request); + if (rval == null) { + GetCoverageRequest request = new GetCoverageRequest(); + request.setModelName(modelName); + Object obj = ThriftClient.sendRequest(request); - if (obj != null) { - if (obj instanceof GridCoverage) { - rval = (GridCoverage) obj; - coverageCache.put(modelName, rval); + if (obj != null) { + if (obj instanceof GridCoverage) { + rval = (GridCoverage) obj; + coverageCache.put(modelName, rval); + } else { + throw new VizException( + "GetCoverageRequest returned object of type [" + + obj.getClass().getName() + + "], expected [" + + GridCoverage.class.getName() + "]"); + } + } + } + return rval; + } + } + + public Map getCoverages(Collection modelNames) + throws VizException { + Map coverages = new HashMap(); + List toRequest = new ArrayList(); + synchronized (coverageCache) { + for (String modelName : modelNames) { + GridCoverage coverage = coverageCache.get(modelName); + if (coverage == null) { + toRequest.add(modelName); } else { - throw new VizException( - "GetCoverageRequest returned object of type [" - + obj.getClass().getName() - + "], expected [" - + GridCoverage.class.getName() + "]"); + coverages.put(modelName, coverage); + } + } + if (!toRequest.isEmpty()) { + GetCoveragesRequest request = new GetCoveragesRequest(); + request.setModelNames(toRequest); + List list = (List) ThriftClient.sendRequest(request); + for (int i = 0; i < list.size(); i++) { + coverageCache.put(toRequest.get(i), + (GridCoverage) list.get(i)); + coverages.put(toRequest.get(i), (GridCoverage) list.get(i)); } } } - - return rval; + return coverages; } /** diff --git a/cave/com.raytheon.viz.volumebrowser/src/com/raytheon/viz/volumebrowser/datacatalog/AbstractInventoryDataCatalog.java b/cave/com.raytheon.viz.volumebrowser/src/com/raytheon/viz/volumebrowser/datacatalog/AbstractInventoryDataCatalog.java index 298771fa88..bdc23ad46b 100644 --- a/cave/com.raytheon.viz.volumebrowser/src/com/raytheon/viz/volumebrowser/datacatalog/AbstractInventoryDataCatalog.java +++ b/cave/com.raytheon.viz.volumebrowser/src/com/raytheon/viz/volumebrowser/datacatalog/AbstractInventoryDataCatalog.java @@ -45,6 +45,7 @@ import com.raytheon.uf.viz.core.level.LevelMapping; import com.raytheon.uf.viz.core.level.LevelMappingFactory; import com.raytheon.uf.viz.core.level.LevelUtilities; import com.raytheon.uf.viz.derivparam.inv.AbstractInventory; +import com.raytheon.viz.volumebrowser.vbui.DataListsProdTableComp.DataSelection; import com.raytheon.viz.volumebrowser.vbui.MenuItemManager; import com.raytheon.viz.volumebrowser.vbui.SelectedData; import com.raytheon.viz.volumebrowser.vbui.VBMenuBarItemsMgr.ViewMenu; @@ -172,6 +173,7 @@ public abstract class AbstractInventoryDataCatalog extends AbstractDataCatalog { while (levelStr != null) { // Convert levels into planes. Level level = LevelFactory.getInstance().getLevel(levelStr); + if (levels3D.contains(level)) { for (String plane : get3DPlanes(sourcesToProcess)) { request.addAvailablePlane(plane); @@ -181,10 +183,13 @@ public abstract class AbstractInventoryDataCatalog extends AbstractDataCatalog { + level.getMasterLevel().getName()); LevelMapping lm = LevelMappingFactory.getInstance() .getLevelMappingForLevel(level); + if (lm != null) { request.addAvailablePlane(lm.getKey()); } + levelStr = levelQueue.poll(); + } if (request.isCanceled()) { Thread thread = inventoryJob.getThread(); @@ -233,7 +238,10 @@ public abstract class AbstractInventoryDataCatalog extends AbstractDataCatalog { } catch (InterruptedException e) { statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(), e); } - return new ArrayList(returnQueue); + List result = new ArrayList(returnQueue); + result.retainAll(MenuItemManager.getInstance() + .getMapOfKeys(DataSelection.SOURCES).keySet()); + return result; } public List getSupportedSources() { @@ -267,6 +275,7 @@ public abstract class AbstractInventoryDataCatalog extends AbstractDataCatalog { if (plane.startsWith("spatial-")) { levels = LevelUtilities.getOrderedSetOfStandardLevels(plane .replace("spatial-", "")); + } else { LevelMapping lm = lmf.getLevelMappingForKey(plane); if (lm != null) { diff --git a/cave/com.raytheon.viz.volumebrowser/src/com/raytheon/viz/volumebrowser/datacatalog/GribDataCatalog.java b/cave/com.raytheon.viz.volumebrowser/src/com/raytheon/viz/volumebrowser/datacatalog/GribDataCatalog.java index 361986a151..aa46799050 100644 --- a/cave/com.raytheon.viz.volumebrowser/src/com/raytheon/viz/volumebrowser/datacatalog/GribDataCatalog.java +++ b/cave/com.raytheon.viz.volumebrowser/src/com/raytheon/viz/volumebrowser/datacatalog/GribDataCatalog.java @@ -181,6 +181,7 @@ public class GribDataCatalog extends AbstractInventoryDataCatalog { List selectedLevels = new ArrayList(lmf .getLevelMappingForKey(catalogEntry.selectedPlanesKey) .getLevels()); + RequestConstraint masterRC = new RequestConstraint(null, ConstraintType.IN); RequestConstraint oneRC = new RequestConstraint(null, @@ -353,19 +354,19 @@ public class GribDataCatalog extends AbstractInventoryDataCatalog { */ @Override protected Collection get3DLevels() { + ArrayList all = new ArrayList(); NavigableSet tilts = LevelUtilities .getOrderedSetOfStandardLevels("TILT"); - NavigableSet pres = LevelUtilities - .getOrderedSetOfStandardLevels("MB"); - NavigableSet theta = LevelUtilities - .getOrderedSetOfStandardLevels("K"); - ArrayList all = new ArrayList(); - if (pres != null) { - all.addAll(pres); - } if (tilts != null) { all.addAll(tilts); } + NavigableSet pres = LevelUtilities + .getOrderedSetOfStandardLevels("MB"); + if (pres != null) { + all.addAll(pres); + } + NavigableSet theta = LevelUtilities + .getOrderedSetOfStandardLevels("K"); if (theta != null) { all.addAll(theta); } @@ -409,11 +410,20 @@ public class GribDataCatalog extends AbstractInventoryDataCatalog { && lons.isEmpty()) { return null; } + List sources = getSupportedSourcesInternal(); + Map coverages = new HashMap(); + try { + coverages = CoverageUtils.getInstance().getCoverages(sources); + } catch (VizException e) { + statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(), e); + } Set fileredSources = new HashSet(); - for (String source : getSupportedSourcesInternal()) { + for (String source : sources) { try { - GridCoverage coverage = CoverageUtils.getInstance() - .getCoverage(source); + GridCoverage coverage = coverages.get(source); + if (coverage == null) { + coverage = CoverageUtils.getInstance().getCoverage(source); + } if (coverage == null) { fileredSources.add(source); continue; @@ -489,11 +499,19 @@ public class GribDataCatalog extends AbstractInventoryDataCatalog { return results; } ToolsDataManager tdm = ToolsDataManager.getInstance(); + Map coverages = new HashMap(); + try { + coverages = CoverageUtils.getInstance().getCoverages(sources); + } catch (VizException e) { + statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(), e); + } Set validPlanes = new HashSet(sources.size()); for (String source : sources) { try { - GridCoverage coverage = CoverageUtils.getInstance() - .getCoverage(source); + GridCoverage coverage = coverages.get(source); + if (coverage == null) { + coverage = CoverageUtils.getInstance().getCoverage(source); + } if (coverage == null) { Set results = new HashSet(); results.addAll(MenuItemManager.getInstance() diff --git a/edexOsgi/com.raytheon.edex.plugin.grib/res/spring/grib-request.xml b/edexOsgi/com.raytheon.edex.plugin.grib/res/spring/grib-request.xml index cf55a193c8..4bbbb1e9cc 100644 --- a/edexOsgi/com.raytheon.edex.plugin.grib/res/spring/grib-request.xml +++ b/edexOsgi/com.raytheon.edex.plugin.grib/res/spring/grib-request.xml @@ -16,6 +16,12 @@ + + + + + + \ No newline at end of file diff --git a/edexOsgi/com.raytheon.edex.plugin.grib/src/com/raytheon/edex/plugin/grib/handler/GetCoveragesHandler.java b/edexOsgi/com.raytheon.edex.plugin.grib/src/com/raytheon/edex/plugin/grib/handler/GetCoveragesHandler.java new file mode 100644 index 0000000000..4778a56a99 --- /dev/null +++ b/edexOsgi/com.raytheon.edex.plugin.grib/src/com/raytheon/edex/plugin/grib/handler/GetCoveragesHandler.java @@ -0,0 +1,58 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.edex.plugin.grib.handler; + +import java.util.ArrayList; +import java.util.List; + +import com.raytheon.edex.plugin.grib.spatial.GribSpatialCache; +import com.raytheon.uf.common.dataplugin.grib.request.GetCoveragesRequest; +import com.raytheon.uf.common.dataplugin.grib.spatial.projections.GridCoverage; +import com.raytheon.uf.common.serialization.comm.IRequestHandler; + +/** + * + * TODO Add Description + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Dec 20, 2011            bsteffen     Initial creation
+ * 
+ *
+ * + * @author bsteffen + * @version 1.0 + */ +public class GetCoveragesHandler implements + IRequestHandler { + + @Override + public List handleRequest(GetCoveragesRequest request) { + List result = new ArrayList(); + for (String modelName : request.getModelNames()) { + result.add(GribSpatialCache.getInstance().getGrid(modelName)); + } + return result; + } +} diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.grib/src/com/raytheon/uf/common/dataplugin/grib/request/GetCoveragesRequest.java b/edexOsgi/com.raytheon.uf.common.dataplugin.grib/src/com/raytheon/uf/common/dataplugin/grib/request/GetCoveragesRequest.java new file mode 100644 index 0000000000..5d538a3118 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.grib/src/com/raytheon/uf/common/dataplugin/grib/request/GetCoveragesRequest.java @@ -0,0 +1,58 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.common.dataplugin.grib.request; + +import java.util.List; + +import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; +import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; +import com.raytheon.uf.common.serialization.comm.IServerRequest; + +/** + * + * TODO Add Description + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Dec 20, 2011            bsteffen     Initial creation
+ * 
+ *
+ * + * @author bsteffen + * @version 1.0 + */ +@DynamicSerialize +public class GetCoveragesRequest implements IServerRequest { + @DynamicSerializeElement + private List modelNames; + + public List getModelNames() { + return modelNames; + } + + public void setModelNames(List modelNames) { + this.modelNames = modelNames; + } + +}