Issue #183 change grib data catalog to use bulk requests to retrieve all coverages at once.

Change-Id: I99f3dfc7900b57b074062155a4e7a65851f70170

Former-commit-id: e76323dc97 [formerly d90356d266] [formerly 113722a596] [formerly e76323dc97 [formerly d90356d266] [formerly 113722a596] [formerly dfd869c1c1 [formerly 113722a596 [formerly 3cdfc8f73cce96c00b7ac37a1e34e9d051b628b6]]]]
Former-commit-id: dfd869c1c1
Former-commit-id: 573108f2c5 [formerly f2ff133117] [formerly 09449968f9cc4d3fac177bbbf9f58f412fe658c0 [formerly f082727112]]
Former-commit-id: 9649bd78778fa82525127c827ee65f451beab1a7 [formerly 7335f90bfb]
Former-commit-id: 0d9cbcd09e

cave/com.raytheon.viz.grid/src/com/raytheon/viz/grid/inv/GridInventory.java

@@ -152,6 +152,7 @@ public class GridInventory extends AbstractInventory implements
         for (Map<String, RequestConstraint> constraints : constraintsToTry) {
             evaluateRequestConstraints(constraints);
         }
+
     }
 
     private DataTree getTreeFromEdex() {
@@ -349,6 +350,7 @@ public class GridInventory extends AbstractInventory implements
      * @param newGridTree
      */
     private void initAliasModels(DataTree newGridTree) {
+        Set<String> allAliasModels = new HashSet<String>();
         sourceAliases.clear();
         GribModelLookup lookup = GribModelLookup.getInstance();
         for (String modelName : newGridTree.getSources()) {
@@ -357,6 +359,8 @@ public class GridInventory extends AbstractInventory implements
                 SourceNode source = newGridTree.getSourceNode(modelName);
                 SourceNode dest = newGridTree.getSourceNode(model.getAlias());
                 if (source != null && dest != null) {
+                    allAliasModels.add(source.getValue());
+                    allAliasModels.add(dest.getValue());
                     List<String> aliases = sourceAliases.get(dest.getValue());
                     if (aliases == null) {
                         aliases = new ArrayList<String>();
@@ -367,28 +371,29 @@ public class GridInventory extends AbstractInventory implements
                 }
             }
         }
-        for (Entry<String, List<String>> aliases : sourceAliases.entrySet()) {
+        // Requesting coverages all at once is more efficient
-            Collections.sort(aliases.getValue(), new Comparator<String>() {
+        try {
+            final Map<String, GridCoverage> coverages = CoverageUtils
+                    .getInstance().getCoverages(allAliasModels);
 
-                @Override
+            for (Entry<String, List<String>> aliases : sourceAliases.entrySet()) {
-                public int compare(String model1, String model2) {
+                Collections.sort(aliases.getValue(), new Comparator<String>() {
-                    try {
+
-                        GridCoverage coverage1 = CoverageUtils.getInstance()
+                    @Override
-                                .getCoverage(model1);
+                    public int compare(String model1, String model2) {
+                        GridCoverage coverage1 = coverages.get(model1);
                         Integer res1 = coverage1.getNx() * coverage1.getNy();
-                        GridCoverage coverage2 = CoverageUtils.getInstance()
+                        GridCoverage coverage2 = coverages.get(model2);
-                                .getCoverage(model2);
                         Integer res2 = coverage2.getNx() * coverage2.getNy();
                         return res2.compareTo(res1);
-                    } catch (VizException e) {
-                        statusHandler.handle(Priority.PROBLEM,
-                                "Unable to create model aliases, problems with "
-                                        + model1 + " and " + model2, e);
-                        return 0;
                     }
-                }
 
-            });
+                });
+            }
+        } catch (VizException e) {
+            statusHandler.handle(Priority.PROBLEM,
+                    "Unable to create model aliases", e);
+            return;
         }
     }
 
@@ -449,6 +454,28 @@ public class GridInventory extends AbstractInventory implements
 
     public List<Integer> getPerts(Map<String, RequestConstraint> query)
             throws VizException {
+        RequestConstraint nameRC = query.get(MODEL_NAME_QUERY);
+        if (nameRC == null) {
+            // Only bother grabbing nodes with perts
+            nameRC = new RequestConstraint(null, ConstraintType.IN);
+            nameRC.setConstraintValueList(modelsWithPerts
+                    .toArray(new String[0]));
+            query = new HashMap<String, RequestConstraint>(query);
+            query.put(MODEL_NAME_QUERY, nameRC);
+        } else {
+            boolean hasPerts = false;
+            for (String modelName : modelsWithPerts) {
+                if (nameRC.evaluate(modelName)) {
+                    hasPerts = true;
+                    break;
+                }
+            }
+            // If this query is not valid for any models with perts then it has
+            // no perts, don't bother with a query.
+            if (!hasPerts) {
+                return Collections.emptyList();
+            }
+        }
         Set<Integer> perts = new HashSet<Integer>();
         for (AbstractRequestableLevelNode node : evaluateRequestConstraints(query)) {
             perts.addAll(getPerts(node));
@@ -459,7 +486,6 @@ public class GridInventory extends AbstractInventory implements
 
     protected static List<Integer> getPerts(AbstractRequestableLevelNode node)
             throws VizException {
-
         if (node instanceof GribRequestableLevelNode) {
             GribRequestableLevelNode gNode = (GribRequestableLevelNode) node;
             if (gNode.getPerts() != null) {

cave/com.raytheon.viz.grid/src/com/raytheon/viz/grid/util/CoverageUtils.java

@@ -22,8 +22,11 @@ package com.raytheon.viz.grid.util;
 import java.awt.RenderingHints;
 import java.awt.image.Raster;
 import java.awt.image.RenderedImage;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import javax.media.jai.BorderExtender;
@@ -40,6 +43,7 @@ import org.geotools.coverage.processing.Operations;
 import org.opengis.geometry.Envelope;
 
 import com.raytheon.uf.common.dataplugin.grib.request.GetCoverageRequest;
+import com.raytheon.uf.common.dataplugin.grib.request.GetCoveragesRequest;
 import com.raytheon.uf.common.dataplugin.grib.spatial.projections.GridCoverage;
 import com.raytheon.uf.common.datastorage.records.FloatDataRecord;
 import com.raytheon.uf.common.geospatial.MapUtil;
@@ -79,28 +83,56 @@ public class CoverageUtils {
     }
 
     public GridCoverage getCoverage(String modelName) throws VizException {
-        GridCoverage rval = coverageCache.get(modelName);
+        synchronized (coverageCache) {
+            GridCoverage rval = coverageCache.get(modelName);
 
-        if (rval == null) {
+            if (rval == null) {
-            GetCoverageRequest request = new GetCoverageRequest();
+                GetCoverageRequest request = new GetCoverageRequest();
-            request.setModelName(modelName);
+                request.setModelName(modelName);
-            Object obj = ThriftClient.sendRequest(request);
+                Object obj = ThriftClient.sendRequest(request);
 
-            if (obj != null) {
+                if (obj != null) {
-                if (obj instanceof GridCoverage) {
+                    if (obj instanceof GridCoverage) {
-                    rval = (GridCoverage) obj;
+                        rval = (GridCoverage) obj;
-                    coverageCache.put(modelName, rval);
+                        coverageCache.put(modelName, rval);
+                    } else {
+                        throw new VizException(
+                                "GetCoverageRequest returned object of type ["
+                                        + obj.getClass().getName()
+                                        + "], expected ["
+                                        + GridCoverage.class.getName() + "]");
+                    }
+                }
+            }
+            return rval;
+        }
+    }
+
+    public Map<String, GridCoverage> getCoverages(Collection<String> modelNames)
+            throws VizException {
+        Map<String, GridCoverage> coverages = new HashMap<String, GridCoverage>();
+        List<String> toRequest = new ArrayList<String>();
+        synchronized (coverageCache) {
+            for (String modelName : modelNames) {
+                GridCoverage coverage = coverageCache.get(modelName);
+                if (coverage == null) {
+                    toRequest.add(modelName);
                 } else {
-                    throw new VizException(
+                    coverages.put(modelName, coverage);
-                            "GetCoverageRequest returned object of type ["
+                }
-                                    + obj.getClass().getName()
+            }
-                                    + "], expected ["
+            if (!toRequest.isEmpty()) {
-                                    + GridCoverage.class.getName() + "]");
+                GetCoveragesRequest request = new GetCoveragesRequest();
+                request.setModelNames(toRequest);
+                List<?> list = (List<?>) ThriftClient.sendRequest(request);
+                for (int i = 0; i < list.size(); i++) {
+                    coverageCache.put(toRequest.get(i),
+                            (GridCoverage) list.get(i));
+                    coverages.put(toRequest.get(i), (GridCoverage) list.get(i));
                 }
             }
         }
-
+        return coverages;
-        return rval;
     }
 
     /**

cave/com.raytheon.viz.volumebrowser/src/com/raytheon/viz/volumebrowser/datacatalog/AbstractInventoryDataCatalog.java

@@ -45,6 +45,7 @@ import com.raytheon.uf.viz.core.level.LevelMapping;
 import com.raytheon.uf.viz.core.level.LevelMappingFactory;
 import com.raytheon.uf.viz.core.level.LevelUtilities;
 import com.raytheon.uf.viz.derivparam.inv.AbstractInventory;
+import com.raytheon.viz.volumebrowser.vbui.DataListsProdTableComp.DataSelection;
 import com.raytheon.viz.volumebrowser.vbui.MenuItemManager;
 import com.raytheon.viz.volumebrowser.vbui.SelectedData;
 import com.raytheon.viz.volumebrowser.vbui.VBMenuBarItemsMgr.ViewMenu;
@@ -172,6 +173,7 @@ public abstract class AbstractInventoryDataCatalog extends AbstractDataCatalog {
             while (levelStr != null) {
                 // Convert levels into planes.
                 Level level = LevelFactory.getInstance().getLevel(levelStr);
+
                 if (levels3D.contains(level)) {
                     for (String plane : get3DPlanes(sourcesToProcess)) {
                         request.addAvailablePlane(plane);
@@ -181,10 +183,13 @@ public abstract class AbstractInventoryDataCatalog extends AbstractDataCatalog {
                         + level.getMasterLevel().getName());
                 LevelMapping lm = LevelMappingFactory.getInstance()
                         .getLevelMappingForLevel(level);
+
                 if (lm != null) {
                     request.addAvailablePlane(lm.getKey());
                 }
+
                 levelStr = levelQueue.poll();
+
             }
             if (request.isCanceled()) {
                 Thread thread = inventoryJob.getThread();
@@ -233,7 +238,10 @@ public abstract class AbstractInventoryDataCatalog extends AbstractDataCatalog {
         } catch (InterruptedException e) {
             statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(), e);
         }
-        return new ArrayList<String>(returnQueue);
+        List<String> result = new ArrayList<String>(returnQueue);
+        result.retainAll(MenuItemManager.getInstance()
+                .getMapOfKeys(DataSelection.SOURCES).keySet());
+        return result;
     }
 
     public List<String> getSupportedSources() {
@@ -267,6 +275,7 @@ public abstract class AbstractInventoryDataCatalog extends AbstractDataCatalog {
                 if (plane.startsWith("spatial-")) {
                     levels = LevelUtilities.getOrderedSetOfStandardLevels(plane
                             .replace("spatial-", ""));
+
                 } else {
                     LevelMapping lm = lmf.getLevelMappingForKey(plane);
                     if (lm != null) {

cave/com.raytheon.viz.volumebrowser/src/com/raytheon/viz/volumebrowser/datacatalog/GribDataCatalog.java

@@ -181,6 +181,7 @@ public class GribDataCatalog extends AbstractInventoryDataCatalog {
                 List<Level> selectedLevels = new ArrayList<Level>(lmf
                         .getLevelMappingForKey(catalogEntry.selectedPlanesKey)
                         .getLevels());
+
                 RequestConstraint masterRC = new RequestConstraint(null,
                         ConstraintType.IN);
                 RequestConstraint oneRC = new RequestConstraint(null,
@@ -353,19 +354,19 @@ public class GribDataCatalog extends AbstractInventoryDataCatalog {
      */
     @Override
     protected Collection<? extends Level> get3DLevels() {
+        ArrayList<Level> all = new ArrayList<Level>();
         NavigableSet<Level> tilts = LevelUtilities
                 .getOrderedSetOfStandardLevels("TILT");
-        NavigableSet<Level> pres = LevelUtilities
-                .getOrderedSetOfStandardLevels("MB");
-        NavigableSet<Level> theta = LevelUtilities
-                .getOrderedSetOfStandardLevels("K");
-        ArrayList<Level> all = new ArrayList<Level>();
-        if (pres != null) {
-            all.addAll(pres);
-        }
         if (tilts != null) {
             all.addAll(tilts);
         }
+        NavigableSet<Level> pres = LevelUtilities
+                .getOrderedSetOfStandardLevels("MB");
+        if (pres != null) {
+            all.addAll(pres);
+        }
+        NavigableSet<Level> theta = LevelUtilities
+                .getOrderedSetOfStandardLevels("K");
         if (theta != null) {
             all.addAll(theta);
         }
@@ -409,11 +410,20 @@ public class GribDataCatalog extends AbstractInventoryDataCatalog {
                 && lons.isEmpty()) {
             return null;
         }
+        List<String> sources = getSupportedSourcesInternal();
+        Map<String, GridCoverage> coverages = new HashMap<String, GridCoverage>();
+        try {
+            coverages = CoverageUtils.getInstance().getCoverages(sources);
+        } catch (VizException e) {
+            statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(), e);
+        }
         Set<String> fileredSources = new HashSet<String>();
-        for (String source : getSupportedSourcesInternal()) {
+        for (String source : sources) {
             try {
-                GridCoverage coverage = CoverageUtils.getInstance()
+                GridCoverage coverage = coverages.get(source);
-                        .getCoverage(source);
+                if (coverage == null) {
+                    coverage = CoverageUtils.getInstance().getCoverage(source);
+                }
                 if (coverage == null) {
                     fileredSources.add(source);
                     continue;
@@ -489,11 +499,19 @@ public class GribDataCatalog extends AbstractInventoryDataCatalog {
             return results;
         }
         ToolsDataManager tdm = ToolsDataManager.getInstance();
+        Map<String, GridCoverage> coverages = new HashMap<String, GridCoverage>();
+        try {
+            coverages = CoverageUtils.getInstance().getCoverages(sources);
+        } catch (VizException e) {
+            statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(), e);
+        }
         Set<String> validPlanes = new HashSet<String>(sources.size());
         for (String source : sources) {
             try {
-                GridCoverage coverage = CoverageUtils.getInstance()
+                GridCoverage coverage = coverages.get(source);
-                        .getCoverage(source);
+                if (coverage == null) {
+                    coverage = CoverageUtils.getInstance().getCoverage(source);
+                }
                 if (coverage == null) {
                     Set<String> results = new HashSet<String>();
                     results.addAll(MenuItemManager.getInstance()

edexOsgi/com.raytheon.edex.plugin.grib/res/spring/grib-request.xml

@@ -17,5 +17,11 @@
   		<constructor-arg ref="getCoverageHandler"/>
   	</bean>
   	
+   <bean id="getCoveragesHandler" class="com.raytheon.edex.plugin.grib.handler.GetCoveragesHandler" />
+   <bean factory-bean="handlerRegistry" factory-method="register">
+      <constructor-arg value="com.raytheon.uf.common.dataplugin.grib.request.GetCoveragesRequest"/>
+  	  <constructor-arg ref="getCoveragesHandler"/>
+   </bean>
+  
   
  </beans>

edexOsgi/com.raytheon.edex.plugin.grib/src/com/raytheon/edex/plugin/grib/handler/GetCoveragesHandler.java

@@ -0,0 +1,58 @@
+/**
+ * This software was developed and / or modified by Raytheon Company,
+ * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+ * 
+ * U.S. EXPORT CONTROLLED TECHNICAL DATA
+ * This software product contains export-restricted data whose
+ * export/transfer/disclosure is restricted by U.S. law. Dissemination
+ * to non-U.S. persons whether in the United States or abroad requires
+ * an export license or other authorization.
+ * 
+ * Contractor Name:        Raytheon Company
+ * Contractor Address:     6825 Pine Street, Suite 340
+ *                         Mail Stop B8
+ *                         Omaha, NE 68106
+ *                         402.291.0100
+ * 
+ * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+ * further licensing information.
+ **/
+package com.raytheon.edex.plugin.grib.handler;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.raytheon.edex.plugin.grib.spatial.GribSpatialCache;
+import com.raytheon.uf.common.dataplugin.grib.request.GetCoveragesRequest;
+import com.raytheon.uf.common.dataplugin.grib.spatial.projections.GridCoverage;
+import com.raytheon.uf.common.serialization.comm.IRequestHandler;
+
+/**
+ * 
+ * TODO Add Description
+ * 
+ * <pre>
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Dec 20, 2011            bsteffen     Initial creation
+ * 
+ * </pre>
+ * 
+ * @author bsteffen
+ * @version 1.0
+ */
+public class GetCoveragesHandler implements
+        IRequestHandler<GetCoveragesRequest> {
+
+    @Override
+    public List<GridCoverage> handleRequest(GetCoveragesRequest request) {
+        List<GridCoverage> result = new ArrayList<GridCoverage>();
+        for (String modelName : request.getModelNames()) {
+            result.add(GribSpatialCache.getInstance().getGrid(modelName));
+        }
+        return result;
+    }
+}

edexOsgi/com.raytheon.uf.common.dataplugin.grib/src/com/raytheon/uf/common/dataplugin/grib/request/GetCoveragesRequest.java

@@ -0,0 +1,58 @@
+/**
+ * This software was developed and / or modified by Raytheon Company,
+ * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+ * 
+ * U.S. EXPORT CONTROLLED TECHNICAL DATA
+ * This software product contains export-restricted data whose
+ * export/transfer/disclosure is restricted by U.S. law. Dissemination
+ * to non-U.S. persons whether in the United States or abroad requires
+ * an export license or other authorization.
+ * 
+ * Contractor Name:        Raytheon Company
+ * Contractor Address:     6825 Pine Street, Suite 340
+ *                         Mail Stop B8
+ *                         Omaha, NE 68106
+ *                         402.291.0100
+ * 
+ * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+ * further licensing information.
+ **/
+package com.raytheon.uf.common.dataplugin.grib.request;
+
+import java.util.List;
+
+import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
+import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
+import com.raytheon.uf.common.serialization.comm.IServerRequest;
+
+/**
+ * 
+ * TODO Add Description
+ * 
+ * <pre>
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Dec 20, 2011            bsteffen     Initial creation
+ * 
+ * </pre>
+ * 
+ * @author bsteffen
+ * @version 1.0
+ */
+@DynamicSerialize
+public class GetCoveragesRequest implements IServerRequest {
+    @DynamicSerializeElement
+    private List<String> modelNames;
+
+    public List<String> getModelNames() {
+        return modelNames;
+    }
+
+    public void setModelNames(List<String> modelNames) {
+        this.modelNames = modelNames;
+    }
+
+}