xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #183 change grib data catalog to use bulk requests to retrieve all coverages at once.

Browse source 
Change-Id: I99f3dfc7900b57b074062155a4e7a65851f70170

Former-commit-id: 3cdfc8f73cce96c00b7ac37a1e34e9d051b628b6
This commit is contained in:
Ben Steffensmeier 2012-01-27 14:48:40 -06:00
parent ab1273cc02
commit 113722a596
7 changed files with 254 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

60
cave/com.raytheon.viz.grid/src/com/raytheon/viz/grid/inv/GridInventory.java
View file

@ -152,6 +152,7 @@ public class GridInventory extends AbstractInventory implements
for (Map<String, RequestConstraint> constraints : constraintsToTry) {
evaluateRequestConstraints(constraints);
}
}
private DataTree getTreeFromEdex() {
@ -349,6 +350,7 @@ public class GridInventory extends AbstractInventory implements
* @param newGridTree
*/
private void initAliasModels(DataTree newGridTree) {
Set<String> allAliasModels = new HashSet<String>();
sourceAliases.clear();
GribModelLookup lookup = GribModelLookup.getInstance();
for (String modelName : newGridTree.getSources()) {
@ -357,6 +359,8 @@ public class GridInventory extends AbstractInventory implements
SourceNode source = newGridTree.getSourceNode(modelName);
SourceNode dest = newGridTree.getSourceNode(model.getAlias());
if (source != null && dest != null) {
allAliasModels.add(source.getValue());
allAliasModels.add(dest.getValue());
List<String> aliases = sourceAliases.get(dest.getValue());
if (aliases == null) {
aliases = new ArrayList<String>();
@ -367,28 +371,29 @@ public class GridInventory extends AbstractInventory implements
}
}
}
for (Entry<String, List<String>> aliases : sourceAliases.entrySet()) {
Collections.sort(aliases.getValue(), new Comparator<String>() {
// Requesting coverages all at once is more efficient
try {
final Map<String, GridCoverage> coverages = CoverageUtils
.getInstance().getCoverages(allAliasModels);
@Override
public int compare(String model1, String model2) {
try {
GridCoverage coverage1 = CoverageUtils.getInstance()
.getCoverage(model1);
for (Entry<String, List<String>> aliases : sourceAliases.entrySet()) {
Collections.sort(aliases.getValue(), new Comparator<String>() {
@Override
public int compare(String model1, String model2) {
GridCoverage coverage1 = coverages.get(model1);
Integer res1 = coverage1.getNx() * coverage1.getNy();
GridCoverage coverage2 = CoverageUtils.getInstance()
.getCoverage(model2);
GridCoverage coverage2 = coverages.get(model2);
Integer res2 = coverage2.getNx() * coverage2.getNy();
return res2.compareTo(res1);
} catch (VizException e) {
statusHandler.handle(Priority.PROBLEM,
"Unable to create model aliases, problems with "
+ model1 + " and " + model2, e);
return 0;
}
}
});
});
}
} catch (VizException e) {
statusHandler.handle(Priority.PROBLEM,
"Unable to create model aliases", e);
return;
}
}
@ -449,6 +454,28 @@ public class GridInventory extends AbstractInventory implements
public List<Integer> getPerts(Map<String, RequestConstraint> query)
throws VizException {
RequestConstraint nameRC = query.get(MODEL_NAME_QUERY);
if (nameRC == null) {
// Only bother grabbing nodes with perts
nameRC = new RequestConstraint(null, ConstraintType.IN);
nameRC.setConstraintValueList(modelsWithPerts
.toArray(new String[0]));
query = new HashMap<String, RequestConstraint>(query);
query.put(MODEL_NAME_QUERY, nameRC);
} else {
boolean hasPerts = false;
for (String modelName : modelsWithPerts) {
if (nameRC.evaluate(modelName)) {
hasPerts = true;
break;
}
}
// If this query is not valid for any models with perts then it has
// no perts, don't bother with a query.
if (!hasPerts) {
return Collections.emptyList();
}
}
Set<Integer> perts = new HashSet<Integer>();
for (AbstractRequestableLevelNode node : evaluateRequestConstraints(query)) {
perts.addAll(getPerts(node));
@ -459,7 +486,6 @@ public class GridInventory extends AbstractInventory implements
protected static List<Integer> getPerts(AbstractRequestableLevelNode node)
throws VizException {
if (node instanceof GribRequestableLevelNode) {
GribRequestableLevelNode gNode = (GribRequestableLevelNode) node;
if (gNode.getPerts() != null) {

64
cave/com.raytheon.viz.grid/src/com/raytheon/viz/grid/util/CoverageUtils.java
View file

@ -22,8 +22,11 @@ package com.raytheon.viz.grid.util;
import java.awt.RenderingHints;
import java.awt.image.Raster;
import java.awt.image.RenderedImage;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.media.jai.BorderExtender;
@ -40,6 +43,7 @@ import org.geotools.coverage.processing.Operations;
import org.opengis.geometry.Envelope;
import com.raytheon.uf.common.dataplugin.grib.request.GetCoverageRequest;
import com.raytheon.uf.common.dataplugin.grib.request.GetCoveragesRequest;
import com.raytheon.uf.common.dataplugin.grib.spatial.projections.GridCoverage;
import com.raytheon.uf.common.datastorage.records.FloatDataRecord;
import com.raytheon.uf.common.geospatial.MapUtil;
@ -79,28 +83,56 @@ public class CoverageUtils {
}
public GridCoverage getCoverage(String modelName) throws VizException {
GridCoverage rval = coverageCache.get(modelName);
synchronized (coverageCache) {
GridCoverage rval = coverageCache.get(modelName);
if (rval == null) {
GetCoverageRequest request = new GetCoverageRequest();
request.setModelName(modelName);
Object obj = ThriftClient.sendRequest(request);
if (rval == null) {
GetCoverageRequest request = new GetCoverageRequest();
request.setModelName(modelName);
Object obj = ThriftClient.sendRequest(request);
if (obj != null) {
if (obj instanceof GridCoverage) {
rval = (GridCoverage) obj;
coverageCache.put(modelName, rval);
if (obj != null) {
if (obj instanceof GridCoverage) {
rval = (GridCoverage) obj;
coverageCache.put(modelName, rval);
} else {
throw new VizException(
"GetCoverageRequest returned object of type ["
+ obj.getClass().getName()
+ "], expected ["
+ GridCoverage.class.getName() + "]");
}
}
}
return rval;
}
}
public Map<String, GridCoverage> getCoverages(Collection<String> modelNames)
throws VizException {
Map<String, GridCoverage> coverages = new HashMap<String, GridCoverage>();
List<String> toRequest = new ArrayList<String>();
synchronized (coverageCache) {
for (String modelName : modelNames) {
GridCoverage coverage = coverageCache.get(modelName);
if (coverage == null) {
toRequest.add(modelName);
} else {
throw new VizException(
"GetCoverageRequest returned object of type ["
+ obj.getClass().getName()
+ "], expected ["
+ GridCoverage.class.getName() + "]");
coverages.put(modelName, coverage);
}
}
if (!toRequest.isEmpty()) {
GetCoveragesRequest request = new GetCoveragesRequest();
request.setModelNames(toRequest);
List<?> list = (List<?>) ThriftClient.sendRequest(request);
for (int i = 0; i < list.size(); i++) {
coverageCache.put(toRequest.get(i),
(GridCoverage) list.get(i));
coverages.put(toRequest.get(i), (GridCoverage) list.get(i));
}
}
}
return rval;
return coverages;
}
/**

11
cave/com.raytheon.viz.volumebrowser/src/com/raytheon/viz/volumebrowser/datacatalog/AbstractInventoryDataCatalog.java
View file

@ -45,6 +45,7 @@ import com.raytheon.uf.viz.core.level.LevelMapping;
import com.raytheon.uf.viz.core.level.LevelMappingFactory;
import com.raytheon.uf.viz.core.level.LevelUtilities;
import com.raytheon.uf.viz.derivparam.inv.AbstractInventory;
import com.raytheon.viz.volumebrowser.vbui.DataListsProdTableComp.DataSelection;
import com.raytheon.viz.volumebrowser.vbui.MenuItemManager;
import com.raytheon.viz.volumebrowser.vbui.SelectedData;
import com.raytheon.viz.volumebrowser.vbui.VBMenuBarItemsMgr.ViewMenu;
@ -172,6 +173,7 @@ public abstract class AbstractInventoryDataCatalog extends AbstractDataCatalog {
while (levelStr != null) {
// Convert levels into planes.
Level level = LevelFactory.getInstance().getLevel(levelStr);
if (levels3D.contains(level)) {
for (String plane : get3DPlanes(sourcesToProcess)) {
request.addAvailablePlane(plane);
@ -181,10 +183,13 @@ public abstract class AbstractInventoryDataCatalog extends AbstractDataCatalog {
+ level.getMasterLevel().getName());
LevelMapping lm = LevelMappingFactory.getInstance()
.getLevelMappingForLevel(level);
if (lm != null) {
request.addAvailablePlane(lm.getKey());
}
levelStr = levelQueue.poll();
}
if (request.isCanceled()) {
Thread thread = inventoryJob.getThread();
@ -233,7 +238,10 @@ public abstract class AbstractInventoryDataCatalog extends AbstractDataCatalog {
} catch (InterruptedException e) {
statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(), e);
}
return new ArrayList<String>(returnQueue);
List<String> result = new ArrayList<String>(returnQueue);
result.retainAll(MenuItemManager.getInstance()
.getMapOfKeys(DataSelection.SOURCES).keySet());
return result;
}
public List<String> getSupportedSources() {
@ -267,6 +275,7 @@ public abstract class AbstractInventoryDataCatalog extends AbstractDataCatalog {
if (plane.startsWith("spatial-")) {
levels = LevelUtilities.getOrderedSetOfStandardLevels(plane
.replace("spatial-", ""));
} else {
LevelMapping lm = lmf.getLevelMappingForKey(plane);
if (lm != null) {

44
cave/com.raytheon.viz.volumebrowser/src/com/raytheon/viz/volumebrowser/datacatalog/GribDataCatalog.java
View file

@ -181,6 +181,7 @@ public class GribDataCatalog extends AbstractInventoryDataCatalog {
List<Level> selectedLevels = new ArrayList<Level>(lmf
.getLevelMappingForKey(catalogEntry.selectedPlanesKey)
.getLevels());
RequestConstraint masterRC = new RequestConstraint(null,
ConstraintType.IN);
RequestConstraint oneRC = new RequestConstraint(null,
@ -353,19 +354,19 @@ public class GribDataCatalog extends AbstractInventoryDataCatalog {
*/
@Override
protected Collection<? extends Level> get3DLevels() {
ArrayList<Level> all = new ArrayList<Level>();
NavigableSet<Level> tilts = LevelUtilities
.getOrderedSetOfStandardLevels("TILT");
NavigableSet<Level> pres = LevelUtilities
.getOrderedSetOfStandardLevels("MB");
NavigableSet<Level> theta = LevelUtilities
.getOrderedSetOfStandardLevels("K");
ArrayList<Level> all = new ArrayList<Level>();
if (pres != null) {
all.addAll(pres);
}
if (tilts != null) {
all.addAll(tilts);
}
NavigableSet<Level> pres = LevelUtilities
.getOrderedSetOfStandardLevels("MB");
if (pres != null) {
all.addAll(pres);
}
NavigableSet<Level> theta = LevelUtilities
.getOrderedSetOfStandardLevels("K");
if (theta != null) {
all.addAll(theta);
}
@ -409,11 +410,20 @@ public class GribDataCatalog extends AbstractInventoryDataCatalog {
&& lons.isEmpty()) {
return null;
}
List<String> sources = getSupportedSourcesInternal();
Map<String, GridCoverage> coverages = new HashMap<String, GridCoverage>();
try {
coverages = CoverageUtils.getInstance().getCoverages(sources);
} catch (VizException e) {
statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(), e);
}
Set<String> fileredSources = new HashSet<String>();
for (String source : getSupportedSourcesInternal()) {
for (String source : sources) {
try {
GridCoverage coverage = CoverageUtils.getInstance()
.getCoverage(source);
GridCoverage coverage = coverages.get(source);
if (coverage == null) {
coverage = CoverageUtils.getInstance().getCoverage(source);
}
if (coverage == null) {
fileredSources.add(source);
continue;
@ -489,11 +499,19 @@ public class GribDataCatalog extends AbstractInventoryDataCatalog {
return results;
}
ToolsDataManager tdm = ToolsDataManager.getInstance();
Map<String, GridCoverage> coverages = new HashMap<String, GridCoverage>();
try {
coverages = CoverageUtils.getInstance().getCoverages(sources);
} catch (VizException e) {
statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(), e);
}
Set<String> validPlanes = new HashSet<String>(sources.size());
for (String source : sources) {
try {
GridCoverage coverage = CoverageUtils.getInstance()
.getCoverage(source);
GridCoverage coverage = coverages.get(source);
if (coverage == null) {
coverage = CoverageUtils.getInstance().getCoverage(source);
}
if (coverage == null) {
Set<String> results = new HashSet<String>();
results.addAll(MenuItemManager.getInstance()

6
edexOsgi/com.raytheon.edex.plugin.grib/res/spring/grib-request.xml
View file

@ -16,6 +16,12 @@
<constructor-arg value="com.raytheon.uf.common.dataplugin.grib.request.GetCoverageRequest"/>
<constructor-arg ref="getCoverageHandler"/>
</bean>
<bean id="getCoveragesHandler" class="com.raytheon.edex.plugin.grib.handler.GetCoveragesHandler" />
<bean factory-bean="handlerRegistry" factory-method="register">
<constructor-arg value="com.raytheon.uf.common.dataplugin.grib.request.GetCoveragesRequest"/>
<constructor-arg ref="getCoveragesHandler"/>
</bean>
</beans>

58
edexOsgi/com.raytheon.edex.plugin.grib/src/com/raytheon/edex/plugin/grib/handler/GetCoveragesHandler.java Normal file
View file

@ -0,0 +1,58 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.edex.plugin.grib.handler;
import java.util.ArrayList;
import java.util.List;
import com.raytheon.edex.plugin.grib.spatial.GribSpatialCache;
import com.raytheon.uf.common.dataplugin.grib.request.GetCoveragesRequest;
import com.raytheon.uf.common.dataplugin.grib.spatial.projections.GridCoverage;
import com.raytheon.uf.common.serialization.comm.IRequestHandler;
/**
*
* TODO Add Description
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Dec 20, 2011 bsteffen Initial creation
*
* </pre>
*
* @author bsteffen
* @version 1.0
*/
public class GetCoveragesHandler implements
IRequestHandler<GetCoveragesRequest> {
@Override
public List<GridCoverage> handleRequest(GetCoveragesRequest request) {
List<GridCoverage> result = new ArrayList<GridCoverage>();
for (String modelName : request.getModelNames()) {
result.add(GribSpatialCache.getInstance().getGrid(modelName));
}
return result;
}
}

58
edexOsgi/com.raytheon.uf.common.dataplugin.grib/src/com/raytheon/uf/common/dataplugin/grib/request/GetCoveragesRequest.java Normal file
View file

@ -0,0 +1,58 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.dataplugin.grib.request;
import java.util.List;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
import com.raytheon.uf.common.serialization.comm.IServerRequest;
/**
*
* TODO Add Description
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Dec 20, 2011 bsteffen Initial creation
*
* </pre>
*
* @author bsteffen
* @version 1.0
*/
@DynamicSerialize
public class GetCoveragesRequest implements IServerRequest {
@DynamicSerializeElement
private List<String> modelNames;
public List<String> getModelNames() {
return modelNames;
}
public void setModelNames(List<String> modelNames) {
this.modelNames = modelNames;
}
}