From eb821d758026b6584a164df5d9680de7f66743fc Mon Sep 17 00:00:00 2001 From: Ben Steffensmeier Date: Wed, 16 Jan 2013 10:38:23 -0600 Subject: [PATCH] Issue #1495 enable ensemble subset requests in opendap. Change-Id: I5c74b23f6c5c31b397855f102ee08f9c5737ac1a Former-commit-id: ddaf9e6c69221600deedfbaf991a3936d0bb711e --- .../common/ui/UserSelectComp.java | 1 + .../subset/GriddedSubsetManagerDlg.java | 5 +- .../datadelivery/registry/Ensemble.java | 196 +++++++++++++----- .../datadelivery/registry/GriddedDataSet.java | 22 ++ .../registry/GriddedDataSetMetaData.java | 15 -- .../datadelivery/registry/Parameter.java | 13 -- .../datadelivery/registry/Subscription.java | 13 ++ .../retrieval/util/DataSizeUtils.java | 27 ++- .../retrieval/xml/RetrievalAttribute.java | 13 ++ .../adapters/GridMetadataAdapter.java | 37 ++-- .../opendap/OpenDAPMetaDataParser.java | 12 +- .../opendap/OpenDAPParseUtility.java | 31 +-- .../opendap/OpenDAPRequestBuilder.java | 19 +- .../opendap/OpenDAPRetrievalGenerator.java | 66 ++++-- .../retrieval/response/OpenDAPTranslator.java | 49 +++-- .../util/ResponseProcessingUtilities.java | 4 +- .../util/RetrievalGeneratorUtilities.java | 38 ++-- 17 files changed, 369 insertions(+), 192 deletions(-) diff --git a/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/common/ui/UserSelectComp.java b/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/common/ui/UserSelectComp.java index a05d1b40f0..4c3f0b995c 100644 --- a/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/common/ui/UserSelectComp.java +++ b/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/common/ui/UserSelectComp.java @@ -413,6 +413,7 @@ public class UserSelectComp extends Composite implements IUpdate, IDisplay, u.setEnvelope(groupDefinition.getEnvelope()); u.setNumFcstHours(subscription.getTime() .getSelectedTimeIndices().size()); + u.setNumEnsembleMembers(subscription.getEnsemble()); u.determineNumberRequestedGrids(subscription.getParameter()); Coverage cov = new GriddedCoverage(); diff --git a/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/subset/GriddedSubsetManagerDlg.java b/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/subset/GriddedSubsetManagerDlg.java index 08310c7fdf..442698ad35 100644 --- a/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/subset/GriddedSubsetManagerDlg.java +++ b/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/subset/GriddedSubsetManagerDlg.java @@ -186,6 +186,9 @@ public class GriddedSubsetManagerDlg time.setSelectedTimeIndices(fcstIndices); subscription.setTime(time); + + subscription.setEnsemble(dataSet.getEnsemble()); + return subscription; } @@ -270,7 +273,7 @@ public class GriddedSubsetManagerDlg // Get the temporal data int numFcstHours = this.timingTabControls.getSelectedFcstHours().length; dataSize.setNumFcstHours(numFcstHours); - + dataSize.setNumEnsembleMembers(dataSet.getEnsemble()); // Get the Areal data ReferencedEnvelope envelope = this.spatialTabControls.getEnvelope(); diff --git a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Ensemble.java b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Ensemble.java index d53cd8625a..3d9047cea0 100644 --- a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Ensemble.java +++ b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Ensemble.java @@ -1,5 +1,9 @@ package com.raytheon.uf.common.datadelivery.registry; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; @@ -30,67 +34,161 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; @DynamicSerialize public class Ensemble implements ISerializableObject { + /** + * names of the various ensemble members + */ + @DynamicSerializeElement + @XmlAttribute + private List members; + + /** + * which members to request. + */ + @DynamicSerializeElement + @XmlAttribute + private List selectedMembers; + public Ensemble() { } - /** - * ensemble models set this - */ - @DynamicSerializeElement - @XmlAttribute - private String name; + public Ensemble(Ensemble other) { + if (other.members != null) { + members = new ArrayList(other.members); + } + if (other.selectedMembers != null) { + selectedMembers = new ArrayList(other.selectedMembers); + } + } + + public List getMembers() { + return members; + } + + public void setMembers(List members) { + this.members = members; + } + + public List getSelectedMembers() { + return selectedMembers; + } + + public void setSelectedMembers(List selectedMembers) { + this.selectedMembers = selectedMembers; + } + + public int getMemberCount() { + if (members == null) { + // if no members are defined then assume a single unnamed member. + return 1; + } else { + return members.size(); + } + } + + public int getSelectedMemberCount() { + if (members == null) { + // if no members are defined then assume a single unnamed member + // which is automatically selected + return 1; + } else if (selectedMembers == null) { + return 0; + } else { + return selectedMembers.size(); + } + } + + public boolean hasSelection() { + return selectedMembers != null && !selectedMembers.isEmpty(); + } /** - * ensemble models set this + * + * Get the range of indices of the selected members, inclusively. This + * method should usually be used after a split operation to guarantee that + * selected ensembles are consecutive + * + * @return a int[2] representing the start and end indices of the ensemble + * members + * @throws IllegalStateException + * if the selected member are non consecutive. */ - @DynamicSerializeElement - @XmlAttribute - private String length; + public int[] getSelectedRange() { + int[] result = new int[2]; + int[] indexes = getSortedIndexes(); + if (indexes == null) { + return result; + } + result[0] = indexes[0]; + result[1] = result[0] - 1; + for (int index : indexes) { + result[1] += 1; + if (result[1] != index) { + throw new IllegalStateException( + "Cannot get selected range for nonconsecutive ensemble members\nMembers " + + members + "\nSelected" + selectedMembers); + } + } + return result; + } /** - * ensemble models set this + * Split this ensemble into multiple Ensembles each with consecutive + * members. Used in cases where multiple ensembles can be requested at once + * if they are consecutive. + * + * @param maxEnsembles + * The mazximum number of consecutive members in any Ensemble + * @return */ - @DynamicSerializeElement - @XmlAttribute - private String init; - - /** - * ensemble models set this - */ - @DynamicSerializeElement - @XmlAttribute - private Integer size; - - public void setName(String name) { - this.name = name; + public List split(int maxEnsembles) { + int[] indexes = getSortedIndexes(); + if (indexes == null) { + return Arrays.asList(new Ensemble(this)); + } + List result = new ArrayList(); + List selected = new ArrayList(selectedMembers.size()); + int start = indexes[0]; + int end = start - 1; + for (int index : indexes) { + end += 1; + if (index != end || (end - start) >= maxEnsembles) { + // Either we have run out of consecutive indexes or we have the + // max consecutive. so split off and start the next one. + Ensemble e = new Ensemble(this); + e.setSelectedMembers(new ArrayList(selected)); + selected.clear(); + result.add(e); + start = index; + end = start; + } + selected.add(members.get(index)); + } + Ensemble e = new Ensemble(this); + e.setSelectedMembers(selected); + result.add(e); + return result; } - public String getName() { - return name; + private int[] getSortedIndexes() { + if (!hasSelection()) { + return null; + } + int[] indexes = new int[selectedMembers.size()]; + int c = 0; + for (String selected : selectedMembers) { + int index = members.indexOf(selected); + if (index >= 0) { + indexes[c++] = index; + } + } + if (c == 0) { + return null; + } else if (c < indexes.length) { + indexes = Arrays.copyOf(indexes, c); + } + Arrays.sort(indexes); + return indexes; } - public void setSize(Integer size) { - this.size = size; - } - - public Integer getSize() { - return size; - } - - public void setLength(String length) { - this.length = length; - } - - public String getLength() { - return length; - } - - public void setInit(String init) { - this.init = init; - } - - public String getInit() { - return init; - } } diff --git a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/GriddedDataSet.java b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/GriddedDataSet.java index 69dc1aaba9..39c50ecf39 100644 --- a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/GriddedDataSet.java +++ b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/GriddedDataSet.java @@ -20,6 +20,7 @@ package com.raytheon.uf.common.datadelivery.registry; import java.util.HashSet; +import java.util.List; import java.util.Set; import javax.xml.bind.annotation.XmlAccessType; @@ -59,6 +60,10 @@ public abstract class GriddedDataSet extends DataSet { @DynamicSerializeElement protected Set forecastHours = new HashSet(); + @DynamicSerializeElement + @XmlElement + private Ensemble ensemble; + /** * @return the cycles */ @@ -89,6 +94,14 @@ public abstract class GriddedDataSet extends DataSet { this.forecastHours = forecastHours; } + public Ensemble getEnsemble() { + return ensemble; + } + + public void setEnsemble(Ensemble ensemble) { + this.ensemble = ensemble; + } + /** * {@inheritDoc} */ @@ -100,6 +113,15 @@ public abstract class GriddedDataSet extends DataSet { GriddedDataSet other = (GriddedDataSet) toCombine; this.getCycles().addAll(other.getCycles()); this.getForecastHours().addAll(other.getForecastHours()); + if (this.getEnsemble() != null) { + List mine = this.getEnsemble().getSelectedMembers(); + List theirs = other.getEnsemble().getSelectedMembers(); + if (mine == null) { + ensemble.setSelectedMembers(theirs); + } else if (theirs != null) { + mine.addAll(theirs); + } + } } } } diff --git a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/GriddedDataSetMetaData.java b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/GriddedDataSetMetaData.java index bd620a29c4..70466a535c 100644 --- a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/GriddedDataSetMetaData.java +++ b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/GriddedDataSetMetaData.java @@ -6,7 +6,6 @@ import java.util.Map; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; -import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlSeeAlso; import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; @@ -60,13 +59,6 @@ public abstract class GriddedDataSetMetaData extends @SlotAttribute(CYCLE_SLOT) private int cycle = NO_CYCLE; - /** - * ensemble models set this - */ - @DynamicSerializeElement - @XmlElement - private Ensemble ensemble; - public void setLevelTypes(Map levelTypes) { this.levelTypes = levelTypes; } @@ -92,11 +84,4 @@ public abstract class GriddedDataSetMetaData extends return cycle; } - public void setEnsemble(Ensemble ensemble) { - this.ensemble = ensemble; - } - - public Ensemble getEnsemble() { - return ensemble; - } } diff --git a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Parameter.java b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Parameter.java index d21c2dfa57..f92747fedd 100644 --- a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Parameter.java +++ b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Parameter.java @@ -92,10 +92,6 @@ public class Parameter implements ISerializableObject, Serializable { @DynamicSerializeElement private String fillValue; - @XmlAttribute - @DynamicSerializeElement - private Integer ensemble; - @XmlAttribute @DynamicSerializeElement private String baseType; @@ -124,7 +120,6 @@ public class Parameter implements ISerializableObject, Serializable { this.dataType = copy.dataType; this.missingValue = copy.missingValue; this.fillValue = copy.fillValue; - this.ensemble = copy.ensemble; this.baseType = copy.baseType; // deep copy @@ -265,14 +260,6 @@ public class Parameter implements ISerializableObject, Serializable { return levels; } - public void setEnsemble(Integer ensemble) { - this.ensemble = ensemble; - } - - public Integer getEnsemble() { - return ensemble; - } - public void setFillValue(String fillValue) { this.fillValue = fillValue; } diff --git a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Subscription.java b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Subscription.java index a7eb9581e0..38f07a3ec7 100644 --- a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Subscription.java +++ b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Subscription.java @@ -243,6 +243,10 @@ public class Subscription implements ISerializableObject, Serializable { @DynamicSerializeElement private ArrayList parameter; + @XmlElement + @DynamicSerializeElement + private Ensemble ensemble; + @XmlAttribute @DynamicSerializeElement private boolean deleted; @@ -920,4 +924,13 @@ public class Subscription implements ISerializableObject, Serializable { public int getLatencyInMinutes() { return latencyInMinutes; } + + public Ensemble getEnsemble() { + return ensemble; + } + + public void setEnsemble(Ensemble ensemble) { + this.ensemble = ensemble; + } + } diff --git a/edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/util/DataSizeUtils.java b/edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/util/DataSizeUtils.java index 0c14d7da1c..0d5b656b19 100644 --- a/edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/util/DataSizeUtils.java +++ b/edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/util/DataSizeUtils.java @@ -26,6 +26,7 @@ import org.geotools.geometry.jts.ReferencedEnvelope; import com.raytheon.uf.common.datadelivery.registry.Coverage; import com.raytheon.uf.common.datadelivery.registry.DataSet; +import com.raytheon.uf.common.datadelivery.registry.Ensemble; import com.raytheon.uf.common.datadelivery.registry.GriddedCoverage; import com.raytheon.uf.common.datadelivery.registry.GriddedDataSet; import com.raytheon.uf.common.datadelivery.registry.Levels; @@ -102,6 +103,8 @@ public class DataSizeUtils { /** Number of forecast hours */ private int numFcstHours = 0; + private int numEnsembleMembers = 0; + /** Envelope */ private ReferencedEnvelope envelope = null; @@ -153,6 +156,7 @@ public class DataSizeUtils { public long getDataSetSizeInBytes() { long l = numRequestedGrids * numFcstHours + * numEnsembleMembers * dataSet.getServiceType().getRequestBytesPerParameterPerLevel( numberOfGridCells); return l; @@ -181,8 +185,15 @@ public class DataSizeUtils { long numCells = griddedCov.getGridCoverage().getNx() * griddedCov.getGridCoverage().getNy(); // Default to 1 forecast hour if not a gridded data set - long fcstHrs = dataSet instanceof GriddedDataSet ? ((GriddedDataSet) dataSet) - .getForecastHours().size() : 1; + long numEns = 1; + long fcstHrs = 1; + if (dataSet instanceof GriddedDataSet) { + GriddedDataSet gDataSet = (GriddedDataSet) dataSet; + fcstHrs = gDataSet.getForecastHours().size(); + if (gDataSet.getEnsemble() != null) { + numEns = gDataSet.getEnsemble().getMemberCount(); + } + } Map paramMap = dataSet.getParameters(); // get the number of grids available @@ -196,7 +207,8 @@ public class DataSizeUtils { numGridsAvailable += (numLevels > 0 ? numLevels : 1); } - fullSize = fcstHrs + fullSize = numEns + * fcstHrs * numGridsAvailable * dataSet.getServiceType() .getRequestBytesPerParameterPerLevel( @@ -295,4 +307,13 @@ public class DataSizeUtils { this.numRequestedGrids = numGrids; } + + public void setNumEnsembleMembers(Ensemble ensemble) { + if (ensemble == null) { + this.numEnsembleMembers = 1; + } else { + this.numEnsembleMembers = ensemble.getSelectedMemberCount(); + } + + } } diff --git a/edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/xml/RetrievalAttribute.java b/edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/xml/RetrievalAttribute.java index c5f48a70ff..860d87acee 100644 --- a/edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/xml/RetrievalAttribute.java +++ b/edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/xml/RetrievalAttribute.java @@ -29,6 +29,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; import com.raytheon.uf.common.datadelivery.registry.Coverage; import com.raytheon.uf.common.datadelivery.registry.CoverageAdapter; +import com.raytheon.uf.common.datadelivery.registry.Ensemble; import com.raytheon.uf.common.datadelivery.registry.Parameter; import com.raytheon.uf.common.datadelivery.registry.Time; import com.raytheon.uf.common.serialization.ISerializableObject; @@ -100,6 +101,10 @@ public class RetrievalAttribute implements ISerializableObject, Serializable { @DynamicSerializeElement private Time time; + @XmlElement + @DynamicSerializeElement + private Ensemble ensemble; + @XmlElement(name = "provider") @DynamicSerializeElement private String provider; @@ -156,4 +161,12 @@ public class RetrievalAttribute implements ISerializableObject, Serializable { return subName; } + public Ensemble getEnsemble() { + return ensemble; + } + + public void setEnsemble(Ensemble ensemble) { + this.ensemble = ensemble; + } + } diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/metadata/adapters/GridMetadataAdapter.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/metadata/adapters/GridMetadataAdapter.java index 3e8f524583..7649d934b3 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/metadata/adapters/GridMetadataAdapter.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/metadata/adapters/GridMetadataAdapter.java @@ -20,6 +20,8 @@ package com.raytheon.uf.edex.datadelivery.retrieval.metadata.adapters; import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import com.raytheon.edex.util.Util; import com.raytheon.uf.common.datadelivery.registry.GriddedCoverage; @@ -60,11 +62,18 @@ public class GridMetadataAdapter extends AbstractMetadataAdapter { Level[] levels = getLevels(attXML); int size = levels.length; + List ensembles = null; + if (attXML.getEnsemble() != null && attXML.getEnsemble().hasSelection()) { + ensembles = attXML.getEnsemble().getSelectedMembers(); + size *= ensembles.size(); + } else { + ensembles = Arrays.asList((String) null); + } + if (attXML.getTime().getSelectedTimeIndices() != null) { if (levels.length > 1 || attXML.getTime().getSelectedTimeIndices().size() > 1) { - size = levels.length - * attXML.getTime().getSelectedTimeIndices().size(); + size *= attXML.getTime().getSelectedTimeIndices().size(); } } @@ -80,20 +89,21 @@ public class GridMetadataAdapter extends AbstractMetadataAdapter { } if (attXML.getTime().getSelectedTimeIndices() != null) { - - for (int i = 0; i < attXML.getTime().getSelectedTimeIndices() - .size(); i++) { - for (int j = 0; j < levels.length; j++) { - int bin = (levels.length * i) + j; - pdos[bin] = populateGridRecord(attXML.getSubName(), - attXML.getParameter(), - levels[j], gridCoverage); + int bin = 0; + for (String ensemble : ensembles) { + for (int i = 0; i < attXML.getTime().getSelectedTimeIndices() + .size(); i++) { + for (int j = 0; j < levels.length; j++) { + pdos[bin++] = populateGridRecord(attXML.getSubName(), + attXML.getParameter(), levels[j], ensemble, + gridCoverage); + } } } } else { pdos[0] = populateGridRecord(attXML.getSubName(), - attXML.getParameter(), levels[0], + attXML.getParameter(), levels[0], ensembles.get(0), gridCoverage); } @@ -108,10 +118,9 @@ public class GridMetadataAdapter extends AbstractMetadataAdapter { * @return */ private GridRecord populateGridRecord(String name, Parameter parm, - Level level, - GridCoverage gridCoverage) { + Level level, String ensembleId, GridCoverage gridCoverage) { return ResponseProcessingUtilities.getGridRecord(name, parm, level, - gridCoverage); + ensembleId, gridCoverage); } /** diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPMetaDataParser.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPMetaDataParser.java index af804b230a..1cf52b060b 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPMetaDataParser.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPMetaDataParser.java @@ -37,6 +37,7 @@ import com.raytheon.uf.common.datadelivery.registry.DataSet; import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData; import com.raytheon.uf.common.datadelivery.registry.DataType; import com.raytheon.uf.common.datadelivery.registry.GriddedCoverage; +import com.raytheon.uf.common.datadelivery.registry.GriddedDataSet; import com.raytheon.uf.common.datadelivery.registry.GriddedDataSetMetaData; import com.raytheon.uf.common.datadelivery.registry.Levels; import com.raytheon.uf.common.datadelivery.registry.OpenDapGriddedDataSet; @@ -179,7 +180,8 @@ class OpenDAPMetaDataParser extends MetaDataParser { * @param dataDateFormat * @return */ - private Map getParameters(DAS das, DataSet dataSet, + private Map getParameters(DAS das, + GriddedDataSet dataSet, GriddedDataSetMetaData gdsmd, Link link, Collection collection, String dataDateFormat) { @@ -336,7 +338,8 @@ class OpenDAPMetaDataParser extends MetaDataParser { if (das.getAttributeTable(ens) != null) { try { AttributeTable at = das.getAttributeTable(ens); - gdsmd.setEnsemble(OpenDAPParseUtility.getInstance().parseEnsemble(at)); + dataSet.setEnsemble(OpenDAPParseUtility.getInstance() + .parseEnsemble(at)); } catch (Exception en) { logParsingException(ens, "Ensemble", collectionName, url); } @@ -414,11 +417,6 @@ class OpenDAPMetaDataParser extends MetaDataParser { levMin, levMax)); parm.addLevelType(type); - // set if an ensemble member - if (gdsmd.getEnsemble() != null) { - parm.setEnsemble(gdsmd.getEnsemble().getSize()); - } - parameters.put(name, parm); } catch (Exception le) { diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPParseUtility.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPParseUtility.java index 37c865d6c4..77a59a67ba 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPParseUtility.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPParseUtility.java @@ -52,6 +52,8 @@ final class OpenDAPParseUtility { private static final Pattern QUOTES_PATTERN = Pattern.compile("\""); + private static final Pattern COMMA_PATTERN = Pattern.compile(","); + /** Singleton instance of this class */ private static OpenDAPParseUtility instance = null; @@ -252,35 +254,16 @@ final class OpenDAPParseUtility { */ public Ensemble parseEnsemble(AttributeTable table) { - String stime = serviceConfig.getConstantValue("TIMEINIT"); - String slength = serviceConfig.getConstantValue("LENGTH"); String sname = serviceConfig.getConstantValue("NAME"); - int size = new Integer(trim(table.getAttribute( - serviceConfig.getConstantValue("SIZE")).getValueAt(0))) - .intValue(); - String name = null; - String length = null; - String tinit = null; - - if (table.getAttribute(slength) != null) { - length = table.getAttribute(slength).getValueAt(0); - } + Ensemble ens = new Ensemble(); if (table.getAttribute(sname) != null) { - name = table.getAttribute(sname).getValueAt(0); + String name = trim(table.getAttribute(sname).getValueAt(0)); + String[] members = COMMA_PATTERN.split(name); + ens.setMembers(Arrays.asList(members)); } - - if (table.getAttribute(stime) != null) { - tinit = table.getAttribute(stime).getValueAt(0); - } - - Ensemble ens = new Ensemble(); - ens.setSize(size); - ens.setLength(length); - ens.setName(name); - ens.setInit(tinit); - return ens; + } /** diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPRequestBuilder.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPRequestBuilder.java index 2aa40d5e24..f917f94bc0 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPRequestBuilder.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPRequestBuilder.java @@ -22,6 +22,7 @@ package com.raytheon.uf.edex.datadelivery.retrieval.opendap; import com.google.common.annotations.VisibleForTesting; import com.raytheon.uf.common.datadelivery.registry.Coverage; +import com.raytheon.uf.common.datadelivery.registry.Ensemble; import com.raytheon.uf.common.datadelivery.registry.GriddedCoverage; import com.raytheon.uf.common.datadelivery.registry.Levels; import com.raytheon.uf.common.datadelivery.registry.Parameter; @@ -216,17 +217,17 @@ class OpenDAPRequestBuilder extends RequestBuilder { */ public String processEnsemble() { - StringBuilder buf = new StringBuilder(); - - if (getRetrievalAttribute().getParameter().getEnsemble() != null) { - buf.append("[" - + (getRetrievalAttribute().getParameter().getEnsemble() - 1) - + "]"); + if (getRetrievalAttribute().getEnsemble() != null) { + Ensemble e = getRetrievalAttribute().getEnsemble(); + int[] range = e.getSelectedRange(); + if (range[0] == range[1]) { + return "[" + range[0] + "]"; + } else { + return "[" + range[0] + ":" + range[1] + "]"; + } } else { - buf.append(""); + return ""; } - - return buf.toString(); } @Override diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPRetrievalGenerator.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPRetrievalGenerator.java index 40fbe7ffdb..9ae6f3f8eb 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPRetrievalGenerator.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPRetrievalGenerator.java @@ -22,14 +22,15 @@ package com.raytheon.uf.edex.datadelivery.retrieval.opendap; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import com.google.common.annotations.VisibleForTesting; import com.raytheon.uf.common.datadelivery.registry.Coverage; import com.raytheon.uf.common.datadelivery.registry.DataSet; import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData; +import com.raytheon.uf.common.datadelivery.registry.Ensemble; import com.raytheon.uf.common.datadelivery.registry.GriddedCoverage; import com.raytheon.uf.common.datadelivery.registry.GriddedDataSetMetaData; import com.raytheon.uf.common.datadelivery.registry.Levels; @@ -265,13 +266,14 @@ class OpenDAPRetrievalGenerator extends RetrievalGenerator { * @param parm * @return */ - protected HashMap> getGridDuplicates( + protected Map> getGridDuplicates( String name, - Parameter parm, ArrayList times, ArrayList levels, + Parameter parm, List times, List levels, + List ensembleMembers, GriddedCoverage cov) { return RetrievalGeneratorUtilities.findGridDuplicates(name, times, - levels, + levels, ensembleMembers, parm, cov.getRequestGridCoverage()); } @@ -286,6 +288,13 @@ class OpenDAPRetrievalGenerator extends RetrievalGenerator { List retrievals = new ArrayList(); Subscription sub = bundle.getSubscription(); + if (sub.getEnsemble() != null && !sub.getEnsemble().hasSelection()) { + // TODO remove this once the UI allows you to make an ensemble + // selection. + sub.getEnsemble() + .setSelectedMembers(sub.getEnsemble().getMembers()); + } + int sfactor = getSizingFactor(getDimensionalSize(sub.getCoverage())); sub = removeDuplicates(sub); @@ -319,6 +328,13 @@ class OpenDAPRetrievalGenerator extends RetrievalGenerator { return Collections.emptyList(); } + List ensembles = null; + if (sub.getEnsemble() == null) { + ensembles = Arrays.asList((Ensemble) null); + } else { + ensembles = sub.getEnsemble().split(1); + } + for (List timeSequence : subTime.getTimeSequences(sfactor)) { for (Parameter param : sub.getParameter()) { @@ -332,9 +348,11 @@ class OpenDAPRetrievalGenerator extends RetrievalGenerator { sub.getTime()); for (Time time : times) { - Retrieval retrieval = getRetrieval(sub, bundle, - param, paramLevels, time); - retrievals.add(retrieval); + for (Ensemble ensemble : ensembles) { + Retrieval retrieval = getRetrieval(sub, bundle, + param, paramLevels, time, ensemble); + retrievals.add(retrieval); + } } } else { @@ -350,9 +368,12 @@ class OpenDAPRetrievalGenerator extends RetrievalGenerator { // and time for (Time time : times) { for (Levels level : levels) { - Retrieval retrieval = getRetrieval(sub, - bundle, param, level, time); - retrievals.add(retrieval); + for (Ensemble ensemble : ensembles) { + Retrieval retrieval = getRetrieval(sub, + bundle, param, level, time, + ensemble); + retrievals.add(retrieval); + } } } } @@ -375,7 +396,7 @@ class OpenDAPRetrievalGenerator extends RetrievalGenerator { * @return */ private Retrieval getRetrieval(Subscription sub, SubscriptionBundle bundle, - Parameter param, Levels level, Time time) { + Parameter param, Levels level, Time time, Ensemble ensemble) { Retrieval retrieval = new Retrieval(); retrieval.setSubscriptionName(sub.getName()); @@ -404,6 +425,7 @@ class OpenDAPRetrievalGenerator extends RetrievalGenerator { lparam.setLevels(level); att.setTime(time); att.setParameter(lparam); + att.setEnsemble(ensemble); att.setSubName(retrieval.getSubscriptionName()); att.setPlugin(pt.getPlugin()); att.setProvider(sub.getProvider()); @@ -488,7 +510,6 @@ class OpenDAPRetrievalGenerator extends RetrievalGenerator { param.setBaseType(origParm.getBaseType()); param.setDataType(origParm.getDataType()); param.setDefinition(origParm.getDefinition()); - param.setEnsemble(origParm.getEnsemble()); param.setFillValue(origParm.getFillValue()); param.setLevelType(origParm.getLevelType()); param.setMissingValue(origParm.getMissingValue()); @@ -536,6 +557,13 @@ class OpenDAPRetrievalGenerator extends RetrievalGenerator { int sfactor = getSizingFactor(getDimensionalSize(sub.getCoverage())); + List ensembles = null; + if (sub.getEnsemble() != null && sub.getEnsemble().hasSelection()) { + ensembles = sub.getEnsemble().getSelectedMembers(); + } else { + ensembles = Arrays.asList((String) null); + } + for (List timeSequence : sub.getTime().getTimeSequences( sfactor)) { @@ -557,14 +585,14 @@ class OpenDAPRetrievalGenerator extends RetrievalGenerator { ArrayList levels = ResponseProcessingUtilities .getOpenDAPGridLevels(param.getLevels()); - HashMap> dups = getGridDuplicates( + Map> dups = getGridDuplicates( sub.getName(), - param, times, levels, + param, times, levels, ensembles, (GriddedCoverage) sub.getCoverage()); for (int i = 0; i < times.size(); i++) { DataTime dtime = times.get(i); - ArrayList levDups = dups.get(dtime); + List levDups = dups.get(dtime); if (levDups != null) { // single level, remove the time @@ -594,14 +622,14 @@ class OpenDAPRetrievalGenerator extends RetrievalGenerator { .getOpenDAPGridLevels(level); } - HashMap> dups = getGridDuplicates( - sub.getName(), - param, times, plevels, + Map> dups = getGridDuplicates( + sub.getName(), param, times, plevels, + ensembles, ((GriddedCoverage) sub.getCoverage())); for (int i = 0; i < times.size(); i++) { DataTime dtime = times.get(i); - ArrayList levDups = dups.get(dtime); + List levDups = dups.get(dtime); if (levDups != null) { diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/response/OpenDAPTranslator.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/response/OpenDAPTranslator.java index c3c790be13..75b4c933a0 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/response/OpenDAPTranslator.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/response/OpenDAPTranslator.java @@ -23,6 +23,7 @@ package com.raytheon.uf.edex.datadelivery.retrieval.response; import java.util.ArrayList; import java.util.Arrays; import java.util.Enumeration; +import java.util.List; import com.raytheon.uf.common.datadelivery.registry.GriddedCoverage; import com.raytheon.uf.common.datadelivery.retrieval.xml.RetrievalAttribute; @@ -149,33 +150,41 @@ public class OpenDAPTranslator extends RetrievalTranslator { PrimitiveVector pm = darray.getPrimitiveVector(); float[] values = (float[]) pm.getInternalStorage(); + List ensembles = null; + if (attXML.getEnsemble() != null && attXML.getEnsemble().hasSelection()) { + ensembles = attXML.getEnsemble().getSelectedMembers(); + } else { + ensembles = Arrays.asList((String) null); + } + // time dependencies int start = 0; - PluginDataObject[] records = new PluginDataObject[numLevels * numTimes]; + PluginDataObject[] records = new PluginDataObject[numLevels * numTimes + * ensembles.size()]; - for (int i = 0; i < times.size(); i++) { + int bin = 0; + for (int i = 0; i < ensembles.size(); i++) { + for (DataTime dataTime : times) { + for (int j = 0; j < numLevels; j++) { + PluginDataObject record = getPdo(bin); + record.setDataTime(dataTime); + record.constructDataURI(); - DataTime dataTime = times.get(i); + int end = start + gridSize; - for (int j = 0; j < numLevels; j++) { - int bin = (numLevels * i) + j; - PluginDataObject record = getPdo(bin); - record.setDataTime(dataTime); - record.constructDataURI(); + float[] subValues = Arrays.copyOfRange(values, start, end); - int end = start + gridSize; + subValues = GridMetadataAdapter.adjustGrid(nx, ny, + subValues, Float.parseFloat(attXML.getParameter() + .getMissingValue()), true); - float[] subValues = Arrays.copyOfRange(values, start, end); - - subValues = GridMetadataAdapter.adjustGrid(nx, ny, subValues, - Float.parseFloat(attXML.getParameter() - .getMissingValue()), true); - - record.setMessageData(subValues); - record.setOverwriteAllowed(true); - records[bin] = record; - statusHandler.info("Creating record: " + record.getDataURI()); - start = end; + record.setMessageData(subValues); + record.setOverwriteAllowed(true); + records[bin++] = record; + statusHandler.info("Creating record: " + + record.getDataURI()); + start = end; + } } } diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/util/ResponseProcessingUtilities.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/util/ResponseProcessingUtilities.java index bdd987f7b6..1250b30a4f 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/util/ResponseProcessingUtilities.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/util/ResponseProcessingUtilities.java @@ -61,7 +61,7 @@ public class ResponseProcessingUtilities { .getHandler(ResponseProcessingUtilities.class); public static GridRecord getGridRecord(String name, Parameter parm, - Level level, + Level level, String ensembleId, GridCoverage gridCoverage) { com.raytheon.uf.common.parameter.Parameter parameter = new com.raytheon.uf.common.parameter.Parameter(); @@ -75,7 +75,7 @@ public class ResponseProcessingUtilities { record.setLevel(level); record.setParameter(parameter); record.setDatasetId(name); - + record.setEnsembleId(ensembleId); return record; } diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/util/RetrievalGeneratorUtilities.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/util/RetrievalGeneratorUtilities.java index 31da7a43aa..09f717150a 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/util/RetrievalGeneratorUtilities.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/util/RetrievalGeneratorUtilities.java @@ -21,6 +21,8 @@ package com.raytheon.uf.edex.datadelivery.retrieval.util; import java.util.ArrayList; import java.util.HashMap; +import java.util.List; +import java.util.Map; import com.raytheon.uf.common.datadelivery.registry.Parameter; import com.raytheon.uf.common.dataplugin.PluginException; @@ -87,35 +89,39 @@ public class RetrievalGeneratorUtilities { * @param cov * @return */ - public static HashMap> findGridDuplicates( - String name, - ArrayList times, ArrayList levels, Parameter parm, + public static Map> findGridDuplicates(String name, + List times, List levels, + List ensembleMembers, Parameter parm, GridCoverage cov) { - HashMap> dups = new HashMap>(); + HashMap> dups = new HashMap>(); for (DataTime time : times) { - ArrayList levDups = dups.get(time); + List levDups = dups.get(time); if (levDups == null) { levDups = new ArrayList(); } for (Level level : levels) { - try { + for (String ensembleMember : ensembleMembers) { + try { - GridRecord rec = ResponseProcessingUtilities.getGridRecord( - name, parm, level, cov); - rec.setDataTime(time); - rec.constructDataURI(); - boolean isDup = findDuplicateUri(rec.getDataURI(), "grid"); - if (isDup) { - levDups.add(level); + GridRecord rec = ResponseProcessingUtilities + .getGridRecord(name, parm, level, + ensembleMember, cov); + rec.setDataTime(time); + rec.constructDataURI(); + boolean isDup = findDuplicateUri(rec.getDataURI(), + "grid"); + if (isDup) { + levDups.add(level); + } + } catch (PluginException e) { + statusHandler.handle(Priority.PROBLEM, + e.getLocalizedMessage(), e); } - } catch (PluginException e) { - statusHandler.handle(Priority.PROBLEM, - e.getLocalizedMessage(), e); } }