Merge branch '13.2.1_delivery' into development

Conflicts:
	cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/BaseGfePyController.java
	cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/procedures/ProcedureJob.java

Former-commit-id: a0bcc2276fb95097cfc10869f0aa1ee48189fc3a

cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/common/ui/UserSelectComp.java

@@ -413,6 +413,7 @@ public class UserSelectComp extends Composite implements IUpdate, IDisplay,
                 u.setEnvelope(groupDefinition.getEnvelope());
                 u.setNumFcstHours(subscription.getTime()
                         .getSelectedTimeIndices().size());
+                u.setNumEnsembleMembers(subscription.getEnsemble());
                 u.determineNumberRequestedGrids(subscription.getParameter());
 
                 Coverage cov = new GriddedCoverage();

cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/SendToServerSubscriptionNotificationService.java

@@ -44,6 +44,7 @@ import com.raytheon.uf.common.status.UFStatus.Priority;
  * ------------ ---------- ----------- --------------------------
  * Jan 04, 2013 1441       djohnson     Initial creation
  * Jan 17, 2013 1501       djohnson     Route to datadelivery.
+ * Jan 21, 2013 1501       djohnson     Include subscription on all requests.
  * 
  * </pre>
  * 
@@ -182,6 +183,7 @@ public class SendToServerSubscriptionNotificationService implements
         req.setCategory("Subscription Approval Denied");
         req.setPriority(2);
         req.setId(subscription.getId());
+        req.setSubscription(subscription);
 
         sendRequest(req);
     }
@@ -214,6 +216,7 @@ public class SendToServerSubscriptionNotificationService implements
         req.setCategory("Subscription");
         req.setPriority(3);
         req.setMessage(subscription.getName() + " Activated");
+        req.setSubscription(subscription);
 
         sendRequest(req);
     }
@@ -229,6 +232,7 @@ public class SendToServerSubscriptionNotificationService implements
         req.setCategory("Subscription");
         req.setPriority(3);
         req.setMessage(subscription.getName() + " Deactivated");
+        req.setSubscription(subscription);
 
         sendRequest(req);
     }

cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/SubscriptionManagerDlg.java

@@ -58,7 +58,6 @@ import com.raytheon.uf.common.registry.handler.RegistryObjectHandlers;
 import com.raytheon.uf.common.status.IUFStatusHandler;
 import com.raytheon.uf.common.status.UFStatus;
 import com.raytheon.uf.common.status.UFStatus.Priority;
-import com.raytheon.uf.viz.core.VizApp;
 import com.raytheon.uf.viz.core.auth.UserController;
 import com.raytheon.uf.viz.core.exception.VizException;
 import com.raytheon.uf.viz.datadelivery.actions.DataBrowserAction;
@@ -112,6 +111,8 @@ import com.raytheon.viz.ui.presenter.IDisplay;
  * Jan 02, 2013 1441       djohnson   Add ability to delete groups.
  * Jan 03, 2013 1437       bgonzale   Moved configuration file management code to SubscriptionManagerConfigDlg
  *                                    and SubscriptionConfigurationManager.
+ * Jan 21, 2013 1501       djohnson   Only send notification if subscription was actually activated/deactivated,
+ *                                    remove race condition of GUI thread updating the table after notification.
  * </pre>
  * 
  * @author mpduff
@@ -864,34 +865,23 @@ public class SubscriptionManagerDlg extends CaveSWTDialog implements
                                         SWT.OK, sub.getName() + " Activated",
                                         response.getMessageToDisplay());
                             }
-                            updatedList.add(sub);
-
-                            if (activate) {
-                                subscriptionNotificationService
-                                        .sendSubscriptionActivatedNotification(
-                                                sub,
-                                                username);
-
-                            } else {
-                                subscriptionNotificationService
-                                        .sendSubscriptionDeactivatedNotification(
-                                                sub, username);
 
+                            if (!response.isAllowFurtherEditing()) {
+                                if (activate) {
+                                    subscriptionNotificationService
+                                            .sendSubscriptionActivatedNotification(
+                                                    sub, username);
+                                } else {
+                                    subscriptionNotificationService
+                                            .sendSubscriptionDeactivatedNotification(
+                                                    sub, username);
+                                }
                             }
                         } catch (RegistryHandlerException e) {
                             statusHandler.handle(Priority.PROBLEM,
                                     "Error processing request.", e);
                         }
                     }
-
-                    VizApp.runAsync(new Runnable() {
-                        @Override
-                        public void run() {
-                            if (isDisposed() == false) {
-                                tableComp.updateTable(updatedList);
-                            }
-                        }
-                    });
                 }
             }
         } catch (VizException e) {

cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/subset/GriddedEnsembleSubsetTab.java

@@ -0,0 +1,177 @@
+/**
+ * This software was developed and / or modified by Raytheon Company,
+ * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+ *
+ * U.S. EXPORT CONTROLLED TECHNICAL DATA
+ * This software product contains export-restricted data whose
+ * export/transfer/disclosure is restricted by U.S. law. Dissemination
+ * to non-U.S. persons whether in the United States or abroad requires
+ * an export license or other authorization.
+ *
+ * Contractor Name:        Raytheon Company
+ * Contractor Address:     6825 Pine Street, Suite 340
+ *                         Mail Stop B8
+ *                         Omaha, NE 68106
+ *                         402.291.0100
+ *
+ * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+ * further licensing information.
+ **/
+package com.raytheon.uf.viz.datadelivery.subscription.subset;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.eclipse.swt.SWT;
+import org.eclipse.swt.layout.GridData;
+import org.eclipse.swt.layout.GridLayout;
+import org.eclipse.swt.widgets.Composite;
+import org.eclipse.swt.widgets.Group;
+
+import com.raytheon.uf.common.datadelivery.registry.Ensemble;
+import com.raytheon.uf.common.datadelivery.registry.Subscription;
+import com.raytheon.uf.common.util.CollectionUtil;
+import com.raytheon.uf.viz.datadelivery.subscription.subset.xml.SubsetXML;
+import com.raytheon.viz.ui.widgets.duallist.DualList;
+import com.raytheon.viz.ui.widgets.duallist.DualListConfig;
+import com.raytheon.viz.ui.widgets.duallist.IUpdate;
+
+/**
+ * 
+ * TODO Add Description
+ * 
+ * <pre>
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Jan 3, 2013            bsteffen     Initial creation
+ * 
+ * </pre>
+ * 
+ * @author bsteffen
+ * @version 1.0
+ */
+public class GriddedEnsembleSubsetTab {
+
+    private static final String NAME = "Ensemble Members";
+
+    private final Set<IDataSize> listeners = new HashSet<IDataSize>();
+
+    private final Ensemble ensemble;
+
+    private DualList dualList;
+
+    private boolean modified;
+
+    public GriddedEnsembleSubsetTab(Composite parentComp, Ensemble ensemble) {
+        this.ensemble = ensemble;
+        init(parentComp);
+    }
+
+    private void init(Composite parentComp) {
+        GridLayout gl = new GridLayout(1, false);
+        GridData gd = new GridData(SWT.FILL, SWT.DEFAULT, true, false);
+
+        gl.horizontalSpacing = 0;
+        gl.verticalSpacing = 0;
+        gl.marginWidth = 0;
+        gl.marginHeight = 0;
+
+        Group group = new Group(parentComp, SWT.NONE);
+        group.setText(getName());
+        group.setLayout(gl);
+        group.setLayoutData(gd);
+
+        DualListConfig dualListConfig = new DualListConfig();
+        dualListConfig.setAvailableListLabel("Available Members:");
+        dualListConfig.setSelectedListLabel("Selected Memebers:");
+        dualListConfig.setListHeight(125);
+        dualListConfig.setListWidth(175);
+        dualListConfig.setShowUpDownBtns(false);
+        dualListConfig.setFullList(ensemble.getMembers());
+        dualList = new DualList(group, SWT.NONE, dualListConfig,
+                new IUpdate() {
+
+            @Override
+            public void selectionChanged() {
+                modified = true;
+                notifyListeners();
+            }
+
+            @Override
+            public void hasEntries(boolean entries) {
+
+            }
+        });
+    }
+
+    public String getName() {
+        return NAME;
+    }
+
+    public Ensemble getEnsembleWithSelection() {
+        Ensemble ensemble = new Ensemble(this.ensemble);
+        ensemble.setSelectedMembers(Arrays.asList(dualList
+                .getSelectedListItems()));
+        return ensemble;
+    }
+
+    private void loadFromEnsemble(Ensemble ensemble) {
+        dualList.clearSelection();
+        if (ensemble != null && ensemble.getSelectedMembers() != null) {
+            dualList.selectItems(ensemble.getSelectedMembers().toArray(
+                    new String[0]));
+        }
+    }
+
+    public void populateSubscription(Subscription subscription) {
+        subscription.setEnsemble(getEnsembleWithSelection());
+    }
+
+    public void loadFromSubscription(Subscription subscription) {
+        loadFromEnsemble(subscription.getEnsemble());
+    }
+
+    public void populateSubsetXML(SubsetXML<?> subsetXml) {
+        subsetXml.setEnsemble(getEnsembleWithSelection());
+    }
+
+    public void loadFromSubsetXML(SubsetXML<?> subsetXml) {
+        loadFromEnsemble(subsetXml.getEnsemble());
+    }
+
+    public boolean isValid() {
+        return !CollectionUtil.isNullOrEmpty(dualList.getSelectedListItems());
+    }
+
+    public boolean isModified() {
+        return modified;
+    }
+
+    public void setModified(boolean modified) {
+        this.modified = modified;
+    }
+
+    public void addListener(IDataSize listener) {
+        synchronized (this.listeners) {
+            listeners.add(listener);
+        }
+    }
+
+    protected void notifyListeners() {
+        Collection<IDataSize> listeners;
+        synchronized (this.listeners) {
+            listeners = new ArrayList<IDataSize>(
+                    this.listeners);
+        }
+        for (IDataSize listener : listeners) {
+            listener.updateDataSize();
+        }
+    }
+
+}

cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/subset/GriddedSubsetManagerDlg.java

@@ -24,6 +24,7 @@ import java.io.StringWriter;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -34,13 +35,18 @@ import java.util.TreeSet;
 import javax.xml.bind.JAXBException;
 
 import org.eclipse.swt.SWT;
+import org.eclipse.swt.layout.GridData;
+import org.eclipse.swt.layout.GridLayout;
 import org.eclipse.swt.widgets.Composite;
 import org.eclipse.swt.widgets.Shell;
+import org.eclipse.swt.widgets.TabFolder;
+import org.eclipse.swt.widgets.TabItem;
 import org.geotools.geometry.jts.ReferencedEnvelope;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Ordering;
 import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData;
+import com.raytheon.uf.common.datadelivery.registry.Ensemble;
 import com.raytheon.uf.common.datadelivery.registry.GriddedDataSet;
 import com.raytheon.uf.common.datadelivery.registry.GriddedDataSetMetaData;
 import com.raytheon.uf.common.datadelivery.registry.Subscription;
@@ -77,6 +83,7 @@ import com.raytheon.uf.viz.datadelivery.utils.DataDeliveryUtils;
  * Dec 10, 2012 1259       bsteffen     Switch Data Delivery from LatLon to referenced envelopes.
  * Jan 04, 2013 1299       djohnson     Add logging of invalid forecast hour information if it occurs again.
  * Jan 04, 2013 1420       mpduff       Pass cycles in for rules.
+ * Jan 18, 2013 1414       bsteffen     Add ensemble tab.
  * 
  * </pre>
  * 
@@ -115,6 +122,8 @@ public class GriddedSubsetManagerDlg
 
     private DataSetMetaData metaData;
 
+    private GriddedEnsembleSubsetTab ensembleTab;
+
     /**
      * Constructor.
      * 
@@ -150,6 +159,82 @@ public class GriddedSubsetManagerDlg
         super(shell, dataSet);
     }
 
+    @Override
+    protected void createGridTabs(TabFolder tabFolder) {
+        super.createGridTabs(tabFolder);
+        Ensemble e = dataSet.getEnsemble();
+        if (e != null && e.getMembers() != null) {
+            TabItem ensembleTabItem = new TabItem(tabFolder, SWT.NONE, 2);
+            Composite ensembleComp = new Composite(tabFolder, SWT.NONE);
+            ensembleComp.setLayout(new GridLayout(1, false));
+            ensembleComp.setLayoutData(new GridData(SWT.CENTER, SWT.DEFAULT,
+                    true, false));
+            ensembleTabItem.setControl(ensembleComp);
+            ensembleTab = new GriddedEnsembleSubsetTab(ensembleComp,
+                    dataSet.getEnsemble());
+            ensembleTab.addListener(this);
+            ensembleTabItem.setText(ensembleTab.getName());
+        }
+    }
+
+    @Override
+    protected Collection<String> getInvalidTabs() {
+        Collection<String> invalidTabs = super.getInvalidTabs();
+        if (ensembleTab != null && !ensembleTab.isValid()) {
+            invalidTabs.add(ensembleTab.getName());
+        }
+        return invalidTabs;
+    }
+
+    @Override
+    protected void populateSubsetXML(SubsetXML<SpecificDateTimeXML> subset) {
+        super.populateSubsetXML(subset);
+        if (ensembleTab != null) {
+            ensembleTab.populateSubsetXML(subset);
+        }
+    }
+
+    @Override
+    protected void loadFromSubsetXML(SubsetXML<SpecificDateTimeXML> subsetXml) {
+        super.loadFromSubsetXML(subsetXml);
+        if (ensembleTab != null) {
+            ensembleTab.loadFromSubsetXML(subsetXml);
+        }
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see
+     * com.raytheon.uf.viz.datadelivery.subscription.subset.SubsetManagerDlg
+     * #loadFromSubscription
+     * (com.raytheon.uf.common.datadelivery.registry.Subscription)
+     */
+    @Override
+    protected void loadFromSubscription(Subscription subscription) {
+        super.loadFromSubscription(subscription);
+        if (ensembleTab != null) {
+            ensembleTab.loadFromSubscription(subscription);
+        }
+    }
+
+    @Override
+    protected boolean isDirty() {
+        boolean modified = super.isDirty();
+        if (!modified && ensembleTab != null) {
+            modified = ensembleTab.isModified();
+        }
+        return modified;
+    }
+
+    @Override
+    protected void setClean() {
+        super.setClean();
+        if (ensembleTab != null) {
+            ensembleTab.setModified(false);
+        }
+    }
+
     /**
      * {@inheritDoc}
      */
@@ -186,6 +271,11 @@ public class GriddedSubsetManagerDlg
 
         time.setSelectedTimeIndices(fcstIndices);
         subscription.setTime(time);
+
+        if (ensembleTab != null) {
+            ensembleTab.populateSubscription(subscription);
+        }
+
         return subscription;
     }
 
@@ -193,7 +283,7 @@ public class GriddedSubsetManagerDlg
      * {@inheritDoc}
      */
     @Override
-    protected SpecificDateTimeXML getTimeXml() {
+    protected SpecificDateTimeXML getTimeXmlFromSubscription() {
         SpecificDateTimeXML timeXml = new SpecificDateTimeXML();
         Time time = this.subscription.getTime();
         List<Integer> cycleTimes = time.getCycleTimes();
@@ -270,7 +360,12 @@ public class GriddedSubsetManagerDlg
         // Get the temporal data
         int numFcstHours = this.timingTabControls.getSelectedFcstHours().length;
         dataSize.setNumFcstHours(numFcstHours);
-
+        if (ensembleTab != null) {
+            dataSize.setNumEnsembleMembers(ensembleTab
+                    .getEnsembleWithSelection());
+        } else {
+            dataSize.setNumEnsembleMembers(dataSet.getEnsemble());
+        }
         // Get the Areal data
         ReferencedEnvelope envelope = this.spatialTabControls.getEnvelope();
 

cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/subset/SubsetManagerDlg.java

@@ -20,6 +20,7 @@
 package com.raytheon.uf.viz.datadelivery.subscription.subset;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
@@ -137,9 +138,6 @@ public abstract class SubsetManagerDlg<DATASET extends DataSet, PRESENTER extend
     private final IUFStatusHandler statusHandler = UFStatus
             .getHandler(SubsetManagerDlg.class);
 
-    /** Tab test map */
-    private final Map<String, String> tabTextMap = new HashMap<String, String>();
-
     /** Subset Name text box */
     private Text nameText;
 
@@ -180,15 +178,6 @@ public abstract class SubsetManagerDlg<DATASET extends DataSet, PRESENTER extend
     /** Edit flag */
     private boolean create = true;
 
-    /** Flag for vertical levels/parameters dirty */
-    private boolean verticalParameterDirty = false;
-
-    /** Flag for vertical date/cycle/forecast dirty. */
-    private boolean dateDirty = false;
-
-    /** Flag for spatial dirty */
-    private boolean spatialDirty = false;
-
     /** Subscription object */
     protected Subscription subscription;
 
@@ -201,14 +190,11 @@ public abstract class SubsetManagerDlg<DATASET extends DataSet, PRESENTER extend
     /** Subset manager constant */
     private final String DD_SUBSET_MANAGER = "Data Delivery Subset Manager - ";
 
-    /** Priority Tab constant */
+    private final String VERTICAL_TAB = "Vertical Levels/Parameters";
-    private final String VERTICAL_TAB = "verticalTab";
 
-    /** Latency Tab constant */
+    private final String TIMING_TAB = "Forecast Hours";
-    private final String TIMING_TAB = "timingTab";
 
-    /** Routing Tab constant */
+    private final String SPATIAL_TAB = "Spatial";
-    private final String SPATIAL_TAB = "spatialTab";
 
     private final ISubscriptionService subscriptionService = DataDeliveryServices
             .getSubscriptionService();
@@ -291,14 +277,17 @@ public abstract class SubsetManagerDlg<DATASET extends DataSet, PRESENTER extend
      */
     @Override
     protected void initializeComponents(Shell shell) {
-        getTabText();
 
         createTabFolder();
         createInfoComp();
         createButtons();
 
         if (loadDataSet) {
-            populate();
+            if (subsetXml != null) {
+                loadFromSubsetXML(subsetXml);
+            } else if (subscription != null) {
+                loadFromSubscription(subscription);
+            }
         }
 
         shell.addShellListener(new ShellAdapter() {
@@ -357,13 +346,6 @@ public abstract class SubsetManagerDlg<DATASET extends DataSet, PRESENTER extend
         // shell.setMinimumSize(shell.getSize());
     }
 
-    /** Get a map of tab text values */
-    private void getTabText() {
-        tabTextMap.put(VERTICAL_TAB, "Vertical Levels/Parameters");
-        tabTextMap.put(TIMING_TAB, "Forecast Hours");
-        tabTextMap.put(SPATIAL_TAB, "Spatial");
-    }
-
     /** Create the tabs */
     private void createTabs(TabFolder tabFolder) {
         createGridTabs(tabFolder);
@@ -376,12 +358,12 @@ public abstract class SubsetManagerDlg<DATASET extends DataSet, PRESENTER extend
     }
 
     /** Create the grid tab */
-    private void createGridTabs(TabFolder tabFolder) {
+    protected void createGridTabs(TabFolder tabFolder) {
         GridData gd = new GridData(SWT.CENTER, SWT.DEFAULT, true, false);
         GridLayout gl = new GridLayout(1, false);
 
         TabItem verticalTab = new TabItem(tabFolder, SWT.NONE);
-        verticalTab.setText(tabTextMap.get(VERTICAL_TAB));
+        verticalTab.setText(VERTICAL_TAB);
         verticalTab.setData("valid", false);
         Composite vertComp = new Composite(tabFolder, SWT.NONE);
         vertComp.setLayout(gl);
@@ -393,7 +375,7 @@ public abstract class SubsetManagerDlg<DATASET extends DataSet, PRESENTER extend
         gl = new GridLayout(1, false);
 
         TabItem timingTab = new TabItem(tabFolder, SWT.NONE);
-        timingTab.setText(tabTextMap.get(TIMING_TAB));
+        timingTab.setText(TIMING_TAB);
         timingTab.setData("valid", false);
         Composite timingComp = new Composite(tabFolder, SWT.NONE);
         timingComp.setLayout(gl);
@@ -407,7 +389,7 @@ public abstract class SubsetManagerDlg<DATASET extends DataSet, PRESENTER extend
         gl = new GridLayout(1, false);
 
         TabItem spatialTab = new TabItem(tabFolder, SWT.NONE);
-        spatialTab.setText(tabTextMap.get(SPATIAL_TAB));
+        spatialTab.setText(SPATIAL_TAB);
         Composite spatialComp = new Composite(tabFolder, SWT.NONE);
         spatialComp.setLayout(gl);
         spatialComp.setLayoutData(gd);
@@ -710,37 +692,43 @@ public abstract class SubsetManagerDlg<DATASET extends DataSet, PRESENTER extend
             return false;
         }
 
-        // Don't get the last tab, saved subsets
+        Collection<String> invalidTabs = getInvalidTabs();
-        Map<String, Boolean> tabsValidMap = new HashMap<String, Boolean>();
+
+        if (!invalidTabs.isEmpty()) {
+            StringBuilder message = new StringBuilder(
+                    "The following tabs do not have valid entries:\n\n");
+            for (String tab : invalidTabs) {
+                message.append(tab + "\n");
+            }
+            DataDeliveryUtils.showMessage(shell, getStyle(), "Invalid Entries",
+                    message.toString());
+            return false;
+        }
+
+        return true;
+    }
+
+    protected Collection<String> getInvalidTabs() {
+        Collection<String> invalidTabs = new ArrayList<String>(3);
 
         // Get the tabs to validate
         // TODO Hardcoding the tabs for now, fix this later
 
         // Validate the vertical tab
-        tabsValidMap.put(VERTICAL_TAB, vTab.isValid());
+        if (!vTab.isValid()) {
+            invalidTabs.add(VERTICAL_TAB);
+        }
 
-        tabsValidMap.put(TIMING_TAB, timingTabControls.isValid());
+        if (!timingTabControls.isValid()) {
+            invalidTabs.add(TIMING_TAB);
+        }
 
         // Next is spatial subset tab
-        tabsValidMap.put(SPATIAL_TAB, spatialTabControls.isValid());
+        if (!spatialTabControls.isValid()) {
-
+            invalidTabs.add(SPATIAL_TAB);
-        StringBuilder buf = new StringBuilder(
-                "The following tabs do not have valid entries:\n\n");
-        boolean showMsg = false;
-        for (String tab : tabsValidMap.keySet()) {
-            if (!tabsValidMap.get(tab)) {
-                buf.append(tabTextMap.get(tab) + "\n");
-                showMsg = true;
-            }
         }
 
-        if (showMsg) {
+        return invalidTabs;
-            DataDeliveryUtils.showMessage(shell, getStyle(), "Invalid Entries",
-                    buf.toString());
-            return false;
-        }
-
-        return true;
     }
 
     /**
@@ -806,6 +794,15 @@ public abstract class SubsetManagerDlg<DATASET extends DataSet, PRESENTER extend
         }
 
         SubsetXML<TIMEXML> subset = new SubsetXML<TIMEXML>();
+        populateSubsetXML(subset);
+
+        // Have all the info, now save the file
+        SubsetFileManager.getInstance().saveSubset(subset, this.shell);
+        setClean();
+        subsetTab.enableButtons(nameText);
+    }
+
+    protected void populateSubsetXML(SubsetXML<TIMEXML> subset) {
         subset.setBaseSubsetName(nameText.getText());
         subset.setDatasetName(dataSet.getDataSetName());
         subset.setProviderName(dataSet.getProviderName());
@@ -821,12 +818,6 @@ public abstract class SubsetManagerDlg<DATASET extends DataSet, PRESENTER extend
         // finally the date/cycle/forecast data
         TIMEXML time = timingTabControls.getSaveInfo();
         subset.setTime(time);
-
-        // Have all the info, now save the file
-        SubsetFileManager.getInstance().saveSubset(subset, this.shell);
-
-        setClean();
-        subsetTab.enableButtons(nameText);
     }
 
     /**
@@ -844,58 +835,36 @@ public abstract class SubsetManagerDlg<DATASET extends DataSet, PRESENTER extend
         SubsetXML<TIMEXML> loadedSubsetXml = (SubsetXML<TIMEXML>) SubsetFileManager
                 .getInstance().loadSubset(subsetName);
 
-        updateSelections(loadedSubsetXml);
+        loadFromSubsetXML(loadedSubsetXml);
     }
 
-    /**
+    protected void loadFromSubsetXML(SubsetXML<TIMEXML> subsetXml) {
-     * Populate the dialog controls.
-     */
-    private void populate() {
-        if (subsetXml == null) {
-            if (subscription == null) {
-                return;
-            }
-            populateSubset();
-        }
-
-        AreaXML area = subsetXml.getArea();
-        spatialTabControls.setDataSet(this.dataSet);
-        spatialTabControls.populate(area);
-
         ArrayList<VerticalXML> vertList = subsetXml.getVerticalList();
         vTab.populate(vertList, dataSet);
 
         TIMEXML time = subsetXml.getTime();
         this.timingTabControls.populate(time, dataSet);
-        this.nameText.setText(subsetXml.getBaseSubsetName());
+
+        if (this.subsetXml == subsetXml) {
+            // only populate area and name if subsetXml is loading from initial
+            // load, not from the saved subsets tab.
+            AreaXML area = subsetXml.getArea();
+            spatialTabControls.setDataSet(this.dataSet);
+            spatialTabControls.populate(area);
+
+            this.nameText.setText(subsetXml.getBaseSubsetName());
+        }
     }
 
-    /**
+    protected void loadFromSubscription(Subscription subscription) {
-     * Update selections with from the loadedSubsetXML object.
+        this.nameText.setText(this.subscription.getName());
-     */
-    private void updateSelections(SubsetXML<TIMEXML> loadedSubsetXml) {
-        ArrayList<VerticalXML> vertList = loadedSubsetXml.getVerticalList();
-        vTab.updateSettings(vertList);
-
-        TIMEXML time = loadedSubsetXml.getTime();
-        timingTabControls.updateSettings(time);
-    }
-
-    /**
-     * Populate the subset object.
-     */
-    private void populateSubset() {
-        subsetXml = new SubsetXML<TIMEXML>();
-        subsetXml.setDatasetName(this.dataSet.getDataSetName());
-        subsetXml.setProviderName(this.dataSet.getProviderName());
-        subsetXml.setSubsetName(this.subscription.getName());
 
         // Cycle time
-        TIMEXML timeXml = getTimeXml();
+        TIMEXML timeXml = getTimeXmlFromSubscription();
 
         timeXml.setLatestData(true);
 
-        subsetXml.setTime(timeXml);
+        this.timingTabControls.populate(timeXml, dataSet);
 
         // Area
         AreaXML area = new AreaXML();
@@ -911,7 +880,8 @@ public abstract class SubsetManagerDlg<DATASET extends DataSet, PRESENTER extend
             area.setEnvelope(envelope);
         }
 
-        subsetXml.setArea(area);
+        spatialTabControls.setDataSet(this.dataSet);
+        spatialTabControls.populate(area);
 
         // Vertical/Parameters
         Map<String, VerticalXML> levelMap = new HashMap<String, VerticalXML>();
@@ -946,9 +916,9 @@ public abstract class SubsetManagerDlg<DATASET extends DataSet, PRESENTER extend
             }
         }
 
-        for (VerticalXML v : levelMap.values()) {
+        ArrayList<VerticalXML> vertList = new ArrayList<VerticalXML>(
-            subsetXml.addVertical(v);
+                levelMap.values());
-        }
+        vTab.populate(vertList, dataSet);
     }
 
     /**
@@ -956,25 +926,22 @@ public abstract class SubsetManagerDlg<DATASET extends DataSet, PRESENTER extend
      * 
      * @return The time object
      */
-    protected abstract TIMEXML getTimeXml();
+    protected abstract TIMEXML getTimeXmlFromSubscription();
 
     /**
      * If any mods have been made to the composite selections, set dirty true.
      */
-    private boolean isDirty() {
+    protected boolean isDirty() {
 
-        verticalParameterDirty = vTab.isDirty();
+        if (vTab.isDirty()) {
-        if (verticalParameterDirty) {
             return true;
         }
 
-        dateDirty = timingTabControls.isDirty();
+        if (timingTabControls.isDirty()) {
-        if (dateDirty) {
             return true;
         }
 
-        spatialDirty = spatialTabControls.isSpatialDirty();
+        if (spatialTabControls.isSpatialDirty()) {
-        if (spatialDirty) {
             return true;
         }
 
@@ -984,7 +951,7 @@ public abstract class SubsetManagerDlg<DATASET extends DataSet, PRESENTER extend
     /**
      * Reset the dirty flags.
      */
-    private void setClean() {
+    protected void setClean() {
         vTab.setClean();
         timingTabControls.setDirty(false);
         spatialTabControls.setSpatialDirty(false);

cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/subset/VerticalSubsetTab.java

@@ -471,7 +471,7 @@ public class VerticalSubsetTab extends SubsetTab implements
      * @param dataSet
      *            The DataSetMetaData object
      */
-    public void populate(ArrayList<VerticalXML> vertList, DataSet dataSet) {
+    public void populate(List<VerticalXML> vertList, DataSet dataSet) {
         this.dataSet = dataSet;
         createExpandBarItems();
 

cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/subset/xml/SubsetXML.java

@@ -29,6 +29,7 @@ import javax.xml.bind.annotation.XmlElements;
 import javax.xml.bind.annotation.XmlRootElement;
 
 import com.raytheon.edex.util.Util;
+import com.raytheon.uf.common.datadelivery.registry.Ensemble;
 import com.raytheon.uf.viz.datadelivery.common.xml.AreaXML;
 import com.raytheon.uf.viz.datadelivery.common.xml.IDisplayXml;
 
@@ -65,6 +66,9 @@ public class SubsetXML<TIMEXML extends TimeXML> implements IDisplayXml {
 
     @XmlElement(name = "area", type = AreaXML.class)
     protected AreaXML area;
+
+    @XmlElement
+    protected Ensemble ensemble;
     
     @XmlElements({ @XmlElement(name = "vertical", type = VerticalXML.class) })
     protected ArrayList<VerticalXML> verticalList = new ArrayList<VerticalXML>();
@@ -86,6 +90,21 @@ public class SubsetXML<TIMEXML extends TimeXML> implements IDisplayXml {
         this.subsetName = subsetName;
     }
 
+    /**
+     * @return the ensemble
+     */
+    public Ensemble getEnsemble() {
+        return ensemble;
+    }
+
+    /**
+     * @param ensemble
+     *            the ensemble to set
+     */
+    public void setEnsemble(Ensemble ensemble) {
+        this.ensemble = ensemble;
+    }
+
     /**
      * @return the area
      */

cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/BaseGfePyController.java

@@ -42,6 +42,8 @@ import com.raytheon.viz.gfe.smartscript.FieldDefinition;
  * ------------ ---------- ----------- --------------------------
  * Nov 10, 2008            njensen     Initial creation
  * Jan 08, 2013  1486      dgilling    Refactor based on PythonScriptController.
+ * Jan 18, 2013            njensen     Added garbageCollect()
+ * 
  * </pre>
  * 
  * @author njensen
@@ -161,4 +163,20 @@ public abstract class BaseGfePyController extends PythonScriptController {
         }
         return varDict;
     }
+    }
+
+    /**
+     * Runs the python garbage collector. This should be run at the end of a
+     * procedure or tool in case the custom python used tk. If the python used
+     * tk and it is not garbage collected, errors about invalid threads may
+     * occur when the garbage collector runs in another python interpreter.
+     */
+    public void garbageCollect() {
+        try {
+            jep.eval("import gc");
+            jep.eval("gc.collect()");
+        } catch (JepException e) {
+            statusHandler.handle(Priority.PROBLEM,
+                    "Error garbage collecting GFE python interpreter", e);
+        }
 }

cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/procedures/ProcedureJob.java

@@ -57,6 +57,7 @@ import com.raytheon.viz.gfe.jobs.AsyncProgressJob;
  * ------------ ---------- ----------- --------------------------
  * Oct 8, 2009             njensen     Initial creation
  * Jan 8, 2013  1486       dgilling    Support changes to BaseGfePyController.
+ * Jan 18, 2013 1509       njensen     Garbage collect after running procedure
  * 
  * </pre>
  * 
@@ -396,6 +397,7 @@ public class ProcedureJob extends AbstractQueueJob<ProcedureRequest> {
             statusHandler.handle(Priority.PROBLEM, "Error executing procedure "
                     + procedureName, e);
         } finally {
+            controller.garbageCollect();
             progressJob.done(pjStatus);
         }
     }

cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/smarttool/script/SmartToolJob.java

@@ -53,6 +53,7 @@ import com.raytheon.viz.gfe.smarttool.Tool;
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * Jan 19, 2010            njensen     Initial creation
+ * Jan 18, 2013    1509  njensen  Garbage collect after running tool
  * 
  * </pre>
  * 
@@ -164,6 +165,7 @@ public class SmartToolJob extends AbstractQueueJob<SmartToolRequest> {
                                         "Error in smart tool", e);
                                 throw e;
                             } finally {
+                                python.garbageCollect();
                                 progressJob.done(pjResult);
                                 req = request;
                                 request = null;

edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Ensemble.java

@@ -1,5 +1,9 @@
 package com.raytheon.uf.common.datadelivery.registry;
 
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlAttribute;
@@ -30,67 +34,161 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
 @DynamicSerialize
 public class Ensemble implements ISerializableObject {
 
+    /**
+     * names of the various ensemble members
+     */
+    @DynamicSerializeElement
+    @XmlAttribute
+    private List<String> members;
+
+    /**
+     * which members to request.
+     */
+    @DynamicSerializeElement
+    @XmlAttribute
+    private List<String> selectedMembers;
+
     public Ensemble() {
 
     }
 
-    /**
+    public Ensemble(Ensemble other) {
-     * ensemble models set this
+        if (other.members != null) {
-     */
+            members = new ArrayList<String>(other.members);
-    @DynamicSerializeElement
+        }
-    @XmlAttribute
+        if (other.selectedMembers != null) {
-    private String name;
+            selectedMembers = new ArrayList<String>(other.selectedMembers);
+        }
+    }
+
+    public List<String> getMembers() {
+        return members;
+    }
+
+    public void setMembers(List<String> members) {
+        this.members = members;
+    }
+
+    public List<String> getSelectedMembers() {
+        return selectedMembers;
+    }
+
+    public void setSelectedMembers(List<String> selectedMembers) {
+        this.selectedMembers = selectedMembers;
+    }
+
+    public int getMemberCount() {
+        if (members == null) {
+            // if no members are defined then assume a single unnamed member.
+            return 1;
+        } else {
+            return members.size();
+        }
+    }
+
+    public int getSelectedMemberCount() {
+        if (members == null) {
+            // if no members are defined then assume a single unnamed member
+            // which is automatically selected
+            return 1;
+        } else if (selectedMembers == null) {
+            return 0;
+        } else {
+            return selectedMembers.size();
+        }
+    }
+
+    public boolean hasSelection() {
+        return selectedMembers != null && !selectedMembers.isEmpty();
+    }
 
     /**
-     * ensemble models set this
+     * 
+     * Get the range of indices of the selected members, inclusively. This
+     * method should usually be used after a split operation to guarantee that
+     * selected ensembles are consecutive
+     * 
+     * @return a int[2] representing the start and end indices of the ensemble
+     *         members
+     * @throws IllegalStateException
+     *             if the selected member are non consecutive.
      */
-    @DynamicSerializeElement
+    public int[] getSelectedRange() {
-    @XmlAttribute
+        int[] result = new int[2];
-    private String length;
+        int[] indexes = getSortedIndexes();
+        if (indexes == null) {
+            return result;
+        }
+        result[0] = indexes[0];
+        result[1] = result[0] - 1;
+        for (int index : indexes) {
+            result[1] += 1;
+            if (result[1] != index) {
+                throw new IllegalStateException(
+                        "Cannot get selected range for nonconsecutive ensemble members\nMembers "
+                                + members + "\nSelected" + selectedMembers);
+            }
+        }
+        return result;
+    }
 
     /**
-     * ensemble models set this
+     * Split this ensemble into multiple Ensembles each with consecutive
+     * members. Used in cases where multiple ensembles can be requested at once
+     * if they are consecutive.
+     * 
+     * @param maxEnsembles
+     *            The mazximum number of consecutive members in any Ensemble
+     * @return
      */
-    @DynamicSerializeElement
+    public List<Ensemble> split(int maxEnsembles) {
-    @XmlAttribute
+        int[] indexes = getSortedIndexes();
-    private String init;
+        if (indexes == null) {
-
+            return Arrays.asList(new Ensemble(this));
-    /**
+        }
-     * ensemble models set this
+        List<Ensemble> result = new ArrayList<Ensemble>();
-     */
+        List<String> selected = new ArrayList<String>(selectedMembers.size());
-    @DynamicSerializeElement
+        int start = indexes[0];
-    @XmlAttribute
+        int end = start - 1;
-    private Integer size;
+        for (int index : indexes) {
-
+            end += 1;
-    public void setName(String name) {
+            if (index != end || (end - start) >= maxEnsembles) {
-        this.name = name;
+                // Either we have run out of consecutive indexes or we have the
+                // max consecutive. so split off and start the next one.
+                Ensemble e = new Ensemble(this);
+                e.setSelectedMembers(new ArrayList<String>(selected));
+                selected.clear();
+                result.add(e);
+                start = index;
+                end = start;
+            }
+            selected.add(members.get(index));
+        }
+        Ensemble e = new Ensemble(this);
+        e.setSelectedMembers(selected);
+        result.add(e);
+        return result;
     }
 
-    public String getName() {
+    private int[] getSortedIndexes() {
-        return name;
+        if (!hasSelection()) {
+            return null;
+        }
+        int[] indexes = new int[selectedMembers.size()];
+        int c = 0;
+        for (String selected : selectedMembers) {
+            int index = members.indexOf(selected);
+            if (index >= 0) {
+                indexes[c++] = index;
+            }
+        }
+        if (c == 0) {
+            return null;
+        } else if (c < indexes.length) {
+            indexes = Arrays.copyOf(indexes, c);
+        }
+        Arrays.sort(indexes);
+        return indexes;
     }
 
-    public void setSize(Integer size) {
-        this.size = size;
-    }
-
-    public Integer getSize() {
-        return size;
-    }
-
-    public void setLength(String length) {
-        this.length = length;
-    }
-
-    public String getLength() {
-        return length;
-    }
-
-    public void setInit(String init) {
-        this.init = init;
-    }
-
-    public String getInit() {
-        return init;
-    }
 }

edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/GriddedDataSet.java

@@ -20,6 +20,7 @@
 package com.raytheon.uf.common.datadelivery.registry;
 
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 
 import javax.xml.bind.annotation.XmlAccessType;
@@ -59,6 +60,10 @@ public abstract class GriddedDataSet extends DataSet {
     @DynamicSerializeElement
     protected Set<Integer> forecastHours = new HashSet<Integer>();
 
+    @DynamicSerializeElement
+    @XmlElement
+    private Ensemble ensemble;
+
     /**
      * @return the cycles
      */
@@ -89,6 +94,14 @@ public abstract class GriddedDataSet extends DataSet {
         this.forecastHours = forecastHours;
     }
 
+    public Ensemble getEnsemble() {
+        return ensemble;
+    }
+
+    public void setEnsemble(Ensemble ensemble) {
+        this.ensemble = ensemble;
+    }
+
     /**
      * {@inheritDoc}
      */
@@ -100,6 +113,15 @@ public abstract class GriddedDataSet extends DataSet {
             GriddedDataSet other = (GriddedDataSet) toCombine;
             this.getCycles().addAll(other.getCycles());
             this.getForecastHours().addAll(other.getForecastHours());
+            if (this.getEnsemble() != null) {
+                List<String> mine = this.getEnsemble().getSelectedMembers();
+                List<String> theirs = other.getEnsemble().getSelectedMembers();
+                if (mine == null) {
+                    ensemble.setSelectedMembers(theirs);
+                } else if (theirs != null) {
+                    mine.addAll(theirs);
+                }
+            }
         }
     }
 }

edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/GriddedDataSetMetaData.java

@@ -6,7 +6,6 @@ import java.util.Map;
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlAttribute;
-import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlSeeAlso;
 import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 
@@ -60,13 +59,6 @@ public abstract class GriddedDataSetMetaData extends
     @SlotAttribute(CYCLE_SLOT)
     private int cycle = NO_CYCLE;
 
-    /**
-     * ensemble models set this
-     */
-    @DynamicSerializeElement
-    @XmlElement
-    private Ensemble ensemble;
-
     public void setLevelTypes(Map<DataLevelType, Levels> levelTypes) {
         this.levelTypes = levelTypes;
     }
@@ -92,11 +84,4 @@ public abstract class GriddedDataSetMetaData extends
         return cycle;
     }
 
-    public void setEnsemble(Ensemble ensemble) {
-        this.ensemble = ensemble;
-    }
-
-    public Ensemble getEnsemble() {
-        return ensemble;
-    }
 }

edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Parameter.java

@@ -92,10 +92,6 @@ public class Parameter implements ISerializableObject, Serializable {
     @DynamicSerializeElement
     private String fillValue;
 
-    @XmlAttribute
-    @DynamicSerializeElement
-    private Integer ensemble;
-
     @XmlAttribute
     @DynamicSerializeElement
     private String baseType;
@@ -124,7 +120,6 @@ public class Parameter implements ISerializableObject, Serializable {
         this.dataType = copy.dataType;
         this.missingValue = copy.missingValue;
         this.fillValue = copy.fillValue;
-        this.ensemble = copy.ensemble;
         this.baseType = copy.baseType;
 
         // deep copy
@@ -265,14 +260,6 @@ public class Parameter implements ISerializableObject, Serializable {
         return levels;
     }
 
-    public void setEnsemble(Integer ensemble) {
-        this.ensemble = ensemble;
-    }
-
-    public Integer getEnsemble() {
-        return ensemble;
-    }
-
     public void setFillValue(String fillValue) {
         this.fillValue = fillValue;
     }

edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Subscription.java

@@ -243,6 +243,10 @@ public class Subscription implements ISerializableObject, Serializable {
     @DynamicSerializeElement
     private ArrayList<Parameter> parameter;
 
+    @XmlElement
+    @DynamicSerializeElement
+    private Ensemble ensemble;
+
     @XmlAttribute
     @DynamicSerializeElement
     private boolean deleted;
@@ -920,4 +924,13 @@ public class Subscription implements ISerializableObject, Serializable {
     public int getLatencyInMinutes() {
         return latencyInMinutes;
     }
+
+    public Ensemble getEnsemble() {
+        return ensemble;
+    }
+
+    public void setEnsemble(Ensemble ensemble) {
+        this.ensemble = ensemble;
+    }
+
 }

edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/util/DataSizeUtils.java

@@ -26,6 +26,7 @@ import org.geotools.geometry.jts.ReferencedEnvelope;
 
 import com.raytheon.uf.common.datadelivery.registry.Coverage;
 import com.raytheon.uf.common.datadelivery.registry.DataSet;
+import com.raytheon.uf.common.datadelivery.registry.Ensemble;
 import com.raytheon.uf.common.datadelivery.registry.GriddedCoverage;
 import com.raytheon.uf.common.datadelivery.registry.GriddedDataSet;
 import com.raytheon.uf.common.datadelivery.registry.Levels;
@@ -102,6 +103,8 @@ public class DataSizeUtils {
     /** Number of forecast hours */
     private int numFcstHours = 0;
 
+    private int numEnsembleMembers = 0;
+
     /** Envelope */
     private ReferencedEnvelope envelope = null;
 
@@ -153,6 +156,7 @@ public class DataSizeUtils {
     public long getDataSetSizeInBytes() {
         long l = numRequestedGrids
                 * numFcstHours
+                * numEnsembleMembers
                 * dataSet.getServiceType().getRequestBytesPerParameterPerLevel(
                         numberOfGridCells);
         return l;
@@ -181,8 +185,15 @@ public class DataSizeUtils {
                     long numCells = griddedCov.getGridCoverage().getNx()
                             * griddedCov.getGridCoverage().getNy();
                     // Default to 1 forecast hour if not a gridded data set
-                    long fcstHrs = dataSet instanceof GriddedDataSet ? ((GriddedDataSet) dataSet)
+                    long numEns = 1;
-                            .getForecastHours().size() : 1;
+                    long fcstHrs = 1;
+                    if (dataSet instanceof GriddedDataSet) {
+                        GriddedDataSet gDataSet = (GriddedDataSet) dataSet;
+                        fcstHrs = gDataSet.getForecastHours().size();
+                        if (gDataSet.getEnsemble() != null) {
+                            numEns = gDataSet.getEnsemble().getMemberCount();
+                        }
+                    }
                     Map<String, Parameter> paramMap = dataSet.getParameters();
 
                     // get the number of grids available
@@ -196,7 +207,8 @@ public class DataSizeUtils {
                         numGridsAvailable += (numLevels > 0 ? numLevels : 1);
                     }
 
-                    fullSize = fcstHrs
+                    fullSize = numEns
+                            * fcstHrs
                             * numGridsAvailable
                             * dataSet.getServiceType()
                                     .getRequestBytesPerParameterPerLevel(
@@ -295,4 +307,13 @@ public class DataSizeUtils {
 
         this.numRequestedGrids = numGrids;
     }
+
+    public void setNumEnsembleMembers(Ensemble ensemble) {
+        if (ensemble == null) {
+            this.numEnsembleMembers = 1;
+        } else {
+            this.numEnsembleMembers = ensemble.getSelectedMemberCount();
+        }
+
+    }
 }

edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/util/LookupManager.java

@@ -42,8 +42,10 @@ import com.raytheon.uf.common.serialization.SerializationUtil;
 import com.raytheon.uf.common.status.IUFStatusHandler;
 import com.raytheon.uf.common.status.UFStatus;
 import com.raytheon.uf.common.status.UFStatus.Priority;
+import com.raytheon.uf.common.util.CollectionUtil;
 import com.raytheon.uf.common.util.ServiceLoaderUtil;
 
+
 /**
  * Lookup table manager
  * 
@@ -54,6 +56,7 @@ import com.raytheon.uf.common.util.ServiceLoaderUtil;
  * ------------ ---------- ----------- --------------------------
  * Mar 7, 2011    357      dhladky     Initial creation
  * Oct 27, 2012   1163     dhladky     Improved, dynamically create files, Added Units
+ * Jan 18, 2013   1513     dhladky     Level lookup refit.
  * 
  * </pre>
  * 
@@ -381,10 +384,9 @@ public class LookupManager {
      * @throws Exception
      */
     public void modifyLevelLookups(String modelName, double dz, float min,
-            float max) throws Exception {
+            float max, List<Double> levs) throws Exception {
 
         LevelLookup ll = null;
-        List<Double> levs = null;
 
         if (levelLookupExists(modelName)) {
             ll = getLevelsFromFile(modelName);
@@ -397,25 +399,33 @@ public class LookupManager {
             ll = new LevelLookup();
         }
 
-        if (levs == null) {
+        boolean gen = false;
+
+        
+        if (CollectionUtil.isNullOrEmpty(levs)) {
             ll.setLevelXml(new ArrayList<Double>());
             levs = ll.getLevelXml();
+            gen = true;
+        } else {
+            ll.setLevelXml(levs);
         }
 
-        int diff = (int) (max - min);
+        if (gen) {
-        int total = (int) Math.abs((diff / dz));
+            int diff = (int) (max - min);
-        // These add simple place holder level
+            int total = (int) Math.abs((diff / dz));
-        // identifiers. It is up to the admin
+            // These add simple place holder level
-        // to add the real values
+            // identifiers. It is up to the admin
-        if (diff < 0) {
+            // to add the real values
-            for (int i = 0; i <= total; i++) {
+            if (diff < 0) {
-                double lev = max + i * dz;
+                for (int i = 0; i <= total; i++) {
-                levs.add(lev);
+                    double lev = max + i * dz;
-            }
+                    levs.add(lev);
-        } else {
+                }
-            for (int i = 0; i <= total; i++) {
+            } else {
-                double lev = min + i * dz;
+                for (int i = 0; i <= total; i++) {
-                levs.add(lev);
+                    double lev = min + i * dz;
+                    levs.add(lev);
+                }
             }
         }
 
@@ -551,4 +561,6 @@ public class LookupManager {
 
         return unitXml;
     }
+    
+    
 }

edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/xml/RetrievalAttribute.java

@@ -29,6 +29,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 
 import com.raytheon.uf.common.datadelivery.registry.Coverage;
 import com.raytheon.uf.common.datadelivery.registry.CoverageAdapter;
+import com.raytheon.uf.common.datadelivery.registry.Ensemble;
 import com.raytheon.uf.common.datadelivery.registry.Parameter;
 import com.raytheon.uf.common.datadelivery.registry.Time;
 import com.raytheon.uf.common.serialization.ISerializableObject;
@@ -100,6 +101,10 @@ public class RetrievalAttribute implements ISerializableObject, Serializable {
     @DynamicSerializeElement
     private Time time;
 
+    @XmlElement
+    @DynamicSerializeElement
+    private Ensemble ensemble;
+
     @XmlElement(name = "provider")
     @DynamicSerializeElement
     private String provider;
@@ -156,4 +161,12 @@ public class RetrievalAttribute implements ISerializableObject, Serializable {
         return subName;
     }
 
+    public Ensemble getEnsemble() {
+        return ensemble;
+    }
+
+    public void setEnsemble(Ensemble ensemble) {
+        this.ensemble = ensemble;
+    }
+
 }

edexOsgi/com.raytheon.uf.edex.datadelivery.event/src/com/raytheon/uf/edex/datadelivery/event/handler/SubscriptionNotificationHandler.java

@@ -46,6 +46,7 @@ import com.raytheon.uf.common.status.UFStatus;
  * Sep 06, 2012     687    mpduff     Send a SubscriptionNotificationResponse object.
  * Sep 24, 2012    1157    mpduff     Changed to use BaseSubscriptionNotificationRequest.
  * Jan 17, 2013 1501       djohnson     If a subscription is still in the registry, use it for the notification response.
+ * Jan 21, 2013 1501       djohnson     Throw an exception if subscription is not provided on the request.
  * </pre>
  * 
  * @author mpduff
@@ -97,6 +98,13 @@ public class SubscriptionNotificationHandler<T extends Subscription> extends
         final IBaseSubscriptionHandler<T> subscriptionHandler = response
                 .getSubscriptionHandler();
         final T requestSubscription = request.getSubscription();
+
+        if (requestSubscription == null) {
+            throw new IllegalArgumentException(
+                    "Unable to send a notification for a null subscription!",
+                    new NullPointerException("requestSubscription"));
+        }
+
         final T registryVersion = subscriptionHandler
                 .getByName(requestSubscription.getName());
 

edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/OPENDAPServiceConfig.xml

@@ -74,6 +74,7 @@
     <constant name="RESOLUTION" value="resolution"/>
     <constant name="DATE_COMPARE_FORMAT" value="EEE MMM dd hhmmss zzz yyyy"/>
     <constant name="META_DATA_SUFFIX" value=".info"/>
+    <constant name="DATA_SUFFIX" value=".dods"/>
     <constant name="FILL_VALUE" value="_FillValue"/>
     <constant name="MAX_RETRIEVAL_SIZE" value="4500000" />
     <constant name="KELVIN" value="k" />

edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/NOMADS-harvester.xml

@@ -41,13 +41,13 @@
 		<crawlDir>/awips2/crawl</crawlDir>
 		<dateFormat>HHddMMMyyyy</dateFormat>
 		<ignore>ruc</ignore>
-		<ignore>rap32</ignore>
+		<ignore>rap_f</ignore>
         <ignore>\.das$</ignore> 
         <ignore>\.dds$</ignore>
         <ignore>help$</ignore>
-        <ignore>fens_all</ignore>
+        <ignore>fens\d\d\d</ignore>
-        <ignore>cmcens_all</ignore>
+        <ignore>cmcens[cp]\d\d</ignore>
-        <ignore>gep_all</ignore>
+        <ignore>ge[cp]\d\d</ignore>
 		<!-- seed scan once a day at 12z -->
 		<seedScan>0 0 12 * * ?</seedScan>
 		<!-- main scan every 12 minutes -->

edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/lookups/cmcensLevelLookup.xml

@@ -1,13 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<LevelLookup>
-    <Level>1000.0</Level>
-    <Level>925.0</Level>
-    <Level>850.0</Level>
-    <Level>700.0</Level>
-    <Level>500.0</Level>
-    <Level>250.0</Level>
-    <Level>200.0</Level>
-    <Level>100.0</Level>
-    <Level>50.0</Level>
-    <Level>10.0</Level>
-</LevelLookup>

edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/lookups/fensLevelLookup.xml

@@ -1,13 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<LevelLookup>
-    <Level>1000.0</Level>
-    <Level>925.0</Level>
-    <Level>850.0</Level>
-    <Level>700.0</Level>
-    <Level>500.0</Level>
-    <Level>250.0</Level>
-    <Level>200.0</Level>
-    <Level>100.0</Level>
-    <Level>50.0</Level>
-    <Level>10.0</Level>
-</LevelLookup>

edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/lookups/gensLevelLookup.xml

@@ -1,29 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<LevelLookup>
-    <Level>1000.0</Level>
-    <Level>975.0</Level>
-    <Level>950.0</Level>
-    <Level>925.0</Level>
-    <Level>900.0</Level>
-    <Level>850.0</Level>
-    <Level>800.0</Level>
-    <Level>750.0</Level>
-    <Level>700.0</Level>
-    <Level>650.0</Level>
-    <Level>600.0</Level>
-    <Level>550.0</Level>
-    <Level>500.0</Level>
-    <Level>450.0</Level>
-    <Level>400.0</Level>
-    <Level>350.0</Level>
-    <Level>300.0</Level>
-    <Level>250.0</Level>
-    <Level>200.0</Level>
-    <Level>150.0</Level>
-    <Level>100.0</Level>
-    <Level>70.0</Level>
-    <Level>50.0</Level>
-    <Level>30.0</Level>
-    <Level>20.0</Level>
-    <Level>10.0</Level>
-</LevelLookup>

edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/lookups/gens_bcLevelLookup.xml

@@ -1,10 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<LevelLookup>
-    <Level>1000.0</Level>
-    <Level>925.0</Level>
-    <Level>850.0</Level>
-    <Level>600.0</Level>
-    <Level>500.0</Level>
-    <Level>250.0</Level>
-    <Level>200.0</Level>
-</LevelLookup>

edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/lookups/gfsLevelLookup.xml

@@ -1,29 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<LevelLookup>
-    <Level>1000.0</Level>
-    <Level>975.0</Level>
-    <Level>950.0</Level>
-    <Level>925.0</Level>
-    <Level>900.0</Level>
-    <Level>850.0</Level>
-    <Level>800.0</Level>
-    <Level>750.0</Level>
-    <Level>700.0</Level>
-    <Level>650.0</Level>
-    <Level>600.0</Level>
-    <Level>550.0</Level>
-    <Level>500.0</Level>
-    <Level>450.0</Level>
-    <Level>400.0</Level>
-    <Level>350.0</Level>
-    <Level>300.0</Level>
-    <Level>250.0</Level>
-    <Level>200.0</Level>
-    <Level>150.0</Level>
-    <Level>100.0</Level>
-    <Level>70.0</Level>
-    <Level>50.0</Level>
-    <Level>30.0</Level>
-    <Level>20.0</Level>
-    <Level>10.0</Level>
-</LevelLookup>

edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/lookups/gfs_2p5LevelLookup.xml

@@ -1,29 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<LevelLookup>
-    <Level>1000.0</Level>
-    <Level>975.0</Level>
-    <Level>950.0</Level>
-    <Level>925.0</Level>
-    <Level>900.0</Level>
-    <Level>850.0</Level>
-    <Level>800.0</Level>
-    <Level>750.0</Level>
-    <Level>700.0</Level>
-    <Level>650.0</Level>
-    <Level>600.0</Level>
-    <Level>550.0</Level>
-    <Level>500.0</Level>
-    <Level>450.0</Level>
-    <Level>400.0</Level>
-    <Level>350.0</Level>
-    <Level>300.0</Level>
-    <Level>250.0</Level>
-    <Level>200.0</Level>
-    <Level>150.0</Level>
-    <Level>100.0</Level>
-    <Level>70.0</Level>
-    <Level>50.0</Level>
-    <Level>30.0</Level>
-    <Level>20.0</Level>
-    <Level>10.0</Level>
-</LevelLookup>

edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/lookups/gfs_hdLevelLookup.xml

@@ -1,50 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<LevelLookup>
-    <Level>1000.0</Level>
-    <Level>975.0</Level>
-    <Level>950.0</Level>
-    <Level>925.0</Level>
-    <Level>900.0</Level>
-    <Level>875.0</Level>
-    <Level>850.0</Level>
-    <Level>825.0</Level>
-    <Level>800.0</Level>
-    <Level>775.0</Level>
-    <Level>750.0</Level>
-    <Level>725.0</Level>
-    <Level>700.0</Level>
-    <Level>675.0</Level>
-    <Level>650.0</Level>
-    <Level>625.0</Level>
-    <Level>600.0</Level>
-    <Level>575.0</Level>
-    <Level>550.0</Level>
-    <Level>525.0</Level>
-    <Level>500.0</Level>
-    <Level>475.0</Level>
-    <Level>450.0</Level>
-    <Level>425.0</Level>
-    <Level>400.0</Level>
-    <Level>375.0</Level>
-    <Level>350.0</Level>
-    <Level>325.0</Level>
-    <Level>300.0</Level>
-    <Level>275.0</Level>
-    <Level>250.0</Level>
-    <Level>225.0</Level>
-    <Level>200.0</Level>
-    <Level>175.0</Level>
-    <Level>150.0</Level>
-    <Level>125.0</Level>
-    <Level>100.0</Level>
-    <Level>70.0</Level>
-    <Level>50.0</Level>
-    <Level>30.0</Level>
-    <Level>20.0</Level>
-    <Level>10.0</Level>
-    <Level>7.0</Level>
-    <Level>5.0</Level>
-    <Level>3.0</Level>
-    <Level>2.0</Level>
-    <Level>1.0</Level>
-</LevelLookup>

edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/lookups/hireswLevelLookup.xml

@@ -1,13 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<LevelLookup>
-    <Level>1000.0</Level>
-    <Level>925.0</Level>
-    <Level>850.0</Level>
-    <Level>700.0</Level>
-    <Level>600.0</Level>
-    <Level>500.0</Level>
-    <Level>400.0</Level>
-    <Level>300.0</Level>
-    <Level>250.0</Level>
-    <Level>200.0</Level>
-</LevelLookup>

edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/lookups/naefs_bcLevelLookup.xml

@@ -1,10 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<LevelLookup>
-    <Level>1000.0</Level>
-    <Level>925.0</Level>
-    <Level>850.0</Level>
-    <Level>700.0</Level>
-    <Level>500.0</Level>
-    <Level>250.0</Level>
-    <Level>200.0</Level>
-</LevelLookup>

edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/lookups/namLevelLookup.xml

@@ -1,45 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<LevelLookup>
-    <Level>1000.0</Level>
-    <Level>975.0</Level>
-    <Level>950.0</Level>
-    <Level>925.0</Level>
-    <Level>900.0</Level>
-    <Level>875.0</Level>
-    <Level>850.0</Level>
-    <Level>825.0</Level>
-    <Level>800.0</Level>
-    <Level>775.0</Level>
-    <Level>750.0</Level>
-    <Level>725.0</Level>
-    <Level>700.0</Level>
-    <Level>675.0</Level>
-    <Level>650.0</Level>
-    <Level>625.0</Level>
-    <Level>600.0</Level>
-    <Level>575.0</Level>
-    <Level>550.0</Level>
-    <Level>525.0</Level>
-    <Level>500.0</Level>
-    <Level>475.0</Level>
-    <Level>450.0</Level>
-    <Level>425.0</Level>
-    <Level>400.0</Level>
-    <Level>375.0</Level>
-    <Level>350.0</Level>
-    <Level>325.0</Level>
-    <Level>300.0</Level>
-    <Level>275.0</Level>
-    <Level>250.0</Level>
-    <Level>225.0</Level>
-    <Level>200.0</Level>
-    <Level>175.0</Level>
-    <Level>150.0</Level>
-    <Level>125.0</Level>
-    <Level>100.0</Level>
-    <Level>75.0</Level>
-    <Level>50.0</Level>
-    <Level>20.0</Level>
-    <Level>10.0</Level>
-    <Level>5.0</Level>
-</LevelLookup>

edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/lookups/ofsLevelLookup.xml

@@ -1,43 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<LevelLookup>
-    <Level>6000.0</Level>
-    <Level>5500.0</Level>
-    <Level>5000.0</Level>
-    <Level>4500.0</Level>
-    <Level>4000.0</Level>
-    <Level>3500.0</Level>
-    <Level>3000.0</Level>
-    <Level>2500.0</Level>
-    <Level>2000.0</Level>
-    <Level>1500.0</Level>
-    <Level>1000.0</Level>
-    <Level>950.0</Level>
-    <Level>900.0</Level>
-    <Level>850.0</Level>
-    <Level>800.0</Level>
-    <Level>750.0</Level>
-    <Level>700.0</Level>
-    <Level>650.0</Level>
-    <Level>600.0</Level>
-    <Level>550.0</Level>
-    <Level>500.0</Level>
-    <Level>450.0</Level>
-    <Level>400.0</Level>
-    <Level>350.0</Level>
-    <Level>300.0</Level>
-    <Level>250.0</Level>
-    <Level>200.0</Level>
-    <Level>150.0</Level>
-    <Level>100.0</Level>
-    <Level>90.0</Level>
-    <Level>80.0</Level>
-    <Level>70.0</Level>
-    <Level>60.0</Level>
-    <Level>50.0</Level>
-    <Level>30.0</Level>
-    <Level>20.0</Level>
-    <Level>10.0</Level>
-    <Level>6.0</Level>
-    <Level>3.0</Level>
-    <Level>0.0</Level>
-</LevelLookup>

edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/lookups/rapLevelLookup.xml

@@ -1,40 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<LevelLookup>
-    <Level>1000.0</Level>
-    <Level>975.0</Level>
-    <Level>950.0</Level>
-    <Level>925.0</Level>
-    <Level>900.0</Level>
-    <Level>875.0</Level>
-    <Level>850.0</Level>
-    <Level>825.0</Level>
-    <Level>800.0</Level>
-    <Level>775.0</Level>
-    <Level>750.0</Level>
-    <Level>725.0</Level>
-    <Level>700.0</Level>
-    <Level>675.0</Level>
-    <Level>650.0</Level>
-    <Level>625.0</Level>
-    <Level>600.0</Level>
-    <Level>575.0</Level>
-    <Level>550.0</Level>
-    <Level>525.0</Level>
-    <Level>500.0</Level>
-    <Level>475.0</Level>
-    <Level>450.0</Level>
-    <Level>425.0</Level>
-    <Level>400.0</Level>
-    <Level>375.0</Level>
-    <Level>350.0</Level>
-    <Level>325.0</Level>
-    <Level>300.0</Level>
-    <Level>275.0</Level>
-    <Level>250.0</Level>
-    <Level>225.0</Level>
-    <Level>200.0</Level>
-    <Level>175.0</Level>
-    <Level>150.0</Level>
-    <Level>125.0</Level>
-    <Level>100.0</Level>
-</LevelLookup>

edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/lookups/rtofsLevelLookup.xml

@@ -1,4 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<LevelLookup>
-    <Level>1.0</Level>
-</LevelLookup>

edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/lookups/rucLevelLookup.xml

@@ -1,40 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<LevelLookup>
-    <Level>1000.0</Level>
-    <Level>975.0</Level>
-    <Level>950.0</Level>
-    <Level>925.0</Level>
-    <Level>900.0</Level>
-    <Level>875.0</Level>
-    <Level>850.0</Level>
-    <Level>825.0</Level>
-    <Level>800.0</Level>
-    <Level>775.0</Level>
-    <Level>750.0</Level>
-    <Level>725.0</Level>
-    <Level>700.0</Level>
-    <Level>675.0</Level>
-    <Level>650.0</Level>
-    <Level>625.0</Level>
-    <Level>600.0</Level>
-    <Level>575.0</Level>
-    <Level>550.0</Level>
-    <Level>525.0</Level>
-    <Level>500.0</Level>
-    <Level>475.0</Level>
-    <Level>450.0</Level>
-    <Level>425.0</Level>
-    <Level>400.0</Level>
-    <Level>375.0</Level>
-    <Level>350.0</Level>
-    <Level>325.0</Level>
-    <Level>300.0</Level>
-    <Level>275.0</Level>
-    <Level>250.0</Level>
-    <Level>225.0</Level>
-    <Level>200.0</Level>
-    <Level>175.0</Level>
-    <Level>150.0</Level>
-    <Level>125.0</Level>
-    <Level>100.0</Level>
-</LevelLookup>

edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/lookups/srefLevelLookup.xml

@@ -1,42 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<LevelLookup>
-    <Level>1000.0</Level>
-    <Level>975.0</Level>
-    <Level>950.0</Level>
-    <Level>925.0</Level>
-    <Level>900.0</Level>
-    <Level>875.0</Level>
-    <Level>850.0</Level>
-    <Level>825.0</Level>
-    <Level>800.0</Level>
-    <Level>775.0</Level>
-    <Level>750.0</Level>
-    <Level>725.0</Level>
-    <Level>700.0</Level>
-    <Level>675.0</Level>
-    <Level>650.0</Level>
-    <Level>625.0</Level>
-    <Level>600.0</Level>
-    <Level>575.0</Level>
-    <Level>550.0</Level>
-    <Level>525.0</Level>
-    <Level>500.0</Level>
-    <Level>475.0</Level>
-    <Level>450.0</Level>
-    <Level>425.0</Level>
-    <Level>400.0</Level>
-    <Level>375.0</Level>
-    <Level>350.0</Level>
-    <Level>325.0</Level>
-    <Level>300.0</Level>
-    <Level>275.0</Level>
-    <Level>250.0</Level>
-    <Level>225.0</Level>
-    <Level>200.0</Level>
-    <Level>175.0</Level>
-    <Level>150.0</Level>
-    <Level>125.0</Level>
-    <Level>100.0</Level>
-    <Level>75.0</Level>
-    <Level>50.0</Level>
-</LevelLookup>

edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/lookups/sref_bcLevelLookup.xml

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<LevelLookup>
-    <Level>1000.0</Level>
-    <Level>850.0</Level>
-    <Level>700.0</Level>
-    <Level>500.0</Level>
-    <Level>250.0</Level>
-</LevelLookup>

edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/metadata/adapters/GridMetadataAdapter.java

@@ -20,6 +20,8 @@
 package com.raytheon.uf.edex.datadelivery.retrieval.metadata.adapters;
 
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 
 import com.raytheon.edex.util.Util;
 import com.raytheon.uf.common.datadelivery.registry.GriddedCoverage;
@@ -60,11 +62,18 @@ public class GridMetadataAdapter extends AbstractMetadataAdapter {
         Level[] levels = getLevels(attXML);
         int size = levels.length;
 
+        List<String> ensembles = null;
+        if (attXML.getEnsemble() != null && attXML.getEnsemble().hasSelection()) {
+            ensembles = attXML.getEnsemble().getSelectedMembers();
+            size *= ensembles.size();
+        } else {
+            ensembles = Arrays.asList((String) null);
+        }
+
         if (attXML.getTime().getSelectedTimeIndices() != null) {
             if (levels.length > 1
                     || attXML.getTime().getSelectedTimeIndices().size() > 1) {
-                size = levels.length
+                size *= attXML.getTime().getSelectedTimeIndices().size();
-                        * attXML.getTime().getSelectedTimeIndices().size();
             }
         }
 
@@ -80,20 +89,21 @@ public class GridMetadataAdapter extends AbstractMetadataAdapter {
         }
 
         if (attXML.getTime().getSelectedTimeIndices() != null) {
-
+            int bin = 0;
-            for (int i = 0; i < attXML.getTime().getSelectedTimeIndices()
+            for (String ensemble : ensembles) {
-                    .size(); i++) {
+                for (int i = 0; i < attXML.getTime().getSelectedTimeIndices()
-                for (int j = 0; j < levels.length; j++) {
+                        .size(); i++) {
-                    int bin = (levels.length * i) + j;
+                    for (int j = 0; j < levels.length; j++) {
-                    pdos[bin] = populateGridRecord(attXML.getSubName(),
+                        pdos[bin++] = populateGridRecord(attXML.getSubName(),
-                            attXML.getParameter(),
+                                attXML.getParameter(), levels[j], ensemble,
-                            levels[j], gridCoverage);
+                                gridCoverage);
+                    }
                 }
             }
         } else {
 
             pdos[0] = populateGridRecord(attXML.getSubName(),
-                    attXML.getParameter(), levels[0],
+                    attXML.getParameter(), levels[0], ensembles.get(0),
                     gridCoverage);
 
         }
@@ -108,10 +118,9 @@ public class GridMetadataAdapter extends AbstractMetadataAdapter {
      * @return
      */
     private GridRecord populateGridRecord(String name, Parameter parm,
-            Level level,
+            Level level, String ensembleId, GridCoverage gridCoverage) {
-            GridCoverage gridCoverage) {
         return ResponseProcessingUtilities.getGridRecord(name, parm, level,
-                gridCoverage);
+                ensembleId, gridCoverage);
     }
 
     /**

edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPMetaDataParser.java

@@ -37,6 +37,7 @@ import com.raytheon.uf.common.datadelivery.registry.DataSet;
 import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData;
 import com.raytheon.uf.common.datadelivery.registry.DataType;
 import com.raytheon.uf.common.datadelivery.registry.GriddedCoverage;
+import com.raytheon.uf.common.datadelivery.registry.GriddedDataSet;
 import com.raytheon.uf.common.datadelivery.registry.GriddedDataSetMetaData;
 import com.raytheon.uf.common.datadelivery.registry.Levels;
 import com.raytheon.uf.common.datadelivery.registry.OpenDapGriddedDataSet;
@@ -88,6 +89,7 @@ import dods.dap.DAS;
  * Dec 12, 2012 supplement dhladky      Restored operation of ensembles.
  * Dec 10, 2012 1259       bsteffen     Switch Data Delivery from LatLon to referenced envelopes.
  * Jan 08, 2013            dhladky      Performance enhancements, specific model fixes.
+ * Jan 18, 2013 1513       dhladky      Level look up improvements.
  * 
  * </pre>
  * 
@@ -134,8 +136,10 @@ class OpenDAPMetaDataParser extends MetaDataParser {
                 // create new default lookups
                 if (levelType.equals(LevelType.MB)
                         || levelType.equals(LevelType.SEAB)) {
+
+                    List<Double> levelList = OpenDAPParseUtility.getInstance().parseLevels(gdsmd.getUrl(), serviceConfig.getConstantValue("LEV"));
                     LookupManager.getInstance().modifyLevelLookups(
-                            collectionName, dz, levMin, levMax);
+                            collectionName, dz, levMin, levMax, levelList);
                 }
             }
 
@@ -179,7 +183,8 @@ class OpenDAPMetaDataParser extends MetaDataParser {
      * @param dataDateFormat
      * @return
      */
-    private Map<String, Parameter> getParameters(DAS das, DataSet dataSet,
+    private Map<String, Parameter> getParameters(DAS das,
+            GriddedDataSet dataSet,
             GriddedDataSetMetaData gdsmd, Link link, Collection collection,
             String dataDateFormat) {
 
@@ -336,7 +341,8 @@ class OpenDAPMetaDataParser extends MetaDataParser {
         if (das.getAttributeTable(ens) != null) {
             try {
                 AttributeTable at = das.getAttributeTable(ens);
-                gdsmd.setEnsemble(OpenDAPParseUtility.getInstance().parseEnsemble(at));
+                dataSet.setEnsemble(OpenDAPParseUtility.getInstance()
+                        .parseEnsemble(at));
             } catch (Exception en) {
                 logParsingException(ens, "Ensemble", collectionName, url);
             }
@@ -414,11 +420,6 @@ class OpenDAPMetaDataParser extends MetaDataParser {
                             levMin, levMax));
                     parm.addLevelType(type);
 
-                    // set if an ensemble member
-                    if (gdsmd.getEnsemble() != null) {
-                        parm.setEnsemble(gdsmd.getEnsemble().getSize());
-                    }
-
                     parameters.put(name, parm);
 
                 } catch (Exception le) {
@@ -656,12 +657,12 @@ class OpenDAPMetaDataParser extends MetaDataParser {
             }
 
             DAS das = (DAS) link.getLinks().get(DAP_TYPE.DAS.getDapType());
-            dataSet.setParameters(getParameters(das, dataSet, gdsmd, link,
+            // set url first, used for level lookups
-                    collection, dataDateFormat));
-
             gdsmd.setUrl(link.getUrl().replace(
                     serviceConfig.getConstantValue("META_DATA_SUFFIX"),
                     serviceConfig.getConstantValue("BLANK")));
+            dataSet.setParameters(getParameters(das, dataSet, gdsmd, link,
+                    collection, dataDateFormat));
             Time dataSetTime = gdsmd.getTime();
             if (dataSetTime == null) {
                 throw new IllegalStateException(

edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPParseUtility.java

@@ -21,8 +21,13 @@ import com.raytheon.uf.common.datadelivery.retrieval.xml.UnitLookup;
 import com.raytheon.uf.common.status.IUFStatusHandler;
 import com.raytheon.uf.common.status.UFStatus;
 import com.raytheon.uf.common.status.UFStatus.Priority;
+import com.raytheon.uf.edex.datadelivery.retrieval.util.ConnectionUtil;
 
 import dods.dap.AttributeTable;
+import dods.dap.DArray;
+import dods.dap.DConnect;
+import dods.dap.DataDDS;
+import dods.dap.PrimitiveVector;
 
 /**
  * Constants for working with OpenDAP. This class should remain package-private,
@@ -43,15 +48,18 @@ import dods.dap.AttributeTable;
  * Nov 09, 2012    1163     dhladky     Made pre-load for service config
  * Nov 19, 2012    1166     djohnson    Clean up JAXB representation of registry objects.
  * Jan 08, 2013    1466     dhladky     NCOM dataset name parsing fix.
+ * Jan 18, 2013    1513     dhladky     Level Lookup improvements.
  * </pre>
  * 
  * @author dhladky
  * @version 1.0
  */
-final class OpenDAPParseUtility {
+public final class OpenDAPParseUtility {
 
     private static final Pattern QUOTES_PATTERN = Pattern.compile("\"");
 
+    private static final Pattern COMMA_PATTERN = Pattern.compile(",");
+
     /** Singleton instance of this class */
     private static OpenDAPParseUtility instance = null;
 
@@ -252,35 +260,16 @@ final class OpenDAPParseUtility {
      */
     public Ensemble parseEnsemble(AttributeTable table) {
 
-        String stime = serviceConfig.getConstantValue("TIMEINIT");
-        String slength = serviceConfig.getConstantValue("LENGTH");
         String sname = serviceConfig.getConstantValue("NAME");
-        int size = new Integer(trim(table.getAttribute(
+        Ensemble ens = new Ensemble();
-                serviceConfig.getConstantValue("SIZE")).getValueAt(0)))
-                .intValue();
-        String name = null;
-        String length = null;
-        String tinit = null;
-
-        if (table.getAttribute(slength) != null) {
-            length = table.getAttribute(slength).getValueAt(0);
-        }
 
         if (table.getAttribute(sname) != null) {
-            name = table.getAttribute(sname).getValueAt(0);
+            String name = trim(table.getAttribute(sname).getValueAt(0));
+            String[] members = COMMA_PATTERN.split(name);
+            ens.setMembers(Arrays.asList(members));
         }
-
-        if (table.getAttribute(stime) != null) {
-            tinit = table.getAttribute(stime).getValueAt(0);
-        }
-
-        Ensemble ens = new Ensemble();
-        ens.setSize(size);
-        ens.setLength(length);
-        ens.setName(name);
-        ens.setInit(tinit);
-
         return ens;
+
     }
 
     /**
@@ -353,7 +342,35 @@ final class OpenDAPParseUtility {
         return QUOTES_PATTERN.matcher(val).replaceAll(
                 serviceConfig.getConstantValue("BLANK"));
     }
-
     
+    /**
+     * Parse out the levels from the dods
+     * @param url
+     * @param lev
+     * @return
+     */
+    public List<Double> parseLevels(String url, String lev) {
+
+        List<Double> levels = null;
+
+        try {
+            DConnect connect = ConnectionUtil.getDConnect(url + "?" + lev);
+            DataDDS data = connect.getData(null);
+            DArray array = (DArray) data.getVariable(lev);
+            PrimitiveVector pm = array.getPrimitiveVector();
+            double[] values = (double[]) pm.getInternalStorage();
+            levels = new ArrayList<Double>();
+            for (double value : values) {
+                levels.add(value);
+            }
+
+        } catch (Exception e) {
+            statusHandler.handle(Priority.PROBLEM, "Error downloading/parsing levels: "
+                    + url, e);
+        }
+        
+        return levels;
+
+    }
 
 }

edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPRequestBuilder.java

@@ -22,6 +22,7 @@ package com.raytheon.uf.edex.datadelivery.retrieval.opendap;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.raytheon.uf.common.datadelivery.registry.Coverage;
+import com.raytheon.uf.common.datadelivery.registry.Ensemble;
 import com.raytheon.uf.common.datadelivery.registry.GriddedCoverage;
 import com.raytheon.uf.common.datadelivery.registry.Levels;
 import com.raytheon.uf.common.datadelivery.registry.Parameter;
@@ -216,17 +217,17 @@ class OpenDAPRequestBuilder extends RequestBuilder {
      */
     public String processEnsemble() {
 
-        StringBuilder buf = new StringBuilder();
+        if (getRetrievalAttribute().getEnsemble() != null) {
-
+            Ensemble e = getRetrievalAttribute().getEnsemble();
-        if (getRetrievalAttribute().getParameter().getEnsemble() != null) {
+            int[] range = e.getSelectedRange();
-            buf.append("["
+            if (range[0] == range[1]) {
-                    + (getRetrievalAttribute().getParameter().getEnsemble() - 1)
+                return "[" + range[0] + "]";
-                    + "]");
+            } else {
+                return "[" + range[0] + ":" + range[1] + "]";
+            }
         } else {
-            buf.append("");
+            return "";
         }
-
-        return buf.toString();
     }
 
     @Override

edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPRetrievalGenerator.java

@@ -22,14 +22,15 @@ package com.raytheon.uf.edex.datadelivery.retrieval.opendap;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.raytheon.uf.common.datadelivery.registry.Coverage;
 import com.raytheon.uf.common.datadelivery.registry.DataSet;
 import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData;
+import com.raytheon.uf.common.datadelivery.registry.Ensemble;
 import com.raytheon.uf.common.datadelivery.registry.GriddedCoverage;
 import com.raytheon.uf.common.datadelivery.registry.GriddedDataSetMetaData;
 import com.raytheon.uf.common.datadelivery.registry.Levels;
@@ -265,13 +266,14 @@ class OpenDAPRetrievalGenerator extends RetrievalGenerator {
      * @param parm
      * @return
      */
-    protected HashMap<DataTime, ArrayList<Level>> getGridDuplicates(
+    protected Map<DataTime, List<Level>> getGridDuplicates(
             String name,
-            Parameter parm, ArrayList<DataTime> times, ArrayList<Level> levels,
+            Parameter parm, List<DataTime> times, List<Level> levels,
+            List<String> ensembleMembers,
             GriddedCoverage cov) {
 
         return RetrievalGeneratorUtilities.findGridDuplicates(name, times,
-                levels,
+                levels, ensembleMembers,
                 parm, cov.getRequestGridCoverage());
     }
 
@@ -319,6 +321,13 @@ class OpenDAPRetrievalGenerator extends RetrievalGenerator {
                 return Collections.emptyList();
             }
 
+            List<Ensemble> ensembles = null;
+            if (sub.getEnsemble() == null) {
+                ensembles = Arrays.asList((Ensemble) null);
+            } else {
+                ensembles = sub.getEnsemble().split(1);
+            }
+
             for (List<Integer> timeSequence : subTime.getTimeSequences(sfactor)) {
 
                 for (Parameter param : sub.getParameter()) {
@@ -332,9 +341,11 @@ class OpenDAPRetrievalGenerator extends RetrievalGenerator {
                                 sub.getTime());
 
                         for (Time time : times) {
-                            Retrieval retrieval = getRetrieval(sub, bundle,
+                            for (Ensemble ensemble : ensembles) {
-                                    param, paramLevels, time);
+                                Retrieval retrieval = getRetrieval(sub, bundle,
-                            retrievals.add(retrieval);
+                                        param, paramLevels, time, ensemble);
+                                retrievals.add(retrieval);
+                            }
                         }
 
                     } else {
@@ -350,9 +361,12 @@ class OpenDAPRetrievalGenerator extends RetrievalGenerator {
                             // and time
                             for (Time time : times) {
                                 for (Levels level : levels) {
-                                    Retrieval retrieval = getRetrieval(sub,
+                                    for (Ensemble ensemble : ensembles) {
-                                            bundle, param, level, time);
+                                        Retrieval retrieval = getRetrieval(sub,
-                                    retrievals.add(retrieval);
+                                                bundle, param, level, time,
+                                                ensemble);
+                                        retrievals.add(retrieval);
+                                    }
                                 }
                             }
                         }
@@ -375,7 +389,7 @@ class OpenDAPRetrievalGenerator extends RetrievalGenerator {
      * @return
      */
     private Retrieval getRetrieval(Subscription sub, SubscriptionBundle bundle,
-            Parameter param, Levels level, Time time) {
+            Parameter param, Levels level, Time time, Ensemble ensemble) {
 
         Retrieval retrieval = new Retrieval();
         retrieval.setSubscriptionName(sub.getName());
@@ -404,6 +418,7 @@ class OpenDAPRetrievalGenerator extends RetrievalGenerator {
         lparam.setLevels(level);
         att.setTime(time);
         att.setParameter(lparam);
+        att.setEnsemble(ensemble);
         att.setSubName(retrieval.getSubscriptionName());
         att.setPlugin(pt.getPlugin());
         att.setProvider(sub.getProvider());
@@ -488,7 +503,6 @@ class OpenDAPRetrievalGenerator extends RetrievalGenerator {
         param.setBaseType(origParm.getBaseType());
         param.setDataType(origParm.getDataType());
         param.setDefinition(origParm.getDefinition());
-        param.setEnsemble(origParm.getEnsemble());
         param.setFillValue(origParm.getFillValue());
         param.setLevelType(origParm.getLevelType());
         param.setMissingValue(origParm.getMissingValue());
@@ -536,6 +550,13 @@ class OpenDAPRetrievalGenerator extends RetrievalGenerator {
 
         int sfactor = getSizingFactor(getDimensionalSize(sub.getCoverage()));
 
+        List<String> ensembles = null;
+        if (sub.getEnsemble() != null && sub.getEnsemble().hasSelection()) {
+            ensembles = sub.getEnsemble().getSelectedMembers();
+        } else {
+            ensembles = Arrays.asList((String) null);
+        }
+
         for (List<Integer> timeSequence : sub.getTime().getTimeSequences(
                 sfactor)) {
 
@@ -557,14 +578,14 @@ class OpenDAPRetrievalGenerator extends RetrievalGenerator {
                     ArrayList<Level> levels = ResponseProcessingUtilities
                             .getOpenDAPGridLevels(param.getLevels());
 
-                    HashMap<DataTime, ArrayList<Level>> dups = getGridDuplicates(
+                    Map<DataTime, List<Level>> dups = getGridDuplicates(
                             sub.getName(),
-                            param, times, levels,
+ param, times, levels, ensembles,
                             (GriddedCoverage) sub.getCoverage());
 
                     for (int i = 0; i < times.size(); i++) {
                         DataTime dtime = times.get(i);
-                        ArrayList<Level> levDups = dups.get(dtime);
+                        List<Level> levDups = dups.get(dtime);
 
                         if (levDups != null) {
                             // single level, remove the time
@@ -594,14 +615,14 @@ class OpenDAPRetrievalGenerator extends RetrievalGenerator {
                                     .getOpenDAPGridLevels(level);
                         }
 
-                        HashMap<DataTime, ArrayList<Level>> dups = getGridDuplicates(
+                        Map<DataTime, List<Level>> dups = getGridDuplicates(
-                                sub.getName(),
+                                sub.getName(), param, times, plevels,
-                                param, times, plevels,
+                                ensembles,
                                 ((GriddedCoverage) sub.getCoverage()));
 
                         for (int i = 0; i < times.size(); i++) {
                             DataTime dtime = times.get(i);
-                            ArrayList<Level> levDups = dups.get(dtime);
+                            List<Level> levDups = dups.get(dtime);
 
                             if (levDups != null) {
 

edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/response/OpenDAPTranslator.java

@@ -23,6 +23,7 @@ package com.raytheon.uf.edex.datadelivery.retrieval.response;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Enumeration;
+import java.util.List;
 
 import com.raytheon.uf.common.datadelivery.registry.GriddedCoverage;
 import com.raytheon.uf.common.datadelivery.retrieval.xml.RetrievalAttribute;
@@ -149,33 +150,41 @@ public class OpenDAPTranslator extends RetrievalTranslator {
         PrimitiveVector pm = darray.getPrimitiveVector();
         float[] values = (float[]) pm.getInternalStorage();
 
+        List<String> ensembles = null;
+        if (attXML.getEnsemble() != null && attXML.getEnsemble().hasSelection()) {
+            ensembles = attXML.getEnsemble().getSelectedMembers();
+        } else {
+            ensembles = Arrays.asList((String) null);
+        }
+
         // time dependencies
         int start = 0;
-        PluginDataObject[] records = new PluginDataObject[numLevels * numTimes];
+        PluginDataObject[] records = new PluginDataObject[numLevels * numTimes
+                * ensembles.size()];
 
-        for (int i = 0; i < times.size(); i++) {
+        int bin = 0;
+        for (int i = 0; i < ensembles.size(); i++) {
+            for (DataTime dataTime : times) {
+                for (int j = 0; j < numLevels; j++) {
+                    PluginDataObject record = getPdo(bin);
+                    record.setDataTime(dataTime);
+                    record.constructDataURI();
 
-            DataTime dataTime = times.get(i);
+                    int end = start + gridSize;
 
-            for (int j = 0; j < numLevels; j++) {
+                    float[] subValues = Arrays.copyOfRange(values, start, end);
-                int bin = (numLevels * i) + j;
-                PluginDataObject record = getPdo(bin);
-                record.setDataTime(dataTime);
-                record.constructDataURI();
 
-                int end = start + gridSize;
+                    subValues = GridMetadataAdapter.adjustGrid(nx, ny,
+                            subValues, Float.parseFloat(attXML.getParameter()
+                                    .getMissingValue()), true);
 
-                float[] subValues = Arrays.copyOfRange(values, start, end);
+                    record.setMessageData(subValues);
-
+                    record.setOverwriteAllowed(true);
-                subValues = GridMetadataAdapter.adjustGrid(nx, ny, subValues,
+                    records[bin++] = record;
-                        Float.parseFloat(attXML.getParameter()
+                    statusHandler.info("Creating record: "
-                                .getMissingValue()), true);
+                            + record.getDataURI());
-
+                    start = end;
-                record.setMessageData(subValues);
+                }
-                record.setOverwriteAllowed(true);
-                records[bin] = record;
-                statusHandler.info("Creating record: " + record.getDataURI());
-                start = end;
             }
         }
 

edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/util/ResponseProcessingUtilities.java

@@ -61,7 +61,7 @@ public class ResponseProcessingUtilities {
             .getHandler(ResponseProcessingUtilities.class);
 
     public static GridRecord getGridRecord(String name, Parameter parm,
-            Level level,
+            Level level, String ensembleId,
             GridCoverage gridCoverage) {
 
         com.raytheon.uf.common.parameter.Parameter parameter = new com.raytheon.uf.common.parameter.Parameter();
@@ -75,7 +75,7 @@ public class ResponseProcessingUtilities {
         record.setLevel(level);
         record.setParameter(parameter);
         record.setDatasetId(name);
-
+        record.setEnsembleId(ensembleId);
         return record;
     }
 

edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/util/RetrievalGeneratorUtilities.java

@@ -21,6 +21,8 @@ package com.raytheon.uf.edex.datadelivery.retrieval.util;
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 import com.raytheon.uf.common.datadelivery.registry.Parameter;
 import com.raytheon.uf.common.dataplugin.PluginException;
@@ -87,35 +89,39 @@ public class RetrievalGeneratorUtilities {
      * @param cov
      * @return
      */
-    public static HashMap<DataTime, ArrayList<Level>> findGridDuplicates(
+    public static Map<DataTime, List<Level>> findGridDuplicates(String name,
-            String name,
+            List<DataTime> times, List<Level> levels,
-            ArrayList<DataTime> times, ArrayList<Level> levels, Parameter parm,
+            List<String> ensembleMembers, Parameter parm,
             GridCoverage cov) {
 
-        HashMap<DataTime, ArrayList<Level>> dups = new HashMap<DataTime, ArrayList<Level>>();
+        HashMap<DataTime, List<Level>> dups = new HashMap<DataTime, List<Level>>();
 
         for (DataTime time : times) {
 
-            ArrayList<Level> levDups = dups.get(time);
+            List<Level> levDups = dups.get(time);
 
             if (levDups == null) {
                 levDups = new ArrayList<Level>();
             }
 
             for (Level level : levels) {
-                try {
+                for (String ensembleMember : ensembleMembers) {
+                    try {
 
-                    GridRecord rec = ResponseProcessingUtilities.getGridRecord(
+                        GridRecord rec = ResponseProcessingUtilities
-                            name, parm, level, cov);
+                                .getGridRecord(name, parm, level,
-                    rec.setDataTime(time);
+                                        ensembleMember, cov);
-                    rec.constructDataURI();
+                        rec.setDataTime(time);
-                    boolean isDup = findDuplicateUri(rec.getDataURI(), "grid");
+                        rec.constructDataURI();
-                    if (isDup) {
+                        boolean isDup = findDuplicateUri(rec.getDataURI(),
-                        levDups.add(level);
+                                "grid");
+                        if (isDup) {
+                            levDups.add(level);
+                        }
+                    } catch (PluginException e) {
+                        statusHandler.handle(Priority.PROBLEM,
+                                e.getLocalizedMessage(), e);
                     }
-                } catch (PluginException e) {
-                    statusHandler.handle(Priority.PROBLEM,
-                            e.getLocalizedMessage(), e);
                 }
             }
 

edexOsgi/com.raytheon.uf.tools.cli/impl/capture

@@ -12,6 +12,7 @@ REMOTE_SERVERS_TO_CHECK="dx1f dx3 dx4"
 
 # Flags to control what data capure grabs, to enable flag must be YES, anything else will be considered off.
 RUN_JSTACK="Y"
+JSTACK_ITERATIONS="15"
 RUN_JMAP="Y"
 RUN_QPID_STAT="Y"
 MOVE_ALL_HS_ERR_PID="Y"
@@ -19,6 +20,7 @@ MOVE_ALL_HS_ERR_PID="Y"
 GRAB_REMOTE_TOP="Y"
 GRAB_REMOTE_VMSTAT="Y"
 GRAB_CAVE_AND_ALERTVIZ_LOGS="Y"
+GRAB_SCREENSHOT='Y'
 EDEX_MODE="N"
 FORCE="N"
 TGZ_OUTPUT="Y"
@@ -39,6 +41,9 @@ usage() {
    echo -e "-g {grep string}\tdefault [$grepString]"
    echo "  The grep string used to find the processes"
    echo
+   echo -e "-screen {y/n}\t\tdefault [$GRAB_SCREENSHOT]"
+   echo "  Screen print the current workstation (local user must be running capture)"
+   echo
    echo -e "-s {y/n}\t\tdefault [$RUN_JSTACK]"
    echo "  Run jstack to grab the thread stack information"
    echo
@@ -115,6 +120,18 @@ checkYes() {
    fi
 }
 
+# runs import to grab screen shot of users desktop
+grabScreenShot() {
+   if [ "$GRAB_SCREENSHOT" == "y" ]; then
+      echo "Capturing screen shot of desktop"
+      t1=`date "+%Y%m%d %H:%M:%S"`
+      echo "${t1}: Capturing screen shot of desktop" >> $processFile
+      import -window root -display :0.0 ${dataPath}/screenShot_0.png > ${dataPath}/screenShot_0.log 2>&1 &
+      import -window root -display :0.1 ${dataPath}/screenShot_1.png > ${dataPath}/screenShot_1.log 2>&1 &
+      import -window root -display :0.2 ${dataPath}/screenShot_2.png > ${dataPath}/screenShot_2.log 2>&1 &
+   fi
+}
+
 # runs ssh command to grab top on a remote server, requires auto login to be setup
 grabRemoteTop() {
    if [ "$GRAB_REMOTE_TOP" == "y" ]; then
@@ -143,7 +160,6 @@ grabRemoteVmstat() {
    fi
 }
 
-
 checkForProcsAsOtherUsers() {
    if [ ! -z "$procs" ]; then
       numMyProcs=`echo "$myProcs" | wc -l`
@@ -213,39 +229,104 @@ reasonForCapture() {
    echo $reason >> ${dataPath}/capture_reason.log
 }
 
-# runs jstack in background
+# for a specified pid run jstack a specified number of times in a row
 runJstack() {
    local pid="$1"
-   shift 1
+   local numIterations="$2"
-   local options=$@
+   local options="-l"
-   local prePath="${dataPath}/pid_${pid}_"
+   
-   local t1=`date "+%Y%m%d %H:%M:%S"`
+   if [ "$FORCE" == "y" ]; then
-   local cmd="/awips2/java/bin/jstack $options $pid"
+      options="${options} -F"
-   echo "${t1}: Running command: ${cmd} >> ${prePath}jstack.log 2>&1 &" >> $processFile
-   if [ "$ACCUM" = "y" ]; then
-      echo >> ${prePath}jstack.log
-      echo >> ${prePath}jstack.log
-      echo "Running for $t1" >> ${prePath}jstack.log
    fi
-   $cmd >> ${prePath}jstack.log 2>&1 &
+
+   local cmd="/awips2/java/bin/jstack"
+   local count=1
+   local prePath="${dataPath}/pid_${pid}_"
+   local log=""
+   while [ "$count" -le "$numIterations" ]; do
+      t1=`date "+%Y%m%d %H:%M:%S"`
+      log="${prePath}jstack_${count}.log"
+
+      echo "${t1}: Running command: ${cmd} ${options} ${pid} >> ${log} 2>&1" >> $processFile
+      echo "Running for $t1" >> $log
+      ${cmd} ${options} ${pid} >> ${log} 2>&1
+
+      if [[ "$?" != "0" && $FORCE != "y" ]]; then
+         t1=`date "+%Y%m%d %H:%M:%S"`
+         echo "${t1}: jstack for $pid failed to connect, rerunning with -F" >> $processFile
+         ${cmd} ${options} -F ${pid} >> ${log} 2>&1
+      fi
+      let "count+=1"
+   done
 }
 
-# runs jmap in background
+# Launchs a background process for each PID to pull jstacks
+launchJstacks() {
+   # grab all jstacks
+   if [ "${RUN_JSTACK}" == "y" ]; then
+      if [ ! -z ${cavePid} ]; then
+         echo "Capturing thread stack for pid $cavePid"
+      else 
+         echo "Capturing all process thread stacks"
+      fi
+
+      local count=0
+      while [ "$count" -lt "$numProcs" ]; do
+         runJstack ${pids[$count]} ${JSTACK_ITERATIONS} &
+         let "count+=1"
+      done
+   fi
+}
+
+# runs jmap in background, if it fails will run again with -F
 runJmap() {
    local pid=$1
-   shift 1
-   local options=$@
    local prePath="${dataPath}/pid_${pid}_"
+   local options=""
+
+   if [ "$FORCE" == "y" ]; then
+      options="${options} -F"
+   fi
+
    local t1=`date "+%Y%m%d %H:%M:%S"`
+   local log="${prePath}dump.log"
+   local dumpPath="${prePath}dump"
+
    if [ "$ACCUM" = "y" ]; then
       # accum needs to change hprof by date
       local t2=`date "+%Y%m%d_%H%M%S"`
-      local cmd="/awips2/java/bin/jmap -dump:format=b,file=${prePath}dump_${t2}.hprof $options $pid"
+      dumpPath="${dumpPath}_${t2}.hprof"
    else
-      local cmd="/awips2/java/bin/jmap -dump:format=b,file=${prePath}dump.hprof $options $pid"
+      dumpPath="${dumpPath}.hprof"
+   fi
+
+   local cmd="/awips2/java/bin/jmap -dump:format=b,file=${dumpPath}"
+   echo "${t1}: Running command: $cmd $options $pid >> $log 2>&1 &" >> $processFile
+   $cmd $options $pid >> $log 2>&1 &
+
+   if [[ "$?" != "0" && $FORCE != "y" ]]; then
+      t1=`date "+%Y%m%d %H:%M:%S"`
+      echo "${t1}: jmap for $pid failed to connect, rerunning with -F" >> $processFile
+      $cmd $options -F $pid >> $log 2>&1 &
+   fi
+}
+
+# Launchs a background process for each PID to pull jmap
+launchJmaps() {
+   # grab all jmaps
+   if [ "$RUN_JMAP" == "y" ]; then
+      if [ ! -z ${cavePid} ]; then
+         echo "Capturing process heap dump for pid $cavePid"
+      else 
+         echo "Capturing all Heap Dumps"
+      fi
+
+      local count=0
+      while [ "$count" -lt "$numProcs" ]; do
+         runJmap ${pids[$count]} &
+         let "count+=1"
+      done
    fi
-   echo "${t1}: Running command: $cmd >> ${prePath}dump.log 2>&1 &" >> $processFile
-   $cmd >> ${prePath}dump.log 2>&1 &
 }
 
 # runs qpid-stat 
@@ -253,14 +334,25 @@ runQpidStat() {
    local qpidHost=cp1f 
    local prePath="${dataPath}/"
    local t1=`date "+%Y%m%d %H:%M:%S"`
-   local cmd="/awips2/python/bin/qpid-stat -q -Smsg -L100 ${qpidHost}" 
+   local cmd="/awips2/python/bin/qpid-stat -q -Smsg -L500 ${qpidHost}" 
-   echo "${t1}: Running command: $cmd >> ${prepath}qpid-stat.log 2>&1 &" >> $processFile
+   local log="${prepath}qpid-stat-queues.log"
+   echo "${t1}: Running command: $cmd >> $log 2>&1 &" >> $processFile
    if [ "$ACCUM" = "y" ]; then
-	echo >> ${prePath}qpid-stat.log
+	echo >> $log
-	echo >> ${prePath}qpid-stat.log
+	echo >> $log
-	echo "Running for $t1" >> ${prePath}qpid-stat.log
+	echo "Running for $t1" >> $log
    fi
-   $cmd >> ${prePath}qpid-stat.log 2>&1 &
+   $cmd >> $log 2>&1 &
+
+   log="${prepath}qpid-stat-sessions.log"
+   cmd="/awips2/python/bin/qpid-stat -s -Smsg -L500 ${qpidHost}" 
+   echo "${t1}: Running command: $cmd >> $log 2>&1 &" >> $processFile
+   if [ "$ACCUM" = "y" ]; then
+	echo >> $log
+	echo >> $log
+	echo "Running for $t1" >> $log
+   fi
+   $cmd >> $log 2>&1 &
 }
 
 # runs versions.sh to grab version info
@@ -291,6 +383,7 @@ while [ ! -z "$1" ]; do
       -e) EDEX_MODE="Y"; edexProcs[$edexProcCount]="$1"; shift 1; let "edexProcCount+=1";;
       -a) ACCUM="$1"; shift 1;;
       -v) GRAB_REMOTE_VMSTAT="$1"; shift 1;;
+      -screen) GRAB_SCREENSHOT="$1"; shift 1;;
       -h|*) usage;;
    esac
 done
@@ -308,6 +401,7 @@ checkYes EDEX_MODE $EDEX_MODE
 checkYes TGZ_OUTPUT $TGZ_OUTPUT
 checkYes ACCCUM $ACCCUM
 checkYes RUN_VERSIONS $RUN_VERSIONS
+checkYes GRAB_SCREENSHOT $GRAB_SCREENSHOT
 
 # if PID mode don't grab other hs_err_pids
 if [ ! -z $cavePid ]; then
@@ -383,7 +477,7 @@ if [ "$EDEX_MODE" == "y" ]; then
    grepString="${grepString}) "
 fi
 
-procs=`ps -ef | grep -E "$grepString" | grep -v "grep"`
+procs=`ps -ef | grep -E "$grepString" | grep -v "grep" | grep -v "cave.sh"`
 
 if [ ! -z "$cavePid" ]; then
    # limit cave procs to the requested PID
@@ -399,11 +493,6 @@ echo "" >> $processFile
 
 checkForProcsAsOtherUsers
 
-# get reason for running capture
-if [ "$reason" != "n" ]; then
-   reasonForCapture &
-fi
-
 if [ ! -z "${myProcs}" ]; then
    t1=`date "+%Y%m%d %H:%M:%S"`
    echo "Processes found for user $user, capturing data to $dataPath"
@@ -424,38 +513,9 @@ if [ ! -z "${myProcs}" ]; then
    done
    IFS=$PREV_IFS
 
-   # doing each item in its own loop so we can grab all data for a given type at once
+   launchJstacks
 
-   # grab all jstacks
+   launchJmaps
-   if [ "${RUN_JSTACK}" == "y" ]; then
-      if [ ! -z ${cavePid} ]; then
-         echo "Capturing thread stack for pid $cavePid"
-      else 
-         echo "Capturing all process thread stacks"
-      fi
-
-      count=0
-      while [ "$count" -lt "$numProcs" ]; do
-         if [ "$FORCE" == "y" ]; then
-            runJstack ${pids[$count]} -l -F
-         else
-            runJstack ${pids[$count]} -l
-         fi
-         bPids[$count]=$!
-         let "count+=1"
-      done
-
-      count=0
-      while [ "$count" -lt "$numProcs" ]; do
-         wait ${bPids[$count]}
-         if [ "$?" != "0" ]; then
-            t1=`date "+%Y%m%d %H:%M:%S"`
-            echo "${t1}: jstack for ${pids[$count]} failed to connect, rerunning with -F" >> $processFile
-            runJstack ${pids[$count]} -l -F
-         fi
-         let "count+=1"
-      done
-   fi
 
    runQpidStat
 
@@ -463,36 +523,6 @@ if [ ! -z "${myProcs}" ]; then
 
    grabRemoteVmstat
 
-   # grab all jmaps
-   if [ "$RUN_JMAP" == "y" ]; then
-      if [ ! -z ${cavePid} ]; then
-         echo "Capturing process heap dump for pid $cavePid"
-      else 
-         echo "Capturing all Heap Dumps"
-      fi
-
-      count=0
-      while [ "$count" -lt "$numProcs" ]; do
-         if [ "$FORCE" == "y" ]; then
-            runJmap ${pids[$count]} -F
-         else
-            runJmap ${pids[$count]}
-         fi
-         bPids[$count]=$!
-         let "count+=1"
-      done
-
-      count=0
-      while [ "$count" -lt "$numProcs" ]; do
-         wait ${bPids[$count]}
-         if [ "$?" != "0" ]; then
-            t1=`date "+%Y%m%d %H:%M:%S"`
-            echo "${t1}: jmap for ${pids[$count]} failed to connect, rerunning with -F" >> $processFile
-            runJmap ${pids[$count]} -F
-         fi
-         let "count+=1"
-      done
-   fi
 else
    t1=`date "+%Y%m%d %H:%M:%S"`
    echo "*** NO processes found for user $user, capturing limited data to $dataPath"
@@ -504,6 +534,14 @@ else
    grabRemoteVmstat
 fi
 
+# grab screen shot, spawns background process for each screen
+grabScreenShot
+
+# get reason for running capture
+if [ "$reason" != "n" ]; then
+   reasonForCapture &
+fi
+
 # move all hs_err_pid from user's home directory to capture directory
 if [ "${MOVE_ALL_HS_ERR_PID}" == "y" ]; then
    numErrFiles=`ls ${HOME}/hs_err_pid* 2> /dev/null | wc -l`