Issue #2267 Shared Subscription fixes. DataDelivery Port environment standardization. Logging updates.

Amend: Refactored WMO header applier.
       Fixes for WMO header applier.
       Updated dataDeliveryRetrieval.xml with correct wmo header pattern.
       Removed manual ingest from registry mode.
       Simplified data delivery distribution matching pattern.
       Added serialize test for wfs response.
       Updated comments.
       Moved WfsRetrievalResponse to com.raytheon.uf.edex.datadelivery.retrieval package.
       Added WfsRetrievalResponse to Jaxb unmarshalable classes in DeserializeRetrievedDataFromIngest.
       HibernateBandwidthInitializer.executeAfterRegistryInit(), in NCF mode schedules shared subs.

Change-Id: I0278339701f5f55491120c35d80cbd6d3dcc84f7

Former-commit-id: a6540d69ea [formerly 8bf8efa14520aaba532ca7a2e82e4c1fa24adca8]
Former-commit-id: 21acb75f24

edexOsgi/build.edex/esb/bin/setup.env

@@ -31,14 +31,16 @@ export BROKER_ADDR=localhost
 # setup hdf5 connection if pypies is enabled
 export PYPIES_SERVER=http://localhost:9582
 
+# Registry specific ports
+export EBXML_REGISTRY_WEBSERVER_PORT=8082
+export EBXML_REGISTRY_WEBSERVER_CONFIDENTIAL_PORT=8446
+export EBXML_THRIFT_SERVICE_PORT=9588
 # data delivery specific variables, used below in the localization section
 export NCF_HOST=165.92.30.69
-export NCF_ADDRESS=http://${NCF_HOST}:8084
+export NCF_ADDRESS=http://${NCF_HOST}:${EBXML_REGISTRY_WEBSERVER_PORT}
 export DATADELIVERY_HOST=localhost
-export DATADELIVERY_PORT=9588
 # Currently the registry is hosted on datadelivery, but this could be changed in the future
 export EBXML_REGISTRY_HOST=${DATADELIVERY_HOST}
-export EBXML_REGISTRY_PORT=${DATADELIVERY_PORT}
 export EBXML_REGISTRY_SUBSCRIPTIONS_ENABLED=false
 export EBXML_REGISTRY_FEDERATION_ENABLED=false
 
@@ -48,11 +50,11 @@ export HTTP_PORT=9581
 export HTTP_SERVER=http://localhost:${HTTP_PORT}/services
 export JMS_SERVER=tcp://localhost:5672
 export RADAR_SERVER=tcp://localhost:8813
-export DATADELIVERY_SERVER=http://${DATADELIVERY_HOST}:${DATADELIVERY_PORT}/services
+export DATADELIVERY_SERVER=http://${DATADELIVERY_HOST}:${EBXML_THRIFT_SERVICE_PORT}/services
-export EBXML_REGISTRY_SERVICE=http://${EBXML_REGISTRY_HOST}:${EBXML_REGISTRY_PORT}/services
+export EBXML_REGISTRY_SERVICE=http://${EBXML_REGISTRY_HOST}:${EBXML_THRIFT_SERVICE_PORT}/services
-export EBXML_REGISTRY_LCM_SERVICE=http://${EBXML_REGISTRY_HOST}:${EBXML_REGISTRY_PORT}/lifecycleManager?WSDL
+export EBXML_REGISTRY_LCM_SERVICE=http://${EBXML_REGISTRY_HOST}:${EBXML_THRIFT_SERVICE_PORT}/lifecycleManager?WSDL
-export EBXML_REGISTRY_QUERY_SERVICE=http://${EBXML_REGISTRY_HOST}:${EBXML_REGISTRY_PORT}/queryManager?WSDL
+export EBXML_REGISTRY_QUERY_SERVICE=http://${EBXML_REGISTRY_HOST}:${EBXML_THRIFT_SERVICE_PORT}/queryManager?WSDL
-export NCF_BANDWIDTH_MANAGER_SERVICE=http://${NCF_HOST}:9590/services
+export NCF_BANDWIDTH_MANAGER_SERVICE=http://${NCF_HOST}:${EBXML_THRIFT_SERVICE_PORT}/services
 
 # set the AWIPS II shared directory
 export SHARE_DIR=/awips2/edex/data/share

edexOsgi/build.edex/esb/etc/centralRegistry.sh

@@ -26,9 +26,3 @@ export EDEX_DEBUG_PORT=5011
 export EDEX_JMX_PORT=1622
 export LOG_CONF=logback-registry.xml
 export MGMT_PORT=9607
-
-# Registry specific ports
-export EBXML_REGISTRY_WEBSERVER_PORT=8084
-export EBXML_REGISTRY_WEBSERVER_CONFIDENTIAL_PORT=8446
-export EBXML_QUERY_SERVICE_SOAP_PORT=10146
-export EBXML_THRIFT_SERVICE_PORT=9588

edexOsgi/build.edex/esb/etc/devRegistry.sh

@@ -26,9 +26,3 @@ export EDEX_DEBUG_PORT=5013
 export EDEX_JMX_PORT=1620
 export LOG_CONF=logback-registry.xml
 export MGMT_PORT=9605
-
-# Registry specific ports
-export EBXML_REGISTRY_WEBSERVER_PORT=8082
-export EBXML_REGISTRY_WEBSERVER_CONFIDENTIAL_PORT=8444
-export EBXML_QUERY_SERVICE_SOAP_PORT=10144
-export EBXML_THRIFT_SERVICE_PORT=9588

edexOsgi/build.edex/esb/etc/registry.sh

@@ -26,9 +26,3 @@ export EDEX_DEBUG_PORT=5012
 export EDEX_JMX_PORT=1620
 export LOG_CONF=logback-registry.xml
 export MGMT_PORT=9605
-
-# Registry specific ports
-export EBXML_REGISTRY_WEBSERVER_PORT=8082
-export EBXML_REGISTRY_WEBSERVER_CONFIDENTIAL_PORT=8444
-export EBXML_QUERY_SERVICE_SOAP_PORT=10144
-export EBXML_THRIFT_SERVICE_PORT=9588

edexOsgi/com.raytheon.uf.common.datadelivery.bandwidth/src/com/raytheon/uf/common/datadelivery/bandwidth/BandwidthService.java

@@ -55,6 +55,7 @@ import com.raytheon.uf.common.util.LogUtil;
  * Feb 27, 2013 1644       djohnson     Now abstract, sub-classes provide specific service lookup keys.
  * Jul 18, 2013 1653       mpduff       Add getSubscriptionStatusSummary method.
  * Oct 2,  2013 1797       dhladky      Generics
+ * Oct 01, 2013 2267       bgonzale     Log error response from proposed scheduling.
  * 
  * </pre>
  * 
@@ -202,8 +203,9 @@ public abstract class BandwidthService<T extends Time, C extends Coverage> exten
             LogUtil.logIterable(
                     statusHandler,
                     Priority.PROBLEM,
-                    "Returning null response object, unable to propose scheduling the following subscriptions for bandwidth management:",
+                    "Returning null response object, unable to propose scheduling"
-                    subscriptions);
+                            + "the following subscriptions for bandwidth management:",
+                    subscriptions, e);
             return IProposeScheduleResponse.NULL_OBJECT;
         }
     }

edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/LogUtil.java

@@ -32,6 +32,7 @@ import com.raytheon.uf.common.status.UFStatus.Priority;
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * Oct 26, 2012 1322       djohnson     Initial creation
+ * Oct 01, 2013 2267       bgonzale     Add logging for errors also.
  * 
  * </pre>
  * 
@@ -71,4 +72,35 @@ public final class LogUtil {
         String msg = StringUtil.createMessage(message, iterables, 3);
         statusHandler.handle(priority, msg);
     }
+
+    /**
+     * Logs the message, then each iterable to the output with a new line
+     * separator between each iterable.
+     * 
+     * <pre>
+     * 
+     * Example:
+     * 
+     * This would be the message:
+     *    iterable1
+     *    iterable2
+     * </pre>
+     * 
+     * @param statusHandler
+     *            the status handler
+     * @param priority
+     *            the priority
+     * @param message
+     *            the message
+     * @param iterable
+     *            the iterables
+     * @param error
+     *            the throwable
+     */
+    public static <T> void logIterable(IUFStatusHandler statusHandler,
+            Priority priority, String message, Iterable<T> iterables,
+            Throwable error) {
+        String msg = StringUtil.createMessage(message, iterables, 3);
+        statusHandler.handle(priority, msg, error);
+    }
 }

edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/hibernate/HibernateBandwidthInitializer.java

@@ -34,6 +34,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.RetrievalManager;
  * Jun 25, 2013 2106       djohnson     init() now takes a {@link RetrievalManager} as well.
  * Sep 05, 2013 2330       bgonzale     On WFO registry init, only subscribe to local site subscriptions.
  * Sep 06, 2013 2344       bgonzale     Removed attempt to add to immutable empty set.
+ * Oct 07, 2013 2267       bgonzale     in executeAfterRegistryInit NCF schedules shared subs.
  * 
  * </pre>
  * 
@@ -87,21 +88,22 @@ public class HibernateBandwidthInitializer implements BandwidthInitializer {
         Set<Subscription> activeSubscriptions = new HashSet<Subscription>();
         try {
             final String localOffice = SiteUtil.getSite();
+            final boolean isCentralRegistry = System.getProperty(
+                    "edex.run.mode").equals("centralRegistry");
 
-            // Load active subscriptions for the local office
+            // Load active subscriptions
             for (Subscription sub : findSubscriptionsStrategy
                     .findSubscriptionsToSchedule()) {
                 boolean isShared = (sub instanceof SharedSubscription);
                 boolean isLocalOffice = sub.getOfficeIDs()
                         .contains(localOffice);
 
-                if (!isShared && isLocalOffice) {
+                if ((isCentralRegistry && isShared)
+                        || (!isShared && isLocalOffice)) {
                     activeSubscriptions.add(sub);
                     statusHandler.info("Scheduling Subscription: " + sub);
                 } else {
-                    statusHandler
+                    statusHandler.info("Not Scheduling Subscription: " + sub);
-                            .info("Not Scheduling Non-local Subscription: "
-                                    + sub);
                 }
             }
         } catch (Exception e) {

edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/retrieval/RetrievalManager.java

@@ -38,6 +38,7 @@ import com.raytheon.uf.edex.datadelivery.retrieval.RetrievalManagerNotifyEvent;
  * Jun 13, 2013 2095       djohnson     Can schedule any subclass of BandwidthAllocation.
  * Jun 25, 2013 2106       djohnson     Copy state from another instance, add ability to check for proposed bandwidth throughput changes.
  * Jul 09, 2013 2106       djohnson     Only needs to unregister from the EventBus when used in an EDEX instance, so handled in EdexBandwidthManager.
+ * Oct 03, 2013 2267       bgonzale     Added check for no retrieval plan matching in the proposed retrieval plans.
  * 
  * </pre>
  * 
@@ -269,11 +270,23 @@ public class RetrievalManager {
                 .entrySet()) {
             final RetrievalPlan proposedRetrievalPlan = proposedRetrievalManager.retrievalPlans
                     .get(entry.getKey());
-            if (proposedRetrievalPlan.getDefaultBandwidth() != entry.getValue()
+            if (proposedRetrievalPlan != null && entry.getValue() != null) {
-                    .getDefaultBandwidth()) {
+                if (proposedRetrievalPlan.getDefaultBandwidth() != entry
+                        .getValue().getDefaultBandwidth()) {
                     proposingBandwidthChanges = true;
                     break;
                 }
+            } else {
+                StringBuilder sb = new StringBuilder(
+                        "The ProposedRetrievalPlan, ");
+                sb.append(proposedRetrievalPlan);
+                sb.append(", or the Existing RetrievalPlan, ");
+                sb.append(entry.getKey());
+                sb.append(" : ");
+                sb.append(entry.getValue());
+                sb.append(", is null.  Skipping this check.");
+                statusHandler.info(sb.toString());
+            }
         }
 
         return proposingBandwidthChanges;

edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/META-INF/MANIFEST.MF

@@ -73,4 +73,5 @@ Export-Package: com.raytheon.uf.edex.datadelivery.retrieval;
    dods.dap,
    com.raytheon.uf.common.dataplugin.grid,
    com.raytheon.uf.common.dataplugin.level,
-   com.raytheon.uf.common.datadelivery.registry"
+   com.raytheon.uf.common.datadelivery.registry",
+ com.raytheon.uf.edex.datadelivery.retrieval.wfs

edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/DataDeliveryRetrievalWmoHeaderApplier.java

@@ -23,7 +23,7 @@ import java.text.DateFormat;
 import java.text.Format;
 import java.text.MessageFormat;
 import java.util.Date;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.Map;
 
 import com.raytheon.uf.common.time.util.TimeUtil;
@@ -66,6 +66,9 @@ import com.raytheon.uf.edex.core.props.PropertiesException;
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * Aug 06, 2013 1822       bgonzale     Initial creation
+ * Oct 01, 2013 2267       bgonzale     Pass request parameter instead of components of request.
+ *                                      Fixed ordering of elements in maps and defaults when
+ *                                      element keys are null.
  * 
  * </pre>
  * 
@@ -99,8 +102,9 @@ public class DataDeliveryRetrievalWmoHeaderApplier implements IWmoHeaderApplier
 
     private static Map<String, String> parseMapping(String dataMapping)
             throws PropertiesException {
-        Map<String, String> resultMap = new HashMap<String, String>();
         String[] elementPairs = dataMapping.split(",");
+        Map<String, String> resultMap = new LinkedHashMap<String, String>(
+                elementPairs.length, 1);
 
         for (String elementPair : elementPairs) {
             String[] pair = elementPair.split(":");
@@ -139,21 +143,27 @@ public class DataDeliveryRetrievalWmoHeaderApplier implements IWmoHeaderApplier
         return new Object[] { provider, format, source, date };
     }
 
-    private static String getElement(String dataProvider,
+    /*
+     * if no element found, default to first element, or if empty, default to
+     * null.
+     */
+    private static String getElement(String elementKey,
             Map<String, String> mapping) {
-        String defaultKey = null;
+        String resultKey = null;
         
         for (String key : mapping.keySet()) {
-            if (defaultKey == null) {
+            if (resultKey == null) {
-                defaultKey = key;
+                resultKey = key;
-            }
+                if (elementKey == null) {
-            if (key.startsWith(dataProvider)) {
+                    break;
-                return mapping.get(key);
                 }
             }
-        // if no element found, default to first element, or if empty, default
+            if (key.startsWith(elementKey)) {
-        // to null
+                resultKey = key;
-        return mapping.get(defaultKey);
+                break;
+            }
+        }
+        return mapping.get(resultKey);
     }
 
 }

edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/DeserializeRetrievedDataFromIngest.java

@@ -27,8 +27,8 @@ import com.raytheon.edex.esb.Headers;
 import com.raytheon.uf.common.datadelivery.registry.Coverage;
 import com.raytheon.uf.common.serialization.JAXBManager;
 import com.raytheon.uf.edex.datadelivery.retrieval.opendap.OpenDapRetrievalResponse;
+import com.raytheon.uf.edex.datadelivery.retrieval.wfs.WfsRetrievalResponse;
 import com.raytheon.uf.edex.wmo.message.WMOMessage;
-
 /**
  * Deserializes the retrieved data in a retrievalQueue.
  * 
@@ -41,6 +41,7 @@ import com.raytheon.uf.edex.wmo.message.WMOMessage;
  * Feb 01, 2013 1543       djohnson     Initial creation
  * Mar 05, 2013 1647       djohnson     Remove WMO header.
  * Mar 19, 2013 1794       djohnson     Read from a queue rather than the file system.
+ * Oct 04, 2013 2267       bgonzale     Added WfsRetrieval to unmarshal classes.
  * 
  * </pre>
  * 
@@ -61,7 +62,8 @@ public class DeserializeRetrievedDataFromIngest implements IRetrievalsFinder {
         this.retrievalQueue = retrievalQueue;
         try {
             this.jaxbManager = new JAXBManager(RetrievalResponseXml.class,
-                    OpenDapRetrievalResponse.class, Coverage.class);
+                    OpenDapRetrievalResponse.class, WfsRetrievalResponse.class,
+                    Coverage.class);
         } catch (JAXBException e) {
             throw new ExceptionInInitializerError(e);
         }

edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/IRetrievalPluginDataObjectsProcessor.java

@@ -19,10 +19,9 @@
  **/
 package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
 
-import java.util.Date;
-
 import com.raytheon.uf.common.serialization.SerializationException;
 import com.raytheon.uf.edex.datadelivery.retrieval.adapters.RetrievalAdapter.TranslationException;
+import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord;
 
 /**
  * Processes {@link RetrievalResponseXml} that were generated from a retrieval.
@@ -35,6 +34,7 @@ import com.raytheon.uf.edex.datadelivery.retrieval.adapters.RetrievalAdapter.Tra
  * ------------ ---------- ----------- --------------------------
  * Feb 01, 2013 1543       djohnson     Initial creation
  * Aug 09, 2013 1822       bgonzale     Added parameters to processRetrievedPluginDataObjects.
+ * Oct 01, 2013 2267       bgonzale     Pass request parameter instead of components of request.
  * 
  * </pre>
  * 
@@ -46,24 +46,17 @@ public interface IRetrievalPluginDataObjectsProcessor {
 
     /**
      * Process plugin data objects that were created as a result of a data
-     * delivery retrieval.
+     * delivery retrieval request.
      * 
-     * @param dataProvider
+     * @param request
-     *            NOMADS, MADIS, or PDA
+     *            the request
-     * @param dataFormat
-     *            Binary Grid(OPENDAP), POINT(MADIS), NetCDF4
-     * @param sourceType
-     *            Model, Observation, Satellite
-     * @param date
-     *            the date
      * @param retrievalPluginDataObjects
      *            the retrieval plugin data objects
      * @throws SerializationException
      *             on error with serialization
      * @throws TranslationException
      */
-    void processRetrievedPluginDataObjects(String dataProvider,
+    void processRetrievedPluginDataObjects(RetrievalRequestRecord request,
-            String dataFormat, String sourceType, Date date,
             RetrievalResponseXml retrievalPluginDataObjects)
             throws SerializationException, TranslationException;
 }

edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/RetrievalTask.java

@@ -19,8 +19,6 @@
  **/
 package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
 
-import java.util.Date;
-
 import com.raytheon.uf.common.datadelivery.registry.Network;
 import com.raytheon.uf.common.status.IUFStatusHandler;
 import com.raytheon.uf.common.status.UFStatus;
@@ -45,6 +43,7 @@ import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord;
  * Feb 15, 2013 1543       djohnson     Using xml for retrievals now.
  * Mar 05, 2013 1647       djohnson     Change no retrievals found message to debug.
  * Aug 09, 2013 1822       bgonzale     Added parameters to processRetrievedPluginDataObjects.
+ * Oct 01, 2013 2267       bgonzale     Pass request parameter instead of components of request.
  * 
  * </pre>
  * 
@@ -103,10 +102,8 @@ public class RetrievalTask implements Runnable {
                     request = retrievalDao.getById(retrievalPluginDataObject
                             .getRequestRecord());
                     success = retrievalPluginDataObject.isSuccess();
-                    Date date = request.getInsertTime();
                     retrievedDataProcessor.processRetrievedPluginDataObjects(
-                            request.getProvider(), request.getPlugin(), null,
+                            request, retrievalPluginDataObject);
-                            date, retrievalPluginDataObject);
                 } catch (Exception e) {
                     statusHandler.error(
                             network + " retrieval processing error", e);

edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/SerializeRetrievedDataToDirectory.java

@@ -30,7 +30,9 @@ import com.raytheon.uf.common.datadelivery.registry.Coverage;
 import com.raytheon.uf.common.serialization.JAXBManager;
 import com.raytheon.uf.common.serialization.SerializationException;
 import com.raytheon.uf.common.util.FileUtil;
+import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord;
 import com.raytheon.uf.edex.datadelivery.retrieval.opendap.OpenDapRetrievalResponse;
+import com.raytheon.uf.edex.datadelivery.retrieval.wfs.WfsRetrievalResponse;
 
 /**
  * Serializes the retrieved data to a directory.
@@ -46,6 +48,7 @@ import com.raytheon.uf.edex.datadelivery.retrieval.opendap.OpenDapRetrievalRespo
  * Mar 05, 2013 1647       djohnson     Apply WMO header.
  * Mar 07, 2013 1647       djohnson     Write out as hidden file, then rename.
  * Aug 09, 2013 1822       bgonzale     Added parameters to IWmoHeaderApplier.applyWmoHeader().
+ * Oct 01, 2013 2267       bgonzale     Pass request parameter instead of components of request.
  * 
  * </pre>
  * 
@@ -70,7 +73,8 @@ public class SerializeRetrievedDataToDirectory implements
         this.wmoHeaderWrapper = wmoHeaderWrapper;
         try {
             this.jaxbManager = new JAXBManager(RetrievalResponseXml.class,
-                    OpenDapRetrievalResponse.class, Coverage.class);
+                    OpenDapRetrievalResponse.class, WfsRetrievalResponse.class,
+                    Coverage.class);
         } catch (JAXBException e) {
             throw new ExceptionInInitializerError(e);
         }
@@ -80,8 +84,8 @@ public class SerializeRetrievedDataToDirectory implements
      * {@inheritDoc}
      */
     @Override
-    public void processRetrievedPluginDataObjects(String dataType,
+    public void processRetrievedPluginDataObjects(
-            String dataFormat, String sourceType, Date date,
+            RetrievalRequestRecord request,
             RetrievalResponseXml retrievalPluginDataObjects)
             throws SerializationException {
         retrievalPluginDataObjects.prepareForSerialization();
@@ -94,8 +98,10 @@ public class SerializeRetrievedDataToDirectory implements
 
             final String xml = jaxbManager
                     .marshalToXml(retrievalPluginDataObjects);
-            final String textForFile = wmoHeaderWrapper.applyWmoHeader(
+            final Date date = request.getInsertTime();
-                    dataType, dataFormat, sourceType, date, xml);
+            final String textForFile = wmoHeaderWrapper
+                    .applyWmoHeader(request.getProvider(), request.getPlugin(),
+                    getSourceType(request), date, xml);
 
             // Write as hidden file, this is OS specific, but there is no
             // platform-neutral way to do this with Java
@@ -111,4 +117,18 @@ public class SerializeRetrievedDataToDirectory implements
             throw new SerializationException(e);
         }
     }
+
+    /**
+     * Determine source type from the request.
+     * 
+     * TODO Simple method that is adequate for now. It will need to be updated
+     * for new data from NOMADS, MADIS, and PDA.
+     * 
+     * @return source type string ("MODEL", "OBSERVATION", or "SATELLITE")
+     */
+    private String getSourceType(RetrievalRequestRecord request) {
+        String provider = request.getProvider();
+        return (provider == null || !provider.equalsIgnoreCase("MADIS") ? "MODEL"
+                : "OBSERVATION");
+    }
 }

edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/StoreRetrievedData.java

@@ -19,7 +19,6 @@
  **/
 package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
 
-import java.util.Date;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -63,6 +62,7 @@ import com.raytheon.uf.edex.datadelivery.retrieval.util.RetrievalPersistUtil;
  * Feb 15, 2013 1543       djohnson     Retrieve the retrieval attributes from the database.
  * Aug 09, 2013 1822       bgonzale     Added parameters to processRetrievedPluginDataObjects.
  * Aug 06, 2013 1654       bgonzale     Added AdhocDataRetrievalEvent.
+ * Oct 01, 2013 2267       bgonzale     Pass request parameter instead of components of request.
  * 
  * </pre>
  * 
@@ -95,8 +95,8 @@ public class StoreRetrievedData implements IRetrievalPluginDataObjectsProcessor
      * {@inheritDoc}
      */
     @Override
-    public void processRetrievedPluginDataObjects(String dataProvider,
+    public void processRetrievedPluginDataObjects(
-            String dataFormat, String sourceType, Date date,
+            RetrievalRequestRecord request,
             RetrievalResponseXml retrievalPluginDataObjects)
             throws SerializationException, TranslationException {
         Map<String, PluginDataObject[]> pluginDataObjects = Maps.newHashMap();

edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/wfs/WfsRetrievalResponse.java

@@ -20,6 +20,7 @@ import com.raytheon.uf.edex.datadelivery.retrieval.response.RetrievalResponse;
  * ------------ ---------- ----------- --------------------------
  * May 12, 2013 753        dhladky     Initial creation
  * May 31, 2013 2038       djohnson    Move to correct repo.
+ * Oct 04, 2013 2267       bgonzale    Added default constructor.
  * 
  * </pre>
  * 
@@ -35,8 +36,18 @@ public class WfsRetrievalResponse extends RetrievalResponse {
     @XmlElement
     private String payload;
 
-    public WfsRetrievalResponse(RetrievalAttribute attribute) {
+    /**
+     * Default constructor.
+     */
+    public WfsRetrievalResponse() {
+    }
     
+    /**
+     * Initialization constructor.
+     * 
+     * @param attribute
+     */
+    public WfsRetrievalResponse(RetrievalAttribute attribute) {
     }
 
     @Override

edexOsgi/com.raytheon.uf.edex.plugin.datadelivery.retrieval/utility/edex_static/base/distribution/dataDeliveryRetrieval.xml

@@ -19,8 +19,5 @@
         further_licensing_information.
     -->
 <requestPatterns>
-    <!-- # TODO: What WMO Header should be used for SBN retrievals?
+    <regex>^LZ[ABC][ABC]9[123] KWBC</regex>
-         # The one below is just a random one to get things working..
-    -->
-    <regex>^SMYG10\sLYBM\s280000.*</regex>
 </requestPatterns>

tests/.classpath

@@ -152,5 +152,6 @@
 	<classpathentry combineaccessrules="false" kind="src" path="/com.raytheon.uf.common.wxmath"/>
 	<classpathentry kind="lib" path="lib/org.eclipse.jface_3.8.0.v20120912-135020.jar"/>
 	<classpathentry combineaccessrules="false" kind="src" path="/org.apache.commons.configuration"/>
+	<classpathentry kind="src" path="/com.raytheon.uf.edex.datadelivery.retrieval.wfs"/>
 	<classpathentry kind="output" path="bin"/>
 </classpath>

tests/resources/datadelivery/wfs/wfs_madis_dataset

@@ -0,0 +1,10 @@
+LZAA91 KWBC 032200
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<retrievalResponseXml success="true" xmlns:ns2="com.raytheon.uf.common.datadelivery.registry">
+    <requestRecord index="2" subscriptionName="CimarronKSRapTropo"/>
+    <retrievalResponseWrapper>
+        <openDapRetrievalResponse>
+            <payload>SFRUUC8xLjAgMjAwIE9LClNlcnZlcjogRE9EUy8yLjE4CkNvbnRlbnQtdHlwZTogYXBwbGljYXRpb24vb2N0ZXQtc3RyZWFtCkNvbnRlbnQtRGVzY3JpcHRpb246IGRvZHNfZGF0YQpDb250ZW50LUVuY29kaW5nOiBkZWZsYXRlCgp4nK2Rf0yUdRzHQZjhsVOkICK0J0gKJOOOGwMu4fs8728iAgVekM+Q8OSXqHi34yh3ROMCRUJogAwDdFNMJ+AUSqNF8gxQGxAhJ+luIh5RUoQBG7vkSroblpvVP9h7e2/v7bPX+/P97kOVWmVOupbJEzFWRWqy0h5Eht28meXDFrJVG3arlNogKZOqVKfnZKQmabOy05n1jCQ5abdSawsya1LtsaXgZPkCFsvGKR5pCJYxNvIhLv/H3Nr3sPNfxqo9jy7K/+tVclE+o1GqX7FaGigJkgQGBtlyilSik4tE1PrXMJGdnd3tB16U2Mjep7k7lvLF8jZxW6M/eBye9RHCHof/P0TqPukntVNREc7PFS+G58pGAjhjVxKWsjfgmKvkxlZPcN8lDnAVu5q5xJhK9vbvYta+/eB/8dhqrkPO3mDofnsN+THvo7BahxLFCVTta0Y1GUORtwEJ37TgySPtXFm5CxuT0h5RU7zvb75zXRu62rToOX4VQ0k9GDF9ix/M+ZigpzBV1IuZmQv4aeVODHbX47Q+CLEX+7iaTSq23fM6Kf2og7oNW6hrRSYV547QpauyMO9/AubSSvxaeAHjnmaMd2zDXY0ZFv9D1OFrV8zF++BkhQfcJpo4sRrUSyalbhYFFcVOYe76sxi/L4ZpyQEYw85icHojjI2NGK6XYbp0O11Z9SYNWCeijGwLriQMwy1ZTB0PLqfiveHUPqoDprxsdBu88Zn6PFowjU9rRbhEXHCz0os6Zr5Ive+raXjoGkomz1KPw6M4f20nftnfC/PbfbiafAvHPbqgm1+FVGYF9HwGjg5RdDk3YO5GJvW6NU6lqzfS0LudNPSrDBow1UqX9cnRNr8Np71P4oBTK/waFJwhpYT7/JwGS66tRcqrO3DmQw3u6d2p31MKSgMu09flX1LFu/E0PsJMgyNV1hPY28z6qhTCwkmW20y4tLhDfNEpwuXNMvrRNMJVfdzcSLwJ1xQeMnhkhHCdo91zlnrCGYvjfRRGws2sHY5uOUywbCBLuyKB4HntbH22K0HoM/rungGCuIviyZfKCHZsr3Xbv4mg4Anf9WNOBDVNrRnsJetOF5sF/ozFQbvrisAfa3zjWF2/wFfzdf29QwJf4nTHcs9L4N9rk/n5+Qi8OrMgkfcX+DT3vsISicC/ddnj3BchAh+dmzb8Y4HAR7zQ4uxeLPBSwx8hG8oF3rcwKv2daoH3lFZXNtwUeLHJ1GH4XuDtK16edPhZ2DJLdJ5B038CTMGZrg==</payload>
+        </openDapRetrievalResponse>
+    </retrievalResponseWrapper>
+</retrievalResponseXml>

tests/unit/com/raytheon/uf/edex/datadelivery/retrieval/handlers/DataDeliveryRetrievalWmoHeaderApplierTest.java

@@ -44,6 +44,7 @@ import com.raytheon.uf.edex.wmo.message.WMOMessage;
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * Aug 08, 2013 1822       bgonzale    Initial creation
+ * Oct 01, 2013 2267       bgonzale    Added test for null inputs.
  * 
  * </pre>
  * 
@@ -140,4 +141,17 @@ public class DataDeliveryRetrievalWmoHeaderApplierTest {
         assertThat(message.getMessageBody(), is(equalTo("someData".getBytes())));
     }
 
+    @Test
+    public void createWithNullArgs() {
+        Date date = new Date();
+        setHeader(null, null, null, date);
+        assertThat(wmoHeader.getT1(), is(equalTo('L')));
+        assertThat(wmoHeader.getT2(), is(equalTo('Z')));
+        assertThat(wmoHeader.getA1(), is(equalTo('A')));
+        assertThat(wmoHeader.getA2(), is(equalTo('A')));
+        assertThat(wmoHeader.getIi(), is(equalTo(91)));
+        assertThat(wmoHeader.getCccc(), is(equalTo("KWBC")));
+        assertThat(message.getMessageBody(), is(equalTo("someData".getBytes())));
+    }
+
 }

tests/unit/com/raytheon/uf/edex/datadelivery/retrieval/handlers/DeserializeRetrievedDataFromIngestTest.java

@@ -39,6 +39,7 @@ import com.raytheon.uf.common.serialization.SerializationException;
 import com.raytheon.uf.common.util.FileUtil;
 import com.raytheon.uf.common.util.TestUtil;
 import com.raytheon.uf.common.util.file.FilenameFilters;
+import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord;
 
 /**
  * Test {@link DeserializeRetrievedDataFromIngest}.
@@ -55,6 +56,7 @@ import com.raytheon.uf.common.util.file.FilenameFilters;
  * Mar 05, 2013 1647       djohnson     Pass wmo header strategy to constructor.
  * Mar 19, 2013 1794       djohnson     Read from a queue rather than the file system.
  * Aug 09, 2013 1822       bgonzale     Added parameters to processRetrievedPluginDataObjects.
+ * Oct 01, 2013 2267       bgonzale     Pass request parameter instead of components of request.
  * 
  * </pre>
  * 
@@ -112,10 +114,14 @@ public class DeserializeRetrievedDataFromIngestTest {
             IOException {
         RetrievalResponseXml retrievalPluginDataObjects = RetrievalPluginDataObjectsFixture.INSTANCE
                 .get();
+        RetrievalRequestRecord request = new RetrievalRequestRecord();
+        request.setProvider("");
+        request.setPlugin("");
+        request.setInsertTime(new Date());
 
         new SerializeRetrievedDataToDirectory(directory,
                 new AlwaysSameWmoHeader("SMYG10 LYBM 280000"))
-                .processRetrievedPluginDataObjects("", "", "", new Date(),
+                .processRetrievedPluginDataObjects(request,
                         retrievalPluginDataObjects);
 
         final List<File> files = FileUtil.listFiles(directory,

tests/unit/com/raytheon/uf/edex/datadelivery/retrieval/handlers/RetrievalTaskTest.java

@@ -28,7 +28,6 @@ import static org.mockito.Mockito.mock;
 import java.io.File;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Date;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -88,6 +87,7 @@ import com.raytheon.uf.edex.datadelivery.retrieval.interfaces.IRetrievalResponse
  * Mar 19, 2013 1794       djohnson     RetrievalTasks integrate at a queue.
  * Apr 29, 2013 1910       djohnson     Unregister from EventBus after each test.
  * Aug 09, 2013 1822       bgonzale     Added parameters to processRetrievedPluginDataObjects.
+ * Oct 01, 2013 2267       bgonzale     Pass request parameter instead of components of request.
  * 
  * </pre>
  * 
@@ -110,8 +110,8 @@ public class RetrievalTaskTest {
          * {@inheritDoc}
          */
         @Override
-        public void processRetrievedPluginDataObjects(String dataProvider,
+        public void processRetrievedPluginDataObjects(
-                String dataFormat, String sourceType, Date date,
+                RetrievalRequestRecord request,
                 RetrievalResponseXml retrievalPluginDataObjects)
                 throws SerializationException, TranslationException {
             final List<RetrievalResponseWrapper> retrievalAttributePluginDataObjects = retrievalPluginDataObjects

tests/unit/com/raytheon/uf/edex/datadelivery/retrieval/handlers/SerializeRetrievedDataToDirectoryTest.java

@@ -31,6 +31,7 @@ import org.junit.Test;
 import com.raytheon.uf.common.localization.PathManagerFactoryTest;
 import com.raytheon.uf.common.serialization.SerializationException;
 import com.raytheon.uf.common.util.TestUtil;
+import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord;
 
 /**
  * Test {@link SerializeRetrievedDataToDirectory}.
@@ -45,6 +46,8 @@ import com.raytheon.uf.common.util.TestUtil;
  * Feb 15, 2013 1543       djohnson     Class renames.
  * Mar 05, 2013 1647       djohnson     Pass wmo header strategy to constructor.
  * Aug 09, 2013 1822       bgonzale     Added parameters to processRetrievedPluginDataObjects.
+ * Oct 01, 2013 2267       bgonzale     Pass request parameter instead of components of request.
+ *                                      Add test for wfs retrieval.
  * 
  * </pre>
  * 
@@ -70,9 +73,32 @@ public class SerializeRetrievedDataToDirectoryTest {
 
         RetrievalResponseXml retrievalPluginDataObjects = RetrievalPluginDataObjectsFixture.INSTANCE
                 .get();
+        RetrievalRequestRecord request = new RetrievalRequestRecord();
+        request.setProvider("NOMADS");
+        request.setPlugin("grid");
+        // "Model"
+        request.setInsertTime(new Date());
 
-        service.processRetrievedPluginDataObjects("NOMADS", "GRID", "Model",
+        service.processRetrievedPluginDataObjects(request,
-                new Date(), retrievalPluginDataObjects);
+                retrievalPluginDataObjects);
+
+        assertThat(directory, hasNumberOfFiles(1));
+    }
+
+    @Test
+    public void serializesWfsRetrievedDataToAFileInTheTargetDirectory()
+            throws SerializationException {
+
+        RetrievalResponseXml retrievalPluginDataObjects = WfsRetrievalPluginDataObjectsFixture.INSTANCE
+                .get();
+        RetrievalRequestRecord request = new RetrievalRequestRecord();
+        request.setProvider("MADIS");
+        request.setPlugin("MADIS");
+        // "Model"
+        request.setInsertTime(new Date());
+
+        service.processRetrievedPluginDataObjects(request,
+                retrievalPluginDataObjects);
 
         assertThat(directory, hasNumberOfFiles(1));
     }

tests/unit/com/raytheon/uf/edex/datadelivery/retrieval/handlers/WfsRetrievalPluginDataObjectsFixture.java

@@ -0,0 +1,112 @@
+/**
+ * This software was developed and / or modified by Raytheon Company,
+ * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+ * 
+ * U.S. EXPORT CONTROLLED TECHNICAL DATA
+ * This software product contains export-restricted data whose
+ * export/transfer/disclosure is restricted by U.S. law. Dissemination
+ * to non-U.S. persons whether in the United States or abroad requires
+ * an export license or other authorization.
+ * 
+ * Contractor Name:        Raytheon Company
+ * Contractor Address:     6825 Pine Street, Suite 340
+ *                         Mail Stop B8
+ *                         Omaha, NE 68106
+ *                         402.291.0100
+ * 
+ * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+ * further licensing information.
+ **/
+package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import com.raytheon.uf.common.datadelivery.registry.Coverage;
+import com.raytheon.uf.common.datadelivery.registry.Time;
+import com.raytheon.uf.common.datadelivery.retrieval.xml.RetrievalAttribute;
+import com.raytheon.uf.common.serialization.SerializationException;
+import com.raytheon.uf.common.util.AbstractFixture;
+import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord;
+import com.raytheon.uf.edex.datadelivery.retrieval.interfaces.IRetrievalRequestBuilder;
+import com.raytheon.uf.edex.datadelivery.retrieval.wfs.MockWfsRetrievalAdapter;
+
+/**
+ * Fixture for {@link RetrievalResponseXml} instances.
+ * 
+ * <pre>
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Oct 04, 2013 2267       bgonzale    Initial creation.
+ *                                     Updated to match new processCoverage method.
+ * 
+ * </pre>
+ * 
+ * @author bgonzale
+ * @version 1.0
+ */
+
+public class WfsRetrievalPluginDataObjectsFixture extends
+        AbstractFixture<RetrievalResponseXml> {
+
+    public static final WfsRetrievalPluginDataObjectsFixture INSTANCE = new WfsRetrievalPluginDataObjectsFixture();
+
+    /**
+     * Prevent construction.
+     */
+    private WfsRetrievalPluginDataObjectsFixture() {
+
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public RetrievalResponseXml getInstance(long seedValue, Random random) {
+        RetrievalRequestRecord requestRecord = RetrievalRequestRecordFixture.INSTANCE
+                .get(seedValue);
+        List<RetrievalResponseWrapper> retrievalAttributePluginDataObjects = new ArrayList<RetrievalResponseWrapper>();
+        try {
+            for (final RetrievalAttribute attribute : requestRecord
+                    .getRetrievalObj().getAttributes()) {
+                retrievalAttributePluginDataObjects
+                        .add(new RetrievalResponseWrapper(
+                                new MockWfsRetrievalAdapter()
+                                        .performRequest(new IRetrievalRequestBuilder() {
+                                            @Override
+                                            public String processTime(
+                                                    Time prtXML) {
+                                                return "" + prtXML;
+                                            }
+
+                                            @Override
+                                            public String getRequest() {
+                                                return "request";
+                                            }
+
+                                            @Override
+                                            public RetrievalAttribute getAttribute() {
+                                                return attribute;
+                                            }
+
+                                            @Override
+                                            public String processCoverage(
+                                                    Coverage Coverage) {
+                                                return "noCoverage";
+                                            }
+                                        })));
+            }
+        } catch (SerializationException e) {
+            throw new RuntimeException(e);
+        }
+
+        final RetrievalResponseXml retrievalPluginDataObjects = new RetrievalResponseXml(
+                requestRecord.getId(), retrievalAttributePluginDataObjects);
+        return retrievalPluginDataObjects;
+    }
+    
+}

tests/unit/com/raytheon/uf/edex/datadelivery/retrieval/wfs/MockWfsRetrievalAdapter.java

@@ -0,0 +1,62 @@
+/**
+ * This software was developed and / or modified by Raytheon Company,
+ * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+ * 
+ * U.S. EXPORT CONTROLLED TECHNICAL DATA
+ * This software product contains export-restricted data whose
+ * export/transfer/disclosure is restricted by U.S. law. Dissemination
+ * to non-U.S. persons whether in the United States or abroad requires
+ * an export license or other authorization.
+ * 
+ * Contractor Name:        Raytheon Company
+ * Contractor Address:     6825 Pine Street, Suite 340
+ *                         Mail Stop B8
+ *                         Omaha, NE 68106
+ *                         402.291.0100
+ * 
+ * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+ * further licensing information.
+ **/
+package com.raytheon.uf.edex.datadelivery.retrieval.wfs;
+
+import org.junit.Ignore;
+
+import com.raytheon.uf.common.util.TestUtil;
+import com.raytheon.uf.edex.datadelivery.retrieval.interfaces.IRetrievalRequestBuilder;
+
+
+/**
+ * Overrides specific methods in {@link WfsRetrievalAdapter} that require
+ * external resources.
+ * 
+ * <pre>
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Oct 04, 2013 2267       bgonzale    Initial creation
+ * 
+ * </pre>
+ * 
+ * @author bgonzale
+ * @version 1.0
+ */
+@Ignore
+public class MockWfsRetrievalAdapter extends WfsRetrievalAdapter {
+    /**
+     * Create a WfsRetrievalResponse with a test wfs data file.
+     */
+    @Override
+    public WfsRetrievalResponse performRequest(IRetrievalRequestBuilder request) {
+        String data = new String(TestUtil.readResource(this.getClass(),
+                "/datadelivery/wfs/wfs_madis_dataset"));
+
+        final WfsRetrievalResponse response = new WfsRetrievalResponse(null);
+        response.setAttribute(request.getAttribute());
+        response.setPayLoad(data);
+
+        return response;
+    }
+
+}