Issue #2120 Updates to retrieval to fix time processing

Change-Id: Ibedb988bb4b3806c037f5c8c1363f8233cf7cf2b

Former-commit-id: f2177d48f6 [formerly 5686d945729c28407e16625548050937158f9c65]
Former-commit-id: 6110467a30

edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/BandwidthManager.java

@@ -119,6 +119,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.util.BandwidthUtil;
  * May 20, 2013 1650       djohnson     Add in capability to find required dataset size.
  * Jun 03, 2013 2038       djohnson     Add base functionality to handle point data type subscriptions.
  * Jun 13, 2013 2095       djohnson     Improve bandwidth manager speed, and add performance logging.
+ * JUN 18, 2013 2120       dhladky      Add times to pointtime array
  * </pre>
  * 
  * @author dhladky
@@ -350,13 +351,22 @@ public abstract class BandwidthManager extends
                     // which goes through the retrieval process
                     final Subscription subscription = retrieval
                             .getSubscription();
-                    final Time subTime = subscription.getTime();
+                    subscription.setUrl(dataSetMetaData.getUrl());
-                    subTime.setRequestStartAsDate(earliestRetrievalDataTime);
+                    subscription.setProvider(dataSetMetaData.getProviderName());
-                    subTime.setRequestEndAsDate(latestRetrievalDataTime);
+                    
-
+                    if (subscription.getTime() instanceof PointTime) {
-                    // Now update the retrieval to be ready
+                        final PointTime subTime = (PointTime) subscription
-                    retrieval.setStatus(RetrievalStatus.READY);
+                                .getTime();
-                    bandwidthDaoUtil.update(retrieval);
+                        subTime.setRequestStartAsDate(earliestRetrievalDataTime);
+                        subTime.setRequestEndAsDate(latestRetrievalDataTime);
+                        subTime.setTimes(time.getTimes());
+                        // Now update the retrieval to be ready
+                        retrieval.setStatus(RetrievalStatus.READY);
+                        bandwidthDaoUtil.update(retrieval);
+                    } else {
+                        throw new IllegalArgumentException("Subscription time not PointType! " + subscription.getName());
+                    }
+                    
                 } catch (SerializationException e) {
                     statusHandler.handle(Priority.PROBLEM,
                             e.getLocalizedMessage(), e);

edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/WFSServiceConfig.xml

@@ -35,5 +35,6 @@
 	<constant name="OUTPUTFORMAT" value="text/xml"/>
 	<constant name="FORWARD_SLASH" value="/"/>
 	<constant name="DEFAULT_CRS" value="crs:84"/>
+	<constant name="MAX" value="maxfeatures=3000"/>
 		
 </serviceConfig>

edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval.wfs/src/com/raytheon/uf/edex/datadelivery/retrieval/wfs/WfsRequestBuilder.java

@@ -31,6 +31,7 @@ import com.vividsolutions.jts.geom.Coordinate;
  * May 12, 2013 753        dhladky      created.
  * May 31, 2013 2038       djohnson     Move to correct repo.
  * Jun 11, 2013 1763       dhladky      Made operational.
+ * Jun 18, 2013 2120       dhladky      Added times and max feature processing
  * 
  * </pre>
  * 
@@ -61,6 +62,8 @@ public class WfsRequestBuilder extends RequestBuilder {
     
     public static final String blank = getServiceConfig().getConstantValue("BLANK");
     
+    public static final String max = getServiceConfig().getConstantValue("MAX");
+    
     public static final String ampersand = "&";
     
     private final String wfsURL;
@@ -78,6 +81,8 @@ public class WfsRequestBuilder extends RequestBuilder {
         buffer.append(processCoverage());
         // process the time range you are trying to retrieve
         buffer.append(processTime(getRetrievalAttribute().getTime()));
+        // max feature limit
+        buffer.append(processMax());
         
         this.wfsURL = buffer.toString().trim();
     }
@@ -99,8 +104,7 @@ public class WfsRequestBuilder extends RequestBuilder {
 
         try {
             if (inTime.getEndDate() != null && inTime.getStartDate() != null) {
-            
+
-                
                 Date sDate = inTime.getStartDate();
                 Date eDate = inTime.getEndDate();
                 String endDateString = ogcDateFormat.get().format(eDate);
@@ -167,6 +171,18 @@ public class WfsRequestBuilder extends RequestBuilder {
 
         return blank;
     }
+    
+    /**
+     * constrain to max amount of features
+     * @return
+     */
+    public String processMax() {
+        StringBuilder sb = new StringBuilder();
+        sb.append(ampersand);
+        sb.append(max);
+        
+        return sb.toString();
+    }
 
     @Override
     public String getRequest() {

edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval.wfs/src/com/raytheon/uf/edex/datadelivery/retrieval/wfs/WfsRetrievalAdapter.java

@@ -23,7 +23,7 @@ package com.raytheon.uf.edex.datadelivery.retrieval.wfs;
 import java.util.HashMap;
 import java.util.Map;
 
-import com.raytheon.uf.common.datadelivery.registry.Provider;
+import com.raytheon.uf.common.datadelivery.registry.Connection;
 import com.raytheon.uf.common.datadelivery.retrieval.xml.RetrievalAttribute;
 import com.raytheon.uf.common.dataplugin.PluginDataObject;
 import com.raytheon.uf.common.event.EventBus;
@@ -51,6 +51,7 @@ import com.raytheon.uf.edex.datadelivery.retrieval.util.WfsConnectionUtil;
  * May 12, 2013 753        dhladky      implemented.
  * May 31, 2013 2038       djohnson     Move to correct repo.
  * Jun 03, 2013 1763       dhladky      Readied for retrievals.
+ * Jun 18, 2013 2120       dhladky      Removed provider and fixed time processing.
  * 
  * </pre>
  * 
@@ -62,11 +63,9 @@ public class WfsRetrievalAdapter extends RetrievalAdapter {
 
     private static final IUFStatusHandler statusHandler = UFStatus
             .getHandler(WfsRetrievalAdapter.class);
-    
+
-    private Provider provider;
+    WfsRetrievalAdapter() {
-    
+
-    WfsRetrievalAdapter(Provider provider) {
-        this.provider = provider;
     }
 
     @Override
@@ -118,7 +117,10 @@ public class WfsRetrievalAdapter extends RetrievalAdapter {
 
         String xmlMessage = null;
         try {
-            xmlMessage = WfsConnectionUtil.wfsConnect(request.getRequest(), provider);
+            Connection conn = this.getProviderRetrievalXMl().getConnection();
+            // This is used as the "Realm" in HTTPS connections
+            String providerName = request.getAttribute().getProvider();
+            xmlMessage = WfsConnectionUtil.wfsConnect(request.getRequest(), conn, providerName);
         } catch (Exception e) {
             statusHandler.handle(Priority.ERROR, e.getLocalizedMessage(), e);
             EventBus.publish(new RetrievalEvent(e.getMessage()));

edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval.wfs/src/com/raytheon/uf/edex/datadelivery/retrieval/wfs/WfsRetrievalGenerator.java

@@ -1,25 +1,22 @@
 package com.raytheon.uf.edex.datadelivery.retrieval.wfs;
 
+import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Date;
 import java.util.List;
 
-import com.google.common.annotations.VisibleForTesting;
 import com.raytheon.uf.common.datadelivery.registry.Coverage;
-import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData;
 import com.raytheon.uf.common.datadelivery.registry.DataType;
 import com.raytheon.uf.common.datadelivery.registry.Parameter;
-import com.raytheon.uf.common.datadelivery.registry.PointDataSetMetaData;
 import com.raytheon.uf.common.datadelivery.registry.PointTime;
 import com.raytheon.uf.common.datadelivery.registry.Provider;
 import com.raytheon.uf.common.datadelivery.registry.Provider.ServiceType;
 import com.raytheon.uf.common.datadelivery.registry.ProviderType;
 import com.raytheon.uf.common.datadelivery.registry.Subscription;
 import com.raytheon.uf.common.datadelivery.registry.SubscriptionBundle;
-import com.raytheon.uf.common.datadelivery.registry.handlers.DataDeliveryHandlers;
 import com.raytheon.uf.common.datadelivery.retrieval.xml.Retrieval;
 import com.raytheon.uf.common.datadelivery.retrieval.xml.RetrievalAttribute;
-import com.raytheon.uf.common.registry.handler.RegistryHandlerException;
 import com.raytheon.uf.common.status.IUFStatusHandler;
 import com.raytheon.uf.common.status.UFStatus;
 import com.raytheon.uf.common.status.UFStatus.Priority;
@@ -41,6 +38,7 @@ import com.raytheon.uf.edex.datadelivery.retrieval.adapters.RetrievalAdapter;
  * May 12, 2013 753        dhladky      Implemented
  * May 31, 2013 2038       djohnson     Move to correct repo.
  * Jun 04, 2013 1763       dhladky      Readied for WFS Retrievals.
+ * Jun 18, 2013 2120       dhladky      Fixed times.
  * 
  * </pre>
  * 
@@ -51,11 +49,10 @@ class WfsRetrievalGenerator extends RetrievalGenerator {
 
     private static final IUFStatusHandler statusHandler = UFStatus
             .getHandler(WfsRetrievalGenerator.class);
-    private Provider provider;
+
 
     WfsRetrievalGenerator(Provider provider) {
         super(ServiceType.WFS);
-        this.provider = provider;
     }
 
     @Override
@@ -89,8 +86,6 @@ class WfsRetrievalGenerator extends RetrievalGenerator {
         if (sub != null) {
 
             PointTime subTime = (PointTime) sub.getTime();
-            String retrievalUrl = getRetrievalUrl(sub);
-            sub.setUrl(retrievalUrl);
 
             if (sub.getUrl() == null) {
                 statusHandler
@@ -112,60 +107,6 @@ class WfsRetrievalGenerator extends RetrievalGenerator {
         return retrievals;
     }
 
-    /**
-     * Determines the retrieval URL
-     * 
-     * @param subscription
-     *            the subscription
-     * @return the url for retrieval or null if no retrieval should take place
-     */
-    private static String getRetrievalUrl(Subscription subscription) {
-        String url = subscription.getUrl();
-
-        DataSetMetaData result = null;
-        try {
-            result = DataDeliveryHandlers.getDataSetMetaDataHandler().getById(
-                    url);
-            if (result == null) {
-                throw new RegistryHandlerException(
-                        "Unable to find the dataset by id from its unique url!",
-                        new NullPointerException("DataSetMetaData"));
-            }
-
-            if (satisfiesSubscriptionCriteria(subscription, result)) {
-                return url;
-            }
-        } catch (RegistryHandlerException e) {
-            statusHandler.handle(Priority.PROBLEM,
-                    "Unable to find the dataset by id from its unique url!", e);
-        }
-
-        return null;
-    }
-
-    /**
-     * Determines whether a subscription can be satisified by the dataset
-     * metadata.
-     * 
-     * @param subscription
-     *            the subscription
-     * @param dsmd
-     *            the dataset metadata
-     * @return true if the datasetmetadata will satisfy the subscription
-     */
-    @VisibleForTesting
-    static boolean satisfiesSubscriptionCriteria(Subscription subscription,
-            DataSetMetaData dsmd) {
-
-        if (dsmd instanceof PointDataSetMetaData) {
-            // PointDataSetMetaData data = (PointDataSetMetaData) dsmd;
-            // TODO determine some check for validity of point data sets
-            return true;
-        }
-
-        return false;
-    }
-
     /**
      * Get the retrieval
      * 
@@ -212,7 +153,7 @@ class WfsRetrievalGenerator extends RetrievalGenerator {
             lparam.setLevels(param.getLevels());
             att.setParameter(lparam);
         }
-        
+    
         att.setTime(time);
         att.setSubName(retrieval.getSubscriptionName());
         att.setPlugin(plugin);
@@ -227,12 +168,13 @@ class WfsRetrievalGenerator extends RetrievalGenerator {
      */
     @Override
     protected RetrievalAdapter getServiceRetrievalAdapter() {
-        return new WfsRetrievalAdapter(provider);
+        return new WfsRetrievalAdapter();
     }
 
     @Override
     protected Subscription removeDuplicates(Subscription sub) {
         throw new UnsupportedOperationException("Not implemented for WFS");
     }
+   
 
 }

edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/util/WfsConnectionUtil.java

@@ -7,7 +7,6 @@ import org.apache.http.client.methods.HttpGet;
 import com.raytheon.uf.common.comm.HttpClient;
 import com.raytheon.uf.common.comm.HttpClient.HttpClientResponse;
 import com.raytheon.uf.common.datadelivery.registry.Connection;
-import com.raytheon.uf.common.datadelivery.registry.Provider;
 import com.raytheon.uf.common.status.IUFStatusHandler;
 import com.raytheon.uf.common.status.UFStatus;
 import com.raytheon.uf.common.status.UFStatus.Priority;
@@ -25,6 +24,7 @@ import com.raytheon.uf.common.status.UFStatus.Priority;
  * May 12, 2013 753        dhladky     created.
  * May 31, 2013 1763       dhladky     refined.
  * Jun 17, 2013 2106       djohnson    Use getUnencryptedPassword().
+ * Jun 18, 2013 2120       dhladky     Times fixes and SSL changes
  * 
  * </pre>
  * 
@@ -37,7 +37,8 @@ public class WfsConnectionUtil {
     private static final IUFStatusHandler statusHandler = UFStatus
             .getHandler(WfsConnectionUtil.class);
 
-    public static String wfsConnect(String url, Provider provider) {
+    public static String wfsConnect(String url, Connection conn,
+            String providerName) {
         String xmlResponse = null;
         HttpClient http = null;
 
@@ -45,17 +46,17 @@ public class WfsConnectionUtil {
             // Sets up any proxy info that might be necessary
             // TODO: consider using HTTP POST instead of GET
             ConnectionUtil.getProxyParameters();
+            //url = url.replace("https://dev11:8888", "http://dev05:8085");
             http = HttpClient.getInstance();
             HttpGet get = new HttpGet();
             URI uri = new URI(url);
-            Connection conn = provider.getConnection();
             // check for the need to do a username password auth check
             if (conn != null && conn.getUserName() != null
                     && conn.getPassword() != null) {
-
+                statusHandler.handle(Priority.INFO,
-                http.setCredentials(uri.getHost(), uri.getPort(),
+                        "Attempting credential request: " + providerName);
-                        provider.getName(), conn.getUserName(),
+                http.setCredentials(uri.getHost(), uri.getPort(), providerName,
-                        conn.getUnencryptedPassword());
+                        conn.getUserName(), conn.getUnencryptedPassword());
             }
 
             get.setURI(uri);