From e466f536d0f77554fc9d74a117886c1d32dcaa7c Mon Sep 17 00:00:00 2001 From: Dave Hladky Date: Tue, 18 Jun 2013 16:22:30 -0500 Subject: [PATCH] Issue #2120 Updates to retrieval to fix time processing Change-Id: Ibedb988bb4b3806c037f5c8c1363f8233cf7cf2b Former-commit-id: f2177d48f63efb8d1b01d54a7e85beed43802f0c [formerly 5686d945729c28407e16625548050937158f9c65] Former-commit-id: 6110467a3054df3237a2a4ce483f9a66d7824320 --- .../bandwidth/BandwidthManager.java | 24 +++++-- .../base/datadelivery/WFSServiceConfig.xml | 1 + .../retrieval/wfs/WfsRequestBuilder.java | 20 +++++- .../retrieval/wfs/WfsRetrievalAdapter.java | 16 +++-- .../retrieval/wfs/WfsRetrievalGenerator.java | 72 ++----------------- .../retrieval/util/WfsConnectionUtil.java | 15 ++-- 6 files changed, 60 insertions(+), 88 deletions(-) diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/BandwidthManager.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/BandwidthManager.java index 9dbe299ce2..71246e9c0b 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/BandwidthManager.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/BandwidthManager.java @@ -119,6 +119,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.util.BandwidthUtil; * May 20, 2013 1650 djohnson Add in capability to find required dataset size. * Jun 03, 2013 2038 djohnson Add base functionality to handle point data type subscriptions. * Jun 13, 2013 2095 djohnson Improve bandwidth manager speed, and add performance logging. + * JUN 18, 2013 2120 dhladky Add times to pointtime array * * * @author dhladky @@ -350,13 +351,22 @@ public abstract class BandwidthManager extends // which goes through the retrieval process final Subscription subscription = retrieval .getSubscription(); - final Time subTime = subscription.getTime(); - subTime.setRequestStartAsDate(earliestRetrievalDataTime); - subTime.setRequestEndAsDate(latestRetrievalDataTime); - - // Now update the retrieval to be ready - retrieval.setStatus(RetrievalStatus.READY); - bandwidthDaoUtil.update(retrieval); + subscription.setUrl(dataSetMetaData.getUrl()); + subscription.setProvider(dataSetMetaData.getProviderName()); + + if (subscription.getTime() instanceof PointTime) { + final PointTime subTime = (PointTime) subscription + .getTime(); + subTime.setRequestStartAsDate(earliestRetrievalDataTime); + subTime.setRequestEndAsDate(latestRetrievalDataTime); + subTime.setTimes(time.getTimes()); + // Now update the retrieval to be ready + retrieval.setStatus(RetrievalStatus.READY); + bandwidthDaoUtil.update(retrieval); + } else { + throw new IllegalArgumentException("Subscription time not PointType! " + subscription.getName()); + } + } catch (SerializationException e) { statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(), e); diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/WFSServiceConfig.xml b/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/WFSServiceConfig.xml index 578fc53876..4e8ca8b76e 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/WFSServiceConfig.xml +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/WFSServiceConfig.xml @@ -35,5 +35,6 @@ + \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval.wfs/src/com/raytheon/uf/edex/datadelivery/retrieval/wfs/WfsRequestBuilder.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval.wfs/src/com/raytheon/uf/edex/datadelivery/retrieval/wfs/WfsRequestBuilder.java index c3f441c385..3039cc3688 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval.wfs/src/com/raytheon/uf/edex/datadelivery/retrieval/wfs/WfsRequestBuilder.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval.wfs/src/com/raytheon/uf/edex/datadelivery/retrieval/wfs/WfsRequestBuilder.java @@ -31,6 +31,7 @@ import com.vividsolutions.jts.geom.Coordinate; * May 12, 2013 753 dhladky created. * May 31, 2013 2038 djohnson Move to correct repo. * Jun 11, 2013 1763 dhladky Made operational. + * Jun 18, 2013 2120 dhladky Added times and max feature processing * * * @@ -61,6 +62,8 @@ public class WfsRequestBuilder extends RequestBuilder { public static final String blank = getServiceConfig().getConstantValue("BLANK"); + public static final String max = getServiceConfig().getConstantValue("MAX"); + public static final String ampersand = "&"; private final String wfsURL; @@ -78,6 +81,8 @@ public class WfsRequestBuilder extends RequestBuilder { buffer.append(processCoverage()); // process the time range you are trying to retrieve buffer.append(processTime(getRetrievalAttribute().getTime())); + // max feature limit + buffer.append(processMax()); this.wfsURL = buffer.toString().trim(); } @@ -99,8 +104,7 @@ public class WfsRequestBuilder extends RequestBuilder { try { if (inTime.getEndDate() != null && inTime.getStartDate() != null) { - - + Date sDate = inTime.getStartDate(); Date eDate = inTime.getEndDate(); String endDateString = ogcDateFormat.get().format(eDate); @@ -167,6 +171,18 @@ public class WfsRequestBuilder extends RequestBuilder { return blank; } + + /** + * constrain to max amount of features + * @return + */ + public String processMax() { + StringBuilder sb = new StringBuilder(); + sb.append(ampersand); + sb.append(max); + + return sb.toString(); + } @Override public String getRequest() { diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval.wfs/src/com/raytheon/uf/edex/datadelivery/retrieval/wfs/WfsRetrievalAdapter.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval.wfs/src/com/raytheon/uf/edex/datadelivery/retrieval/wfs/WfsRetrievalAdapter.java index 0f2129a65a..ec151435e9 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval.wfs/src/com/raytheon/uf/edex/datadelivery/retrieval/wfs/WfsRetrievalAdapter.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval.wfs/src/com/raytheon/uf/edex/datadelivery/retrieval/wfs/WfsRetrievalAdapter.java @@ -23,7 +23,7 @@ package com.raytheon.uf.edex.datadelivery.retrieval.wfs; import java.util.HashMap; import java.util.Map; -import com.raytheon.uf.common.datadelivery.registry.Provider; +import com.raytheon.uf.common.datadelivery.registry.Connection; import com.raytheon.uf.common.datadelivery.retrieval.xml.RetrievalAttribute; import com.raytheon.uf.common.dataplugin.PluginDataObject; import com.raytheon.uf.common.event.EventBus; @@ -51,6 +51,7 @@ import com.raytheon.uf.edex.datadelivery.retrieval.util.WfsConnectionUtil; * May 12, 2013 753 dhladky implemented. * May 31, 2013 2038 djohnson Move to correct repo. * Jun 03, 2013 1763 dhladky Readied for retrievals. + * Jun 18, 2013 2120 dhladky Removed provider and fixed time processing. * * * @@ -62,11 +63,9 @@ public class WfsRetrievalAdapter extends RetrievalAdapter { private static final IUFStatusHandler statusHandler = UFStatus .getHandler(WfsRetrievalAdapter.class); - - private Provider provider; - - WfsRetrievalAdapter(Provider provider) { - this.provider = provider; + + WfsRetrievalAdapter() { + } @Override @@ -118,7 +117,10 @@ public class WfsRetrievalAdapter extends RetrievalAdapter { String xmlMessage = null; try { - xmlMessage = WfsConnectionUtil.wfsConnect(request.getRequest(), provider); + Connection conn = this.getProviderRetrievalXMl().getConnection(); + // This is used as the "Realm" in HTTPS connections + String providerName = request.getAttribute().getProvider(); + xmlMessage = WfsConnectionUtil.wfsConnect(request.getRequest(), conn, providerName); } catch (Exception e) { statusHandler.handle(Priority.ERROR, e.getLocalizedMessage(), e); EventBus.publish(new RetrievalEvent(e.getMessage())); diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval.wfs/src/com/raytheon/uf/edex/datadelivery/retrieval/wfs/WfsRetrievalGenerator.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval.wfs/src/com/raytheon/uf/edex/datadelivery/retrieval/wfs/WfsRetrievalGenerator.java index 4f7c8527f4..e6fdd7662e 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval.wfs/src/com/raytheon/uf/edex/datadelivery/retrieval/wfs/WfsRetrievalGenerator.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval.wfs/src/com/raytheon/uf/edex/datadelivery/retrieval/wfs/WfsRetrievalGenerator.java @@ -1,25 +1,22 @@ package com.raytheon.uf.edex.datadelivery.retrieval.wfs; +import java.text.ParseException; import java.util.ArrayList; import java.util.Collections; +import java.util.Date; import java.util.List; -import com.google.common.annotations.VisibleForTesting; import com.raytheon.uf.common.datadelivery.registry.Coverage; -import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData; import com.raytheon.uf.common.datadelivery.registry.DataType; import com.raytheon.uf.common.datadelivery.registry.Parameter; -import com.raytheon.uf.common.datadelivery.registry.PointDataSetMetaData; import com.raytheon.uf.common.datadelivery.registry.PointTime; import com.raytheon.uf.common.datadelivery.registry.Provider; import com.raytheon.uf.common.datadelivery.registry.Provider.ServiceType; import com.raytheon.uf.common.datadelivery.registry.ProviderType; import com.raytheon.uf.common.datadelivery.registry.Subscription; import com.raytheon.uf.common.datadelivery.registry.SubscriptionBundle; -import com.raytheon.uf.common.datadelivery.registry.handlers.DataDeliveryHandlers; import com.raytheon.uf.common.datadelivery.retrieval.xml.Retrieval; import com.raytheon.uf.common.datadelivery.retrieval.xml.RetrievalAttribute; -import com.raytheon.uf.common.registry.handler.RegistryHandlerException; import com.raytheon.uf.common.status.IUFStatusHandler; import com.raytheon.uf.common.status.UFStatus; import com.raytheon.uf.common.status.UFStatus.Priority; @@ -41,6 +38,7 @@ import com.raytheon.uf.edex.datadelivery.retrieval.adapters.RetrievalAdapter; * May 12, 2013 753 dhladky Implemented * May 31, 2013 2038 djohnson Move to correct repo. * Jun 04, 2013 1763 dhladky Readied for WFS Retrievals. + * Jun 18, 2013 2120 dhladky Fixed times. * * * @@ -51,11 +49,10 @@ class WfsRetrievalGenerator extends RetrievalGenerator { private static final IUFStatusHandler statusHandler = UFStatus .getHandler(WfsRetrievalGenerator.class); - private Provider provider; + WfsRetrievalGenerator(Provider provider) { super(ServiceType.WFS); - this.provider = provider; } @Override @@ -89,8 +86,6 @@ class WfsRetrievalGenerator extends RetrievalGenerator { if (sub != null) { PointTime subTime = (PointTime) sub.getTime(); - String retrievalUrl = getRetrievalUrl(sub); - sub.setUrl(retrievalUrl); if (sub.getUrl() == null) { statusHandler @@ -112,60 +107,6 @@ class WfsRetrievalGenerator extends RetrievalGenerator { return retrievals; } - /** - * Determines the retrieval URL - * - * @param subscription - * the subscription - * @return the url for retrieval or null if no retrieval should take place - */ - private static String getRetrievalUrl(Subscription subscription) { - String url = subscription.getUrl(); - - DataSetMetaData result = null; - try { - result = DataDeliveryHandlers.getDataSetMetaDataHandler().getById( - url); - if (result == null) { - throw new RegistryHandlerException( - "Unable to find the dataset by id from its unique url!", - new NullPointerException("DataSetMetaData")); - } - - if (satisfiesSubscriptionCriteria(subscription, result)) { - return url; - } - } catch (RegistryHandlerException e) { - statusHandler.handle(Priority.PROBLEM, - "Unable to find the dataset by id from its unique url!", e); - } - - return null; - } - - /** - * Determines whether a subscription can be satisified by the dataset - * metadata. - * - * @param subscription - * the subscription - * @param dsmd - * the dataset metadata - * @return true if the datasetmetadata will satisfy the subscription - */ - @VisibleForTesting - static boolean satisfiesSubscriptionCriteria(Subscription subscription, - DataSetMetaData dsmd) { - - if (dsmd instanceof PointDataSetMetaData) { - // PointDataSetMetaData data = (PointDataSetMetaData) dsmd; - // TODO determine some check for validity of point data sets - return true; - } - - return false; - } - /** * Get the retrieval * @@ -212,7 +153,7 @@ class WfsRetrievalGenerator extends RetrievalGenerator { lparam.setLevels(param.getLevels()); att.setParameter(lparam); } - + att.setTime(time); att.setSubName(retrieval.getSubscriptionName()); att.setPlugin(plugin); @@ -227,12 +168,13 @@ class WfsRetrievalGenerator extends RetrievalGenerator { */ @Override protected RetrievalAdapter getServiceRetrievalAdapter() { - return new WfsRetrievalAdapter(provider); + return new WfsRetrievalAdapter(); } @Override protected Subscription removeDuplicates(Subscription sub) { throw new UnsupportedOperationException("Not implemented for WFS"); } + } diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/util/WfsConnectionUtil.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/util/WfsConnectionUtil.java index ee4a84508e..b078347c8c 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/util/WfsConnectionUtil.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/util/WfsConnectionUtil.java @@ -7,7 +7,6 @@ import org.apache.http.client.methods.HttpGet; import com.raytheon.uf.common.comm.HttpClient; import com.raytheon.uf.common.comm.HttpClient.HttpClientResponse; import com.raytheon.uf.common.datadelivery.registry.Connection; -import com.raytheon.uf.common.datadelivery.registry.Provider; import com.raytheon.uf.common.status.IUFStatusHandler; import com.raytheon.uf.common.status.UFStatus; import com.raytheon.uf.common.status.UFStatus.Priority; @@ -25,6 +24,7 @@ import com.raytheon.uf.common.status.UFStatus.Priority; * May 12, 2013 753 dhladky created. * May 31, 2013 1763 dhladky refined. * Jun 17, 2013 2106 djohnson Use getUnencryptedPassword(). + * Jun 18, 2013 2120 dhladky Times fixes and SSL changes * * * @@ -37,7 +37,8 @@ public class WfsConnectionUtil { private static final IUFStatusHandler statusHandler = UFStatus .getHandler(WfsConnectionUtil.class); - public static String wfsConnect(String url, Provider provider) { + public static String wfsConnect(String url, Connection conn, + String providerName) { String xmlResponse = null; HttpClient http = null; @@ -45,17 +46,17 @@ public class WfsConnectionUtil { // Sets up any proxy info that might be necessary // TODO: consider using HTTP POST instead of GET ConnectionUtil.getProxyParameters(); + //url = url.replace("https://dev11:8888", "http://dev05:8085"); http = HttpClient.getInstance(); HttpGet get = new HttpGet(); URI uri = new URI(url); - Connection conn = provider.getConnection(); // check for the need to do a username password auth check if (conn != null && conn.getUserName() != null && conn.getPassword() != null) { - - http.setCredentials(uri.getHost(), uri.getPort(), - provider.getName(), conn.getUserName(), - conn.getUnencryptedPassword()); + statusHandler.handle(Priority.INFO, + "Attempting credential request: " + providerName); + http.setCredentials(uri.getHost(), uri.getPort(), providerName, + conn.getUserName(), conn.getUnencryptedPassword()); } get.setURI(uri);