xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #2120 Updates to retrieval to fix time processing

Browse source 
Change-Id: Ibedb988bb4b3806c037f5c8c1363f8233cf7cf2b

Former-commit-id: f2177d48f6 [formerly 5686d945729c28407e16625548050937158f9c65]
Former-commit-id: 6110467a30
This commit is contained in:
Dave Hladky 2013-06-18 16:22:30 -05:00
parent 728f54c6ea
commit e466f536d0
6 changed files with 60 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/BandwidthManager.java
View file

@ -119,6 +119,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.util.BandwidthUtil;
* May 20, 2013 1650 djohnson Add in capability to find required dataset size.
* Jun 03, 2013 2038 djohnson Add base functionality to handle point data type subscriptions.
* Jun 13, 2013 2095 djohnson Improve bandwidth manager speed, and add performance logging.
* JUN 18, 2013 2120 dhladky Add times to pointtime array
* </pre>
*
* @author dhladky
@ -350,13 +351,22 @@ public abstract class BandwidthManager extends
// which goes through the retrieval process
final Subscription subscription = retrieval
.getSubscription();
final Time subTime = subscription.getTime();
subscription.setUrl(dataSetMetaData.getUrl());
subscription.setProvider(dataSetMetaData.getProviderName());
if (subscription.getTime() instanceof PointTime) {
final PointTime subTime = (PointTime) subscription
.getTime();
subTime.setRequestStartAsDate(earliestRetrievalDataTime);
subTime.setRequestEndAsDate(latestRetrievalDataTime);
subTime.setTimes(time.getTimes());
// Now update the retrieval to be ready
retrieval.setStatus(RetrievalStatus.READY);
bandwidthDaoUtil.update(retrieval);
} else {
throw new IllegalArgumentException("Subscription time not PointType! " + subscription.getName());
}
} catch (SerializationException e) {
statusHandler.handle(Priority.PROBLEM,
e.getLocalizedMessage(), e);

1
edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/WFSServiceConfig.xml
View file

@ -35,5 +35,6 @@
<constant name="OUTPUTFORMAT" value="text/xml"/>
<constant name="FORWARD_SLASH" value="/"/>
<constant name="DEFAULT_CRS" value="crs:84"/>
<constant name="MAX" value="maxfeatures=3000"/>
</serviceConfig>

18
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval.wfs/src/com/raytheon/uf/edex/datadelivery/retrieval/wfs/WfsRequestBuilder.java
View file

@ -31,6 +31,7 @@ import com.vividsolutions.jts.geom.Coordinate;
* May 12, 2013 753 dhladky created.
* May 31, 2013 2038 djohnson Move to correct repo.
* Jun 11, 2013 1763 dhladky Made operational.
* Jun 18, 2013 2120 dhladky Added times and max feature processing
*
* </pre>
*
@ -61,6 +62,8 @@ public class WfsRequestBuilder extends RequestBuilder {
public static final String blank = getServiceConfig().getConstantValue("BLANK");
public static final String max = getServiceConfig().getConstantValue("MAX");
public static final String ampersand = "&";
private final String wfsURL;
@ -78,6 +81,8 @@ public class WfsRequestBuilder extends RequestBuilder {
buffer.append(processCoverage());
// process the time range you are trying to retrieve
buffer.append(processTime(getRetrievalAttribute().getTime()));
// max feature limit
buffer.append(processMax());
this.wfsURL = buffer.toString().trim();
}
@ -100,7 +105,6 @@ public class WfsRequestBuilder extends RequestBuilder {
try {
if (inTime.getEndDate() != null && inTime.getStartDate() != null) {
Date sDate = inTime.getStartDate();
Date eDate = inTime.getEndDate();
String endDateString = ogcDateFormat.get().format(eDate);
@ -168,6 +172,18 @@ public class WfsRequestBuilder extends RequestBuilder {
return blank;
}
/**
* constrain to max amount of features
* @return
*/
public String processMax() {
StringBuilder sb = new StringBuilder();
sb.append(ampersand);
sb.append(max);
return sb.toString();
}
@Override
public String getRequest() {
return wfsURL;

12
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval.wfs/src/com/raytheon/uf/edex/datadelivery/retrieval/wfs/WfsRetrievalAdapter.java
View file

@ -23,7 +23,7 @@ package com.raytheon.uf.edex.datadelivery.retrieval.wfs;
import java.util.HashMap;
import java.util.Map;
import com.raytheon.uf.common.datadelivery.registry.Provider;
import com.raytheon.uf.common.datadelivery.registry.Connection;
import com.raytheon.uf.common.datadelivery.retrieval.xml.RetrievalAttribute;
import com.raytheon.uf.common.dataplugin.PluginDataObject;
import com.raytheon.uf.common.event.EventBus;
@ -51,6 +51,7 @@ import com.raytheon.uf.edex.datadelivery.retrieval.util.WfsConnectionUtil;
* May 12, 2013 753 dhladky implemented.
* May 31, 2013 2038 djohnson Move to correct repo.
* Jun 03, 2013 1763 dhladky Readied for retrievals.
* Jun 18, 2013 2120 dhladky Removed provider and fixed time processing.
*
* </pre>
*
@ -63,10 +64,8 @@ public class WfsRetrievalAdapter extends RetrievalAdapter {
private static final IUFStatusHandler statusHandler = UFStatus
.getHandler(WfsRetrievalAdapter.class);
private Provider provider;
WfsRetrievalAdapter() {
WfsRetrievalAdapter(Provider provider) {
this.provider = provider;
}
@Override
@ -118,7 +117,10 @@ public class WfsRetrievalAdapter extends RetrievalAdapter {
String xmlMessage = null;
try {
xmlMessage = WfsConnectionUtil.wfsConnect(request.getRequest(), provider);
Connection conn = this.getProviderRetrievalXMl().getConnection();
// This is used as the "Realm" in HTTPS connections
String providerName = request.getAttribute().getProvider();
xmlMessage = WfsConnectionUtil.wfsConnect(request.getRequest(), conn, providerName);
} catch (Exception e) {
statusHandler.handle(Priority.ERROR, e.getLocalizedMessage(), e);
EventBus.publish(new RetrievalEvent(e.getMessage()));

70
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval.wfs/src/com/raytheon/uf/edex/datadelivery/retrieval/wfs/WfsRetrievalGenerator.java
View file

@ -1,25 +1,22 @@
package com.raytheon.uf.edex.datadelivery.retrieval.wfs;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import com.google.common.annotations.VisibleForTesting;
import com.raytheon.uf.common.datadelivery.registry.Coverage;
import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData;
import com.raytheon.uf.common.datadelivery.registry.DataType;
import com.raytheon.uf.common.datadelivery.registry.Parameter;
import com.raytheon.uf.common.datadelivery.registry.PointDataSetMetaData;
import com.raytheon.uf.common.datadelivery.registry.PointTime;
import com.raytheon.uf.common.datadelivery.registry.Provider;
import com.raytheon.uf.common.datadelivery.registry.Provider.ServiceType;
import com.raytheon.uf.common.datadelivery.registry.ProviderType;
import com.raytheon.uf.common.datadelivery.registry.Subscription;
import com.raytheon.uf.common.datadelivery.registry.SubscriptionBundle;
import com.raytheon.uf.common.datadelivery.registry.handlers.DataDeliveryHandlers;
import com.raytheon.uf.common.datadelivery.retrieval.xml.Retrieval;
import com.raytheon.uf.common.datadelivery.retrieval.xml.RetrievalAttribute;
import com.raytheon.uf.common.registry.handler.RegistryHandlerException;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
@ -41,6 +38,7 @@ import com.raytheon.uf.edex.datadelivery.retrieval.adapters.RetrievalAdapter;
* May 12, 2013 753 dhladky Implemented
* May 31, 2013 2038 djohnson Move to correct repo.
* Jun 04, 2013 1763 dhladky Readied for WFS Retrievals.
* Jun 18, 2013 2120 dhladky Fixed times.
*
* </pre>
*
@ -51,11 +49,10 @@ class WfsRetrievalGenerator extends RetrievalGenerator {
private static final IUFStatusHandler statusHandler = UFStatus
.getHandler(WfsRetrievalGenerator.class);
private Provider provider;
WfsRetrievalGenerator(Provider provider) {
super(ServiceType.WFS);
this.provider = provider;
}
@Override
@ -89,8 +86,6 @@ class WfsRetrievalGenerator extends RetrievalGenerator {
if (sub != null) {
PointTime subTime = (PointTime) sub.getTime();
String retrievalUrl = getRetrievalUrl(sub);
sub.setUrl(retrievalUrl);
if (sub.getUrl() == null) {
statusHandler
@ -112,60 +107,6 @@ class WfsRetrievalGenerator extends RetrievalGenerator {
return retrievals;
}
/**
* Determines the retrieval URL
*
* @param subscription
* the subscription
* @return the url for retrieval or null if no retrieval should take place
*/
private static String getRetrievalUrl(Subscription subscription) {
String url = subscription.getUrl();
DataSetMetaData result = null;
try {
result = DataDeliveryHandlers.getDataSetMetaDataHandler().getById(
url);
if (result == null) {
throw new RegistryHandlerException(
"Unable to find the dataset by id from its unique url!",
new NullPointerException("DataSetMetaData"));
}
if (satisfiesSubscriptionCriteria(subscription, result)) {
return url;
}
} catch (RegistryHandlerException e) {
statusHandler.handle(Priority.PROBLEM,
"Unable to find the dataset by id from its unique url!", e);
}
return null;
}
/**
* Determines whether a subscription can be satisified by the dataset
* metadata.
*
* @param subscription
* the subscription
* @param dsmd
* the dataset metadata
* @return true if the datasetmetadata will satisfy the subscription
*/
@VisibleForTesting
static boolean satisfiesSubscriptionCriteria(Subscription subscription,
DataSetMetaData dsmd) {
if (dsmd instanceof PointDataSetMetaData) {
// PointDataSetMetaData data = (PointDataSetMetaData) dsmd;
// TODO determine some check for validity of point data sets
return true;
}
return false;
}
/**
* Get the retrieval
*
@ -227,7 +168,7 @@ class WfsRetrievalGenerator extends RetrievalGenerator {
*/
@Override
protected RetrievalAdapter getServiceRetrievalAdapter() {
return new WfsRetrievalAdapter(provider);
return new WfsRetrievalAdapter();
}
@Override
@ -235,4 +176,5 @@ class WfsRetrievalGenerator extends RetrievalGenerator {
throw new UnsupportedOperationException("Not implemented for WFS");
}
}

15
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/util/WfsConnectionUtil.java
View file

@ -7,7 +7,6 @@ import org.apache.http.client.methods.HttpGet;
import com.raytheon.uf.common.comm.HttpClient;
import com.raytheon.uf.common.comm.HttpClient.HttpClientResponse;
import com.raytheon.uf.common.datadelivery.registry.Connection;
import com.raytheon.uf.common.datadelivery.registry.Provider;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
@ -25,6 +24,7 @@ import com.raytheon.uf.common.status.UFStatus.Priority;
* May 12, 2013 753 dhladky created.
* May 31, 2013 1763 dhladky refined.
* Jun 17, 2013 2106 djohnson Use getUnencryptedPassword().
* Jun 18, 2013 2120 dhladky Times fixes and SSL changes
*
* </pre>
*
@ -37,7 +37,8 @@ public class WfsConnectionUtil {
private static final IUFStatusHandler statusHandler = UFStatus
.getHandler(WfsConnectionUtil.class);
public static String wfsConnect(String url, Provider provider) {
public static String wfsConnect(String url, Connection conn,
String providerName) {
String xmlResponse = null;
HttpClient http = null;
@ -45,17 +46,17 @@ public class WfsConnectionUtil {
// Sets up any proxy info that might be necessary
// TODO: consider using HTTP POST instead of GET
ConnectionUtil.getProxyParameters();
//url = url.replace("https://dev11:8888", "http://dev05:8085");
http = HttpClient.getInstance();
HttpGet get = new HttpGet();
URI uri = new URI(url);
Connection conn = provider.getConnection();
// check for the need to do a username password auth check
if (conn != null && conn.getUserName() != null
&& conn.getPassword() != null) {
http.setCredentials(uri.getHost(), uri.getPort(),
provider.getName(), conn.getUserName(),
conn.getUnencryptedPassword());
statusHandler.handle(Priority.INFO,
"Attempting credential request: " + providerName);
http.setCredentials(uri.getHost(), uri.getPort(), providerName,
conn.getUserName(), conn.getUnencryptedPassword());
}
get.setURI(uri);