xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #2554 added explicit threading to WFS, ability to send GZIP requests, fixed maxfeatures, fixed adhoc display

Browse source 
Former-commit-id: a9a3973319 [formerly 04c7e2cffc [formerly e79ff905cbe3330dbe22b6663753e1d5e6dbe7bd]]
Former-commit-id: 04c7e2cffc
Former-commit-id: 233238d8df
This commit is contained in:
Dave Hladky 2013-11-21 10:56:25 -06:00
parent 9c2398a7dd
commit 162dcb3f42
8 changed files with 55 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/DataDeliveryProductBrowserDataDefinition.java
View file

@ -32,6 +32,7 @@ import com.raytheon.uf.common.datadelivery.registry.AdhocSubscription;
import com.raytheon.uf.common.datadelivery.registry.Coverage;
import com.raytheon.uf.common.datadelivery.registry.DataType;
import com.raytheon.uf.common.datadelivery.registry.Subscription;
import com.raytheon.uf.common.datadelivery.registry.Subscription.SubscriptionType;
import com.raytheon.uf.common.datadelivery.registry.Utils.SubscriptionStatus;
import com.raytheon.uf.common.datadelivery.registry.handlers.IAdhocSubscriptionHandler;
import com.raytheon.uf.common.datadelivery.registry.handlers.ISubscriptionHandler;
@ -72,6 +73,7 @@ import com.raytheon.viz.pointdata.util.PointDataInventory;
* Sept 22, 2013 2246 dhladky Setup binoffset for time into +-5 min intervals
* Oct 13, 2013 2460 dhladky Added display of Adhoc subscriptions
* Nov 19, 2013 2458 mpduff Only pull subscriptions for the local site
* Nov 21, 2013 2554 dhladky Restored ADHOC's to working.
*
* </pre>
*
@ -401,7 +403,8 @@ public class DataDeliveryProductBrowserDataDefinition
List<Subscription> subList = getSubscriptions();
for (Subscription s : subList) {
if (SubscriptionStatus.ACTIVE.toString().equals(s.getStatus())) {
if (SubscriptionStatus.ACTIVE.toString().equals(s.getStatus())
|| s.getSubscriptionType().equals(SubscriptionType.QUERY)) {
if (s.getDataSetType() == dataType) {
activeSubList.add(s);
subNames.add(s.getName());

3
edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Provider.java
View file

@ -56,6 +56,7 @@ public class Provider {
* Feb 16, 2012 dhladky Initial creation
* Aug 16, 2012 1022 djohnson Add bytesPerParameterRequest.
* Sept 10, 2013 2352 dhladky Changed default size for point overhead
* Nov 20, 2013 2554 dhladky Changed WFS again, added gzipping compensation
*
* </pre>
*
@ -65,7 +66,7 @@ public class Provider {
public enum ServiceType {
// TODO: Only OPENDAP and WFS have the correct amounts
OPENDAP(5000, BYTES_IN_FLOAT), WCS(5000, BYTES_IN_FLOAT), WFS(711724,
OPENDAP(5000, BYTES_IN_FLOAT), WCS(5000, BYTES_IN_FLOAT), WFS(355862,
OneByOneBox), WMS(5000, BYTES_IN_FLOAT), WXXM(5000,
BYTES_IN_FLOAT);

13
edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/util/DataSizeUtils.java
View file

@ -49,6 +49,7 @@ import com.vividsolutions.jts.geom.Coordinate;
* Jun 11, 2013 2021 dhladky WFS semi-scientific sizing.
* Jun 14, 2013 2108 mpduff Abstracted the class.
* Sept 09, 2013 2351 dhladky Fixed incorrect calculation for default pointdata overhead
* Nov 20, 2013 2554 dhladky Generics
*
* </pre>
*
@ -56,7 +57,7 @@ import com.vividsolutions.jts.geom.Coordinate;
* @version 1.0
*/
public abstract class DataSizeUtils<DS extends DataSet> {
public abstract class DataSizeUtils<DS extends DataSet<?, ?>> {
/**
* Factory method to get a DataSizeUtils.
@ -67,8 +68,8 @@ public abstract class DataSizeUtils<DS extends DataSet> {
* The subscription
* @return The correct implementation of DataSizeUtils
*/
public static DataSizeUtils<?> getInstance(DataSet dataSet,
Subscription subscription) {
public static DataSizeUtils<?> getInstance(DataSet<?, ?> dataSet,
Subscription<?, ?> subscription) {
DataSizeUtils<?> dsu = null;
if (subscription.getDataSetType() == DataType.GRID) {
dsu = new GriddedDataSizeUtils((GriddedDataSet) dataSet);
@ -88,7 +89,7 @@ public abstract class DataSizeUtils<DS extends DataSet> {
* ServiceType
* @return dataset size
*/
public static long calculateSize(RetrievalAttribute ra, ServiceType st) {
public static long calculateSize(RetrievalAttribute<?, ?> ra, ServiceType st) {
if (st == ServiceType.OPENDAP) {
if (ra.getCoverage() instanceof GriddedCoverage) {
@ -150,7 +151,7 @@ public abstract class DataSizeUtils<DS extends DataSet> {
* Subscription for calculating the size
* @return Data size in bytes
*/
protected abstract long getDataSetSizeInBytes(Subscription subscription);
protected abstract long getDataSetSizeInBytes(Subscription<?, ?> subscription);
/**
* @return the dataSet
@ -167,7 +168,7 @@ public abstract class DataSizeUtils<DS extends DataSet> {
*
* @return the size in kB
*/
public long getDataSetSizeInKb(Subscription subscription) {
public long getDataSetSizeInKb(Subscription<?, ?> subscription) {
return getDataSetSizeInBytes(subscription) / bytesPerKilobyte;
}

38
edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/util/GriddedDataSizeUtils.java
View file

@ -24,7 +24,6 @@ import java.util.Map;
import org.geotools.geometry.jts.ReferencedEnvelope;
import com.raytheon.uf.common.datadelivery.registry.Coverage;
import com.raytheon.uf.common.datadelivery.registry.Ensemble;
import com.raytheon.uf.common.datadelivery.registry.GriddedCoverage;
import com.raytheon.uf.common.datadelivery.registry.GriddedDataSet;
@ -46,6 +45,7 @@ import com.raytheon.uf.common.util.CollectionUtil;
* ------------ ---------- ----------- --------------------------
* Jun 13, 2013 2108 mpduff Initial creation.
* Sept 25, 2013 1797 dhladky separated time from gridded time
* Nov 20, 2013 2554 dhladky Generics
*
* </pre>
*
@ -74,20 +74,19 @@ public class GriddedDataSizeUtils extends DataSizeUtils<GriddedDataSet> {
*/
private int calculateGridCells(ReferencedEnvelope envelope) {
if (dataSet != null) {
Coverage cov = dataSet.getCoverage();
if (cov instanceof GriddedCoverage) {
GriddedCoverage griddedCov = (GriddedCoverage) cov;
GridCoverage subgridCov = griddedCov
.getRequestGridCoverage(envelope);
if (subgridCov == null) {
subgridCov = griddedCov.getGridCoverage();
}
int nx = subgridCov.getNx();
int ny = subgridCov.getNy();
GriddedCoverage griddedCov = dataSet.getCoverage();
return nx * ny;
GridCoverage subgridCov = griddedCov
.getRequestGridCoverage(envelope);
if (subgridCov == null) {
subgridCov = griddedCov.getGridCoverage();
}
int nx = subgridCov.getNx();
int ny = subgridCov.getNy();
return nx * ny;
}
return 0;
@ -133,19 +132,18 @@ public class GriddedDataSizeUtils extends DataSizeUtils<GriddedDataSet> {
public long getFullSizeInBytes() {
if (dataSet != null) {
if (fullSize == -999) {
GriddedCoverage griddedCov = (GriddedCoverage) dataSet
GriddedCoverage griddedCov = dataSet
.getCoverage();
long numCells = griddedCov.getGridCoverage().getNx()
* griddedCov.getGridCoverage().getNy();
long numEns = 1;
long fcstHrs = 1;
if (dataSet instanceof GriddedDataSet) {
GriddedDataSet gDataSet = dataSet;
fcstHrs = gDataSet.getForecastHours().size();
if (gDataSet.getEnsemble() != null) {
numEns = gDataSet.getEnsemble().getMemberCount();
}
fcstHrs = dataSet.getForecastHours().size();
if (dataSet.getEnsemble() != null) {
numEns = dataSet.getEnsemble().getMemberCount();
}
Map<String, Parameter> paramMap = dataSet.getParameters();
// get the number of grids available
@ -174,7 +172,7 @@ public class GriddedDataSizeUtils extends DataSizeUtils<GriddedDataSet> {
* {@inheritDoc}
*/
@Override
public long getDataSetSizeInBytes(Subscription subscription) {
public long getDataSetSizeInBytes(Subscription<?, ?> subscription) {
final Ensemble ensemble = subscription.getEnsemble();
int numEnsemble = (ensemble == null) ? 1 : ensemble.getMemberCount();

3
edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/util/PointDataSizeUtils.java
View file

@ -37,6 +37,7 @@ import com.vividsolutions.jts.geom.Coordinate;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 13, 2013 2108 mpduff Initial creation.
* Nov 20, 2013 2554 dhladky More Generics
*
* </pre>
*
@ -69,7 +70,7 @@ public class PointDataSizeUtils extends DataSizeUtils<PointDataSet> {
* {@inheritDoc}
*/
@Override
public long getDataSetSizeInBytes(Subscription subscription) {
public long getDataSetSizeInBytes(Subscription<?, ?> subscription) {
return getDataSetSizeInBytes(subscription.getCoverage()
.getRequestEnvelope(),
((PointTime) subscription.getTime()).getInterval());

6
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval.wfs/src/com/raytheon/uf/edex/datadelivery/retrieval/wfs/WfsRequestBuilder.java
View file

@ -37,7 +37,8 @@ import com.vividsolutions.jts.geom.Coordinate;
* end since the times will be correct now.
* Oct 1, 2013 1797 dhladky Generics
* Oct 10, 2013 1797 bgonzale Refactored registry Time objects.
* Oct 28, 2013 2448 dhladky Request start time incorrectly used subscription start time.
* Oct 28, 2013 2448 dhladky Request start time incorrectly used subscription start time.
* Nov 20, 2013 2554 dhladky MaxFeatures was misplaced.
*
* </pre>
*
@ -167,6 +168,7 @@ public class WfsRequestBuilder<T extends Time, C extends Coverage> extends Reque
sb.append("<?xml version=\"1.0\" ?>\n");
sb.append("<wfs:GetFeature service=\"WFS\"\n");
sb.append("version=\"").append(VERSION).append("\"\n");
sb.append("maxFeatures=\"").append(MAX).append("\"\n");
sb.append("outputFormat=\"application/gml+xml; version=3.1\"\n");
sb.append("xmlns:").append(typeName).append("=\"http://").append(typeName).append(".edex.uf.raytheon.com\"\n");
sb.append("xmlns:wfs=\"http://www.opengis.net/wfs\"\n");
@ -174,7 +176,7 @@ public class WfsRequestBuilder<T extends Time, C extends Coverage> extends Reque
sb.append("xmlns:ogc=\"http://www.opengis.net/ogc\"\n");
sb.append("xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n");
sb.append("xsi:schemaLocation=\"http://www.opengis.net/wfs../wfs/").append(VERSION).append("/WFS.xsd\">\n");
sb.append("<wfs:Query typeName=\"").append(typeName).append(":").append(typeName).append("\" maxFeatures=\"").append(MAX).append("\">\n");
sb.append("<wfs:Query typeName=\"").append(typeName).append(":").append(typeName).append("\">\n");
return sb.toString();
}

3
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/util/WfsConnectionUtil.java
View file

@ -33,6 +33,7 @@ import com.raytheon.uf.common.status.UFStatus.Priority;
* Jul 10, 2013 2180 dhladky Updated credential requests
* Aug 23, 2013 2180 mpduff Implement changes to ProviderCredentialsUtil
* Aug 06, 2013 2097 dhladky WFS 2.0 compliance upgrade and switched to POST
* Nov 20, 2013 2554 dhladky Added GZIP capability to WFS requests.
*
* </pre>
*
@ -67,6 +68,8 @@ public class WfsConnectionUtil {
rootUrl = providerConn.getUrl();
http = HttpClient.getInstance();
// accept gzipped data for WFS
http.setGzipResponseHandling(true);
URI uri = new URI(rootUrl);
HttpPost post = new HttpPost(uri);
// check for the need to do a username password auth check

16
edexOsgi/com.raytheon.uf.edex.wfs/res/spring/wfs-ogc-rest-request.xml
View file

@ -4,6 +4,19 @@
http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd">
<bean id="wfsHandler" class="com.raytheon.uf.edex.wfs.WfsHttpHandler">
<!--
Thread the WFS requests, this allows you fine grain control
over how many requests are serviced simultaneously.
<constructor-arg>
<bean id="threadPool" class="org.eclipse.jetty.util.thread.QueuedThreadPool">
<property name="minThreads" value="2"/>
<property name="maxThreads" value="4"/>
</bean>
</constructor-arg>
-->
<constructor-arg>
<list>
<bean class="com.raytheon.uf.edex.wfs.v1_1_0.Wfs1_1_0Provider">
@ -25,7 +38,8 @@
<camelContext id="wfs-rest-camel" xmlns="http://camel.apache.org/schema/spring" errorHandlerRef="errorHandler">
<route id="WfsRest">
<from uri="jetty:http://0.0.0.0:8085/wfs" />
<!-- Set a maximum and minimum number of threads -->
<from uri="jetty:http://0.0.0.0:8085/wfs?maxThreads=4&amp;minThreads=2" />
<bean ref="wfserv" />
</route>