xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #2267 Shared Subscription fixes. DataDelivery Port environment standardization. Logging updates.

Browse source 
Amend: Refactored WMO header applier.
       Fixes for WMO header applier.
       Updated dataDeliveryRetrieval.xml with correct wmo header pattern.
       Removed manual ingest from registry mode.
       Simplified data delivery distribution matching pattern.
       Added serialize test for wfs response.
       Updated comments.
       Moved WfsRetrievalResponse to com.raytheon.uf.edex.datadelivery.retrieval package.
       Added WfsRetrievalResponse to Jaxb unmarshalable classes in DeserializeRetrievedDataFromIngest.
       HibernateBandwidthInitializer.executeAfterRegistryInit(), in NCF mode schedules shared subs.

Change-Id: I0278339701f5f55491120c35d80cbd6d3dcc84f7

Former-commit-id: a6540d69ea [formerly 8bf8efa14520aaba532ca7a2e82e4c1fa24adca8]
Former-commit-id: 21acb75f24
This commit is contained in:
Brad Gonzales 2013-10-01 16:35:56 -05:00
parent 540d0e7b9c
commit dc45b582e1
25 changed files with 383 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
edexOsgi/build.edex/esb/bin/setup.env
View file

@ -31,14 +31,16 @@ export BROKER_ADDR=localhost
# setup hdf5 connection if pypies is enabled
export PYPIES_SERVER=http://localhost:9582
# Registry specific ports
export EBXML_REGISTRY_WEBSERVER_PORT=8082
export EBXML_REGISTRY_WEBSERVER_CONFIDENTIAL_PORT=8446
export EBXML_THRIFT_SERVICE_PORT=9588
# data delivery specific variables, used below in the localization section
export NCF_HOST=165.92.30.69
export NCF_ADDRESS=http://${NCF_HOST}:8084
export NCF_ADDRESS=http://${NCF_HOST}:${EBXML_REGISTRY_WEBSERVER_PORT}
export DATADELIVERY_HOST=localhost
export DATADELIVERY_PORT=9588
# Currently the registry is hosted on datadelivery, but this could be changed in the future
export EBXML_REGISTRY_HOST=${DATADELIVERY_HOST}
export EBXML_REGISTRY_PORT=${DATADELIVERY_PORT}
export EBXML_REGISTRY_SUBSCRIPTIONS_ENABLED=false
export EBXML_REGISTRY_FEDERATION_ENABLED=false
@ -48,11 +50,11 @@ export HTTP_PORT=9581
export HTTP_SERVER=http://localhost:${HTTP_PORT}/services
export JMS_SERVER=tcp://localhost:5672
export RADAR_SERVER=tcp://localhost:8813
export DATADELIVERY_SERVER=http://${DATADELIVERY_HOST}:${DATADELIVERY_PORT}/services
export EBXML_REGISTRY_SERVICE=http://${EBXML_REGISTRY_HOST}:${EBXML_REGISTRY_PORT}/services
export EBXML_REGISTRY_LCM_SERVICE=http://${EBXML_REGISTRY_HOST}:${EBXML_REGISTRY_PORT}/lifecycleManager?WSDL
export EBXML_REGISTRY_QUERY_SERVICE=http://${EBXML_REGISTRY_HOST}:${EBXML_REGISTRY_PORT}/queryManager?WSDL
export NCF_BANDWIDTH_MANAGER_SERVICE=http://${NCF_HOST}:9590/services
export DATADELIVERY_SERVER=http://${DATADELIVERY_HOST}:${EBXML_THRIFT_SERVICE_PORT}/services
export EBXML_REGISTRY_SERVICE=http://${EBXML_REGISTRY_HOST}:${EBXML_THRIFT_SERVICE_PORT}/services
export EBXML_REGISTRY_LCM_SERVICE=http://${EBXML_REGISTRY_HOST}:${EBXML_THRIFT_SERVICE_PORT}/lifecycleManager?WSDL
export EBXML_REGISTRY_QUERY_SERVICE=http://${EBXML_REGISTRY_HOST}:${EBXML_THRIFT_SERVICE_PORT}/queryManager?WSDL
export NCF_BANDWIDTH_MANAGER_SERVICE=http://${NCF_HOST}:${EBXML_THRIFT_SERVICE_PORT}/services
# set the AWIPS II shared directory
export SHARE_DIR=/awips2/edex/data/share

6
edexOsgi/build.edex/esb/etc/centralRegistry.sh
View file

@ -26,9 +26,3 @@ export EDEX_DEBUG_PORT=5011
export EDEX_JMX_PORT=1622
export LOG_CONF=logback-registry.xml
export MGMT_PORT=9607
# Registry specific ports
export EBXML_REGISTRY_WEBSERVER_PORT=8084
export EBXML_REGISTRY_WEBSERVER_CONFIDENTIAL_PORT=8446
export EBXML_QUERY_SERVICE_SOAP_PORT=10146
export EBXML_THRIFT_SERVICE_PORT=9588

6
edexOsgi/build.edex/esb/etc/devRegistry.sh
View file

@ -26,9 +26,3 @@ export EDEX_DEBUG_PORT=5013
export EDEX_JMX_PORT=1620
export LOG_CONF=logback-registry.xml
export MGMT_PORT=9605
# Registry specific ports
export EBXML_REGISTRY_WEBSERVER_PORT=8082
export EBXML_REGISTRY_WEBSERVER_CONFIDENTIAL_PORT=8444
export EBXML_QUERY_SERVICE_SOAP_PORT=10144
export EBXML_THRIFT_SERVICE_PORT=9588

6
edexOsgi/build.edex/esb/etc/registry.sh
View file

@ -26,9 +26,3 @@ export EDEX_DEBUG_PORT=5012
export EDEX_JMX_PORT=1620
export LOG_CONF=logback-registry.xml
export MGMT_PORT=9605
# Registry specific ports
export EBXML_REGISTRY_WEBSERVER_PORT=8082
export EBXML_REGISTRY_WEBSERVER_CONFIDENTIAL_PORT=8444
export EBXML_QUERY_SERVICE_SOAP_PORT=10144
export EBXML_THRIFT_SERVICE_PORT=9588

6
edexOsgi/com.raytheon.uf.common.datadelivery.bandwidth/src/com/raytheon/uf/common/datadelivery/bandwidth/BandwidthService.java
View file

@ -55,6 +55,7 @@ import com.raytheon.uf.common.util.LogUtil;
* Feb 27, 2013 1644 djohnson Now abstract, sub-classes provide specific service lookup keys.
* Jul 18, 2013 1653 mpduff Add getSubscriptionStatusSummary method.
* Oct 2, 2013 1797 dhladky Generics
* Oct 01, 2013 2267 bgonzale Log error response from proposed scheduling.
*
* </pre>
*
@ -202,8 +203,9 @@ public abstract class BandwidthService<T extends Time, C extends Coverage> exten
LogUtil.logIterable(
statusHandler,
Priority.PROBLEM,
"Returning null response object, unable to propose scheduling the following subscriptions for bandwidth management:",
subscriptions);
"Returning null response object, unable to propose scheduling"
+ "the following subscriptions for bandwidth management:",
subscriptions, e);
return IProposeScheduleResponse.NULL_OBJECT;
}
}

32
edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/LogUtil.java
View file

@ -32,6 +32,7 @@ import com.raytheon.uf.common.status.UFStatus.Priority;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Oct 26, 2012 1322 djohnson Initial creation
* Oct 01, 2013 2267 bgonzale Add logging for errors also.
*
* </pre>
*
@ -71,4 +72,35 @@ public final class LogUtil {
String msg = StringUtil.createMessage(message, iterables, 3);
statusHandler.handle(priority, msg);
}
/**
* Logs the message, then each iterable to the output with a new line
* separator between each iterable.
*
* <pre>
*
* Example:
*
* This would be the message:
* iterable1
* iterable2
* </pre>
*
* @param statusHandler
* the status handler
* @param priority
* the priority
* @param message
* the message
* @param iterable
* the iterables
* @param error
* the throwable
*/
public static <T> void logIterable(IUFStatusHandler statusHandler,
Priority priority, String message, Iterable<T> iterables,
Throwable error) {
String msg = StringUtil.createMessage(message, iterables, 3);
statusHandler.handle(priority, msg, error);
}
}

12
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/hibernate/HibernateBandwidthInitializer.java
View file

@ -34,6 +34,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.RetrievalManager;
* Jun 25, 2013 2106 djohnson init() now takes a {@link RetrievalManager} as well.
* Sep 05, 2013 2330 bgonzale On WFO registry init, only subscribe to local site subscriptions.
* Sep 06, 2013 2344 bgonzale Removed attempt to add to immutable empty set.
* Oct 07, 2013 2267 bgonzale in executeAfterRegistryInit NCF schedules shared subs.
*
* </pre>
*
@ -87,21 +88,22 @@ public class HibernateBandwidthInitializer implements BandwidthInitializer {
Set<Subscription> activeSubscriptions = new HashSet<Subscription>();
try {
final String localOffice = SiteUtil.getSite();
final boolean isCentralRegistry = System.getProperty(
"edex.run.mode").equals("centralRegistry");
// Load active subscriptions for the local office
// Load active subscriptions
for (Subscription sub : findSubscriptionsStrategy
.findSubscriptionsToSchedule()) {
boolean isShared = (sub instanceof SharedSubscription);
boolean isLocalOffice = sub.getOfficeIDs()
.contains(localOffice);
if (!isShared && isLocalOffice) {
if ((isCentralRegistry && isShared)
|| (!isShared && isLocalOffice)) {
activeSubscriptions.add(sub);
statusHandler.info("Scheduling Subscription: " + sub);
} else {
statusHandler
.info("Not Scheduling Non-local Subscription: "
+ sub);
statusHandler.info("Not Scheduling Subscription: " + sub);
}
}
} catch (Exception e) {

21
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/retrieval/RetrievalManager.java
View file

@ -38,6 +38,7 @@ import com.raytheon.uf.edex.datadelivery.retrieval.RetrievalManagerNotifyEvent;
* Jun 13, 2013 2095 djohnson Can schedule any subclass of BandwidthAllocation.
* Jun 25, 2013 2106 djohnson Copy state from another instance, add ability to check for proposed bandwidth throughput changes.
* Jul 09, 2013 2106 djohnson Only needs to unregister from the EventBus when used in an EDEX instance, so handled in EdexBandwidthManager.
* Oct 03, 2013 2267 bgonzale Added check for no retrieval plan matching in the proposed retrieval plans.
*
* </pre>
*
@ -269,10 +270,22 @@ public class RetrievalManager {
.entrySet()) {
final RetrievalPlan proposedRetrievalPlan = proposedRetrievalManager.retrievalPlans
.get(entry.getKey());
if (proposedRetrievalPlan.getDefaultBandwidth() != entry.getValue()
.getDefaultBandwidth()) {
proposingBandwidthChanges = true;
break;
if (proposedRetrievalPlan != null && entry.getValue() != null) {
if (proposedRetrievalPlan.getDefaultBandwidth() != entry
.getValue().getDefaultBandwidth()) {
proposingBandwidthChanges = true;
break;
}
} else {
StringBuilder sb = new StringBuilder(
"The ProposedRetrievalPlan, ");
sb.append(proposedRetrievalPlan);
sb.append(", or the Existing RetrievalPlan, ");
sb.append(entry.getKey());
sb.append(" : ");
sb.append(entry.getValue());
sb.append(", is null. Skipping this check.");
statusHandler.info(sb.toString());
}
}

3
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/META-INF/MANIFEST.MF
View file

@ -73,4 +73,5 @@ Export-Package: com.raytheon.uf.edex.datadelivery.retrieval;
dods.dap,
com.raytheon.uf.common.dataplugin.grid,
com.raytheon.uf.common.dataplugin.level,
com.raytheon.uf.common.datadelivery.registry"
com.raytheon.uf.common.datadelivery.registry",
com.raytheon.uf.edex.datadelivery.retrieval.wfs

32
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/DataDeliveryRetrievalWmoHeaderApplier.java
View file

@ -23,7 +23,7 @@ import java.text.DateFormat;
import java.text.Format;
import java.text.MessageFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import com.raytheon.uf.common.time.util.TimeUtil;
@ -66,6 +66,9 @@ import com.raytheon.uf.edex.core.props.PropertiesException;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Aug 06, 2013 1822 bgonzale Initial creation
* Oct 01, 2013 2267 bgonzale Pass request parameter instead of components of request.
* Fixed ordering of elements in maps and defaults when
* element keys are null.
*
* </pre>
*
@ -99,8 +102,9 @@ public class DataDeliveryRetrievalWmoHeaderApplier implements IWmoHeaderApplier
private static Map<String, String> parseMapping(String dataMapping)
throws PropertiesException {
Map<String, String> resultMap = new HashMap<String, String>();
String[] elementPairs = dataMapping.split(",");
Map<String, String> resultMap = new LinkedHashMap<String, String>(
elementPairs.length, 1);
for (String elementPair : elementPairs) {
String[] pair = elementPair.split(":");
@ -139,21 +143,27 @@ public class DataDeliveryRetrievalWmoHeaderApplier implements IWmoHeaderApplier
return new Object[] { provider, format, source, date };
}
private static String getElement(String dataProvider,
/*
* if no element found, default to first element, or if empty, default to
* null.
*/
private static String getElement(String elementKey,
Map<String, String> mapping) {
String defaultKey = null;
String resultKey = null;
for (String key : mapping.keySet()) {
if (defaultKey == null) {
defaultKey = key;
if (resultKey == null) {
resultKey = key;
if (elementKey == null) {
break;
}
}
if (key.startsWith(dataProvider)) {
return mapping.get(key);
if (key.startsWith(elementKey)) {
resultKey = key;
break;
}
}
// if no element found, default to first element, or if empty, default
// to null
return mapping.get(defaultKey);
return mapping.get(resultKey);
}
}

6
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/DeserializeRetrievedDataFromIngest.java
View file

@ -27,8 +27,8 @@ import com.raytheon.edex.esb.Headers;
import com.raytheon.uf.common.datadelivery.registry.Coverage;
import com.raytheon.uf.common.serialization.JAXBManager;
import com.raytheon.uf.edex.datadelivery.retrieval.opendap.OpenDapRetrievalResponse;
import com.raytheon.uf.edex.datadelivery.retrieval.wfs.WfsRetrievalResponse;
import com.raytheon.uf.edex.wmo.message.WMOMessage;
/**
* Deserializes the retrieved data in a retrievalQueue.
*
@ -41,6 +41,7 @@ import com.raytheon.uf.edex.wmo.message.WMOMessage;
* Feb 01, 2013 1543 djohnson Initial creation
* Mar 05, 2013 1647 djohnson Remove WMO header.
* Mar 19, 2013 1794 djohnson Read from a queue rather than the file system.
* Oct 04, 2013 2267 bgonzale Added WfsRetrieval to unmarshal classes.
*
* </pre>
*
@ -61,7 +62,8 @@ public class DeserializeRetrievedDataFromIngest implements IRetrievalsFinder {
this.retrievalQueue = retrievalQueue;
try {
this.jaxbManager = new JAXBManager(RetrievalResponseXml.class,
OpenDapRetrievalResponse.class, Coverage.class);
OpenDapRetrievalResponse.class, WfsRetrievalResponse.class,
Coverage.class);
} catch (JAXBException e) {
throw new ExceptionInInitializerError(e);
}

19
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/IRetrievalPluginDataObjectsProcessor.java
View file

@ -19,10 +19,9 @@
**/
package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
import java.util.Date;
import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.edex.datadelivery.retrieval.adapters.RetrievalAdapter.TranslationException;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord;
/**
* Processes {@link RetrievalResponseXml} that were generated from a retrieval.
@ -35,6 +34,7 @@ import com.raytheon.uf.edex.datadelivery.retrieval.adapters.RetrievalAdapter.Tra
* ------------ ---------- ----------- --------------------------
* Feb 01, 2013 1543 djohnson Initial creation
* Aug 09, 2013 1822 bgonzale Added parameters to processRetrievedPluginDataObjects.
* Oct 01, 2013 2267 bgonzale Pass request parameter instead of components of request.
*
* </pre>
*
@ -46,24 +46,17 @@ public interface IRetrievalPluginDataObjectsProcessor {
/**
* Process plugin data objects that were created as a result of a data
* delivery retrieval.
* delivery retrieval request.
*
* @param dataProvider
* NOMADS, MADIS, or PDA
* @param dataFormat
* Binary Grid(OPENDAP), POINT(MADIS), NetCDF4
* @param sourceType
* Model, Observation, Satellite
* @param date
* the date
* @param request
* the request
* @param retrievalPluginDataObjects
* the retrieval plugin data objects
* @throws SerializationException
* on error with serialization
* @throws TranslationException
*/
void processRetrievedPluginDataObjects(String dataProvider,
String dataFormat, String sourceType, Date date,
void processRetrievedPluginDataObjects(RetrievalRequestRecord request,
RetrievalResponseXml retrievalPluginDataObjects)
throws SerializationException, TranslationException;
}

7
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/RetrievalTask.java
View file

@ -19,8 +19,6 @@
**/
package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
import java.util.Date;
import com.raytheon.uf.common.datadelivery.registry.Network;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
@ -45,6 +43,7 @@ import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord;
* Feb 15, 2013 1543 djohnson Using xml for retrievals now.
* Mar 05, 2013 1647 djohnson Change no retrievals found message to debug.
* Aug 09, 2013 1822 bgonzale Added parameters to processRetrievedPluginDataObjects.
* Oct 01, 2013 2267 bgonzale Pass request parameter instead of components of request.
*
* </pre>
*
@ -103,10 +102,8 @@ public class RetrievalTask implements Runnable {
request = retrievalDao.getById(retrievalPluginDataObject
.getRequestRecord());
success = retrievalPluginDataObject.isSuccess();
Date date = request.getInsertTime();
retrievedDataProcessor.processRetrievedPluginDataObjects(
request.getProvider(), request.getPlugin(), null,
date, retrievalPluginDataObject);
request, retrievalPluginDataObject);
} catch (Exception e) {
statusHandler.error(
network + " retrieval processing error", e);

30
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/SerializeRetrievedDataToDirectory.java
View file

@ -30,7 +30,9 @@ import com.raytheon.uf.common.datadelivery.registry.Coverage;
import com.raytheon.uf.common.serialization.JAXBManager;
import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.common.util.FileUtil;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord;
import com.raytheon.uf.edex.datadelivery.retrieval.opendap.OpenDapRetrievalResponse;
import com.raytheon.uf.edex.datadelivery.retrieval.wfs.WfsRetrievalResponse;
/**
* Serializes the retrieved data to a directory.
@ -46,6 +48,7 @@ import com.raytheon.uf.edex.datadelivery.retrieval.opendap.OpenDapRetrievalRespo
* Mar 05, 2013 1647 djohnson Apply WMO header.
* Mar 07, 2013 1647 djohnson Write out as hidden file, then rename.
* Aug 09, 2013 1822 bgonzale Added parameters to IWmoHeaderApplier.applyWmoHeader().
* Oct 01, 2013 2267 bgonzale Pass request parameter instead of components of request.
*
* </pre>
*
@ -70,7 +73,8 @@ public class SerializeRetrievedDataToDirectory implements
this.wmoHeaderWrapper = wmoHeaderWrapper;
try {
this.jaxbManager = new JAXBManager(RetrievalResponseXml.class,
OpenDapRetrievalResponse.class, Coverage.class);
OpenDapRetrievalResponse.class, WfsRetrievalResponse.class,
Coverage.class);
} catch (JAXBException e) {
throw new ExceptionInInitializerError(e);
}
@ -80,8 +84,8 @@ public class SerializeRetrievedDataToDirectory implements
* {@inheritDoc}
*/
@Override
public void processRetrievedPluginDataObjects(String dataType,
String dataFormat, String sourceType, Date date,
public void processRetrievedPluginDataObjects(
RetrievalRequestRecord request,
RetrievalResponseXml retrievalPluginDataObjects)
throws SerializationException {
retrievalPluginDataObjects.prepareForSerialization();
@ -94,8 +98,10 @@ public class SerializeRetrievedDataToDirectory implements
final String xml = jaxbManager
.marshalToXml(retrievalPluginDataObjects);
final String textForFile = wmoHeaderWrapper.applyWmoHeader(
dataType, dataFormat, sourceType, date, xml);
final Date date = request.getInsertTime();
final String textForFile = wmoHeaderWrapper
.applyWmoHeader(request.getProvider(), request.getPlugin(),
getSourceType(request), date, xml);
// Write as hidden file, this is OS specific, but there is no
// platform-neutral way to do this with Java
@ -111,4 +117,18 @@ public class SerializeRetrievedDataToDirectory implements
throw new SerializationException(e);
}
}
/**
* Determine source type from the request.
*
* TODO Simple method that is adequate for now. It will need to be updated
* for new data from NOMADS, MADIS, and PDA.
*
* @return source type string ("MODEL", "OBSERVATION", or "SATELLITE")
*/
private String getSourceType(RetrievalRequestRecord request) {
String provider = request.getProvider();
return (provider == null || !provider.equalsIgnoreCase("MADIS") ? "MODEL"
: "OBSERVATION");
}
}

6
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/StoreRetrievedData.java
View file

@ -19,7 +19,6 @@
**/
package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@ -63,6 +62,7 @@ import com.raytheon.uf.edex.datadelivery.retrieval.util.RetrievalPersistUtil;
* Feb 15, 2013 1543 djohnson Retrieve the retrieval attributes from the database.
* Aug 09, 2013 1822 bgonzale Added parameters to processRetrievedPluginDataObjects.
* Aug 06, 2013 1654 bgonzale Added AdhocDataRetrievalEvent.
* Oct 01, 2013 2267 bgonzale Pass request parameter instead of components of request.
*
* </pre>
*
@ -95,8 +95,8 @@ public class StoreRetrievedData implements IRetrievalPluginDataObjectsProcessor
* {@inheritDoc}
*/
@Override
public void processRetrievedPluginDataObjects(String dataProvider,
String dataFormat, String sourceType, Date date,
public void processRetrievedPluginDataObjects(
RetrievalRequestRecord request,
RetrievalResponseXml retrievalPluginDataObjects)
throws SerializationException, TranslationException {
Map<String, PluginDataObject[]> pluginDataObjects = Maps.newHashMap();

13
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval.wfs/src/com/raytheon/uf/edex/datadelivery/retrieval/wfs/WfsRetrievalResponse.java → edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/wfs/WfsRetrievalResponse.java
View file

@ -20,6 +20,7 @@ import com.raytheon.uf.edex.datadelivery.retrieval.response.RetrievalResponse;
* ------------ ---------- ----------- --------------------------
* May 12, 2013 753 dhladky Initial creation
* May 31, 2013 2038 djohnson Move to correct repo.
* Oct 04, 2013 2267 bgonzale Added default constructor.
*
* </pre>
*
@ -35,8 +36,18 @@ public class WfsRetrievalResponse extends RetrievalResponse {
@XmlElement
private String payload;
public WfsRetrievalResponse(RetrievalAttribute attribute) {
/**
* Default constructor.
*/
public WfsRetrievalResponse() {
}
/**
* Initialization constructor.
*
* @param attribute
*/
public WfsRetrievalResponse(RetrievalAttribute attribute) {
}
@Override

5
edexOsgi/com.raytheon.uf.edex.plugin.datadelivery.retrieval/utility/edex_static/base/distribution/dataDeliveryRetrieval.xml
View file

@ -19,8 +19,5 @@
further_licensing_information.
-->
<requestPatterns>
<!-- # TODO: What WMO Header should be used for SBN retrievals?
# The one below is just a random one to get things working..
-->
<regex>^SMYG10\sLYBM\s280000.*</regex>
<regex>^LZ[ABC][ABC]9[123] KWBC</regex>
</requestPatterns>

1
tests/.classpath
View file

@ -152,5 +152,6 @@
<classpathentry combineaccessrules="false" kind="src" path="/com.raytheon.uf.common.wxmath"/>
<classpathentry kind="lib" path="lib/org.eclipse.jface_3.8.0.v20120912-135020.jar"/>
<classpathentry combineaccessrules="false" kind="src" path="/org.apache.commons.configuration"/>
<classpathentry kind="src" path="/com.raytheon.uf.edex.datadelivery.retrieval.wfs"/>
<classpathentry kind="output" path="bin"/>
</classpath>

10
tests/resources/datadelivery/wfs/wfs_madis_dataset Normal file
View file

@ -0,0 +1,10 @@
LZAA91 KWBC 032200
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<retrievalResponseXml success="true" xmlns:ns2="com.raytheon.uf.common.datadelivery.registry">
<requestRecord index="2" subscriptionName="CimarronKSRapTropo"/>
<retrievalResponseWrapper>
<openDapRetrievalResponse>
<payload>SFRUUC8xLjAgMjAwIE9LClNlcnZlcjogRE9EUy8yLjE4CkNvbnRlbnQtdHlwZTogYXBwbGljYXRpb24vb2N0ZXQtc3RyZWFtCkNvbnRlbnQtRGVzY3JpcHRpb246IGRvZHNfZGF0YQpDb250ZW50LUVuY29kaW5nOiBkZWZsYXRlCgp4nK2Rf0yUdRzHQZjhsVOkICK0J0gKJOOOGwMu4fs8728iAgVekM+Q8OSXqHi34yh3ROMCRUJogAwDdFNMJ+AUSqNF8gxQGxAhJ+luIh5RUoQBG7vkSroblpvVP9h7e2/v7bPX+/P97kOVWmVOupbJEzFWRWqy0h5Eht28meXDFrJVG3arlNogKZOqVKfnZKQmabOy05n1jCQ5abdSawsya1LtsaXgZPkCFsvGKR5pCJYxNvIhLv/H3Nr3sPNfxqo9jy7K/+tVclE+o1GqX7FaGigJkgQGBtlyilSik4tE1PrXMJGdnd3tB16U2Mjep7k7lvLF8jZxW6M/eBye9RHCHof/P0TqPukntVNREc7PFS+G58pGAjhjVxKWsjfgmKvkxlZPcN8lDnAVu5q5xJhK9vbvYta+/eB/8dhqrkPO3mDofnsN+THvo7BahxLFCVTta0Y1GUORtwEJ37TgySPtXFm5CxuT0h5RU7zvb75zXRu62rToOX4VQ0k9GDF9ix/M+ZigpzBV1IuZmQv4aeVODHbX47Q+CLEX+7iaTSq23fM6Kf2og7oNW6hrRSYV547QpauyMO9/AubSSvxaeAHjnmaMd2zDXY0ZFv9D1OFrV8zF++BkhQfcJpo4sRrUSyalbhYFFcVOYe76sxi/L4ZpyQEYw85icHojjI2NGK6XYbp0O11Z9SYNWCeijGwLriQMwy1ZTB0PLqfiveHUPqoDprxsdBu88Zn6PFowjU9rRbhEXHCz0os6Zr5Ive+raXjoGkomz1KPw6M4f20nftnfC/PbfbiafAvHPbqgm1+FVGYF9HwGjg5RdDk3YO5GJvW6NU6lqzfS0LudNPSrDBow1UqX9cnRNr8Np71P4oBTK/waFJwhpYT7/JwGS66tRcqrO3DmQw3u6d2p31MKSgMu09flX1LFu/E0PsJMgyNV1hPY28z6qhTCwkmW20y4tLhDfNEpwuXNMvrRNMJVfdzcSLwJ1xQeMnhkhHCdo91zlnrCGYvjfRRGws2sHY5uOUywbCBLuyKB4HntbH22K0HoM/rungGCuIviyZfKCHZsr3Xbv4mg4Anf9WNOBDVNrRnsJetOF5sF/ozFQbvrisAfa3zjWF2/wFfzdf29QwJf4nTHcs9L4N9rk/n5+Qi8OrMgkfcX+DT3vsISicC/ddnj3BchAh+dmzb8Y4HAR7zQ4uxeLPBSwx8hG8oF3rcwKv2daoH3lFZXNtwUeLHJ1GH4XuDtK16edPhZ2DJLdJ5B038CTMGZrg==</payload>
</openDapRetrievalResponse>
</retrievalResponseWrapper>
</retrievalResponseXml>

14
tests/unit/com/raytheon/uf/edex/datadelivery/retrieval/handlers/DataDeliveryRetrievalWmoHeaderApplierTest.java
View file

@ -44,6 +44,7 @@ import com.raytheon.uf.edex.wmo.message.WMOMessage;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Aug 08, 2013 1822 bgonzale Initial creation
* Oct 01, 2013 2267 bgonzale Added test for null inputs.
*
* </pre>
*
@ -140,4 +141,17 @@ public class DataDeliveryRetrievalWmoHeaderApplierTest {
assertThat(message.getMessageBody(), is(equalTo("someData".getBytes())));
}
@Test
public void createWithNullArgs() {
Date date = new Date();
setHeader(null, null, null, date);
assertThat(wmoHeader.getT1(), is(equalTo('L')));
assertThat(wmoHeader.getT2(), is(equalTo('Z')));
assertThat(wmoHeader.getA1(), is(equalTo('A')));
assertThat(wmoHeader.getA2(), is(equalTo('A')));
assertThat(wmoHeader.getIi(), is(equalTo(91)));
assertThat(wmoHeader.getCccc(), is(equalTo("KWBC")));
assertThat(message.getMessageBody(), is(equalTo("someData".getBytes())));
}
}

8
tests/unit/com/raytheon/uf/edex/datadelivery/retrieval/handlers/DeserializeRetrievedDataFromIngestTest.java
View file

@ -39,6 +39,7 @@ import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.common.util.FileUtil;
import com.raytheon.uf.common.util.TestUtil;
import com.raytheon.uf.common.util.file.FilenameFilters;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord;
/**
* Test {@link DeserializeRetrievedDataFromIngest}.
@ -55,6 +56,7 @@ import com.raytheon.uf.common.util.file.FilenameFilters;
* Mar 05, 2013 1647 djohnson Pass wmo header strategy to constructor.
* Mar 19, 2013 1794 djohnson Read from a queue rather than the file system.
* Aug 09, 2013 1822 bgonzale Added parameters to processRetrievedPluginDataObjects.
* Oct 01, 2013 2267 bgonzale Pass request parameter instead of components of request.
*
* </pre>
*
@ -112,10 +114,14 @@ public class DeserializeRetrievedDataFromIngestTest {
IOException {
RetrievalResponseXml retrievalPluginDataObjects = RetrievalPluginDataObjectsFixture.INSTANCE
.get();
RetrievalRequestRecord request = new RetrievalRequestRecord();
request.setProvider("");
request.setPlugin("");
request.setInsertTime(new Date());
new SerializeRetrievedDataToDirectory(directory,
new AlwaysSameWmoHeader("SMYG10 LYBM 280000"))
.processRetrievedPluginDataObjects("", "", "", new Date(),
.processRetrievedPluginDataObjects(request,
retrievalPluginDataObjects);
final List<File> files = FileUtil.listFiles(directory,

6
tests/unit/com/raytheon/uf/edex/datadelivery/retrieval/handlers/RetrievalTaskTest.java
View file

@ -28,7 +28,6 @@ import static org.mockito.Mockito.mock;
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@ -88,6 +87,7 @@ import com.raytheon.uf.edex.datadelivery.retrieval.interfaces.IRetrievalResponse
* Mar 19, 2013 1794 djohnson RetrievalTasks integrate at a queue.
* Apr 29, 2013 1910 djohnson Unregister from EventBus after each test.
* Aug 09, 2013 1822 bgonzale Added parameters to processRetrievedPluginDataObjects.
* Oct 01, 2013 2267 bgonzale Pass request parameter instead of components of request.
*
* </pre>
*
@ -110,8 +110,8 @@ public class RetrievalTaskTest {
* {@inheritDoc}
*/
@Override
public void processRetrievedPluginDataObjects(String dataProvider,
String dataFormat, String sourceType, Date date,
public void processRetrievedPluginDataObjects(
RetrievalRequestRecord request,
RetrievalResponseXml retrievalPluginDataObjects)
throws SerializationException, TranslationException {
final List<RetrievalResponseWrapper> retrievalAttributePluginDataObjects = retrievalPluginDataObjects

30
tests/unit/com/raytheon/uf/edex/datadelivery/retrieval/handlers/SerializeRetrievedDataToDirectoryTest.java
View file

@ -31,6 +31,7 @@ import org.junit.Test;
import com.raytheon.uf.common.localization.PathManagerFactoryTest;
import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.common.util.TestUtil;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord;
/**
* Test {@link SerializeRetrievedDataToDirectory}.
@ -45,6 +46,8 @@ import com.raytheon.uf.common.util.TestUtil;
* Feb 15, 2013 1543 djohnson Class renames.
* Mar 05, 2013 1647 djohnson Pass wmo header strategy to constructor.
* Aug 09, 2013 1822 bgonzale Added parameters to processRetrievedPluginDataObjects.
* Oct 01, 2013 2267 bgonzale Pass request parameter instead of components of request.
* Add test for wfs retrieval.
*
* </pre>
*
@ -70,9 +73,32 @@ public class SerializeRetrievedDataToDirectoryTest {
RetrievalResponseXml retrievalPluginDataObjects = RetrievalPluginDataObjectsFixture.INSTANCE
.get();
RetrievalRequestRecord request = new RetrievalRequestRecord();
request.setProvider("NOMADS");
request.setPlugin("grid");
// "Model"
request.setInsertTime(new Date());
service.processRetrievedPluginDataObjects("NOMADS", "GRID", "Model",
new Date(), retrievalPluginDataObjects);
service.processRetrievedPluginDataObjects(request,
retrievalPluginDataObjects);
assertThat(directory, hasNumberOfFiles(1));
}
@Test
public void serializesWfsRetrievedDataToAFileInTheTargetDirectory()
throws SerializationException {
RetrievalResponseXml retrievalPluginDataObjects = WfsRetrievalPluginDataObjectsFixture.INSTANCE
.get();
RetrievalRequestRecord request = new RetrievalRequestRecord();
request.setProvider("MADIS");
request.setPlugin("MADIS");
// "Model"
request.setInsertTime(new Date());
service.processRetrievedPluginDataObjects(request,
retrievalPluginDataObjects);
assertThat(directory, hasNumberOfFiles(1));
}

112
tests/unit/com/raytheon/uf/edex/datadelivery/retrieval/handlers/WfsRetrievalPluginDataObjectsFixture.java Normal file
View file

@ -0,0 +1,112 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import com.raytheon.uf.common.datadelivery.registry.Coverage;
import com.raytheon.uf.common.datadelivery.registry.Time;
import com.raytheon.uf.common.datadelivery.retrieval.xml.RetrievalAttribute;
import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.common.util.AbstractFixture;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord;
import com.raytheon.uf.edex.datadelivery.retrieval.interfaces.IRetrievalRequestBuilder;
import com.raytheon.uf.edex.datadelivery.retrieval.wfs.MockWfsRetrievalAdapter;
/**
* Fixture for {@link RetrievalResponseXml} instances.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Oct 04, 2013 2267 bgonzale Initial creation.
* Updated to match new processCoverage method.
*
* </pre>
*
* @author bgonzale
* @version 1.0
*/
public class WfsRetrievalPluginDataObjectsFixture extends
AbstractFixture<RetrievalResponseXml> {
public static final WfsRetrievalPluginDataObjectsFixture INSTANCE = new WfsRetrievalPluginDataObjectsFixture();
/**
* Prevent construction.
*/
private WfsRetrievalPluginDataObjectsFixture() {
}
/**
* {@inheritDoc}
*/
@Override
public RetrievalResponseXml getInstance(long seedValue, Random random) {
RetrievalRequestRecord requestRecord = RetrievalRequestRecordFixture.INSTANCE
.get(seedValue);
List<RetrievalResponseWrapper> retrievalAttributePluginDataObjects = new ArrayList<RetrievalResponseWrapper>();
try {
for (final RetrievalAttribute attribute : requestRecord
.getRetrievalObj().getAttributes()) {
retrievalAttributePluginDataObjects
.add(new RetrievalResponseWrapper(
new MockWfsRetrievalAdapter()
.performRequest(new IRetrievalRequestBuilder() {
@Override
public String processTime(
Time prtXML) {
return "" + prtXML;
}
@Override
public String getRequest() {
return "request";
}
@Override
public RetrievalAttribute getAttribute() {
return attribute;
}
@Override
public String processCoverage(
Coverage Coverage) {
return "noCoverage";
}
})));
}
} catch (SerializationException e) {
throw new RuntimeException(e);
}
final RetrievalResponseXml retrievalPluginDataObjects = new RetrievalResponseXml(
requestRecord.getId(), retrievalAttributePluginDataObjects);
return retrievalPluginDataObjects;
}
}

62
tests/unit/com/raytheon/uf/edex/datadelivery/retrieval/wfs/MockWfsRetrievalAdapter.java Normal file
View file

@ -0,0 +1,62 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.retrieval.wfs;
import org.junit.Ignore;
import com.raytheon.uf.common.util.TestUtil;
import com.raytheon.uf.edex.datadelivery.retrieval.interfaces.IRetrievalRequestBuilder;
/**
* Overrides specific methods in {@link WfsRetrievalAdapter} that require
* external resources.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Oct 04, 2013 2267 bgonzale Initial creation
*
* </pre>
*
* @author bgonzale
* @version 1.0
*/
@Ignore
public class MockWfsRetrievalAdapter extends WfsRetrievalAdapter {
/**
* Create a WfsRetrievalResponse with a test wfs data file.
*/
@Override
public WfsRetrievalResponse performRequest(IRetrievalRequestBuilder request) {
String data = new String(TestUtil.readResource(this.getClass(),
"/datadelivery/wfs/wfs_madis_dataset"));
final WfsRetrievalResponse response = new WfsRetrievalResponse(null);
response.setAttribute(request.getAttribute());
response.setPayLoad(data);
return response;
}
}