xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #1543 Break apart the retrieval process to support SBN capabilities.

Browse source 
Made interfaces for various aspects of retrieval that are Spring injectable.
Created an initial SBN retrieval task that will watch the /awips2/crawl directory, for proof of concept.

Amend:
  Check whether debug is enabled prior to debug message.
  Move log of request xml inside null check.

Change-Id: I5e3fe835f028cb93c98abce9735d3d59cfc663dc

Former-commit-id: 3b20efeb7d [formerly 120296239cfa021507b4ce9bc3f7333509b1dfb9]
Former-commit-id: 1a17e4a0f5
This commit is contained in:
Dustin Johnson 2013-01-30 13:05:47 -06:00
parent 3ab7c693de
commit b5f69e21d8
50 changed files with 2958 additions and 488 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/approve/SubscriptionDiff.java
View file

@ -53,6 +53,7 @@ import com.vividsolutions.jts.geom.Coordinate;
* Sep 24, 2012 1157 mpduff Use InitialPendingSubsription.
* Dec 10, 2012 1259 bsteffen Switch Data Delivery from LatLon to referenced envelopes.
* Jan 25, 2013 1528 djohnson Compare priorities as primitive ints.
* Jan 30, 2013 1543 djohnson Use List instead of ArrayList.
*
* </pre>
*
@ -239,8 +240,8 @@ public class SubscriptionDiff {
diffMap.put("coverage", true);
}
ArrayList<Parameter> subParamList = sub.getParameter();
ArrayList<Parameter> pendingSubParamList = pendingSub.getParameter();
List<Parameter> subParamList = sub.getParameter();
List<Parameter> pendingSubParamList = pendingSub.getParameter();
ArrayList<String> subParams = new ArrayList<String>();
ArrayList<String> pendingSubParams = new ArrayList<String>();

3
cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/subset/SubsetManagerDlg.java
View file

@ -126,6 +126,7 @@ import com.raytheon.viz.ui.presenter.IDisplay;
* Jan 04, 2012 1420 mpduff Pass the subscription in to the GriddedTimingSelectionDlg.
* Jan 10, 2013 1444 mpduff Fix the loading of saved subsets from the saved subset tab.
* Jan 28, 2013 1530 djohnson Break out long method chaining into local variables for debugging.
* Jan 30, 2013 1543 djohnson Use List instead of ArrayList.
* </pre>
*
* @author mpduff
@ -887,7 +888,7 @@ public abstract class SubsetManagerDlg<DATASET extends DataSet, PRESENTER extend
// Vertical/Parameters
Map<String, VerticalXML> levelMap = new HashMap<String, VerticalXML>();
ArrayList<Parameter> paramaterList = this.subscription.getParameter();
List<Parameter> paramaterList = this.subscription.getParameter();
for (Parameter p : paramaterList) {
for (DataLevelType levelType : p.getLevelType()) {

3
cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/utils/DataDeliveryUtils.java
View file

@ -63,6 +63,7 @@ import com.vividsolutions.jts.geom.Coordinate;
* Jan 10, 2013 1420 mdpuff Added getMaxLatency().
* Jan 14, 2013 1286 djohnson Fix IndexOutOfBounds exception from getMaxLatency.
* Jan 22, 2013 1519 djohnson Correct getMaxLatency() calculations.
* Jan 30, 2013 1543 djohnson Use List instead of ArrayList.
* </pre>
*
* @author mpduff
@ -515,7 +516,7 @@ public class DataDeliveryUtils {
}
fmtStr.append("Parameters:").append(newline);
ArrayList<Parameter> parmArray = sub.getParameter();
List<Parameter> parmArray = sub.getParameter();
for (Parameter p : parmArray) {
fmtStr.append("------ Name: ").append(p.getName()).append(newline);
fmtStr.append("------ Provider Name: ").append(p.getProviderName())

7
edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Subscription.java
View file

@ -4,6 +4,7 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.TimeZone;
import javax.xml.bind.annotation.XmlAccessType;
@ -312,7 +313,7 @@ public class Subscription implements ISerializableObject, Serializable {
@XmlElements({ @XmlElement})
@DynamicSerializeElement
private ArrayList<Parameter> parameter;
private List<Parameter> parameter;
@XmlElement
@DynamicSerializeElement
@ -630,7 +631,7 @@ public class Subscription implements ISerializableObject, Serializable {
* @param parameter
* subscription parameter list
*/
public void setParameter(ArrayList<Parameter> parameter) {
public void setParameter(List<Parameter> parameter) {
this.parameter = parameter;
}
@ -639,7 +640,7 @@ public class Subscription implements ISerializableObject, Serializable {
*
* @return subscription parameter list
*/
public ArrayList<Parameter> getParameter() {
public List<Parameter> getParameter() {
return parameter;
}

138
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/retrieval/SubscriptionRetrievalAgent.java
View file

@ -5,18 +5,15 @@ package com.raytheon.uf.edex.datadelivery.bandwidth.retrieval;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry;
import com.google.common.annotations.VisibleForTesting;
import com.raytheon.uf.common.datadelivery.registry.Network;
import com.raytheon.uf.common.datadelivery.registry.Provider;
import com.raytheon.uf.common.datadelivery.registry.Subscription;
import com.raytheon.uf.common.datadelivery.registry.SubscriptionBundle;
import com.raytheon.uf.common.datadelivery.registry.handlers.DataDeliveryHandlers;
import com.raytheon.uf.common.datadelivery.retrieval.xml.Retrieval;
import com.raytheon.uf.common.datadelivery.retrieval.xml.RetrievalAttribute;
import com.raytheon.uf.common.dataplugin.PluginDataObject;
import com.raytheon.uf.common.registry.handler.RegistryHandlerException;
import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.common.serialization.SerializationUtil;
@ -33,14 +30,8 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.dao.SubscriptionRetrieval;
import com.raytheon.uf.edex.datadelivery.retrieval.RetrievalGenerator;
import com.raytheon.uf.edex.datadelivery.retrieval.RetrievalManagerNotifyEvent;
import com.raytheon.uf.edex.datadelivery.retrieval.ServiceTypeFactory;
import com.raytheon.uf.edex.datadelivery.retrieval.adapters.RetrievalAdapter;
import com.raytheon.uf.edex.datadelivery.retrieval.adapters.RetrievalAdapter.TranslationException;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalDao;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord;
import com.raytheon.uf.edex.datadelivery.retrieval.interfaces.IRetrievalHandler;
import com.raytheon.uf.edex.datadelivery.retrieval.interfaces.IRetrievalRequestBuilder;
import com.raytheon.uf.edex.datadelivery.retrieval.interfaces.IRetrievalResponse;
import com.raytheon.uf.edex.datadelivery.retrieval.util.RetrievalPersistUtil;
import com.raytheon.uf.edex.event.EventBus;
/**
@ -51,17 +42,17 @@ import com.raytheon.uf.edex.event.EventBus;
* SOFTWARE HISTORY
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Aug 27, 2012 726 jspinks Initial release.
* Oct 10, 2012 0726 djohnson Add generics, constants, defaultPriority.
* Nov 26, 2012 dhladky Override default ingest routes based on plugin
* Aug 27, 2012 726 jspinks Initial release.
* Oct 10, 2012 0726 djohnson Add generics, constants, defaultPriority.
* Nov 26, 2012 dhladky Override default ingest routes based on plugin
* Jan 30, 2013 1543 djohnson Should not implement IRetrievalHandler.
*
* </pre>
*
* @version 1.0
*/
public class SubscriptionRetrievalAgent extends
RetrievalAgent<SubscriptionRetrieval> implements
IRetrievalHandler {
RetrievalAgent<SubscriptionRetrieval> {
private static final IUFStatusHandler statusHandler = UFStatus
.getHandler(SubscriptionRetrievalAgent.class);
@ -72,8 +63,6 @@ public class SubscriptionRetrievalAgent extends
private final IBandwidthDao bandwidthDao;
public SubscriptionRetrievalAgent(Network network, String destinationUri,
final Object notifier, int defaultPriority,
RetrievalManager retrievalManager, IBandwidthDao bandwidthDao) {
@ -119,10 +108,7 @@ public class SubscriptionRetrievalAgent extends
retrieval.getIdentifier());
if (retrievalsGenerated) {
// Wake the RetrievalTasks to fetch the data..
EDEXUtil.getMessageProducer().sendAsync(destinationUri, null);
EDEXUtil.getMessageProducer().sendAsync(destinationUri, null);
EDEXUtil.getMessageProducer().sendAsync(destinationUri, null);
EDEXUtil.getMessageProducer().sendAsync(destinationUri, null);
wakeRetrievalTasks();
} else {
// Normally this is the job of the SubscriptionNotifyTask, but if no
// retrievals were generated we have to send it manually
@ -132,6 +118,14 @@ public class SubscriptionRetrievalAgent extends
}
}
@VisibleForTesting
void wakeRetrievalTasks() throws EdexException {
EDEXUtil.getMessageProducer().sendAsync(destinationUri, null);
EDEXUtil.getMessageProducer().sendAsync(destinationUri, null);
EDEXUtil.getMessageProducer().sendAsync(destinationUri, null);
EDEXUtil.getMessageProducer().sendAsync(destinationUri, null);
}
@Override
protected String getAgentType() {
return SUBSCRIPTION_AGENT;
@ -231,8 +225,9 @@ public class SubscriptionRetrievalAgent extends
statusHandler.info("Time to persist requests to db ["
+ timer.getElapsedTime() + "] ms");
} catch (Exception e) {
statusHandler.warn("Subscription: " + subscriptionName
+ " Failed to store to retrievals.");
statusHandler.handle(Priority.WARN, "Subscription: "
+ subscriptionName + " Failed to store to retrievals.",
e);
}
} else {
statusHandler.warn("Subscription: " + subscriptionName
@ -242,103 +237,6 @@ public class SubscriptionRetrievalAgent extends
return retrievalsGenerated;
}
/**
* The actual work gets done here.
*/
@Override
public boolean process(Retrieval retrieval) {
boolean success = true;
if (statusHandler.isPriorityEnabled(Priority.DEBUG)) {
statusHandler.debug("Starting Retrieval: Subscription: "
+ retrieval.getSubscriptionName());
}
RetrievalAdapter pra = ServiceTypeFactory
.retrieveServiceRetrievalAdapter(retrieval.getServiceType());
String adapterClassName = pra.getClass().getName();
try {
pra.setProviderRetrievalXML(retrieval);
// could have multiple retrievals
for (RetrievalAttribute attXML : retrieval.getAttribute()) {
IRetrievalRequestBuilder request = pra
.createRequestMessage(attXML);
statusHandler
.info("Translated provider attribute Request XML: "
+ request.getRequest());
IRetrievalResponse response = null;
if (request != null) {
response = pra.performRequest(request);
HashMap<String, PluginDataObject[]> pdoHash = null;
if (response != null) {
pdoHash = pra.processResponse(response);
if (pdoHash != null && pdoHash.size() > 0) {
// store all types
for (Entry<String, PluginDataObject[]> entry : pdoHash
.entrySet()) {
PluginDataObject[] value = entry.getValue();
if (store(attXML, value)) {
statusHandler.info("Successfully stored: "
+ value.length + " : "
+ adapterClassName + " Plugin : "
+ entry.getKey());
} else {
throw new IllegalStateException(
"Unable to store " + value.length
+ " PDOs to the database!");
}
}
} else {
throw new IllegalStateException(
"No PDO's to store: " + adapterClassName
+ " original: " + attXML.toString());
}
} else {
// null response
throw new IllegalStateException(
"Null response from provider: "
+ adapterClassName + " original: "
+ attXML.toString());
}
}
}
} catch (IllegalStateException e) {
statusHandler.handle(Priority.WARN, e.getLocalizedMessage(), e);
success = false;
} catch (TranslationException e) {
statusHandler.handle(Priority.ERROR, e.getLocalizedMessage(), e);
success = false;
}
return success;
}
/**
* Store PDO's from Provider to EDEX
*/
@Override
public boolean store(RetrievalAttribute attXML, PluginDataObject[] pdos) {
boolean success = false;
String pluginName = pdos[0].getPluginName();
if (pluginName != null) {
success = RetrievalPersistUtil.routePlugin(destinationUri,
pluginName, pdos);
}
return success;
}
private static Provider getProvider(String providerName) {
try {
return DataDeliveryHandlers.getProviderHandler().getByName(

113
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/res/spring/retrieval-datadelivery.xml
View file

@ -1,9 +1,7 @@
<beans
xmlns="http://www.springframework.org/schema/beans"
xmlns:amq="http://activemq.apache.org/schema/core"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://activemq.apache.org/schema/core http://activemq.apache.org/schema/core/activemq-core.xsd
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:util="http://www.springframework.org/schema/util"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.5.xsd
http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd">
<bean id="ddRetrievalDatabaseProperties" class="com.raytheon.uf.edex.database.DatabasePluginProperties">
@ -17,20 +15,109 @@
<bean id="subNotifyTask" class="com.raytheon.uf.edex.datadelivery.retrieval.handlers.SubscriptionNotifyTask" />
<bean id="retrievalTask" class="com.raytheon.uf.edex.datadelivery.retrieval.handlers.RetrievalTask">
<!-- route to send most retrieved records to -->
<constructor-arg value="directvm:dataDeliveryNotify"/>
<constructor-arg ref="subNotifyTask"/>
<!-- A RetrievalTask takes three constructor arguments:
1) How to find retrievals, in this case perform the actual retrieval and return it
2) What to do with found retrievals, in this case process it and send a notification event
3) How to complete retrievals, in this case update the database and send a notification event
-->
<bean id="opsnetRetrievalTask"
class="com.raytheon.uf.edex.datadelivery.retrieval.handlers.RetrievalTask">
<constructor-arg value="OPSNET" />
<constructor-arg>
<bean
class="com.raytheon.uf.edex.datadelivery.retrieval.handlers.PerformRetrievalPluginDataObjectsFinder">
<constructor-arg value="OPSNET" />
</bean>
</constructor-arg>
<constructor-arg>
<bean
class="com.raytheon.uf.edex.datadelivery.retrieval.handlers.NotifyOfPluginDataObjectsDecorator">
<constructor-arg>
<bean
class="com.raytheon.uf.edex.datadelivery.retrieval.handlers.StoreRetrievedData">
<constructor-arg value="directvm:dataDeliveryNotify" />
</bean>
</constructor-arg>
</bean>
</constructor-arg>
<constructor-arg>
<bean
class="com.raytheon.uf.edex.datadelivery.retrieval.handlers.RetrievalResponseCompleter">
<constructor-arg ref="subNotifyTask" />
</bean>
</constructor-arg>
</bean>
<!--
Comment out version above and uncomment this version to test out basic SBN directory drop-off and
pickup from a directory
-->
<!--
<bean id="opsnetRetrievalTask"
class="com.raytheon.uf.edex.datadelivery.retrieval.handlers.RetrievalTask">
<constructor-arg value="OPSNET" />
<constructor-arg>
<bean
class="com.raytheon.uf.edex.datadelivery.retrieval.handlers.PerformRetrievalPluginDataObjectsFinder">
<constructor-arg value="OPSNET" />
</bean>
</constructor-arg>
<constructor-arg>
<bean
class="com.raytheon.uf.edex.datadelivery.retrieval.handlers.SerializeRetrievedDataToDirectory">
<constructor-arg value="/awips2/crawl" />
</bean>
</constructor-arg>
<constructor-arg>
<util:constant
static-field="com.raytheon.uf.edex.datadelivery.retrieval.handlers.IRetrievalResponseCompleter.NULL" />
</constructor-arg>
</bean>
-->
<bean id="sbnRetrievalTask"
class="com.raytheon.uf.edex.datadelivery.retrieval.handlers.RetrievalTask">
<constructor-arg value="SBN" />
<constructor-arg>
<bean
class="com.raytheon.uf.edex.datadelivery.retrieval.handlers.DeserializeRetrievedDataFromDirectory">
<constructor-arg value="/awips2/crawl" />
</bean>
</constructor-arg>
<constructor-arg>
<bean
class="com.raytheon.uf.edex.datadelivery.retrieval.handlers.NotifyOfPluginDataObjectsDecorator">
<constructor-arg>
<bean
class="com.raytheon.uf.edex.datadelivery.retrieval.handlers.StoreRetrievedData">
<constructor-arg value="directvm:dataDeliveryNotify" />
</bean>
</constructor-arg>
</bean>
</constructor-arg>
<constructor-arg>
<bean
class="com.raytheon.uf.edex.datadelivery.retrieval.handlers.RetrievalResponseCompleter">
<constructor-arg ref="subNotifyTask" />
</bean>
</constructor-arg>
</bean>
<bean id="scheduledExecutorService" class="java.util.concurrent.Executors" factory-method="newScheduledThreadPool">
<constructor-arg value="3"/>
</bean>
<bean id="retrievalHandler" class="com.raytheon.uf.edex.datadelivery.retrieval.handlers.RetrievalHandler">
<bean id="retrievalHandler"
class="com.raytheon.uf.edex.datadelivery.retrieval.handlers.RetrievalHandler">
<constructor-arg ref="scheduledExecutorService" />
<constructor-arg ref="retrievalTask"/>
<constructor-arg ref="subNotifyTask"/>
<constructor-arg>
<util:list
value-type="com.raytheon.uf.edex.datadelivery.retrieval.handlers.RetrievalTask.RetrievalTask">
<ref bean="opsnetRetrievalTask" />
<ref bean="sbnRetrievalTask" />
</util:list>
</constructor-arg>
<constructor-arg ref="subNotifyTask" />
</bean>
<camelContext id="dataDeliveryNotify-camel"

51
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/IServiceFactoryLookup.java Normal file
View file

@ -0,0 +1,51 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.retrieval;
import com.raytheon.uf.common.datadelivery.registry.Provider;
/**
* Lookup {@link ServiceFactory} implementations for {@link Provider}s.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 30, 2013 1543 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public interface IServiceFactoryLookup {
/**
* Retrieve the {@link ServiceFactory} for a {@link Provider}.
*
* @param provider
* the provider
* @return the {@link ServiceFactory}
*/
ServiceFactory getProviderServiceFactory(Provider provider);
}

46
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/ServiceTypeFactory.java
View file

@ -21,6 +21,7 @@ package com.raytheon.uf.edex.datadelivery.retrieval;
import com.raytheon.uf.common.datadelivery.registry.Provider;
import com.raytheon.uf.common.datadelivery.registry.Provider.ServiceType;
import com.raytheon.uf.common.util.ServiceLoaderUtil;
import com.raytheon.uf.edex.datadelivery.retrieval.adapters.RetrievalAdapter;
import com.raytheon.uf.edex.datadelivery.retrieval.opendap.OpenDapServiceFactory;
import com.raytheon.uf.edex.datadelivery.retrieval.wcs.WcsServiceFactory;
@ -46,6 +47,35 @@ import com.raytheon.uf.edex.datadelivery.retrieval.wxxm.WxxmServiceFactory;
*/
public final class ServiceTypeFactory {
/**
* Default {@link IServiceFactoryLookup} to be used in production code.
*/
private static class ServiceTypeFactoryLookup implements
IServiceFactoryLookup {
@Override
public ServiceFactory getProviderServiceFactory(Provider provider) {
final ServiceType serviceType = provider.getServiceType();
switch (serviceType) {
case OPENDAP:
return new OpenDapServiceFactory(provider);
case WCS:
return new WcsServiceFactory();
case WFS:
return new WfsServiceFactory();
case WXXM:
return new WxxmServiceFactory();
default:
throw new IllegalArgumentException(String.format(
"No %s available to handle service type [%s]!",
ServiceFactory.class.getSimpleName(), serviceType));
}
}
}
private static final IServiceFactoryLookup SERVICE_FACTORY_LOOKUP = ServiceLoaderUtil
.load(IServiceFactoryLookup.class,
new ServiceTypeFactoryLookup());
private ServiceTypeFactory() {
@ -60,21 +90,7 @@ public final class ServiceTypeFactory {
* @return the factory
*/
public static ServiceFactory retrieveServiceFactory(Provider provider) {
final ServiceType serviceType = provider.getServiceType();
switch (serviceType) {
case OPENDAP:
return new OpenDapServiceFactory(provider);
case WCS:
return new WcsServiceFactory();
case WFS:
return new WfsServiceFactory();
case WXXM:
return new WxxmServiceFactory();
default:
throw new IllegalArgumentException(String.format(
"No %s available to handle service type [%s]!",
ServiceFactory.class.getSimpleName(), serviceType));
}
return SERVICE_FACTORY_LOOKUP.getProviderServiceFactory(provider);
}
/**

4
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/adapters/RetrievalAdapter.java
View file

@ -20,7 +20,7 @@ package com.raytheon.uf.edex.datadelivery.retrieval.adapters;
* further licensing information.
**/
import java.util.HashMap;
import java.util.Map;
import com.raytheon.uf.common.datadelivery.retrieval.xml.Retrieval;
import com.raytheon.uf.common.datadelivery.retrieval.xml.RetrievalAttribute;
@ -59,7 +59,7 @@ public abstract class RetrievalAdapter implements IRetrievalAdapter {
IRetrievalRequestBuilder request);
@Override
public abstract HashMap<String, PluginDataObject[]> processResponse(
public abstract Map<String, PluginDataObject[]> processResponse(
IRetrievalResponse response) throws TranslationException;
/**

56
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/db/RetrievalDao.java
View file

@ -13,6 +13,7 @@ import org.hibernate.Session;
import org.hibernate.Transaction;
import org.hibernate.criterion.Restrictions;
import com.raytheon.uf.common.datadelivery.registry.Network;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.edex.database.DataAccessLayerException;
@ -20,6 +21,23 @@ import com.raytheon.uf.edex.database.dao.CoreDao;
import com.raytheon.uf.edex.database.dao.DaoConfig;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord.State;
/**
*
* DAO for {@link RetrievalRequestRecord} entities.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 30, 2013 1543 djohnson Add SW history.
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public class RetrievalDao extends CoreDao {
private static final transient IUFStatusHandler statusHandler = UFStatus
.getHandler(RetrievalDao.class);
@ -32,9 +50,12 @@ public class RetrievalDao extends CoreDao {
* Returns the next PENDING retrieval request, puts it into a RUNNING state,
* based on current time.
*
* @param network
* the network to constrain requests to
*
* @return
*/
public RetrievalRequestRecord activateNextRetrievalRequest()
public RetrievalRequestRecord activateNextRetrievalRequest(Network network)
throws DataAccessLayerException {
Session sess = null;
Transaction tx = null;
@ -45,22 +66,29 @@ public class RetrievalDao extends CoreDao {
tx = sess.beginTransaction();
final String minPriHql = "select min(rec.priority) from RetrievalRequestRecord rec "
+ "where rec.state = :statePending";
+ "where rec.state = :statePending and rec.network = :network";
final String minInsertHql = "select min(rec.insertTime) from RetrievalRequestRecord rec "
+ "where rec.state = :statePending and rec.priority = :minPri";
+ "where rec.state = :statePending and rec.priority = :minPri and rec.network = :network";
// descending record order to retrieve all for a given subscription
// before moving to the next one if two have the same
// priority/insertTime
final String pkHql = "select rec.id.subscriptionName, min(rec.id.index) from RetrievalRequestRecord rec "
+ "where rec.state = :statePending and rec.priority = :minPri and rec.insertTime = :minInsert "
+ "and rec.network = :network "
+ "group by rec.id.subscriptionName order by min(rec.id.index) desc";
Query minPriQuery = sess.createQuery(minPriHql);
minPriQuery.setParameter("statePending", State.PENDING);
setQueryState(minPriQuery, State.PENDING);
setQueryNetwork(minPriQuery, network);
Query minInsertQuery = sess.createQuery(minInsertHql);
minInsertQuery.setParameter("statePending", State.PENDING);
setQueryState(minInsertQuery, State.PENDING);
setQueryNetwork(minInsertQuery, network);
Query pkQuery = sess.createQuery(pkHql);
pkQuery.setParameter("statePending", State.PENDING);
setQueryState(pkQuery, State.PENDING);
setQueryNetwork(pkQuery, network);
boolean done = false;
while (!done) {
@ -399,4 +427,20 @@ public class RetrievalDao extends CoreDao {
}
}
}
/**
* @param query
* @param state
*/
private void setQueryState(Query query, State state) {
query.setParameter("statePending", state);
}
/**
* @param query
* @param network
*/
private void setQueryNetwork(Query query, Network network) {
query.setParameter("network", network);
}
}

50
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/db/RetrievalRequestRecord.java
View file

@ -28,11 +28,15 @@ import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.Table;
import javax.persistence.Transient;
import com.raytheon.uf.common.datadelivery.registry.Network;
import com.raytheon.uf.common.datadelivery.retrieval.xml.Retrieval;
import com.raytheon.uf.common.datadelivery.retrieval.xml.Retrieval.SubscriptionType;
import com.raytheon.uf.common.dataplugin.persist.IPersistableDataObject;
import com.raytheon.uf.common.serialization.ISerializableObject;
import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.common.serialization.SerializationUtil;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
@ -44,9 +48,10 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* SOFTWARE HISTORY
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* May 09, 2012 rjpeter Initial creation
* Oct 10, 2012 0726 djohnson Add {@link #subRetrievalKey}.
* Nov 26, 2012 1340 dhladky Added additional fields for tracking subscriptions
* May 09, 2012 rjpeter Initial creation
* Oct 10, 2012 0726 djohnson Add {@link #subRetrievalKey}.
* Nov 26, 2012 1340 dhladky Added additional fields for tracking subscriptions
* Jan 30, 2013 1543 djohnson Add PENDING_SBN, give retrieval column a length.
*
* </pre>
*
@ -60,9 +65,8 @@ public class RetrievalRequestRecord implements
IPersistableDataObject<RetrievalRequestRecordPK>, Serializable,
ISerializableObject {
// TODO: Need COMPLETED state?
public enum State {
PENDING, RUNNING, FAILED, COMPLETED
PENDING, RUNNING, FAILED, COMPLETED;
};
private static final long serialVersionUID = 1L;
@ -104,10 +108,13 @@ public class RetrievalRequestRecord implements
@DynamicSerializeElement
private Date insertTime;
@Column(nullable = false)
@Column(nullable = false, length = 100000)
@DynamicSerializeElement
private byte[] retrieval;
@Transient
private Retrieval retrievalObj;
@Column(nullable = false)
@DynamicSerializeElement
private Long subRetrievalKey;
@ -213,4 +220,33 @@ public class RetrievalRequestRecord implements
public String getProvider() {
return provider;
}
}
/**
* Convenience method to set the retrieval byte array from an object.
*
* @param retrieval
* the retrieval
* @throws SerializationException
* on error serializing the retrieval
*/
public void setRetrievalObj(Retrieval retrieval)
throws SerializationException {
this.retrievalObj = retrieval;
this.retrieval = SerializationUtil.transformToThrift(retrieval);
}
/**
* Convenience method to get the retrieval as an object.
*
* @return the retrievalObj
* @throws SerializationException
* on error deserializing the retrieval
*/
public Retrieval getRetrievalObj() throws SerializationException {
if (retrievalObj == null && retrieval != null) {
retrievalObj = SerializationUtil.transformFromThrift(
Retrieval.class, retrieval);
}
return retrievalObj;
}
}

87
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/DeserializeRetrievedDataFromDirectory.java Normal file
View file

@ -0,0 +1,87 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
import java.io.File;
import java.io.FileFilter;
import com.raytheon.uf.common.serialization.SerializationUtil;
import com.raytheon.uf.common.util.CollectionUtil;
import com.raytheon.uf.common.util.FileUtil;
/**
* Deserializes the retrieved data in a directory.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Feb 01, 2013 1543 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public class DeserializeRetrievedDataFromDirectory implements
IRetrievalPluginDataObjectsFinder {
private static final FileFilter NO_DIRECTORIES = new FileFilter() {
@Override
public boolean accept(File pathname) {
return pathname.isFile();
}
};
private final File directory;
/**
* @param directory
*/
public DeserializeRetrievedDataFromDirectory(File directory) {
this.directory = directory;
}
/**
* {@inheritDoc}
*/
@Override
public RetrievalPluginDataObjects findRetrievalPluginDataObjects()
throws Exception {
final File[] files = directory.listFiles(NO_DIRECTORIES);
if (CollectionUtil.isNullOrEmpty(files)) {
return null;
}
final File file = files[0];
try {
return SerializationUtil
.transformFromThrift(RetrievalPluginDataObjects.class,
FileUtil.file2bytes(file));
} finally {
file.delete();
}
}
}

49
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/IRetrievalPluginDataObjectsFinder.java Normal file
View file

@ -0,0 +1,49 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
/**
* Responsible for finding the {@link RetrievalPluginDataObjects} that should be
* processed.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Feb 01, 2013 1543 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public interface IRetrievalPluginDataObjectsFinder {
/**
* Finds the {@link RetrievalPluginDataObjects} that should be processed.
*
* @return the {@link RetrievalPluginDataObjects}
* @throws Exception
*/
RetrievalPluginDataObjects findRetrievalPluginDataObjects()
throws Exception;
}

56
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/IRetrievalPluginDataObjectsProcessor.java Normal file
View file

@ -0,0 +1,56 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
import com.raytheon.uf.common.serialization.SerializationException;
/**
* Processes {@link RetrievalPluginDataObjects} that were generated from a
* retrieval.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Feb 01, 2013 1543 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public interface IRetrievalPluginDataObjectsProcessor {
/**
* Process plugin data objects that were created as a result of a data
* delivery retrieval.
*
* @param retrievalPluginDataObjects
* the retrieval plugin data objects
* @throws SerializationException
* on error with serialization
*/
void processRetrievedPluginDataObjects(
RetrievalPluginDataObjects retrievalPluginDataObjects)
throws Exception;
}

75
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/IRetrievalResponseCompleter.java Normal file
View file

@ -0,0 +1,75 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord;
/**
* Completes retrievals.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 31, 2013 1543 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public interface IRetrievalResponseCompleter {
/**
* Interface for a retrieval response status object.
*/
interface IRetrievalResponseStatus {
/**
* Check whether the retrieval succeeded.
*
* @return the succeeded flag
*/
boolean isSucceeded();
}
// Null object implementation
IRetrievalResponseCompleter NULL = new IRetrievalResponseCompleter() {
@Override
public void completeRetrieval(RetrievalRequestRecord retrieval,
IRetrievalResponseStatus status) {
}
};
/**
* Complete the specified retrieval.
*
* @param retrieval
* the retrieval
* @param succeeded
* true if the retrieval succeeded
*/
void completeRetrieval(RetrievalRequestRecord retrieval,
IRetrievalResponseStatus status);
}

109
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/NotifyOfPluginDataObjectsDecorator.java Normal file
View file

@ -0,0 +1,109 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
import java.util.List;
import com.raytheon.uf.common.datadelivery.event.retrieval.DataRetrievalEvent;
import com.raytheon.uf.common.datadelivery.registry.Provider.ServiceType;
import com.raytheon.uf.common.datadelivery.retrieval.util.DataSizeUtils;
import com.raytheon.uf.common.datadelivery.retrieval.xml.Retrieval;
import com.raytheon.uf.common.datadelivery.retrieval.xml.RetrievalAttribute;
import com.raytheon.uf.common.dataplugin.PluginDataObject;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord;
import com.raytheon.uf.edex.event.EventBus;
/**
* Performs processing on the retrieved plugin data objects, and then sends an
* event about the retrieval.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Feb 01, 2013 1543 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public class NotifyOfPluginDataObjectsDecorator implements
IRetrievalPluginDataObjectsProcessor {
private static final IUFStatusHandler statusHandler = UFStatus
.getHandler(NotifyOfPluginDataObjectsDecorator.class);
private static final EventBus EVENT_BUS = EventBus.getInstance();
private final IRetrievalPluginDataObjectsProcessor retrievedDataProcessor;
public NotifyOfPluginDataObjectsDecorator(
IRetrievalPluginDataObjectsProcessor retrievedDataProcessor) {
this.retrievedDataProcessor = retrievedDataProcessor;
}
@Override
public void processRetrievedPluginDataObjects(
RetrievalPluginDataObjects retrievalPluginDataObjects)
throws Exception {
// TODO: What if one of the records fails to store or serialize, is that
// already handled somewhere?
retrievedDataProcessor
.processRetrievedPluginDataObjects(retrievalPluginDataObjects);
final RetrievalRequestRecord requestRecord = retrievalPluginDataObjects
.getRequestRecord();
final List<RetrievalAttributePluginDataObjects> retrievalAttributePluginDataObjects = retrievalPluginDataObjects
.getRetrievalAttributePluginDataObjects();
for (RetrievalAttributePluginDataObjects pluginDataObjectEntry : retrievalAttributePluginDataObjects) {
RetrievalAttribute attXML = pluginDataObjectEntry.getAttributeXml();
PluginDataObject[] value = pluginDataObjectEntry
.getPluginDataObjects();
if (value.length == 0) {
continue;
}
final String pluginName = value[0].getPluginName();
Retrieval retrieval = requestRecord.getRetrievalObj();
ServiceType serviceType = retrieval.getServiceType();
statusHandler.info("Successfully processed: " + value.length
+ " : " + serviceType + " Plugin : " + pluginName);
DataRetrievalEvent event = new DataRetrievalEvent();
event.setId(retrieval.getSubscriptionName());
event.setOwner(retrieval.getOwner());
event.setNetwork(retrieval.getNetwork().name());
event.setPlugin(pluginName);
event.setProvider(attXML.getProvider());
event.setNumRecords(value.length);
event.setBytes(DataSizeUtils.calculateSize(attXML, serviceType));
EVENT_BUS.publish(event);
}
}
}

199
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/PerformRetrievalPluginDataObjectsFinder.java Normal file
View file

@ -0,0 +1,199 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import com.raytheon.uf.common.datadelivery.registry.Network;
import com.raytheon.uf.common.datadelivery.registry.Provider.ServiceType;
import com.raytheon.uf.common.datadelivery.retrieval.xml.Retrieval;
import com.raytheon.uf.common.datadelivery.retrieval.xml.RetrievalAttribute;
import com.raytheon.uf.common.dataplugin.PluginDataObject;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.time.util.ITimer;
import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.edex.datadelivery.retrieval.ServiceTypeFactory;
import com.raytheon.uf.edex.datadelivery.retrieval.adapters.RetrievalAdapter;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalDao;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord.State;
import com.raytheon.uf.edex.datadelivery.retrieval.interfaces.IRetrievalRequestBuilder;
import com.raytheon.uf.edex.datadelivery.retrieval.interfaces.IRetrievalResponse;
/**
* Performs the actual retrieval, and then returns the plugin data objects for
* processing.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Feb 01, 2013 1543 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public class PerformRetrievalPluginDataObjectsFinder implements
IRetrievalPluginDataObjectsFinder {
private static final IUFStatusHandler statusHandler = UFStatus
.getHandler(PerformRetrievalPluginDataObjectsFinder.class);
private final Network network;
/**
* Constructor.
*
* @param network
*/
public PerformRetrievalPluginDataObjectsFinder(Network network) {
this.network = network;
}
/**
* {@inheritDoc}
*/
@Override
public RetrievalPluginDataObjects findRetrievalPluginDataObjects()
throws Exception {
RetrievalDao dao = new RetrievalDao();
RetrievalPluginDataObjects retVal = null;
ITimer timer = TimeUtil.getTimer();
try {
timer.start();
RetrievalRequestRecord request = dao
.activateNextRetrievalRequest(network);
if (request == null) {
return null;
}
timer.stop();
statusHandler.info("Activation of next retrieval took ["
+ timer.getElapsedTime() + "] ms");
timer.reset();
timer.start();
try {
retVal = process(request);
timer.stop();
statusHandler.info("Retrieval Processing for ["
+ request.getId() + "] took " + timer.getElapsedTime()
+ " ms");
} catch (Exception e) {
statusHandler.error(
"Retrieval Processing failed: [" + request.getId()
+ "]", e);
}
} catch (Exception e) {
statusHandler
.error("Unable to look up next retrieval request at this time.",
e);
}
return retVal;
}
/**
* The actual work gets done here.
*/
private RetrievalPluginDataObjects process(
RetrievalRequestRecord requestRecord) {
requestRecord.setState(State.FAILED);
List<RetrievalAttributePluginDataObjects> retrievalAttributePluginDataObjects = new ArrayList<RetrievalAttributePluginDataObjects>();
try {
Retrieval retrieval = requestRecord.getRetrievalObj();
if (statusHandler.isPriorityEnabled(Priority.DEBUG)) {
statusHandler.debug("Starting Retrieval: Subscription: "
+ retrieval.getSubscriptionName());
}
ServiceType serviceType = retrieval.getServiceType();
RetrievalAdapter pra = ServiceTypeFactory
.retrieveServiceRetrievalAdapter(serviceType);
pra.setProviderRetrievalXML(retrieval);
// Perform the actual retrievals and transforms to plugin data
// objects
final List<RetrievalAttribute> attributes = retrieval
.getAttribute();
for (RetrievalAttribute attXML : attributes) {
IRetrievalRequestBuilder request = pra
.createRequestMessage(attXML);
if (request != null) {
statusHandler
.info("Translated provider attribute Request XML: "
+ request.getRequest());
IRetrievalResponse response = pra.performRequest(request);
if (response != null) {
Map<String, PluginDataObject[]> pdoHash = pra
.processResponse(response);
if (pdoHash != null && !pdoHash.isEmpty()) {
for (Entry<String, PluginDataObject[]> entry : pdoHash
.entrySet()) {
retrievalAttributePluginDataObjects
.add(new RetrievalAttributePluginDataObjects(
attXML, entry.getValue()));
}
requestRecord.setState(State.COMPLETED);
} else {
throw new IllegalStateException(
"No PDO's to store: " + serviceType
+ " original: " + attXML.toString());
}
} else {
// null response
throw new IllegalStateException(
"Null response for service: " + serviceType
+ " original: " + attXML.toString());
}
}
}
} catch (Exception e) {
statusHandler.handle(Priority.WARN, e.getLocalizedMessage(), e);
}
RetrievalPluginDataObjects retrievalPluginDataObject = new RetrievalPluginDataObjects(
requestRecord, retrievalAttributePluginDataObjects);
return retrievalPluginDataObject;
}
}

100
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/RetrievalAttributePluginDataObjects.java Normal file
View file

@ -0,0 +1,100 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
import com.raytheon.uf.common.datadelivery.retrieval.xml.RetrievalAttribute;
import com.raytheon.uf.common.dataplugin.PluginDataObject;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
/**
* Plugin data objects and the retrieval information they are associated with.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Feb 01, 2013 1543 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
@DynamicSerialize
public class RetrievalAttributePluginDataObjects {
@DynamicSerializeElement
private RetrievalAttribute attributeXml;
@DynamicSerializeElement
private PluginDataObject[] pluginDataObjects;
/**
* Constructor.
*/
public RetrievalAttributePluginDataObjects() {
}
/**
* Constructor.
*
* @param attributeXml
* @param pluginDataObjects
*/
public RetrievalAttributePluginDataObjects(RetrievalAttribute attributeXml,
PluginDataObject[] pluginDataObjects) {
this.attributeXml = attributeXml;
this.pluginDataObjects = pluginDataObjects;
}
/**
* @return the attributeXml
*/
public RetrievalAttribute getAttributeXml() {
return attributeXml;
}
/**
* @param attributeXml
* the attributeXml to set
*/
public void setAttributeXml(RetrievalAttribute attributeXml) {
this.attributeXml = attributeXml;
}
/**
* @return the pluginDataObjects
*/
public PluginDataObject[] getPluginDataObjects() {
return pluginDataObjects;
}
/**
* @param pluginDataObjects
* the pluginDataObjects to set
*/
public void setPluginDataObjects(PluginDataObject[] pluginDataObjects) {
this.pluginDataObjects = pluginDataObjects;
}
}

21
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/RetrievalHandler.java
View file

@ -54,7 +54,7 @@ public class RetrievalHandler {
private final ScheduledExecutorService executorService;
private final RetrievalTask retrievalTask;
private final List<RetrievalTask> retrievalTasks;
private final SubscriptionNotifyTask subNotifyTask;
@ -64,23 +64,26 @@ public class RetrievalHandler {
* @param executor
*/
public RetrievalHandler(ScheduledExecutorService executorService,
RetrievalTask retrievalTask, SubscriptionNotifyTask subNotifyTask) {
this(executorService, new RetrievalDao(), retrievalTask, subNotifyTask);
List<RetrievalTask> retrievalTasks,
SubscriptionNotifyTask subNotifyTask) {
this(executorService, new RetrievalDao(), retrievalTasks, subNotifyTask);
}
@VisibleForTesting
RetrievalHandler(ScheduledExecutorService executorService,
RetrievalDao retrievalDao, RetrievalTask retrievalTask,
RetrievalDao retrievalDao, List<RetrievalTask> retrievalTasks,
SubscriptionNotifyTask subNotifyTask) {
this.executorService = executorService;
this.retrievalTask = retrievalTask;
this.retrievalTasks = retrievalTasks;
this.subNotifyTask = subNotifyTask;
// set all Running state retrievals to pending
retrievalDao.resetRunningRetrievalsToPending();
executorService.scheduleWithFixedDelay(retrievalTask, 1, 5,
TimeUnit.MINUTES);
for (RetrievalTask retrievalTask : retrievalTasks) {
executorService.scheduleWithFixedDelay(retrievalTask, 1, 5,
TimeUnit.MINUTES);
}
executorService.scheduleWithFixedDelay(subNotifyTask, 1, 1,
TimeUnit.MINUTES);
}
@ -88,6 +91,8 @@ public class RetrievalHandler {
public void notify(List<String> subscriptions) {
statusHandler.info("Notifying that subscriptions are available.");
executorService.execute(retrievalTask);
for (RetrievalTask retrievalTask : retrievalTasks) {
executorService.execute(retrievalTask);
}
}
}

102
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/RetrievalPluginDataObjects.java Normal file
View file

@ -0,0 +1,102 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
import java.util.List;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord;
/**
* Associates plugin data objects with a retrieval.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Feb 01, 2013 1543 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
@DynamicSerialize
public class RetrievalPluginDataObjects {
@DynamicSerializeElement
private RetrievalRequestRecord requestRecord;
@DynamicSerializeElement
private List<RetrievalAttributePluginDataObjects> retrievalAttributePluginDataObjects;
/**
* Constructor.
*/
public RetrievalPluginDataObjects() {
}
/**
* Constructor.
*
* @param requestRecord
* @param retrievalAttributePluginDataObjects
*/
public RetrievalPluginDataObjects(
RetrievalRequestRecord requestRecord,
List<RetrievalAttributePluginDataObjects> retrievalAttributePluginDataObjects) {
this.requestRecord = requestRecord;
this.retrievalAttributePluginDataObjects = retrievalAttributePluginDataObjects;
}
/**
* @return the requestRecord
*/
public RetrievalRequestRecord getRequestRecord() {
return requestRecord;
}
/**
* @param requestRecord
* the requestRecord to set
*/
public void setRequestRecord(RetrievalRequestRecord requestRecord) {
this.requestRecord = requestRecord;
}
/**
* @return the retrievalAttributePluginDataObjects
*/
public List<RetrievalAttributePluginDataObjects> getRetrievalAttributePluginDataObjects() {
return retrievalAttributePluginDataObjects;
}
/**
* @param retrievalAttributePluginDataObjects
* the retrievalAttributePluginDataObjects to set
*/
public void setRetrievalAttributePluginDataObjects(
List<RetrievalAttributePluginDataObjects> retrievalAttributePluginDataObjects) {
this.retrievalAttributePluginDataObjects = retrievalAttributePluginDataObjects;
}
}

76
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/RetrievalResponseCompleter.java Normal file
View file

@ -0,0 +1,76 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.edex.database.DataAccessLayerException;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalDao;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord;
/**
* Default implementation of {@link IRetrievalResponseCompleter}.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 31, 2013 1543 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public class RetrievalResponseCompleter implements IRetrievalResponseCompleter {
private static final IUFStatusHandler statusHandler = UFStatus
.getHandler(RetrievalResponseCompleter.class);
private final SubscriptionNotifyTask notifyTask;
public RetrievalResponseCompleter(SubscriptionNotifyTask notifyTask) {
this.notifyTask = notifyTask;
}
/**
* {@inheritDoc}
*/
@Override
public void completeRetrieval(RetrievalRequestRecord retrieval,
IRetrievalResponseStatus status) {
RetrievalRequestRecord.State state = status.isSucceeded() ? RetrievalRequestRecord.State.COMPLETED
: RetrievalRequestRecord.State.FAILED;
retrieval.setState(state);
// update database
try {
RetrievalDao dao = new RetrievalDao();
dao.completeRetrievalRequest(retrieval);
notifyTask.checkNotify(retrieval);
} catch (DataAccessLayerException e) {
statusHandler.error("Unable to communicate with the database", e);
}
}
}

60
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/RetrievalResponseStatus.java Normal file
View file

@ -0,0 +1,60 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
import com.raytheon.uf.edex.datadelivery.retrieval.handlers.IRetrievalResponseCompleter.IRetrievalResponseStatus;
/**
* Response from a retrieval.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 31, 2013 1543 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public class RetrievalResponseStatus implements IRetrievalResponseStatus {
private final boolean succeeded;
/**
* Constructor.
*
* @param succeeded
* whether or not the retrieval succeeded
*/
public RetrievalResponseStatus(boolean succeeded) {
this.succeeded = succeeded;
}
/**
* {@inheritDoc}
*/
@Override
public boolean isSucceeded() {
return succeeded;
}
}

216
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/RetrievalTask.java
View file

@ -19,32 +19,11 @@
**/
package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
import java.util.HashMap;
import java.util.Map.Entry;
import com.raytheon.uf.common.datadelivery.event.retrieval.DataRetrievalEvent;
import com.raytheon.uf.common.datadelivery.registry.Provider.ServiceType;
import com.raytheon.uf.common.datadelivery.retrieval.util.DataSizeUtils;
import com.raytheon.uf.common.datadelivery.retrieval.xml.Retrieval;
import com.raytheon.uf.common.datadelivery.retrieval.xml.RetrievalAttribute;
import com.raytheon.uf.common.dataplugin.PluginDataObject;
import com.raytheon.uf.common.serialization.SerializationUtil;
import com.raytheon.uf.common.datadelivery.registry.Network;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.time.util.ITimer;
import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.edex.database.DataAccessLayerException;
import com.raytheon.uf.edex.datadelivery.retrieval.ServiceTypeFactory;
import com.raytheon.uf.edex.datadelivery.retrieval.adapters.RetrievalAdapter;
import com.raytheon.uf.edex.datadelivery.retrieval.adapters.RetrievalAdapter.TranslationException;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalDao;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord;
import com.raytheon.uf.edex.datadelivery.retrieval.interfaces.IRetrievalHandler;
import com.raytheon.uf.edex.datadelivery.retrieval.interfaces.IRetrievalRequestBuilder;
import com.raytheon.uf.edex.datadelivery.retrieval.interfaces.IRetrievalResponse;
import com.raytheon.uf.edex.datadelivery.retrieval.util.RetrievalPersistUtil;
import com.raytheon.uf.edex.event.EventBus;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord.State;
/**
* Inner class to process individual retrievals.
@ -59,193 +38,74 @@ import com.raytheon.uf.edex.event.EventBus;
* Aug 15, 2012 1022 djohnson Moved from inner to class proper.
* Aug 22, 2012 0743 djohnson Continue processing retrievals until there are no more.
* Nov 19, 2012 1166 djohnson Clean up JAXB representation of registry objects.
* Jan 30, 2013 1543 djohnson Constrain to the network retrievals are pulled for.
*
* </pre>
*
* @author dhladky
* @version 1.0
*/
public class RetrievalTask implements IRetrievalHandler, Runnable {
public class RetrievalTask implements Runnable {
private static final IUFStatusHandler statusHandler = UFStatus
.getHandler(RetrievalTask.class);
private static final EventBus EVENT_BUS = EventBus.getInstance();
private final Network network;
private final String generalDestinationUri;
private final IRetrievalPluginDataObjectsProcessor retrievedDataProcessor;
private final SubscriptionNotifyTask notifyTask;
private final IRetrievalResponseCompleter retrievalCompleter;
public RetrievalTask(final String generalDestinationUri,
SubscriptionNotifyTask notifyTask) {
this.generalDestinationUri = generalDestinationUri;
this.notifyTask = notifyTask;
}
private final IRetrievalPluginDataObjectsFinder retrievalDataFinder;
/**
* The actual work gets done here. TODO: Should return a status object.
*/
@Override
public boolean process(Retrieval retrieval) {
boolean success = true;
statusHandler.debug("Starting Retrieval: Subscription: "
+ retrieval.getSubscriptionName());
ServiceType serviceType = retrieval.getServiceType();
try {
RetrievalAdapter pra = ServiceTypeFactory
.retrieveServiceRetrievalAdapter(serviceType);
pra.setProviderRetrievalXML(retrieval);
// could have multiple retrievals
for (RetrievalAttribute attXML : retrieval.getAttribute()) {
IRetrievalRequestBuilder request = pra
.createRequestMessage(attXML);
statusHandler
.info("Translated provider attribute Request XML: "
+ request.getRequest());
IRetrievalResponse response = null;
if (request != null) {
response = pra.performRequest(request);
HashMap<String, PluginDataObject[]> pdoHash = null;
if (response != null) {
pdoHash = pra.processResponse(response);
if (pdoHash != null && pdoHash.size() > 0) {
// store all types
for (Entry<String, PluginDataObject[]> entry : pdoHash
.entrySet()) {
PluginDataObject[] value = entry.getValue();
if (store(attXML, value)) {
statusHandler.info("Successfully stored: "
+ value.length + " : "
+ serviceType + " Plugin : "
+ entry.getKey());
DataRetrievalEvent event = new DataRetrievalEvent();
event.setId(retrieval.getSubscriptionName());
event.setOwner(retrieval.getOwner());
event.setNetwork(retrieval.getNetwork()
.name());
event.setPlugin(entry.getKey());
event.setProvider(attXML.getProvider());
event.setNumRecords(value.length);
event.setBytes(DataSizeUtils.calculateSize(
attXML, serviceType));
EVENT_BUS.publish(event);
} else {
throw new IllegalStateException(
"Unable to store " + value.length
+ " PDOs to the database!");
}
}
} else {
throw new IllegalStateException(
"No PDO's to store: " + serviceType
+ " original: " + attXML.toString());
}
} else {
// null response
throw new IllegalStateException(
"Null response for service: " + serviceType
+ " original: " + attXML.toString());
}
}
}
} catch (IllegalStateException e) {
statusHandler.handle(Priority.WARN, e.getLocalizedMessage(), e);
success = false;
} catch (TranslationException e) {
statusHandler.handle(Priority.ERROR, e.getLocalizedMessage(), e);
success = false;
}
return success;
public RetrievalTask(Network network,
IRetrievalPluginDataObjectsFinder retrievalDataFinder,
IRetrievalPluginDataObjectsProcessor retrievedDataProcessor,
IRetrievalResponseCompleter retrievalCompleter) {
this.network = network;
this.retrievalDataFinder = retrievalDataFinder;
this.retrievedDataProcessor = retrievedDataProcessor;
this.retrievalCompleter = retrievalCompleter;
}
@Override
public void run() {
RetrievalDao dao = new RetrievalDao();
try {
while (true) {
RetrievalRequestRecord request = null;
ITimer timer = TimeUtil.getTimer();
try {
timer.start();
request = dao.activateNextRetrievalRequest();
} catch (Exception e) {
statusHandler
.error("Unable to look up next retrieval request at this time.",
e);
}
// This forces the return from the while loop once there are
// no more retrievals to process
if (request == null) {
statusHandler.info("No Retrievals found.");
return;
}
timer.stop();
statusHandler.info("Activation of next retrieval took ["
+ timer.getElapsedTime() + "] ms");
// process request
boolean success = false;
Retrieval retrieval = null;
RetrievalRequestRecord request = null;
try {
retrieval = SerializationUtil.transformFromThrift(
Retrieval.class, request.getRetrieval());
timer.reset();
timer.start();
success = process(retrieval);
timer.stop();
RetrievalPluginDataObjects retrievalPluginDataObject = retrievalDataFinder
.findRetrievalPluginDataObjects();
// This forces the return from the while loop once there are
// no more retrievals to process
if (retrievalPluginDataObject == null) {
statusHandler.info("No " + network
+ " retrievals found.");
return;
}
statusHandler.info("Retrieval Processing for ["
+ request.getId() + "] took "
+ timer.getElapsedTime() + " ms");
request = retrievalPluginDataObject.getRequestRecord();
success = (request.getState() == State.COMPLETED);
retrievedDataProcessor
.processRetrievedPluginDataObjects(retrievalPluginDataObject);
} catch (Exception e) {
statusHandler.error("Retrieval Processing failed: ["
+ request.getId() + "]", e);
statusHandler.error(
network + " retrieval processing error", e);
}
RetrievalRequestRecord.State state = success ? RetrievalRequestRecord.State.COMPLETED
: RetrievalRequestRecord.State.FAILED;
request.setState(state);
// update database
try {
dao.completeRetrievalRequest(request);
notifyTask.checkNotify(request);
} catch (DataAccessLayerException e) {
statusHandler.error(
"Unable to communicate with the database", e);
if (request != null) {
retrievalCompleter.completeRetrieval(request,
new RetrievalResponseStatus(success));
}
}
} catch (Throwable e) {
// so thread can't die
statusHandler.error("Error caught in retrieval thread", e);
statusHandler.error("Error caught in " + network
+ " retrieval thread", e);
}
}
/**
* Store PDO's from Provider to EDEX
*/
@Override
public boolean store(RetrievalAttribute attXML, PluginDataObject[] pdos) {
// do all of the PDO storage magic...
boolean success = false;
String pluginName = pdos[0].getPluginName();
if (pluginName != null) {
success = RetrievalPersistUtil.routePlugin(generalDestinationUri,
pluginName, pdos);
}
return success;
}
}

74
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/SerializeRetrievedDataToDirectory.java Normal file
View file

@ -0,0 +1,74 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
import java.io.File;
import java.io.IOException;
import java.util.UUID;
import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.common.serialization.SerializationUtil;
import com.raytheon.uf.common.util.FileUtil;
/**
* Serializes the retrieved data to a directory.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Feb 01, 2013 1543 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public class SerializeRetrievedDataToDirectory implements
IRetrievalPluginDataObjectsProcessor {
private final File targetDirectory;
/**
* @param directory
*/
public SerializeRetrievedDataToDirectory(File directory) {
this.targetDirectory = directory;
}
/**
* {@inheritDoc}
*/
@Override
public void processRetrievedPluginDataObjects(
RetrievalPluginDataObjects retrievalPluginDataObjects)
throws SerializationException {
try {
FileUtil.bytes2File(SerializationUtil
.transformToThrift(retrievalPluginDataObjects), new File(
targetDirectory, UUID.randomUUID().toString()));
} catch (IOException e) {
throw new SerializationException(e);
}
}
}

101
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/StoreRetrievedData.java Normal file
View file

@ -0,0 +1,101 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
import java.util.List;
import com.raytheon.uf.common.dataplugin.PluginDataObject;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord;
import com.raytheon.uf.edex.datadelivery.retrieval.util.RetrievalPersistUtil;
/**
* Implementation of {@link IRetrievedDataProcessor} that stores the plugin data
* objects to the database.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 31, 2013 1543 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public class StoreRetrievedData implements IRetrievalPluginDataObjectsProcessor {
private final String generalDestinationUri;
/**
* Constructor.
*
* @param generalDestinationUri
* the destination uri most plugin data will travel through
*/
public StoreRetrievedData(String generalDestinationUri) {
this.generalDestinationUri = generalDestinationUri;
}
/**
* {@inheritDoc}
*/
@Override
public void processRetrievedPluginDataObjects(
RetrievalPluginDataObjects retrievalPluginDataObjects) {
final RetrievalRequestRecord requestRecord = retrievalPluginDataObjects
.getRequestRecord();
final List<RetrievalAttributePluginDataObjects> retrievalAttributePluginDataObjects = retrievalPluginDataObjects
.getRetrievalAttributePluginDataObjects();
for (RetrievalAttributePluginDataObjects pluginDataObjectEntry : retrievalAttributePluginDataObjects) {
PluginDataObject[] value = pluginDataObjectEntry
.getPluginDataObjects();
if (value.length == 0) {
continue;
}
sendToDestinationForStorage(requestRecord, value);
}
}
/**
* Sends the plugin data objects to their configured destination for storage
* to the database.
*/
public boolean sendToDestinationForStorage(
RetrievalRequestRecord requestRecord,
PluginDataObject[] pdos) {
// do all of the PDO storage magic...
boolean success = false;
String pluginName = pdos[0].getPluginName();
if (pluginName != null) {
success = RetrievalPersistUtil.routePlugin(generalDestinationUri,
pluginName, pdos);
}
return success;
}
}

6
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/handlers/SubscriptionNotifyTask.java
View file

@ -19,7 +19,6 @@ import com.raytheon.uf.common.datadelivery.retrieval.xml.Retrieval;
import com.raytheon.uf.common.datadelivery.retrieval.xml.Retrieval.SubscriptionType;
import com.raytheon.uf.common.datadelivery.retrieval.xml.RetrievalAttribute;
import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.common.serialization.SerializationUtil;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
@ -291,9 +290,8 @@ public class SubscriptionNotifyTask implements Runnable {
try {
sb.append("Failed parameters: ");
for (RetrievalRequestRecord failedRec : failedRecs) {
Retrieval retrieval = SerializationUtil
.transformFromThrift(Retrieval.class,
failedRec.getRetrieval());
Retrieval retrieval = failedRec
.getRetrievalObj();
for (RetrievalAttribute att : retrieval
.getAttribute()) {
sb.append(att.getParameter().getName()

6
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/interfaces/IRetrievalAdapter.java
View file

@ -20,10 +20,10 @@ package com.raytheon.uf.edex.datadelivery.retrieval.interfaces;
* further licensing information.
**/
import java.util.HashMap;
import java.util.Map;
import com.raytheon.uf.common.datadelivery.retrieval.xml.RetrievalAttribute;
import com.raytheon.uf.common.datadelivery.retrieval.xml.Retrieval;
import com.raytheon.uf.common.datadelivery.retrieval.xml.RetrievalAttribute;
import com.raytheon.uf.common.dataplugin.PluginDataObject;
/**
@ -52,7 +52,7 @@ public interface IRetrievalAdapter {
public com.raytheon.uf.edex.datadelivery.retrieval.response.RetrievalResponse performRequest(
IRetrievalRequestBuilder requestBuilder);
public HashMap<String, PluginDataObject[]> processResponse(
public Map<String, PluginDataObject[]> processResponse(
IRetrievalResponse response) throws Exception;
public void setProviderRetrievalXML(Retrieval prxml);

44
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/interfaces/IRetrievalHandler.java
View file

@ -1,44 +0,0 @@
package com.raytheon.uf.edex.datadelivery.retrieval.interfaces;
/**
* Interface for Provider Retrieval Adapter
*
* <pre>
*
* SOFTWARE HISTORY
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 07, 2011 dhladky Initial creation
*
* </pre>/
*
* @author dhladky
* @version 1.0
*/
import com.raytheon.uf.common.datadelivery.retrieval.xml.Retrieval;
import com.raytheon.uf.common.datadelivery.retrieval.xml.RetrievalAttribute;
import com.raytheon.uf.common.dataplugin.PluginDataObject;
/**
* Interface for Provider Retrieval Handlers
*
* <pre>
*
* SOFTWARE HISTORY
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 07, 2011 dhladky Initial creation
*
* </pre>
*
* @author dhladky
* @version 1.0
*/
public interface IRetrievalHandler {
public boolean store(RetrievalAttribute attXML, PluginDataObject[] pdos);
public boolean process(Retrieval srxml);
}

6
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/util/ResponseProcessingUtilities.java
View file

@ -48,6 +48,7 @@ import com.raytheon.uf.common.time.DataTime;
* Jan 07, 2011 dhladky Initial creation
* Aug 20, 2012 0743 djohnson Fix cache lookup to use the model name and not hashcode.
* Nov 19, 2012 1166 djohnson Clean up JAXB representation of registry objects.
* Jan 30, 2013 1543 djohnson Log exception stacktrace.
*
* </pre>
*
@ -169,8 +170,9 @@ public class ResponseProcessingUtilities {
levels.add(level);
}
} catch (Exception e) {
statusHandler.error("Couldn't retrieve the levels : "
+ plevels.getLevelType() + " ERROR: " + e);
statusHandler.error(
"Couldn't retrieve the levels : " + plevels.getLevelType(),
e);
}
return levels;

4
edexOsgi/com.raytheon.uf.edex.datadelivery.service/src/com/raytheon/uf/edex/datadelivery/service/verify/BaseSubscriptionVerifier.java
View file

@ -22,6 +22,7 @@ package com.raytheon.uf.edex.datadelivery.service.verify;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import com.raytheon.uf.common.datadelivery.registry.DataSet;
import com.raytheon.uf.common.datadelivery.registry.Parameter;
@ -40,6 +41,7 @@ import com.raytheon.uf.edex.datadelivery.service.verify.SubscriptionIntegrityVer
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Dec 07, 2012 1104 djohnson Initial creation
* Jan 30, 2013 1543 djohnson Use List instead of ArrayList.
*
* </pre>
*
@ -85,7 +87,7 @@ class BaseSubscriptionVerifier implements IVerificationStrategy {
dataSetParams = new ArrayList<Parameter>(0);
}
ArrayList<Parameter> subParams = subscription.getParameter();
List<Parameter> subParams = subscription.getParameter();
if (subParams == null) {
subParams = new ArrayList<Parameter>(0);
}

4
tests/.classpath
View file

@ -71,5 +71,9 @@
<classpathentry combineaccessrules="false" kind="src" path="/com.raytheon.uf.common.datadelivery.service"/>
<classpathentry combineaccessrules="false" kind="src" path="/com.raytheon.uf.common.units"/>
<classpathentry kind="src" path="/com.raytheon.uf.viz.stats"/>
<classpathentry combineaccessrules="false" kind="src" path="/org.dom4j"/>
<classpathentry combineaccessrules="false" kind="src" path="/com.raytheon.uf.edex.plugin.level"/>
<classpathentry combineaccessrules="false" kind="src" path="/com.raytheon.uf.common.dataplugin.level"/>
<classpathentry combineaccessrules="false" kind="src" path="/com.raytheon.uf.common.dataplugin.grid"/>
<classpathentry kind="output" path="bin"/>
</classpath>

1
tests/resources/META-INF/services/com.raytheon.uf.edex.datadelivery.retrieval.IServiceFactoryLookup Normal file
View file

@ -0,0 +1 @@
com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.TestRetrievalServiceFactoryLookup

10
tests/unit/com/raytheon/uf/common/datadelivery/registry/BaseSubscriptionFixture.java
View file

@ -22,7 +22,6 @@ package com.raytheon.uf.common.datadelivery.registry;
import java.util.Date;
import java.util.Random;
import com.google.common.collect.Lists;
import com.raytheon.uf.common.datadelivery.registry.Subscription.SubscriptionPriority;
import com.raytheon.uf.common.registry.ebxml.RegistryUtil;
import com.raytheon.uf.common.time.util.TimeUtil;
@ -39,6 +38,7 @@ import com.raytheon.uf.common.util.AbstractFixture;
* ------------ ---------- ----------- --------------------------
* Sep 27, 2012 1187 djohnson Initial creation
* Oct 16, 2012 0726 djohnson Use other fixtures to get appropriate values.
* Jan 30, 2013 1543 djohnson Add coverage/parameter data.
*
* </pre>
*
@ -61,8 +61,8 @@ public abstract class BaseSubscriptionFixture<T extends Subscription> extends
subscription.setActivePeriodStart(TimeUtil.newDate());
subscription.setActivePeriodEnd(new Date(subscription
.getActivePeriodStart().getTime() + seedValue));
// TODO: Create coverage fixture
// subscription.setCoverage(coverage)
subscription
.setCoverage(GriddedCoverageFixture.INSTANCE.get(seedValue));
subscription
.setDataSetName(OpenDapGriddedDataSetMetaDataFixture.INSTANCE
.get(seedValue).getDataSetName());
@ -73,11 +73,11 @@ public abstract class BaseSubscriptionFixture<T extends Subscription> extends
subscription.setDescription("description" + random.nextInt());
subscription.setFullDataSet(random.nextBoolean());
subscription.setGroupName("group" + random.nextInt());
subscription.setName("name" + random.nextInt());
subscription.setName("name" + seedValue);
subscription.setNotify(random.nextBoolean());
subscription.setOfficeID("officeID" + random.nextInt());
subscription.setOwner("owner" + random.nextInt());
subscription.setParameter(Lists.<Parameter> newArrayList());
subscription.addParameter(ParameterFixture.INSTANCE.get());
// Same priority for all, individual tests needing to test specific
// priorities should set it manually anyway
subscription.setPriority(SubscriptionPriority.NORMAL);

65
tests/unit/com/raytheon/uf/common/datadelivery/registry/ConnectionFixture.java Normal file
View file

@ -0,0 +1,65 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.datadelivery.registry;
import com.raytheon.uf.common.util.AbstractFixture;
/**
* Fixture for {@link Connection}s.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 30, 2013 1543 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public class ConnectionFixture extends AbstractFixture<Connection> {
public static final ConnectionFixture INSTANCE = new ConnectionFixture();
/**
* Private constructor.
*/
private ConnectionFixture() {
}
/**
* {@inheritDoc}
*/
@Override
public Connection get(long seedValue) {
Connection connection = new Connection();
connection.setPassword("somePassword");
connection.setUrl("http://someUrl");
connection.setUserName("someUserName");
return connection;
}
}

81
tests/unit/com/raytheon/uf/common/datadelivery/registry/GriddedCoverageFixture.java Normal file
View file

@ -0,0 +1,81 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.datadelivery.registry;
import com.raytheon.uf.common.gridcoverage.Corner;
import com.raytheon.uf.common.gridcoverage.LatLonGridCoverage;
import com.raytheon.uf.common.gridcoverage.exception.GridCoverageException;
import com.raytheon.uf.common.util.AbstractFixture;
/**
* Fixture for {@link GriddedCoverage}s.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 30, 2013 1543 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public class GriddedCoverageFixture extends AbstractFixture<GriddedCoverage> {
public static final GriddedCoverageFixture INSTANCE = new GriddedCoverageFixture();
/**
* Prevent construction.
*/
private GriddedCoverageFixture() {
}
/**
* {@inheritDoc}
*/
@Override
public GriddedCoverage get(long seedValue) {
LatLonGridCoverage gridCoverage = new LatLonGridCoverage();
gridCoverage.setCrsWKT("Polygon");
gridCoverage.setLa1(10);
gridCoverage.setLo1(-10);
gridCoverage.setDx(1.0);
gridCoverage.setDy(1.0);
gridCoverage.setNx(21);
gridCoverage.setNy(21);
gridCoverage.setSpacingUnit("degree");
gridCoverage.setFirstGridPointCorner(Corner.UpperLeft);
try {
gridCoverage.initialize();
} catch (GridCoverageException e) {
throw new RuntimeException(e);
}
GriddedCoverage coverage = new GriddedCoverage();
coverage.setGridCoverage(gridCoverage);
return coverage;
}
}

13
tests/unit/com/raytheon/uf/common/datadelivery/registry/LevelsFixture.java
View file

@ -19,6 +19,9 @@
**/
package com.raytheon.uf.common.datadelivery.registry;
import java.util.Arrays;
import com.raytheon.uf.common.datadelivery.registry.DataLevelType.LevelType;
import com.raytheon.uf.common.util.AbstractFixture;
/**
@ -31,8 +34,9 @@ import com.raytheon.uf.common.util.AbstractFixture;
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Sep 05, 2012 1102 djohnson Initial creation
* Oct 16, 2012 0726 djohnson Always use OpenDAP service type, use TimeFixture.
* Sep 05, 2012 1102 djohnson Initial creation
* Oct 16, 2012 0726 djohnson Always use OpenDAP service type, use TimeFixture.
* Jan 30, 2013 1543 djohnson Populate attributes.
*
* </pre>
*
@ -56,8 +60,9 @@ public class LevelsFixture extends AbstractFixture<Levels> {
@Override
public Levels get(long seedValue) {
Levels obj = new Levels();
// TODO: Populate attributes
obj.setLevel(Arrays.<Double> asList(1D, 2D, 3D));
obj.setLevelType(LevelType.SFC.getLevelTypeId());
obj.setName(LevelType.getLevelTypeIdName(obj.getLevelType()));
return obj;
}

8
tests/unit/com/raytheon/uf/common/datadelivery/registry/ProviderFixture.java
View file

@ -36,8 +36,9 @@ import com.raytheon.uf.common.util.AbstractFixture;
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Sep 5, 2012 1102 djohnson Initial creation
* Nov 19, 2012 1166 djohnson Clean up JAXB representation of registry objects.
* Sep 05, 2012 1102 djohnson Initial creation
* Nov 19, 2012 1166 djohnson Clean up JAXB representation of registry objects.
* Jan 30, 2013 1543 djohnson Add connection data.
*
* </pre>
*
@ -63,8 +64,7 @@ public class ProviderFixture extends AbstractFixture<Provider> {
Random random = new Random(seedValue);
Provider provider = new Provider();
// TODO: ConnectionFixture
// provider.setConnection(ConnectionFixture.INSTANCE.get(seedValue));
provider.setConnection(ConnectionFixture.INSTANCE.get(seedValue));
provider.setErrorResponsePattern("error");
provider.setName("providerName" + seedValue);
// TODO: ProjectionFixture

13
tests/unit/com/raytheon/uf/common/datadelivery/registry/SubscriptionBuilder.java
View file

@ -35,6 +35,7 @@ import com.raytheon.uf.common.time.util.TimeUtil;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 07, 2013 1453 djohnson Initial creation
* Jan 30, 2013 1543 djohnson Add ability to specify route.
*
* </pre>
*
@ -82,6 +83,8 @@ public class SubscriptionBuilder {
private String url = "http://someurl";
private Network route = Network.OPSNET;
/**
* Constructor.
*/
@ -112,6 +115,7 @@ public class SubscriptionBuilder {
subscription.setSubscriptionStart(subscriptionStart);
subscription.setSubscriptionEnd(subscriptionEnd);
subscription.setUrl(url);
subscription.setRoute(route);
subscription.setId(RegistryUtil.getRegistryObjectKey(subscription));
@ -289,4 +293,13 @@ public class SubscriptionBuilder {
return this;
}
/**
* @param route
* the route to set
* @return the {@link SubscriptionBuilder}
*/
public SubscriptionBuilder withRoute(Network route) {
this.route = route;
return this;
}
}

86
tests/unit/com/raytheon/uf/common/util/DateMatchers.java Normal file
View file

@ -0,0 +1,86 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.util;
import java.text.DateFormat;
import java.util.Date;
import org.hamcrest.Description;
import org.hamcrest.TypeSafeMatcher;
import org.junit.Ignore;
/**
* Date matchers for JUnit/Hamcrest, intentionally package-private.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Feb 01, 2013 1543 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
@Ignore
final class DateMatchers {
/**
* Prevent construction.
*/
private DateMatchers() {
}
static class DateFormatMatcher extends TypeSafeMatcher<Date> {
private final DateFormat format;
private final String expectedDateFormat;
private String actualFormatted;
DateFormatMatcher(DateFormat format, Date expected) {
this.format = format;
this.expectedDateFormat = format.format(expected);
}
/**
* {@inheritDoc}
*/
@Override
public void describeTo(Description description) {
description.appendText("a formatted date of ");
description.appendValue(expectedDateFormat);
description.appendText(" not ");
description.appendValue(actualFormatted);
}
/**
* {@inheritDoc}
*/
@Override
public boolean matchesSafely(Date item) {
actualFormatted = format.format(item);
return expectedDateFormat.equals(actualFormatted);
}
}
}

90
tests/unit/com/raytheon/uf/common/util/FileMatchers.java Normal file
View file

@ -0,0 +1,90 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.util;
import java.io.File;
import org.hamcrest.Description;
import org.hamcrest.TypeSafeMatcher;
import org.junit.Ignore;
/**
* File matchers for JUnit/Hamcrest, intentionally package-private.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Feb 01, 2013 1543 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
@Ignore
final class FileMatchers {
/**
* Prevent construction.
*/
private FileMatchers() {
}
static class DirectoryNumberOfFilesMatcher extends
TypeSafeMatcher<File> {
private final int expectedNumberOfFiles;
private int actualNumberOfFiles;
/**
* @param expectedNumberOfFiles
*/
DirectoryNumberOfFilesMatcher(int numberOfFiles) {
this.expectedNumberOfFiles = numberOfFiles;
}
/**
* {@inheritDoc}
*/
@Override
public void describeTo(Description description) {
description.appendText("a directory with ");
description.appendValue(expectedNumberOfFiles);
description.appendText((expectedNumberOfFiles == 1) ? " file"
: " files");
description.appendText(" not ");
description.appendValue(actualNumberOfFiles);
}
/**
* {@inheritDoc}
*/
@Override
protected boolean matchesSafely(File item) {
actualNumberOfFiles = item.listFiles().length;
return item.isDirectory()
&& actualNumberOfFiles == expectedNumberOfFiles;
}
}
}

47
tests/unit/com/raytheon/uf/common/util/Matchers.java
View file

@ -19,14 +19,16 @@
**/
package com.raytheon.uf.common.util;
import java.text.DateFormat;
import java.io.File;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.TimeZone;
import org.hamcrest.Description;
import org.hamcrest.Matcher;
import org.junit.internal.matchers.TypeSafeMatcher;
import org.junit.Ignore;
import com.raytheon.uf.common.util.DateMatchers.DateFormatMatcher;
import com.raytheon.uf.common.util.FileMatchers.DirectoryNumberOfFilesMatcher;
/**
* Custom Hamcrest matcher implementations.
@ -44,38 +46,9 @@ import org.junit.internal.matchers.TypeSafeMatcher;
* @author djohnson
* @version 1.0
*/
@Ignore
public final class Matchers {
private static class DateFormatMatcher extends TypeSafeMatcher<Date> {
private final DateFormat format;
private final String expectedDateFormat;
private DateFormatMatcher(DateFormat format, Date expected) {
this.format = format;
this.expectedDateFormat = format.format(expected);
}
/**
* {@inheritDoc}
*/
@Override
public void describeTo(Description description) {
description.appendText("a formatted date of ").appendText(
expectedDateFormat);
}
/**
* {@inheritDoc}
*/
@Override
public boolean matchesSafely(Date item) {
return format.format(item).equals(expectedDateFormat);
}
}
private static final SimpleDateFormat YYYY_MM_DD_FORMAT = new SimpleDateFormat(
"yyyyMMdd");
static {
@ -89,4 +62,12 @@ public final class Matchers {
return new DateFormatMatcher(YYYY_MM_DD_FORMAT, expected);
}
public static <T> Matcher<File> hasNumberOfFiles(final int numberOfFiles) {
return new DirectoryNumberOfFilesMatcher(numberOfFiles);
}
public static <T> Matcher<File> hasNoFiles() {
return hasNumberOfFiles(0);
}
}

292
tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/retrieval/MockServiceFactory.java Normal file
View file

@ -0,0 +1,292 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.bandwidth.retrieval;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import org.junit.Ignore;
import com.google.common.collect.Maps;
import com.raytheon.uf.common.datadelivery.registry.Collection;
import com.raytheon.uf.common.datadelivery.registry.DataSet;
import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData;
import com.raytheon.uf.common.datadelivery.registry.GriddedCoverageFixture;
import com.raytheon.uf.common.datadelivery.registry.OpenDapGriddedDataSetMetaDataFixture;
import com.raytheon.uf.common.datadelivery.registry.Provider;
import com.raytheon.uf.common.datadelivery.registry.Provider.ServiceType;
import com.raytheon.uf.common.datadelivery.registry.Subscription;
import com.raytheon.uf.common.datadelivery.registry.SubscriptionBundle;
import com.raytheon.uf.common.datadelivery.registry.Time;
import com.raytheon.uf.common.datadelivery.retrieval.xml.Retrieval;
import com.raytheon.uf.common.datadelivery.retrieval.xml.RetrievalAttribute;
import com.raytheon.uf.common.dataplugin.PluginDataObject;
import com.raytheon.uf.common.dataplugin.grid.GridRecord;
import com.raytheon.uf.edex.datadelivery.retrieval.IExtractMetaData;
import com.raytheon.uf.edex.datadelivery.retrieval.IParseMetaData;
import com.raytheon.uf.edex.datadelivery.retrieval.LinkStore;
import com.raytheon.uf.edex.datadelivery.retrieval.RetrievalGenerator;
import com.raytheon.uf.edex.datadelivery.retrieval.ServiceFactory;
import com.raytheon.uf.edex.datadelivery.retrieval.adapters.RetrievalAdapter;
import com.raytheon.uf.edex.datadelivery.retrieval.interfaces.IRetrievalRequestBuilder;
import com.raytheon.uf.edex.datadelivery.retrieval.interfaces.IRetrievalResponse;
import com.raytheon.uf.edex.datadelivery.retrieval.response.RetrievalResponse;
/**
* {@link ServiceFactory} that doesn't do much, just create a fake retrieval.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 30, 2013 1543 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
@Ignore
public class MockServiceFactory extends RetrievalGenerator implements
ServiceFactory, IExtractMetaData, IParseMetaData {
/**
* Just returns a retrieval.
*/
public class MockRetrievalAdapter extends RetrievalAdapter implements
IRetrievalRequestBuilder {
/**
* {@inheritDoc}
*/
@Override
public IRetrievalRequestBuilder createRequestMessage(
RetrievalAttribute prxml) {
return this;
}
/**
* {@inheritDoc}
*/
@Override
public RetrievalResponse performRequest(IRetrievalRequestBuilder request) {
return new RetrievalResponse(request.getAttribute());
}
/**
* {@inheritDoc}
*/
@Override
public Map<String, PluginDataObject[]> processResponse(
IRetrievalResponse response) throws TranslationException {
final Map<String, PluginDataObject[]> map = Maps.newHashMap();
map.put("grid", new PluginDataObject[] { new GridRecord() });
return map;
}
/**
* {@inheritDoc}
*/
@Override
public String processTime(Time prtXML) {
return "someTime";
}
/**
* {@inheritDoc}
*/
@Override
public String processCoverage() {
return "someCoverage";
}
/**
* {@inheritDoc}
*/
@Override
public String getRequest() {
return "someRequest";
}
/**
* {@inheritDoc}
*/
@Override
public RetrievalAttribute getAttribute() {
return new RetrievalAttribute();
}
}
/**
* @param serviceType
*/
public MockServiceFactory(ServiceType serviceType) {
super(serviceType);
}
/**
* @param provider
*/
public MockServiceFactory(Provider provider) {
this(provider.getServiceType());
}
/**
* {@inheritDoc}
*/
@Override
public IExtractMetaData getExtractor() {
return this;
}
/**
* {@inheritDoc}
*/
@Override
public IParseMetaData getParser(Date lastUpdate) {
return this;
}
/**
* {@inheritDoc}
*/
@Override
public RetrievalGenerator getRetrievalGenerator() {
return this;
}
/**
* {@inheritDoc}
*/
@Override
public List<DataSetMetaData> parseMetaData(Provider provider,
LinkStore store, Collection collection, String dataDateFormat) {
return Arrays
.<DataSetMetaData> asList(OpenDapGriddedDataSetMetaDataFixture.INSTANCE
.get());
}
/**
* {@inheritDoc}
*/
@Override
public void storeMetaData(List<DataSetMetaData> metaDatas, DataSet dataSet) {
}
/**
* {@inheritDoc}
*/
@Override
public Map<String, Object> extractMetaData() throws Exception {
return Collections.emptyMap();
}
/**
* {@inheritDoc}
*/
@Override
public boolean checkLastUpdate(Date lastUpdate) {
return true;
}
/**
* {@inheritDoc}
*/
@Override
public void setDataDate() throws Exception {
}
/**
* {@inheritDoc}
*/
@Override
public Date getDataDate() {
return new Date();
}
/**
* {@inheritDoc}
*/
@Override
public void setUrl(String url) {
}
/**
* {@inheritDoc}
*/
@Override
public List<Retrieval> buildRetrieval(SubscriptionBundle bundle) {
final Subscription subscription = bundle.getSubscription();
Retrieval retrieval = new Retrieval();
retrieval.setConnection(bundle.getConnection());
retrieval.setNetwork(subscription.getRoute());
retrieval.setOwner(subscription.getOwner());
retrieval.setProviderType(bundle.getProvider().getProviderType()
.iterator().next());
retrieval.setServiceType(this.getServiceType());
retrieval.setSubscriptionName(subscription.getName());
retrieval.setSubscriptionType(getSubscriptionType(subscription));
final ArrayList<RetrievalAttribute> attributes = new ArrayList<RetrievalAttribute>();
attributes.add(getAttribute(bundle));
attributes.add(getAttribute(bundle));
retrieval.setAttribute(attributes);
return Arrays.asList(retrieval);
}
/**
* @param bundle
* @return
*/
private RetrievalAttribute getAttribute(SubscriptionBundle bundle) {
RetrievalAttribute attribute = new RetrievalAttribute();
attribute.setCoverage(GriddedCoverageFixture.INSTANCE.get());
attribute.setSubName(bundle.getSubscription().getName());
attribute.setProvider(bundle.getProvider().getName());
return attribute;
}
/**
* {@inheritDoc}
*/
@Override
protected RetrievalAdapter getServiceRetrievalAdapter() {
return new MockRetrievalAdapter();
}
/**
* {@inheritDoc}
*/
@Override
protected Subscription removeDuplicates(Subscription sub) {
return sub;
}
}

132
tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/retrieval/SubscriptionRetrievalAgentTest.java Normal file
View file

@ -0,0 +1,132 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.bandwidth.retrieval;
import static org.hamcrest.Matchers.emptyCollectionOf;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
import static org.junit.Assert.assertThat;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.util.List;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import com.raytheon.uf.common.datadelivery.registry.Network;
import com.raytheon.uf.common.datadelivery.registry.OpenDapGriddedDataSetMetaDataFixture;
import com.raytheon.uf.common.datadelivery.registry.ProviderFixture;
import com.raytheon.uf.common.datadelivery.registry.Subscription;
import com.raytheon.uf.common.datadelivery.registry.SubscriptionBuilder;
import com.raytheon.uf.common.datadelivery.registry.handlers.DataDeliveryHandlers;
import com.raytheon.uf.common.localization.PathManagerFactoryTest;
import com.raytheon.uf.common.registry.handler.RegistryHandlerException;
import com.raytheon.uf.common.registry.handler.RegistryObjectHandlersUtil;
import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.edex.core.EdexException;
import com.raytheon.uf.edex.database.DataAccessLayerException;
import com.raytheon.uf.edex.database.dao.DatabaseUtil;
import com.raytheon.uf.edex.datadelivery.bandwidth.dao.IBandwidthDao;
import com.raytheon.uf.edex.datadelivery.bandwidth.dao.SubscriptionRetrieval;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalDao;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord.State;
/**
* Test {@link SubscriptionRetrievalAgent}.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 30, 2013 1543 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public class SubscriptionRetrievalAgentTest {
@Before
public void setUp() throws RegistryHandlerException {
DatabaseUtil.start();
PathManagerFactoryTest.initLocalization();
RegistryObjectHandlersUtil.initMocks();
when(DataDeliveryHandlers.getProviderHandler().getByName(anyString()))
.thenReturn(ProviderFixture.INSTANCE.get());
when(
DataDeliveryHandlers.getDataSetMetaDataHandler().getById(
anyString())).thenReturn(
OpenDapGriddedDataSetMetaDataFixture.INSTANCE.get());
}
@After
public void tearDown() {
DatabaseUtil.shutdown();
}
@Test
public void opsnetRoutedSubscriptionsCreateRetrievalsInPendingState()
throws EdexException, SerializationException {
testRetrievalIsPlacedInCorrectState(Network.OPSNET, State.PENDING);
}
@Test
public void sbnRoutedSubscriptionsCreateRetrievalsInPendingState()
throws EdexException, SerializationException {
testRetrievalIsPlacedInCorrectState(Network.SBN, State.PENDING);
}
private void testRetrievalIsPlacedInCorrectState(final Network route,
final State expectedState) throws SerializationException,
EdexException, DataAccessLayerException {
Subscription subscription = new SubscriptionBuilder().withRoute(route)
.build();
final SubscriptionRetrieval subscriptionRetrieval = new SubscriptionRetrieval();
subscriptionRetrieval.setSubscription(subscription);
subscriptionRetrieval.setNetwork(subscription.getRoute());
IBandwidthDao bandwidthDao = mock(IBandwidthDao.class);
SubscriptionRetrievalAgent agent = new SubscriptionRetrievalAgent(
route, "someUri", new Object(), 1, null, bandwidthDao) {
@Override
void wakeRetrievalTasks() throws EdexException {
// Do nothing
}
};
agent.processAllocation(subscriptionRetrieval);
RetrievalDao dao = new RetrievalDao();
final List<RetrievalRequestRecord> requests = dao
.getRequests(subscription.getName());
assertThat(requests,
is(not(emptyCollectionOf(RetrievalRequestRecord.class))));
final RetrievalRequestRecord request = requests.iterator().next();
assertThat(request.getState(), is(expectedState));
}
}

55
tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/retrieval/TestRetrievalServiceFactoryLookup.java Normal file
View file

@ -0,0 +1,55 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.bandwidth.retrieval;
import org.junit.Ignore;
import com.raytheon.uf.common.datadelivery.registry.Provider;
import com.raytheon.uf.edex.datadelivery.retrieval.IServiceFactoryLookup;
import com.raytheon.uf.edex.datadelivery.retrieval.ServiceFactory;
/**
* Implementation of {@link IServiceFactoryLookup} that doesn't really
* go out to the internet.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 30, 2013 1543 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
@Ignore
public class TestRetrievalServiceFactoryLookup implements
IServiceFactoryLookup {
/**
* {@inheritDoc}
*/
@Override
public ServiceFactory getProviderServiceFactory(Provider provider) {
return new MockServiceFactory(provider);
}
}

110
tests/unit/com/raytheon/uf/edex/datadelivery/retrieval/handlers/DeserializeRetrievedDataFromDirectoryTest.java Normal file
View file

@ -0,0 +1,110 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
import static com.raytheon.uf.common.util.Matchers.hasNoFiles;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
import java.io.File;
import org.junit.Test;
import com.raytheon.uf.common.util.TestUtil;
/**
* Test {@link DeserializeRetrievedDataFromDirectory}.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Feb 01, 2013 1543 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public class DeserializeRetrievedDataFromDirectoryTest {
private final File directory = TestUtil
.setupTestClassDir(DeserializeRetrievedDataFromDirectoryTest.class);
private final DeserializeRetrievedDataFromDirectory service = new DeserializeRetrievedDataFromDirectory(
directory);
@Test
public void deserializesRetrievedDataFromAFileInTheTargetDirectory()
throws Exception {
RetrievalPluginDataObjects retrievalPluginDataObjects = RetrievalPluginDataObjectsFixture.INSTANCE
.get();
new SerializeRetrievedDataToDirectory(directory)
.processRetrievedPluginDataObjects(retrievalPluginDataObjects);
final RetrievalPluginDataObjects restored = service
.findRetrievalPluginDataObjects();
// Just make sure the grid record URI is the same, that's good enough
// for our purposes
assertThat(restored.getRetrievalAttributePluginDataObjects().get(0)
.getPluginDataObjects()[0].getDataURI(),
is(equalTo(retrievalPluginDataObjects
.getRetrievalAttributePluginDataObjects().get(0)
.getPluginDataObjects()[0].getDataURI())));
}
@Test
public void deletesFileAfterRetrievingFromTheTargetDirectory()
throws Exception {
RetrievalPluginDataObjects retrievalPluginDataObjects = RetrievalPluginDataObjectsFixture.INSTANCE
.get();
new SerializeRetrievedDataToDirectory(directory)
.processRetrievedPluginDataObjects(retrievalPluginDataObjects);
service.findRetrievalPluginDataObjects();
assertThat(directory, hasNoFiles());
}
@Test
public void ignoresSubDirectories() throws Exception {
new File(directory, "subDir1").mkdirs();
service.findRetrievalPluginDataObjects();
}
@Test
public void returnsNullWhenNoFileInTheTargetDirectory() throws Exception {
final RetrievalPluginDataObjects restored = service
.findRetrievalPluginDataObjects();
assertNull(restored);
}
}

77
tests/unit/com/raytheon/uf/edex/datadelivery/retrieval/handlers/RetrievalHandlerTest.java
View file

@ -19,11 +19,10 @@
**/
package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
import static org.junit.Assert.assertFalse;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@ -31,10 +30,6 @@ import java.util.concurrent.TimeUnit;
import org.junit.BeforeClass;
import org.junit.Test;
import com.raytheon.uf.common.datadelivery.registry.Provider.ServiceType;
import com.raytheon.uf.common.datadelivery.retrieval.xml.Retrieval;
import com.raytheon.uf.common.datadelivery.retrieval.xml.RetrievalAttribute;
import com.raytheon.uf.common.localization.PathManagerFactoryTest;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalDao;
import com.raytheon.uf.edex.event.EventBusTest;
@ -50,6 +45,7 @@ import com.raytheon.uf.edex.event.EventBusTest;
* Jul 06, 2012 740 djohnson Initial creation
* Aug 09. 2012 1022 djohnson Changes to RetrievalHandler.
* Nov 19, 2012 1166 djohnson Clean up JAXB representation of registry objects.
* Jan 30, 2013 1543 djohnson RetrievalTask now requires a Network.
*
* </pre>
*
@ -69,8 +65,11 @@ public class RetrievalHandlerTest {
private final SubscriptionNotifyTask subNotifyTask = mock(SubscriptionNotifyTask.class);
private final IRetrievalResponseCompleter retrievalCompleter = mock(IRetrievalResponseCompleter.class);
private final RetrievalHandler handler = new RetrievalHandler(
executorService, mockDao, retrievalTask, subNotifyTask);
executorService, mockDao, Arrays.asList(retrievalTask),
subNotifyTask);
@BeforeClass
public static void classSetUp() {
@ -82,31 +81,45 @@ public class RetrievalHandlerTest {
verify(mockDao).resetRunningRetrievalsToPending();
}
@Test
public void testIllegalStateExceptionThrownDuringProcessWillReturnFalse() {
PathManagerFactoryTest.initLocalization();
Retrieval retrieval = new Retrieval() {
private static final long serialVersionUID = 1109443017002028345L;
@Override
public ArrayList<RetrievalAttribute> getAttribute() {
throw new IllegalStateException(EXCEPTION_MESSAGE);
}
/**
* {@inheritDoc}
*/
@Override
public ServiceType getServiceType() {
return ServiceType.OPENDAP;
}
};
RetrievalTask task = new RetrievalTask("someDestination", subNotifyTask);
assertFalse("Expected false when an IllegalStateException was thrown!",
task.process(retrieval));
}
// FIXME: Create PerformRetrievalPluginDataObjectsFinderTest and move this
// test there
// @Test
// public void testIllegalStateExceptionThrownDuringProcessWillReturnFalse()
// throws SerializationException {
//
// PathManagerFactoryTest.initLocalization();
// Retrieval retrieval = new Retrieval() {
// private static final long serialVersionUID = 1109443017002028345L;
//
// @Override
// public ArrayList<RetrievalAttribute> getAttribute() {
// throw new IllegalStateException(EXCEPTION_MESSAGE);
// }
//
// /**
// * {@inheritDoc}
// */
// @Override
// public ServiceType getServiceType() {
// return ServiceType.OPENDAP;
// }
// };
// RetrievalRequestRecord record = new RetrievalRequestRecord();
// try {
// record.setRetrievalObj(retrieval);
// } catch (NullPointerException npe) {
// // This is expected because we create an anonymous retrieval
// // instance, and can't dynamically serialize it
// }
//
// RetrievalTask task = new RetrievalTask(
// new PerformRetrievalPluginDataObjectsFinder(Network.OPSNET),
// new NotifyOfPluginDataObjectsDecorator(
// mock(IRetrievedDataProcessor.class)),
// retrievalCompleter);
// assertFalse("Expected false when an IllegalStateException was thrown!",
// task.process(record));
// }
@Test
public void testOnNotifyOfSubscriptionsARetrievalTaskIsExecuted() {

87
tests/unit/com/raytheon/uf/edex/datadelivery/retrieval/handlers/RetrievalPluginDataObjectsFixture.java Normal file
View file

@ -0,0 +1,87 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
import java.util.ArrayList;
import java.util.List;
import com.raytheon.uf.common.datadelivery.retrieval.xml.RetrievalAttribute;
import com.raytheon.uf.common.dataplugin.PluginDataObject;
import com.raytheon.uf.common.dataplugin.grid.GridRecord;
import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.common.util.AbstractFixture;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord;
/**
* Fixture for {@link RetrievalPluginDataObjects} instances.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Feb 01, 2013 1543 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public class RetrievalPluginDataObjectsFixture extends
AbstractFixture<RetrievalPluginDataObjects> {
public static final RetrievalPluginDataObjectsFixture INSTANCE = new RetrievalPluginDataObjectsFixture();
/**
* Prevent construction.
*/
private RetrievalPluginDataObjectsFixture() {
}
/**
* {@inheritDoc}
*/
@Override
public RetrievalPluginDataObjects get(long seedValue) {
RetrievalRequestRecord requestRecord = RetrievalRequestRecordFixture.INSTANCE
.get(seedValue);
List<RetrievalAttributePluginDataObjects> retrievalAttributePluginDataObjects = new ArrayList<RetrievalAttributePluginDataObjects>();
try {
for (RetrievalAttribute attribute : requestRecord.getRetrievalObj()
.getAttribute()) {
// TODO: GridRecordFixture
final GridRecord gridRecord = new GridRecord();
gridRecord.setDataURI("dataUri" + seedValue);
retrievalAttributePluginDataObjects
.add(new RetrievalAttributePluginDataObjects(attribute,
new PluginDataObject[] { gridRecord }));
}
} catch (SerializationException e) {
throw new RuntimeException(e);
}
final RetrievalPluginDataObjects retrievalPluginDataObjects = new RetrievalPluginDataObjects(requestRecord,
retrievalAttributePluginDataObjects);
return retrievalPluginDataObjects;
}
}

103
tests/unit/com/raytheon/uf/edex/datadelivery/retrieval/handlers/RetrievalRequestRecordFixture.java Normal file
View file

@ -0,0 +1,103 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
import com.raytheon.uf.common.datadelivery.registry.Provider;
import com.raytheon.uf.common.datadelivery.registry.ProviderFixture;
import com.raytheon.uf.common.datadelivery.registry.Subscription;
import com.raytheon.uf.common.datadelivery.registry.SubscriptionBundle;
import com.raytheon.uf.common.datadelivery.registry.SubscriptionFixture;
import com.raytheon.uf.common.datadelivery.retrieval.xml.Retrieval.SubscriptionType;
import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.common.serialization.SerializationUtil;
import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.common.util.AbstractFixture;
import com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.MockServiceFactory;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecordPK;
/**
* Fixture for {@link RetrievalRequestRecord}.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 30, 2013 1543 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public class RetrievalRequestRecordFixture extends
AbstractFixture<RetrievalRequestRecord> {
public static final RetrievalRequestRecordFixture INSTANCE = new RetrievalRequestRecordFixture();
/**
* Private constructor.
*/
private RetrievalRequestRecordFixture() {
}
/**
* {@inheritDoc}
*/
@Override
public RetrievalRequestRecord get(long seedValue) {
Subscription subscription = SubscriptionFixture.INSTANCE.get(seedValue);
final Provider provider = ProviderFixture.INSTANCE.get(seedValue);
SubscriptionBundle bundle = new SubscriptionBundle();
bundle.setBundleId(subscription.getSubscriptionId());
bundle.setPriority(1);
bundle.setProvider(provider);
bundle.setConnection(provider.getConnection());
bundle.setSubscription(subscription);
RetrievalRequestRecord rec = new RetrievalRequestRecord(
subscription.getName(), 0, seedValue);
rec.setId(new RetrievalRequestRecordPK(subscription.getName(), 0));
rec.setOwner(subscription.getOwner());
rec.setPriority(1);
rec.setInsertTime(TimeUtil.newDate());
rec.setNetwork(subscription.getRoute());
rec.setProvider(subscription.getProvider());
rec.setPlugin("grib");
rec.setSubscriptionType(SubscriptionType.SUBSCRIBED);
rec.setState(RetrievalRequestRecord.State.PENDING);
try {
rec.setRetrieval(SerializationUtil
.transformToThrift(new MockServiceFactory(provider)
.buildRetrieval(bundle).iterator().next()));
} catch (SerializationException e) {
throw new RuntimeException(e);
}
return rec;
}
}

238
tests/unit/com/raytheon/uf/edex/datadelivery/retrieval/handlers/RetrievalTaskTest.java Normal file
View file

@ -0,0 +1,238 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.mock;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import com.google.common.eventbus.Subscribe;
import com.raytheon.uf.common.datadelivery.event.retrieval.DataRetrievalEvent;
import com.raytheon.uf.common.datadelivery.registry.Network;
import com.raytheon.uf.common.dataplugin.PluginDataObject;
import com.raytheon.uf.common.localization.PathManagerFactoryTest;
import com.raytheon.uf.common.registry.handler.RegistryHandlerException;
import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.edex.database.DataAccessLayerException;
import com.raytheon.uf.edex.database.dao.DatabaseUtil;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalDao;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord;
import com.raytheon.uf.edex.datadelivery.retrieval.db.RetrievalRequestRecord.State;
import com.raytheon.uf.edex.event.EventBus;
import com.raytheon.uf.edex.event.EventBusTest;
/**
* Test {@link RetrievalTask}.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 30, 2013 1543 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public class RetrievalTaskTest {
/**
* Places the plugin data object into a collection for inspection.
*/
public static class PlaceInCollectionProcessor implements
IRetrievalPluginDataObjectsProcessor {
public final List<PluginDataObject> pluginDataObjects = new ArrayList<PluginDataObject>();
/**
* {@inheritDoc}
*/
@Override
public void processRetrievedPluginDataObjects(
RetrievalPluginDataObjects retrievalPluginDataObjects)
throws Exception {
final List<RetrievalAttributePluginDataObjects> retrievalAttributePluginDataObjects = retrievalPluginDataObjects
.getRetrievalAttributePluginDataObjects();
for (RetrievalAttributePluginDataObjects pluginDataObjectEntry : retrievalAttributePluginDataObjects) {
PluginDataObject[] value = pluginDataObjectEntry
.getPluginDataObjects();
pluginDataObjects.addAll(Arrays.asList(value));
}
}
}
private final RetrievalRequestRecord opsnetRetrieval = RetrievalRequestRecordFixture.INSTANCE
.get(1);
private final RetrievalRequestRecord sbnRetrieval = RetrievalRequestRecordFixture.INSTANCE
.get(2);
private RetrievalDao dao;
private final PlaceInCollectionProcessor retrievedDataProcessor = new PlaceInCollectionProcessor();
private final List<DataRetrievalEvent> eventsReceived = new ArrayList<DataRetrievalEvent>();
@BeforeClass
public static void classSetUp() {
EventBusTest.initSynchronous();
}
@Before
public void setUp() throws RegistryHandlerException {
DatabaseUtil.start();
PathManagerFactoryTest.initLocalization();
opsnetRetrieval.setNetwork(Network.OPSNET);
sbnRetrieval.setNetwork(Network.SBN);
dao = new RetrievalDao();
EventBus.getInstance().register(this);
}
@After
public void tearDown() {
DatabaseUtil.shutdown();
}
@Test
public void processesRetrievalForItsSpecifiedNetwork()
throws DataAccessLayerException {
stageRetrievals();
runRetrievalTask();
verifyCorrectStateForRetrieval(opsnetRetrieval, State.COMPLETED);
}
@Test
public void storesPluginDataObjectsForItsSpecifiedNetwork()
throws DataAccessLayerException, SerializationException {
stageRetrievals();
runRetrievalTask();
assertThat(retrievedDataProcessor.pluginDataObjects.size(),
is(equalTo(opsnetRetrieval.getRetrievalObj().getAttribute()
.size())));
}
@Test
public void dataRetrievalEventIsSentForItsSpecifiedNetwork()
throws Exception {
stageRetrievals();
runRetrievalTask();
assertThat(eventsReceived.size(), is(equalTo(opsnetRetrieval
.getRetrievalObj().getAttribute().size())));
// TODO: Is there a way to distinguish between the events sent by the
// separate retrieval attributes, e.g. to make sure each attribute sent
// an event and not one attribute sent two?
}
// TODO: Add tests for one retrieval failing and another succeeding, make
// sure correct events are sent and correct number of plugin data objects
// generated
@Test
public void doesNotProcessRetrievalForAnotherNetwork()
throws DataAccessLayerException {
stageRetrievals();
runRetrievalTask();
verifyCorrectStateForRetrieval(sbnRetrieval, State.PENDING);
}
/**
* Stage the retrievals in the database.
*/
private void stageRetrievals() {
dao.create(opsnetRetrieval);
dao.create(sbnRetrieval);
}
/**
* Run the actual retrieval task.
*/
private void runRetrievalTask() {
// Create required strategies for finding, processing, and completing
// retrievals
final IRetrievalPluginDataObjectsFinder retrievalDataFinder = new PerformRetrievalPluginDataObjectsFinder(
Network.OPSNET);
final IRetrievalPluginDataObjectsProcessor retrievalPluginDataObjectsProcessor = new NotifyOfPluginDataObjectsDecorator(
retrievedDataProcessor);
final IRetrievalResponseCompleter retrievalCompleter = new RetrievalResponseCompleter(
mock(SubscriptionNotifyTask.class));
new RetrievalTask(Network.OPSNET, retrievalDataFinder,
retrievalPluginDataObjectsProcessor, retrievalCompleter).run();
}
/**
* Verify the retrieval record is in the expected state.
*
* @param retrieval
* the retrieval
* @param state
* the expected state
* @throws DataAccessLayerException
*/
private void verifyCorrectStateForRetrieval(
RetrievalRequestRecord retrieval, State state)
throws DataAccessLayerException {
RetrievalRequestRecord recordInDb = dao
.getRequests(retrieval.getId().getSubscriptionName())
.iterator().next();
assertThat(recordInDb.getState(), is(equalTo(state)));
}
/**
* This method will be invoked by the EventBus when a data retrieval event
* is sent.
*
* @param event
* the event
*/
@Subscribe
public void receivedDataDeliveryEvent(DataRetrievalEvent event) {
eventsReceived.add(event);
}
}

67
tests/unit/com/raytheon/uf/edex/datadelivery/retrieval/handlers/SerializeRetrievedDataToDirectoryTest.java Normal file
View file

@ -0,0 +1,67 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.retrieval.handlers;
import static com.raytheon.uf.common.util.Matchers.hasNumberOfFiles;
import static org.junit.Assert.assertThat;
import java.io.File;
import org.junit.Test;
import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.common.util.TestUtil;
/**
* Test {@link SerializeRetrievedDataToDirectory}.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Feb 01, 2013 1543 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public class SerializeRetrievedDataToDirectoryTest {
private final File directory = TestUtil
.setupTestClassDir(SerializeRetrievedDataToDirectoryTest.class);
private final SerializeRetrievedDataToDirectory service = new SerializeRetrievedDataToDirectory(
directory);
@Test
public void serializesRetrievedDataToAFileInTheTargetDirectory()
throws SerializationException {
RetrievalPluginDataObjects retrievalPluginDataObjects = RetrievalPluginDataObjectsFixture.INSTANCE
.get();
service.processRetrievedPluginDataObjects(retrievalPluginDataObjects);
assertThat(directory, hasNumberOfFiles(1));
}
}