From 8cb100327756b2b07068fe234f984c682afbd9cc Mon Sep 17 00:00:00 2001 From: Dustin Johnson Date: Wed, 5 Jun 2013 11:15:48 -0500 Subject: [PATCH] Issue #2038 Add support for point data type subscriptions to BandwidthManager Amend: Peer review comments, improve time considerations for Point subscriptions Change-Id: I9dde7f7da9dab9c8f3163a87b1ebe64303fc7dc2 Former-commit-id: c677ae32d75079493ce84428c03632c63968d7f0 [formerly 7c4a6261fcb168774d4215d44a20207dc9abd053 [formerly f7ed3ec86188635eb069878ac490af06ea98e4a2]] Former-commit-id: 7c4a6261fcb168774d4215d44a20207dc9abd053 Former-commit-id: 63f8ff76c3ff7448d83f68b3c8d5f71b61b0b53c --- .../datadelivery/filter/MetaDataManager.java | 4 +- .../subscription/SubscriptionService.java | 58 +-- .../subset/PointTimeSubsetTab.java | 17 +- deltaScripts/13.5.1/updateProviderType.sh | 31 ++ deltaScripts/13.5.1/updateProviderType.xsl | 15 + .../bandwidth/IBandwidthRequest.java | 1 - .../datadelivery/registry/Collection.java | 7 +- .../DataDeliveryRegistryObjectTypes.java | 3 + .../datadelivery/registry/GriddedTime.java | 6 +- .../datadelivery/registry/PointTime.java | 16 +- .../datadelivery/registry/Provider.java | 71 +--- .../datadelivery/registry/ProviderType.java | 127 ++++++ .../registry/SubscriptionBundle.java | 11 +- .../uf/common/datadelivery/registry/Time.java | 6 +- .../datadelivery/retrieval/xml/Retrieval.java | 12 +- .../bandwidth-datadelivery-edex-impl.xml | 5 + .../res/spring/bandwidth-datadelivery.xml | 10 +- .../bandwidth/BandwidthManager.java | 363 +++++++++++------- .../bandwidth/IBandwidthManager.java | 13 - .../bandwidth/InMemoryBandwidthDao.java | 112 ++++-- .../bandwidth/dao/IBandwidthDao.java | 55 ++- .../hibernate/HibernateBandwidthDao.java | 27 +- .../hibernate/ISubscriptionRetrievalDao.java | 36 +- .../hibernate/SubscriptionRetrievalDao.java | 92 ++++- .../retrieval/SubscriptionRetrievalAgent.java | 6 +- .../bandwidth/util/BandwidthDaoUtil.java | 96 +++-- ...roviderDataTypeAvailabilityCalculator.java | 128 ++++++ .../crawler/FileCommunicationStrategy.java | 3 +- .../harvester/MADIS-harvester.xml | 2 +- .../harvester/NOMADS-harvester.xml | 80 ++-- .../harvester/OGC-harvester.xml.sample | 2 +- .../retrieval/RetrievalGenerationHandler.java | 10 +- .../opendap/OpenDAPRetrievalGenerator.java | 22 +- .../bandwidth/BandwidthManagerIntTest.java | 125 +++++- .../ebxml/dao/AbstractRegistryTest.java | 4 +- ...idth-datadelivery-integrationtest-impl.xml | 7 + .../registry/BaseSubscriptionFixture.java | 3 +- .../registry/PointDataSetMetaDataFixture.java | 82 ++++ .../registry/PointTimeFixture.java | 63 +++ .../registry/ProviderFixture.java | 6 +- .../registry/WFSPointDataSetFixture.java | 70 ++++ .../com/raytheon/uf/common/util/TestUtil.java | 4 + .../bandwidth/AbstractBandwidthDaoTest.java | 206 ++++++++-- .../bandwidth/HibernateBandwidthDaoTest.java | 6 + .../bandwidth/util/BandwidthDaoUtilTest.java | 71 +++- ...derDataTypeAvailabilityCalculatorTest.java | 111 ++++++ .../MockOpenDapRetrievalGenerator.java | 4 +- .../AbstractSubscriptionServiceTest.java | 23 +- .../SubscriptionServiceStoreAdhocTest.java | 12 + .../CreateSubscriptionPresenterTest.java | 6 +- 50 files changed, 1742 insertions(+), 508 deletions(-) create mode 100644 deltaScripts/13.5.1/updateProviderType.sh create mode 100644 deltaScripts/13.5.1/updateProviderType.xsl create mode 100644 edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/ProviderType.java create mode 100644 edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/ProviderDataTypeAvailabilityCalculator.java create mode 100644 tests/unit/com/raytheon/uf/common/datadelivery/registry/PointDataSetMetaDataFixture.java create mode 100644 tests/unit/com/raytheon/uf/common/datadelivery/registry/PointTimeFixture.java create mode 100644 tests/unit/com/raytheon/uf/common/datadelivery/registry/WFSPointDataSetFixture.java create mode 100644 tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/util/ProviderDataTypeAvailabilityCalculatorTest.java diff --git a/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/filter/MetaDataManager.java b/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/filter/MetaDataManager.java index 93c145e731..d8036611bd 100644 --- a/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/filter/MetaDataManager.java +++ b/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/filter/MetaDataManager.java @@ -37,7 +37,7 @@ import com.raytheon.uf.common.datadelivery.registry.DataLevelType.LevelType; import com.raytheon.uf.common.datadelivery.registry.DataSet; import com.raytheon.uf.common.datadelivery.registry.GriddedDataSet; import com.raytheon.uf.common.datadelivery.registry.Provider; -import com.raytheon.uf.common.datadelivery.registry.Provider.ProviderType; +import com.raytheon.uf.common.datadelivery.registry.ProviderType; import com.raytheon.uf.common.datadelivery.registry.handlers.DataDeliveryHandlers; import com.raytheon.uf.common.registry.handler.RegistryHandlerException; import com.raytheon.uf.common.status.IUFStatusHandler; @@ -171,7 +171,7 @@ public class MetaDataManager { .getAll()) { for (ProviderType type : provider.getProviderType()) { - typeSet.add(type.toString()); + typeSet.add(type.getDataType().toString()); } } } catch (RegistryHandlerException e) { diff --git a/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/SubscriptionService.java b/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/SubscriptionService.java index 706ee33493..30c489d5a7 100644 --- a/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/SubscriptionService.java +++ b/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/SubscriptionService.java @@ -575,37 +575,39 @@ public class SubscriptionService implements ISubscriptionService { throws RegistryHandlerException { for (Subscription subscription : subscriptions) { - final ISubscriptionHandler subscriptionHandler = DataDeliveryHandlers - .getSubscriptionHandler(); - final List potentialDuplicates = subscriptionHandler - .getActiveByDataSetAndProvider( - subscription.getDataSetName(), - subscription.getProvider()); - List overlappingSubscriptions = Lists.newArrayList(); - for (Subscription potentialDuplicate : potentialDuplicates) { - final ISubscriptionOverlapResponse overlapResponse = subscriptionOverlapService - .isOverlapping(potentialDuplicate, subscription); - final String potentialDuplicateName = potentialDuplicate - .getName(); - if (overlapResponse.isDuplicate()) { - return new SubscriptionServiceResult(true, - "This subscription would be an exact duplicate of " - + potentialDuplicateName); + if (!(subscription instanceof AdhocSubscription)) { + final ISubscriptionHandler subscriptionHandler = DataDeliveryHandlers + .getSubscriptionHandler(); + final List potentialDuplicates = subscriptionHandler + .getActiveByDataSetAndProvider( + subscription.getDataSetName(), + subscription.getProvider()); + List overlappingSubscriptions = Lists.newArrayList(); + for (Subscription potentialDuplicate : potentialDuplicates) { + final ISubscriptionOverlapResponse overlapResponse = subscriptionOverlapService + .isOverlapping(potentialDuplicate, subscription); + final String potentialDuplicateName = potentialDuplicate + .getName(); + if (overlapResponse.isDuplicate()) { + return new SubscriptionServiceResult(true, + "This subscription would be an exact duplicate of " + + potentialDuplicateName); + } + if (overlapResponse.isOverlapping()) { + overlappingSubscriptions.add(potentialDuplicateName); + } } - if (overlapResponse.isOverlapping()) { - overlappingSubscriptions.add(potentialDuplicateName); + if (!overlappingSubscriptions.isEmpty()) { + Collections.sort(overlappingSubscriptions); + forceApplyPrompt + .displayMessage( + displayTextStrategy, + StringUtil + .createMessage( + ISubscriptionOverlapService.OVERLAPPING_SUBSCRIPTIONS, + overlappingSubscriptions)); } } - if (!overlappingSubscriptions.isEmpty()) { - Collections.sort(overlappingSubscriptions); - forceApplyPrompt - .displayMessage( - displayTextStrategy, - StringUtil - .createMessage( - ISubscriptionOverlapService.OVERLAPPING_SUBSCRIPTIONS, - overlappingSubscriptions)); - } } try { diff --git a/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/subset/PointTimeSubsetTab.java b/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/subset/PointTimeSubsetTab.java index 4606769110..4431194428 100644 --- a/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/subset/PointTimeSubsetTab.java +++ b/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/subset/PointTimeSubsetTab.java @@ -19,6 +19,10 @@ **/ package com.raytheon.uf.viz.datadelivery.subscription.subset; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + import org.eclipse.swt.SWT; import org.eclipse.swt.layout.GridData; import org.eclipse.swt.layout.GridLayout; @@ -28,6 +32,7 @@ import org.eclipse.swt.widgets.Group; import org.eclipse.swt.widgets.Label; import org.eclipse.swt.widgets.Shell; +import com.raytheon.uf.common.datadelivery.registry.PointTime; import com.raytheon.uf.viz.datadelivery.subscription.subset.presenter.IPointDataTimingSubsetView; /** @@ -40,6 +45,7 @@ import com.raytheon.uf.viz.datadelivery.subscription.subset.presenter.IPointData * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * May 29, 2013 223 mpduff Initial creation. + * Jun 06, 2013 2038 djohnson Place refresh intervals into PointTime so BandwidthManager has access. * * * @@ -51,7 +57,7 @@ public class PointTimeSubsetTab extends DataTimingSubsetTab implements IPointDataTimingSubsetView { /** Data Retrieval Intervals */ - private final String[] INTERVALS = new String[] { "5", "10", "15", "30" }; + private final String[] INTERVALS; /** Retrieval Interval Selection Combo */ private Combo intervalCombo; @@ -69,6 +75,15 @@ public class PointTimeSubsetTab extends DataTimingSubsetTab implements public PointTimeSubsetTab(Composite parentComp, IDataSize callback, Shell shell) { super(parentComp, callback, shell); + + // Use the string version of the refresh intervals found in PointTime + final List allowedRefreshIntervals = new ArrayList(); + for (Iterator iter = PointTime.getAllowedRefreshIntervals() + .iterator(); iter.hasNext();) { + allowedRefreshIntervals.add(Integer.toString(iter.next())); + } + INTERVALS = allowedRefreshIntervals + .toArray(new String[allowedRefreshIntervals.size()]); } /* diff --git a/deltaScripts/13.5.1/updateProviderType.sh b/deltaScripts/13.5.1/updateProviderType.sh new file mode 100644 index 0000000000..8c88d57409 --- /dev/null +++ b/deltaScripts/13.5.1/updateProviderType.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +XSLT_SCRIPT="updateProviderType.xsl" +# ensure that the xslt script is present +if [ ! -f ${XSLT_SCRIPT} ]; then + echo "ERROR: the required xslt script - ${XSLT_SCRIPT} was not found." + echo "FATAL: the update has failed!" + exit 1 +fi + +echo "INFO: update started - updating ProviderType to be a class proper" + +# Update subscription manager configuration files +for FILE in `find /awips2/edex/data/utility/common_static -iname \*-harvester.xml` +do + cp $FILE $FILE.bak + xsltproc ${XSLT_SCRIPT} ${FILE}.bak > ${FILE} + + # Make sure each command succeeds + if [ $? -ne 0 ]; then + echo "FATAL: the update has failed!" + exit 1 + fi + + # Delete the md5 file + rm $FILE.md5 +done + +echo "INFO: the update has completed successfully!" + +exit 0 diff --git a/deltaScripts/13.5.1/updateProviderType.xsl b/deltaScripts/13.5.1/updateProviderType.xsl new file mode 100644 index 0000000000..0d0b32b9a6 --- /dev/null +++ b/deltaScripts/13.5.1/updateProviderType.xsl @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/edexOsgi/com.raytheon.uf.common.datadelivery.bandwidth/src/com/raytheon/uf/common/datadelivery/bandwidth/IBandwidthRequest.java b/edexOsgi/com.raytheon.uf.common.datadelivery.bandwidth/src/com/raytheon/uf/common/datadelivery/bandwidth/IBandwidthRequest.java index 7e55d113de..d055c296dd 100644 --- a/edexOsgi/com.raytheon.uf.common.datadelivery.bandwidth/src/com/raytheon/uf/common/datadelivery/bandwidth/IBandwidthRequest.java +++ b/edexOsgi/com.raytheon.uf.common.datadelivery.bandwidth/src/com/raytheon/uf/common/datadelivery/bandwidth/IBandwidthRequest.java @@ -33,7 +33,6 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; public class IBandwidthRequest extends AbstractPrivilegedRequest { public static enum RequestType { - METADATA_UPDATE, // Get the current retrieval plan for the // specified begin and end time. RETRIEVAL_PLAN, SHOW_ALLOCATION, diff --git a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Collection.java b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Collection.java index fddc811c98..9e4b732674 100644 --- a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Collection.java +++ b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Collection.java @@ -15,7 +15,6 @@ import javax.xml.bind.annotation.XmlEnumValue; import javax.xml.bind.annotation.XmlRootElement; import com.raytheon.uf.common.datadelivery.registry.Projection.ProjectionType; -import com.raytheon.uf.common.datadelivery.registry.Provider.ProviderType; import com.raytheon.uf.common.serialization.ISerializableObject; import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; @@ -96,7 +95,7 @@ public class Collection implements ISerializableObject { @XmlAttribute @DynamicSerializeElement - private ProviderType dataType; + private DataType dataType; @XmlAttribute @DynamicSerializeElement @@ -170,7 +169,7 @@ public class Collection implements ISerializableObject { return format; } - public ProviderType getDataType() { + public DataType getDataType() { return dataType; } @@ -353,7 +352,7 @@ public class Collection implements ISerializableObject { return ignore; } - public void setDataType(ProviderType dataType) { + public void setDataType(DataType dataType) { this.dataType = dataType; } diff --git a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/DataDeliveryRegistryObjectTypes.java b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/DataDeliveryRegistryObjectTypes.java index b1dd779a4a..ac882f809c 100644 --- a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/DataDeliveryRegistryObjectTypes.java +++ b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/DataDeliveryRegistryObjectTypes.java @@ -58,4 +58,7 @@ public final class DataDeliveryRegistryObjectTypes { public static final String DATASET = RegistryUtil .getObjectType(DataSet.class); + + public static final String PROVIDER = RegistryUtil + .getObjectType(Provider.class); } diff --git a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/GriddedTime.java b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/GriddedTime.java index 9500aa8ffe..b812fce1fd 100644 --- a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/GriddedTime.java +++ b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/GriddedTime.java @@ -52,6 +52,7 @@ import com.raytheon.uf.common.status.UFStatus.Priority; * Jan 15, 2011 dhladky Initial creation * Jul 24, 2012 955 djohnson Use List instead of ArrayList. * Jun 04, 2013 223 mpduff Cleanup. + * Jun 06, 2013 2038 djohnson Remove throws ParseException. * * * @@ -512,8 +513,7 @@ public class GriddedTime extends Time implements ISerializableObject, * @throws ParseException */ @Override - public void setRequestStartAsDate(Date requestStartDate) - throws ParseException { + public void setRequestStartAsDate(Date requestStartDate) { this.requestStartDate = requestStartDate; if (requestStartDate != null && getFormat() != null) { SimpleDateFormat dateFormat = new SimpleDateFormat(getFormat()); @@ -548,7 +548,7 @@ public class GriddedTime extends Time implements ISerializableObject, * @throws ParseException */ @Override - public void setRequestEndAsDate(Date requestEndDate) throws ParseException { + public void setRequestEndAsDate(Date requestEndDate) { this.requestEndDate = requestEndDate; if (requestEndDate != null && getFormat() != null) { SimpleDateFormat dateFormat = new SimpleDateFormat(getFormat()); diff --git a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/PointTime.java b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/PointTime.java index d8ec412bb5..6b0f0fb13e 100644 --- a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/PointTime.java +++ b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/PointTime.java @@ -21,8 +21,10 @@ package com.raytheon.uf.common.datadelivery.registry; **/ import java.io.Serializable; +import java.util.Arrays; import java.util.Date; import java.util.List; +import java.util.SortedSet; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -30,6 +32,7 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlElements; import javax.xml.bind.annotation.XmlRootElement; +import com.google.common.collect.Sets; import com.raytheon.uf.common.serialization.ISerializableObject; import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; @@ -56,9 +59,6 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; public class PointTime extends Time implements ISerializableObject, Serializable { - /** - * - */ private static final long serialVersionUID = 234624356321L; @XmlElement @@ -128,4 +128,14 @@ public class PointTime extends Time implements ISerializableObject, public void setInterval(int interval) { this.interval = interval; } + + /** + * Get the allowed refresh intervals. This should be a configurable value at + * some point. + * + * @return the allowed refresh intervals + */ + public static SortedSet getAllowedRefreshIntervals() { + return Sets.newTreeSet(Arrays.asList(5, 10, 15, 30)); + } } diff --git a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Provider.java b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Provider.java index 1813b0df31..a50828c0d6 100644 --- a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Provider.java +++ b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Provider.java @@ -3,15 +3,12 @@ package com.raytheon.uf.common.datadelivery.registry; import static com.google.common.base.Preconditions.checkNotNull; import java.util.List; -import java.util.regex.Pattern; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlElements; -import javax.xml.bind.annotation.XmlEnum; -import javax.xml.bind.annotation.XmlEnumValue; import javax.xml.bind.annotation.XmlRootElement; import com.raytheon.uf.common.registry.annotations.RegistryObject; @@ -52,58 +49,6 @@ import com.raytheon.uf.common.time.domain.api.IDuration; @RegistryObject({ "name" }) public class Provider implements ISerializableObject { - /** - * Enumeration of provider types, we know so far - * - * 
-     * 
-     * SOFTWARE HISTORY
-     * 
-     * Date         Ticket#    Engineer    Description
-     * ------------ ---------- ----------- --------------------------
-     * Feb 16, 2012            dhladky      Initial creation
-     * Nov 19, 2012 1166       djohnson     Clean up JAXB representation of registry objects.
-     * Jan 14, 2013 1286       djohnson     Extracted {@link IDuration}.
-     * 
-     *
- * - * @author dhladky - * @version 1.0 - */ - @XmlEnum - public enum ProviderType { - @XmlEnumValue(ProviderType.GRID_STRING_VALUE) - GRID(ProviderType.GRID_STRING_VALUE, "grid"), @XmlEnumValue(ProviderType.POINT_STRING_VALUE) - POINT(ProviderType.POINT_STRING_VALUE, "point"); - - private static final String GRID_STRING_VALUE = "Grid"; - - private static final String POINT_STRING_VALUE = "Point"; - - private final String providerType; - - private final String plugin; - - private ProviderType(String name, String plugin) { - providerType = name; - this.plugin = plugin; - } - - @Override - public String toString() { - return providerType; - } - - /** - * Return the name of the plugin responsible for the type of data. - * - * @return the plugin name - */ - public String getPlugin() { - return plugin; - } - } - /** * Service Type * @@ -146,9 +91,6 @@ public class Provider implements ISerializableObject { private static final Integer BYTES_IN_FLOAT = Float.SIZE / Byte.SIZE; - private static final Pattern POSTED_FILE_DELAY_PATTERN = Pattern - .compile("\\s*(\\d+)\\s+([^\\s]+)\\s*"); - @XmlAttribute(name = "name", required = true) @DynamicSerializeElement @SlotAttribute @@ -323,4 +265,17 @@ public class Provider implements ISerializableObject { this.timeBetweenCrawlRequests = timeBetweenCrawlRequests; } + /** + * @param dataSetType + * @return + */ + public ProviderType getProviderType(DataType dataSetType) { + for (ProviderType providerType : getProviderType()) { + if (providerType.getDataType().equals(dataSetType)) { + return providerType; + } + } + return null; + } + } diff --git a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/ProviderType.java b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/ProviderType.java new file mode 100644 index 0000000000..4d18e348fa --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/ProviderType.java @@ -0,0 +1,127 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.common.datadelivery.registry; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlRootElement; + +import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; +import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; + +/** + * Provider configuration for a supported data type. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Jun 07, 2013 2038       djohnson     Initial creation
+ * 
+ *
+ * + * @author djohnson + * @version 1.0 + */ +@XmlRootElement +@XmlAccessorType(XmlAccessType.NONE) +@DynamicSerialize +public class ProviderType { + + @XmlAttribute + @DynamicSerializeElement + private DataType dataType; + + @XmlAttribute + @DynamicSerializeElement + private String plugin; + + @XmlAttribute + @DynamicSerializeElement + private int availabilityDelay; + + /** + * Constructor. + */ + public ProviderType() { + } + + /** + * Convenience constructor. + * + * @param dataType + * @param plugin + * @param availabilityDelay + */ + public ProviderType(DataType dataType, String plugin, int availabilityDelay) { + this.dataType = dataType; + this.plugin = plugin; + this.availabilityDelay = availabilityDelay; + } + + /** + * @return the dataType + */ + public DataType getDataType() { + return dataType; + } + + /** + * @param dataType + * the dataType to set + */ + public void setDataType(DataType dataType) { + this.dataType = dataType; + } + + /** + * @return the plugin + */ + public String getPlugin() { + return plugin; + } + + /** + * @param plugin + * the plugin to set + */ + public void setPlugin(String plugin) { + this.plugin = plugin; + } + + /** + * @return the availabilityDelay + */ + public int getAvailabilityDelay() { + return availabilityDelay; + } + + /** + * @param availabilityDelay + * the availabilityDelay to set + */ + public void setAvailabilityDelay(int availabilityDelay) { + this.availabilityDelay = availabilityDelay; + } + +} diff --git a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/SubscriptionBundle.java b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/SubscriptionBundle.java index 8c486d2c87..ef5d544445 100644 --- a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/SubscriptionBundle.java +++ b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/SubscriptionBundle.java @@ -2,7 +2,6 @@ package com.raytheon.uf.common.datadelivery.registry; import java.util.List; -import com.raytheon.uf.common.datadelivery.registry.Provider.ProviderType; import com.raytheon.uf.common.serialization.ISerializableObject; import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; @@ -107,21 +106,19 @@ public class SubscriptionBundle implements ISerializableObject { * * @return the type */ - public ProviderType getDataType() { - ProviderType pt = null; + public DataType getDataType() { if (subscription != null) { if (subscription.getCoverage() instanceof GriddedCoverage) { - pt = ProviderType.GRID; + return DataType.GRID; } // TODO: Add more data types, currently defaulting to POINT only if // not a GriddedCoverage, when there could be other data types than // just Grid/Point else { - pt = ProviderType.POINT; + return DataType.POINT; } } - - return pt; + return null; } } diff --git a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Time.java b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Time.java index f1dce5bb5d..6c531bbf2d 100644 --- a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Time.java +++ b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/Time.java @@ -55,6 +55,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; * Sep 28, 2012 1187 djohnson {@link #setEndDate(Date)} was incorrectly changing the start date. * Nov 19, 2012 1166 djohnson Clean up JAXB representation of registry objects. * Jun 04, 2013 223 mpduff Added interval field. + * Jun 06, 2013 2038 djohnson Remove throws ParseException. * * * @@ -599,8 +600,7 @@ public class Time implements ISerializableObject, Serializable { * @return * @throws ParseException */ - public void setRequestStartAsDate(Date requestStartDate) - throws ParseException { + public void setRequestStartAsDate(Date requestStartDate) { this.requestStartDate = requestStartDate; if (requestStartDate != null && getFormat() != null) { SimpleDateFormat dateFormat = new SimpleDateFormat(getFormat()); @@ -633,7 +633,7 @@ public class Time implements ISerializableObject, Serializable { * @return * @throws ParseException */ - public void setRequestEndAsDate(Date requestEndDate) throws ParseException { + public void setRequestEndAsDate(Date requestEndDate) { this.requestEndDate = requestEndDate; if (requestEndDate != null && getFormat() != null) { SimpleDateFormat dateFormat = new SimpleDateFormat(getFormat()); diff --git a/edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/xml/Retrieval.java b/edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/xml/Retrieval.java index 237d6d4342..9c47bf08aa 100644 --- a/edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/xml/Retrieval.java +++ b/edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/xml/Retrieval.java @@ -33,8 +33,8 @@ import javax.xml.bind.annotation.XmlEnumValue; import javax.xml.bind.annotation.XmlRootElement; import com.raytheon.uf.common.datadelivery.registry.Connection; +import com.raytheon.uf.common.datadelivery.registry.DataType; import com.raytheon.uf.common.datadelivery.registry.Network; -import com.raytheon.uf.common.datadelivery.registry.Provider.ProviderType; import com.raytheon.uf.common.datadelivery.registry.Provider.ServiceType; import com.raytheon.uf.common.serialization.ISerializableObject; import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; @@ -83,7 +83,7 @@ public class Retrieval implements ISerializableObject, Serializable { @XmlElement @DynamicSerializeElement - private ProviderType providerType; + private DataType dataType; @XmlElement @DynamicSerializeElement @@ -171,12 +171,12 @@ public class Retrieval implements ISerializableObject, Serializable { this.subscriptionName = subscriptionName; } - public void setProviderType(ProviderType providerType) { - this.providerType = providerType; + public void setDataType(DataType providerType) { + this.dataType = providerType; } - public ProviderType getProviderType() { - return providerType; + public DataType getDataType() { + return dataType; } public void setSubscriptionType(SubscriptionType subscriptionType) { diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/res/spring/bandwidth-datadelivery-edex-impl.xml b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/res/spring/bandwidth-datadelivery-edex-impl.xml index 292e7cf6ca..dd1eb3df79 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/res/spring/bandwidth-datadelivery-edex-impl.xml +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/res/spring/bandwidth-datadelivery-edex-impl.xml @@ -44,6 +44,11 @@ + + + + diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/res/spring/bandwidth-datadelivery.xml b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/res/spring/bandwidth-datadelivery.xml index 2a2dbc45c4..074b05a3ae 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/res/spring/bandwidth-datadelivery.xml +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/res/spring/bandwidth-datadelivery.xml @@ -1,6 +1,6 @@ + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd"> @@ -39,12 +39,6 @@ - - - - diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/BandwidthManager.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/BandwidthManager.java index 39003a155e..86e161f68b 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/BandwidthManager.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/BandwidthManager.java @@ -7,7 +7,6 @@ import java.util.Calendar; import java.util.Collections; import java.util.Date; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.SortedSet; @@ -21,6 +20,7 @@ import java.util.regex.Pattern; import org.springframework.context.support.ClassPathXmlApplicationContext; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.eventbus.AllowConcurrentEvents; import com.google.common.eventbus.Subscribe; @@ -35,11 +35,14 @@ import com.raytheon.uf.common.datadelivery.bandwidth.data.BandwidthGraphData; import com.raytheon.uf.common.datadelivery.registry.AdhocSubscription; import com.raytheon.uf.common.datadelivery.registry.DataDeliveryRegistryObjectTypes; import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData; +import com.raytheon.uf.common.datadelivery.registry.DataType; import com.raytheon.uf.common.datadelivery.registry.GriddedDataSetMetaData; import com.raytheon.uf.common.datadelivery.registry.Network; +import com.raytheon.uf.common.datadelivery.registry.PointDataSetMetaData; +import com.raytheon.uf.common.datadelivery.registry.PointTime; +import com.raytheon.uf.common.datadelivery.registry.SiteSubscription; import com.raytheon.uf.common.datadelivery.registry.Subscription; import com.raytheon.uf.common.datadelivery.registry.Time; -import com.raytheon.uf.common.datadelivery.registry.SiteSubscription; import com.raytheon.uf.common.datadelivery.registry.handlers.DataDeliveryHandlers; import com.raytheon.uf.common.event.EventBus; import com.raytheon.uf.common.registry.event.InsertRegistryEvent; @@ -52,6 +55,7 @@ import com.raytheon.uf.common.status.UFStatus; import com.raytheon.uf.common.status.UFStatus.Priority; import com.raytheon.uf.common.time.util.ITimer; import com.raytheon.uf.common.time.util.TimeUtil; +import com.raytheon.uf.common.util.CollectionUtil; import com.raytheon.uf.common.util.FileUtil; import com.raytheon.uf.common.util.IFileModifiedWatcher; import com.raytheon.uf.common.util.LogUtil; @@ -110,6 +114,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.util.BandwidthUtil; * Mar 28, 2013 1841 djohnson Subscription is now UserSubscription. * May 02, 2013 1910 djohnson Shutdown proposed bandwidth managers in a finally. * May 20, 2013 1650 djohnson Add in capability to find required dataset size. + * Jun 03, 2013 2038 djohnson Add base functionality to handle point data type subscriptions. * * * @author dhladky @@ -252,22 +257,102 @@ public abstract class BandwidthManager extends } /** - * {@inheritDoc} + * Process a {@link GriddedDataSetMetaData} that was received from the event + * bus. + * + * @param dataSetMetaData + * the metadadata */ - @Override @Subscribe - public void updateDataSetMetaData(DataSetMetaData dataSetMetaData) + public void updateGriddedDataSetMetaData( + GriddedDataSetMetaData dataSetMetaData) throws ParseException { - if (dataSetMetaData instanceof GriddedDataSetMetaData) { - GriddedDataSetMetaData gdsmd = (GriddedDataSetMetaData) dataSetMetaData; + // Daily/Hourly/Monthly datasets + if (dataSetMetaData.getCycle() == GriddedDataSetMetaData.NO_CYCLE) { + updateDataSetMetaDataWithoutCycle(dataSetMetaData); + } + // Regular cycle containing datasets + else { + updateDataSetMetaDataWithCycle(dataSetMetaData); + } + } - // Daily/Hourly/Monthly datasets - if (gdsmd.getCycle() == GriddedDataSetMetaData.NO_CYCLE) { - updateDataSetMetaDataWithoutCycle(gdsmd); - } - // Regular cycle containing datasets - else { - updateDataSetMetaDataWithCycle(gdsmd); + /** + * Process a {@link PointDataSetMetaData} that was received from the event + * bus. + * + * @param dataSetMetaData + * the metadadata + */ + @Subscribe + public void updatePointDataSetMetaData(PointDataSetMetaData dataSetMetaData) { + // TODO: Change PointDataSetMetaData to only be able to use PointTime + // objects + final PointTime time = (PointTime) dataSetMetaData.getTime(); + final String providerName = dataSetMetaData.getProviderName(); + final String dataSetName = dataSetMetaData.getDataSetName(); + final Date pointTimeStart = time.getStartDate(); + final Date pointTimeEnd = time.getEndDate(); + + final SortedSet allowedRefreshIntervals = PointTime + .getAllowedRefreshIntervals(); + final long maxAllowedRefreshIntervalInMillis = TimeUtil.MILLIS_PER_MINUTE + * allowedRefreshIntervals.last(); + final long minAllowedRefreshIntervalInMillis = TimeUtil.MILLIS_PER_MINUTE + * allowedRefreshIntervals.first(); + + // Find any retrievals ranging from those with the minimum refresh + // interval to the maximum refresh interval + final Date startDate = new Date(pointTimeStart.getTime() + + minAllowedRefreshIntervalInMillis); + final Date endDate = new Date(pointTimeEnd.getTime() + + maxAllowedRefreshIntervalInMillis); + + final SortedSet subscriptionRetrievals = bandwidthDao + .getSubscriptionRetrievals(providerName, dataSetName, + RetrievalStatus.SCHEDULED, startDate, endDate); + + if (!CollectionUtil.isNullOrEmpty(subscriptionRetrievals)) { + for (SubscriptionRetrieval retrieval : subscriptionRetrievals) { + // Now check and make sure that at least one of the times falls + // in their retrieval range, their latency is the retrieval + // interval + final int retrievalInterval = retrieval + .getSubscriptionLatency(); + + // This is the latest time on the data we care about, once the + // retrieval is signaled to go it retrieves everything up to + // its start time + final Date latestRetrievalDataTime = retrieval.getStartTime() + .getTime(); + // This is the earliest possible time this retrieval cares about + final Date earliestRetrievalDataTime = new Date( + latestRetrievalDataTime.getTime() + - (TimeUtil.MILLIS_PER_MINUTE * retrievalInterval)); + + // If the end is before any times we care about or the start is + // after the latest times we care about, skip it + if (pointTimeEnd.before(earliestRetrievalDataTime) + || pointTimeStart.after(latestRetrievalDataTime)) { + continue; + } + + try { + // Update the retrieval times on the subscription object + // which goes through the retrieval process + final Subscription subscription = retrieval + .getSubscription(); + final Time subTime = subscription.getTime(); + subTime.setRequestStartAsDate(earliestRetrievalDataTime); + subTime.setRequestEndAsDate(latestRetrievalDataTime); + + // Now update the retrieval to be ready + retrieval.setStatus(RetrievalStatus.READY); + bandwidthDaoUtil.update(retrieval); + } catch (SerializationException e) { + statusHandler.handle(Priority.PROBLEM, + e.getLocalizedMessage(), e); + } } } } @@ -420,14 +505,50 @@ public abstract class BandwidthManager extends private List schedule(Subscription subscription, SortedSet cycles) { - List unscheduled = new ArrayList(); SortedSet retrievalTimes = bandwidthDaoUtil .getRetrievalTimes(subscription, cycles); + return scheduleSubscriptionForRetrievalTimes(subscription, + retrievalTimes); + } + + /** + * Schedules a subscription that specifies a retrieval interval, rather than + * cycle times. + * + * @param subscription + * the subscription + * @param retrievalInterval + * the retrieval interval + * @return the list of unscheduled subscriptions + */ + private List schedule(Subscription subscription, + int retrievalInterval) { + SortedSet retrievalTimes = bandwidthDaoUtil + .getRetrievalTimes(subscription, retrievalInterval); + + return scheduleSubscriptionForRetrievalTimes(subscription, + retrievalTimes); + } + + /** + * Schedule the given subscription for the specified retrieval times. + * + * @param subscription + * the subscription + * @param retrievalTimes + * the retrieval times + * @return the unscheduled subscriptions + */ + private List scheduleSubscriptionForRetrievalTimes( + Subscription subscription, SortedSet retrievalTimes) { + if (retrievalTimes.isEmpty()) { - return unscheduled; + return Collections.emptyList(); } + List unscheduled = Lists.newArrayList(); + for (Calendar retrievalTime : retrievalTimes) { // Retrieve all the current subscriptions by provider, dataset name @@ -604,9 +725,26 @@ public abstract class BandwidthManager extends */ @Override public List schedule(Subscription subscription) { - SortedSet cycles = new TreeSet(subscription.getTime() - .getCycleTimes()); - List unscheduled = schedule(subscription, cycles); + // TODO: In 13.6.1 pull out all of the subscription stuff into a + // separate plugin, BandwidthManager should not work with Subscription + // objects directly, it should have extension plugins that can allocate + // bandwidth in their own types (e.g. registry syncing should be able to + // sync into the bandwidth management infrastructure if required) + List unscheduled; + + final DataType dataSetType = subscription.getDataSetType(); + switch (dataSetType) { + case GRID: + unscheduled = handleGridded(subscription); + break; + case POINT: + unscheduled = handlePoint(subscription); + break; + default: + throw new IllegalArgumentException( + "The BandwidthManager doesn't know how to treat subscriptions with data type [" + + dataSetType + "]!"); + } unscheduleSubscriptionsForAllocations(unscheduled); @@ -737,137 +875,78 @@ public abstract class BandwidthManager extends // it's active, attempt to add it.. if (bandwidthSubscriptions.isEmpty() && subscription.isActive() && !subscription.isUnscheduled()) { - final boolean subscribedToCycles = !subscription.getTime() - .getCycleTimes().isEmpty(); - final boolean useMostRecentDataSetUpdate = !subscribedToCycles; - - // The subscription has cycles, so we can allocate bandwidth at - // expected times - List unscheduled = Collections.emptyList(); - if (subscribedToCycles) { - unscheduled = schedule(subscription); - } - - // Create an adhoc subscription based on the new subscription, - // and set it to retrieve the most recent cycle (or most recent - // url if a daily product) - if (subscription instanceof SiteSubscription) { - AdhocSubscription adhoc = new AdhocSubscription( - (SiteSubscription) subscription); - adhoc = bandwidthDaoUtil.setAdhocMostRecentUrlAndTime( - adhoc, useMostRecentDataSetUpdate); - - if (adhoc == null) { - statusHandler - .info(String - .format("There wasn't applicable most recent dataset metadata to use for new subscription [%s]. " - + "No adhoc requested.", - subscription.getName())); - } else { - unscheduled = schedule(adhoc); - } - } else { - statusHandler - .warn("Unable to create adhoc queries for shared subscriptions at this point. This functionality should be added in the future..."); - } - return unscheduled; + return schedule(subscription); } else if (!subscription.isActive() || subscription.isUnscheduled()) { // See if the subscription was inactivated or unscheduled.. // Need to remove BandwidthReservations for this // subscription. return remove(bandwidthSubscriptions); } else { - - // Compare the 'updated' Subscription with the stored - // SubscriptionDaos to determine - // if the changes made to the Subscription would affect - // BandwidthReservations - // already in place for this subscription. - - Subscription old = bandwidthSubscriptions.get(0) - .getSubscription(); - - // Check to see if estimated size changed. If there was a change - // to - // which parameters or levels or coverage or forecast hours, - // those - // don't effect BandwidthReservations so there is no need to - // change the - // RetrievalPlan as long as the size stays the same - - if (BandwidthUtil.subscriptionRequiresReschedule(subscription, - old)) { - - // OK, have to remove the old Subscriptions and add the new - // ones.. - List unscheduled = remove(bandwidthSubscriptions); - // No need to check anything else since all the - // BandwidthSubscription's have been replaced. - unscheduled.addAll(schedule(subscription)); - return unscheduled; - } - - List unscheduled = new ArrayList(); - // Check that the cycles in both subscriptions are the same - SortedSet newCycles = new TreeSet( - subscription.getTime().getCycleTimes()); - SortedSet oldCycles = new TreeSet(old - .getTime().getCycleTimes()); - - if (newCycles.size() != oldCycles.size() - || !newCycles.containsAll(oldCycles) - || !oldCycles.containsAll(newCycles)) { - // Cycle times have changed, reschedule. - - // Create a Set of the common elements.. - Set commonCycles = Sets - .union(oldCycles, newCycles); - - // Remove the common elements from the old cycles, these - // need to be removed from the RetrievalPlan.. - oldCycles.removeAll(commonCycles); - - // Remove the common elements from the new cycles, these - // need to be added to the RetrievalPlan.. - newCycles.removeAll(commonCycles); - - // Remove the old cycles, add the new ones... - if (!oldCycles.isEmpty()) { - // Create a List of SubscriptionDaos that need to be - // removed.. - List bandwidthSubscriptionToRemove = new ArrayList(); - BandwidthSubscription bandwidthSubscription = null; - Iterator itr = bandwidthSubscriptions - .iterator(); - while (itr.hasNext()) { - bandwidthSubscription = itr.next(); - if (oldCycles.contains(bandwidthSubscription - .getCycle())) { - bandwidthSubscriptionToRemove - .add(bandwidthSubscription); - itr.remove(); - } - } - unscheduled - .addAll(remove(bandwidthSubscriptionToRemove)); - } - - if (!newCycles.isEmpty()) { - unscheduled.addAll(schedule(subscription, newCycles)); - } - } - - // Update the remaining dao's with the current subscription... - for (BandwidthSubscription bandwidthSubscription : bandwidthSubscriptions) { - bandwidthSubscription.setSubscription(subscription); - bandwidthDao.update(bandwidthSubscription); - } - + // Normal update, unschedule old allocations and create new ones + List unscheduled = remove(bandwidthSubscriptions); + unscheduled.addAll(schedule(subscription)); return unscheduled; } } } + /** + * Handle scheduling point data type subscriptions. + * + * @param subscription + * the subscription + * @return the list of unscheduled subscriptions + */ + private List handlePoint(Subscription subscription) { + return schedule(subscription, + ((PointTime) subscription.getTime()).getInterval()); + } + + /** + * Handle scheduling grid data type subscriptions. + * + * @param subscription + * the subscription + * @return the list of unscheduled subscriptions + */ + private List handleGridded(Subscription subscription) { + final List cycles = subscription.getTime().getCycleTimes(); + final boolean subscribedToCycles = !CollectionUtil + .isNullOrEmpty(cycles); + final boolean useMostRecentDataSetUpdate = !subscribedToCycles; + + // The subscription has cycles, so we can allocate bandwidth at + // expected times + List unscheduled = Collections.emptyList(); + if (subscribedToCycles) { + unscheduled = schedule(subscription, Sets.newTreeSet(cycles)); + } + + // Create an adhoc subscription based on the new subscription, + // and set it to retrieve the most recent cycle (or most recent + // url if a daily product) + if (subscription instanceof SiteSubscription) { + AdhocSubscription adhoc = new AdhocSubscription( + (SiteSubscription) subscription); + adhoc = bandwidthDaoUtil.setAdhocMostRecentUrlAndTime(adhoc, + useMostRecentDataSetUpdate); + + if (adhoc == null) { + statusHandler + .info(String + .format("There wasn't applicable most recent dataset metadata to use for new subscription [%s]. " + + "No adhoc requested.", + subscription.getName())); + } else { + unscheduled = schedule(adhoc); + } + } else { + statusHandler + .warn("Unable to create adhoc queries for shared subscriptions at this point. This functionality should be added in the future..."); + } + return unscheduled; + } + /** * {@inheritDoc} * @@ -1092,13 +1171,7 @@ public abstract class BandwidthManager extends boolean setBandwidth = setBandwidth(requestNetwork, bandwidth); response = setBandwidth; break; - case METADATA_UPDATE: - DataSetMetaData r = request.getDataSetMetaData(); - updateDataSetMetaData(r); - break; - case SHOW_ALLOCATION: - break; case SHOW_BUCKET: diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/IBandwidthManager.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/IBandwidthManager.java index 6bf59e86ea..7d749abe4b 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/IBandwidthManager.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/IBandwidthManager.java @@ -19,13 +19,11 @@ **/ package com.raytheon.uf.edex.datadelivery.bandwidth; -import java.text.ParseException; import java.util.List; import com.google.common.eventbus.AllowConcurrentEvents; import com.google.common.eventbus.Subscribe; import com.raytheon.uf.common.datadelivery.registry.AdhocSubscription; -import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData; import com.raytheon.uf.common.datadelivery.registry.Subscription; import com.raytheon.uf.common.registry.event.InsertRegistryEvent; import com.raytheon.uf.common.registry.event.RemoveRegistryEvent; @@ -64,17 +62,6 @@ public interface IBandwidthManager { @Subscribe void registryEventListener(InsertRegistryEvent re); - /** - * Persist the DataSetMetaData update and store the necessary statistical - * data's to generate the predictive update time for a dataset. - * - * @param dataSetMetaData - * @throws ParseException - */ - @Subscribe - void updateDataSetMetaData(DataSetMetaData dataSetMetaData) - throws ParseException; - /** * When a Subscription is removed from the Registry, a RemoveRegistryEvent * is generated and forwarded to this method to remove the necessary diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/InMemoryBandwidthDao.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/InMemoryBandwidthDao.java index 2b3303d8ee..54f74876b3 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/InMemoryBandwidthDao.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/InMemoryBandwidthDao.java @@ -21,11 +21,16 @@ package com.raytheon.uf.edex.datadelivery.bandwidth; import java.util.ArrayList; import java.util.Calendar; +import java.util.Comparator; +import java.util.Date; import java.util.Iterator; import java.util.List; +import java.util.SortedSet; +import java.util.TreeSet; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicLong; +import com.google.common.collect.Sets; import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData; import com.raytheon.uf.common.datadelivery.registry.Network; import com.raytheon.uf.common.datadelivery.registry.Subscription; @@ -55,6 +60,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.util.BandwidthUtil; * ------------ ---------- ----------- -------------------------- * Oct 24, 2012 1286 djohnson Initial creation * Dec 12, 2012 1286 djohnson Use concurrent lists to avoid concurrent modification exceptions. + * Jun 03, 2013 2038 djohnson Add method to get subscription retrievals by provider, dataset, and status. * * * @@ -87,8 +93,8 @@ class InMemoryBandwidthDao implements IBandwidthDao { .hasNext();) { BandwidthAllocation current = iter.next(); if ((current instanceof SubscriptionRetrieval) - && ((SubscriptionRetrieval) current).getBandwidthSubscription() - .getId() == subscriptionId) { + && ((SubscriptionRetrieval) current) + .getBandwidthSubscription().getId() == subscriptionId) { continue; } @@ -158,8 +164,8 @@ class InMemoryBandwidthDao implements IBandwidthDao { * {@inheritDoc} */ @Override - public List getBandwidthDataSetUpdate(String providerName, - String dataSetName) { + public List getBandwidthDataSetUpdate( + String providerName, String dataSetName) { ArrayList results = clone(bandwidthDataSetUpdates); for (Iterator iter = results.iterator(); iter @@ -180,10 +186,10 @@ class InMemoryBandwidthDao implements IBandwidthDao { * {@inheritDoc} */ @Override - public List getBandwidthDataSetUpdate(String providerName, - String dataSetName, Calendar baseReferenceTime) { - List results = getBandwidthDataSetUpdate(providerName, - dataSetName); + public List getBandwidthDataSetUpdate( + String providerName, String dataSetName, Calendar baseReferenceTime) { + List results = getBandwidthDataSetUpdate( + providerName, dataSetName); for (Iterator iter = results.iterator(); iter .hasNext();) { @@ -205,7 +211,8 @@ class InMemoryBandwidthDao implements IBandwidthDao { public List getDeferred(Network network, Calendar endTime) { List results = getBandwidthAllocations(network); - for (Iterator iter = results.iterator(); iter.hasNext();) { + for (Iterator iter = results.iterator(); iter + .hasNext();) { BandwidthAllocation current = iter.next(); if (RetrievalStatus.DEFERRED.equals(current.getStatus()) && !current.getEndTime().after(endTime)) { @@ -252,7 +259,8 @@ class InMemoryBandwidthDao implements IBandwidthDao { * {@inheritDoc} */ @Override - public List getBandwidthSubscription(Subscription subscription) { + public List getBandwidthSubscription( + Subscription subscription) { return getBandwidthSubscriptionByRegistryId(subscription.getId()); } @@ -263,8 +271,8 @@ class InMemoryBandwidthDao implements IBandwidthDao { public List getBandwidthSubscriptionByRegistryId( String registryId) { final ArrayList results = clone(bandwidthSubscriptions); - for (Iterator iter = results - .iterator(); iter.hasNext();) { + for (Iterator iter = results.iterator(); iter + .hasNext();) { final BandwidthSubscription current = iter.next(); if (registryId.equals(current.getRegistryId())) { continue; @@ -299,8 +307,7 @@ class InMemoryBandwidthDao implements IBandwidthDao { List results = new ArrayList( getSubscriptionRetrievals(provider, dataSetName)); List subscriptionsMatching = getBandwidthSubscriptions( - provider, - dataSetName, baseReferenceTime); + provider, dataSetName, baseReferenceTime); OUTER: for (Iterator iter = results.iterator(); iter .hasNext();) { @@ -367,12 +374,12 @@ class InMemoryBandwidthDao implements IBandwidthDao { * {@inheritDoc} */ @Override - public List getBandwidthSubscriptions(String provider, - String dataSetName, Calendar baseReferenceTime) { + public List getBandwidthSubscriptions( + String provider, String dataSetName, Calendar baseReferenceTime) { List bandwidthSubscriptions = getBandwidthSubscriptions(); - for (Iterator iter = bandwidthSubscriptions.iterator(); iter - .hasNext();) { + for (Iterator iter = bandwidthSubscriptions + .iterator(); iter.hasNext();) { BandwidthSubscription current = iter.next(); if (provider.equals(current.getProvider()) && dataSetName.equals(current.getDataSetName()) @@ -405,10 +412,12 @@ class InMemoryBandwidthDao implements IBandwidthDao { * {@inheritDoc} */ @Override - public BandwidthSubscription newBandwidthSubscription(Subscription subscription, - Calendar baseReferenceTime) throws SerializationException { - BandwidthSubscription entity = BandwidthUtil.getSubscriptionDaoForSubscription( - subscription, baseReferenceTime); + public BandwidthSubscription newBandwidthSubscription( + Subscription subscription, Calendar baseReferenceTime) + throws SerializationException { + BandwidthSubscription entity = BandwidthUtil + .getSubscriptionDaoForSubscription(subscription, + baseReferenceTime); update(entity); @@ -429,8 +438,8 @@ class InMemoryBandwidthDao implements IBandwidthDao { .hasNext();) { BandwidthAllocation current = iter.next(); if (current instanceof SubscriptionRetrieval) { - if (((SubscriptionRetrieval) current).getBandwidthSubscription() - .getId() == subscriptionId) { + if (((SubscriptionRetrieval) current) + .getBandwidthSubscription().getId() == subscriptionId) { results.add((SubscriptionRetrieval) current); } } @@ -541,4 +550,59 @@ class InMemoryBandwidthDao implements IBandwidthDao { } } } + + /** + * {@inheritDoc} + */ + @Override + public SortedSet getSubscriptionRetrievals( + String provider, String dataSetName, RetrievalStatus status) { + + final List subscriptionRetrievals = getSubscriptionRetrievals( + provider, dataSetName); + + for (Iterator iter = subscriptionRetrievals + .iterator(); iter.hasNext();) { + SubscriptionRetrieval subRetrieval = iter.next(); + if (!status.equals(subRetrieval.getStatus())) { + iter.remove(); + } + } + + final TreeSet treeSet = Sets + .newTreeSet(new Comparator() { + @Override + public int compare(SubscriptionRetrieval o1, + SubscriptionRetrieval o2) { + return o1.getStartTime().compareTo(o2.getStartTime()); + } + }); + + treeSet.addAll(subscriptionRetrievals); + + return treeSet; + } + + /** + * {@inheritDoc} + */ + @Override + public SortedSet getSubscriptionRetrievals( + String provider, String dataSetName, RetrievalStatus status, + Date earliestDate, Date latestDate) { + + SortedSet results = getSubscriptionRetrievals( + provider, dataSetName, status); + + for (Iterator iter = results + .iterator(); iter.hasNext();) { + SubscriptionRetrieval subRetrieval = iter.next(); + if (earliestDate.after(subRetrieval.getStartTime().getTime()) + || latestDate.before(subRetrieval.getStartTime().getTime())) { + iter.remove(); + } + } + + return results; + } } diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/dao/IBandwidthDao.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/dao/IBandwidthDao.java index da93bf1abf..d1bce30994 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/dao/IBandwidthDao.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/dao/IBandwidthDao.java @@ -20,18 +20,22 @@ package com.raytheon.uf.edex.datadelivery.bandwidth.dao; import java.util.Calendar; +import java.util.Date; import java.util.List; +import java.util.SortedSet; import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData; import com.raytheon.uf.common.datadelivery.registry.Network; import com.raytheon.uf.common.datadelivery.registry.Subscription; import com.raytheon.uf.common.serialization.SerializationException; +import com.raytheon.uf.edex.datadelivery.bandwidth.BandwidthManager; import com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.RetrievalStatus; /** - * Extracted from {@link BandwidthContextFactory} so that {@link BandwidthManager} - * can be run in memory (e.g. for testing proposed bandwidth size limitations - * and informing the user which subscriptions would be unable to be scheduled). + * Extracted from {@link BandwidthContextFactory} so that + * {@link BandwidthManager} can be run in memory (e.g. for testing proposed + * bandwidth size limitations and informing the user which subscriptions would + * be unable to be scheduled). * * 
  * 
@@ -40,6 +44,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.RetrievalStatus;
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * Oct 23, 2012 1286       djohnson     Initial creation
+ * Jun 03, 2013 2038       djohnson     Add method to get subscription retrievals by provider, dataset, and status.
  * 
  *
* @@ -192,6 +197,50 @@ public interface IBandwidthDao { List getSubscriptionRetrievals(String provider, String dataSetName, Calendar baseReferenceTime); + /** + * Get all the subscription retrievals for the specified dataset, with the + * specified status, and ordered by date. + * + * @param provider + * The provider name. + * + * @param dataSetName + * The dataset name. + * + * @param status + * The status + * + * @return the subscription retrievals + */ + SortedSet getSubscriptionRetrievals(String provider, + String dataSetName, RetrievalStatus status); + + /** + * Get all the subscription retrievals for the specified dataset, with the + * specified status, ordered by date, with a start date between the two + * specified dates (inclusive). + * + * @param provider + * The provider name. + * + * @param dataSetName + * The dataset name. + * + * @param status + * The status + * + * @param earliestDate + * the earliest date + * + * @param latestDate + * the latest date + * + * @return the subscription retrievals + */ + SortedSet getSubscriptionRetrievals(String provider, + String dataSetName, RetrievalStatus status, Date earliestDate, + Date latestDate); + /** * Get all the subscription retrievals for the specified dataset and base * reference time. diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/hibernate/HibernateBandwidthDao.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/hibernate/HibernateBandwidthDao.java index 67ca004131..0c6da5ed96 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/hibernate/HibernateBandwidthDao.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/hibernate/HibernateBandwidthDao.java @@ -20,7 +20,9 @@ package com.raytheon.uf.edex.datadelivery.bandwidth.hibernate; import java.util.Calendar; +import java.util.Date; import java.util.List; +import java.util.SortedSet; import org.hibernate.dialect.Dialect; import org.hibernate.jdbc.Work; @@ -52,6 +54,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.util.BandwidthUtil; * Feb 07, 2013 1543 djohnson Moved session management context to CoreDao. * Feb 11, 2013 1543 djohnson Use Spring transactions. * Feb 13, 2013 1543 djohnson Converted into a service, created new DAOs as required. + * Jun 03, 2013 2038 djohnson Add method to get subscription retrievals by provider, dataset, and status. * * * @@ -204,6 +207,28 @@ public class HibernateBandwidthDao implements IBandwidthDao { dataSetName); } + /** + * {@inheritDoc} + */ + @Override + public SortedSet getSubscriptionRetrievals( + String provider, String dataSetName, RetrievalStatus status) { + return subscriptionRetrievalDao.getByProviderDataSetAndStatus(provider, + dataSetName, status); + } + + /** + * {@inheritDoc} + */ + @Override + public SortedSet getSubscriptionRetrievals( + String provider, String dataSetName, RetrievalStatus status, + Date earliestDate, Date latestDate) { + return this.subscriptionRetrievalDao + .getByProviderDataSetStatusAndDateRange(provider, dataSetName, + status, earliestDate, latestDate); + } + /** * {@inheritDoc} */ @@ -339,7 +364,7 @@ public class HibernateBandwidthDao implements IBandwidthDao { * The unit of work to do. */ public void doWork(Work work) { - subscriptionRetrievalDao.doWork(work); + subscriptionRetrievalDao.executeWork(work); } /** diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/hibernate/ISubscriptionRetrievalDao.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/hibernate/ISubscriptionRetrievalDao.java index 7f45519acc..a47166ee70 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/hibernate/ISubscriptionRetrievalDao.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/hibernate/ISubscriptionRetrievalDao.java @@ -20,12 +20,15 @@ package com.raytheon.uf.edex.datadelivery.bandwidth.hibernate; import java.util.Calendar; +import java.util.Date; import java.util.List; +import java.util.SortedSet; import org.hibernate.dialect.Dialect; import org.hibernate.jdbc.Work; import com.raytheon.uf.edex.datadelivery.bandwidth.dao.SubscriptionRetrieval; +import com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.RetrievalStatus; /** * DAO for {@link SubscriptionRetrieval} instances. @@ -37,6 +40,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.dao.SubscriptionRetrieval; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Feb 13, 2013 1543 djohnson Initial creation + * Jun 03, 2013 2038 djohnson Add method to get subscription retrievals by provider, dataset, and status. * * * @@ -67,21 +71,45 @@ interface ISubscriptionRetrievalDao extends List getByProviderDataSet(String provider, String dataSetName); + /** + * Get by provider, dataset, and retrieval status. The results will be + * ordered by start date. + * + * @param provider + * @param dataSetName + * @param status + * @return + */ + SortedSet getByProviderDataSetAndStatus( + String provider, String dataSetName, RetrievalStatus status); + + /** + * Get by provider, dataset, retrieval status, and a date range. The results + * will be ordered by start date. + * + * @param provider + * @param dataSetName + * @param status + * @param earliestDate + * @param latestDate + * @return + */ + SortedSet getByProviderDataSetStatusAndDateRange( + String provider, String dataSetName, RetrievalStatus status, + Date earliestDate, Date latestDate); + /** * Do arbitrary work. * * @param work * work */ - // TODO: It would be nice to remove this method, if possible - void doWork(Work work); + void executeWork(Work work); /** * Get the hibernate dialect. * * @return */ - // TODO: It would be nice to remove this method, if possible Dialect getDialect(); - } \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/hibernate/SubscriptionRetrievalDao.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/hibernate/SubscriptionRetrievalDao.java index 2d352ca8b7..6ea3474bd9 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/hibernate/SubscriptionRetrievalDao.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/hibernate/SubscriptionRetrievalDao.java @@ -20,11 +20,15 @@ package com.raytheon.uf.edex.datadelivery.bandwidth.hibernate; import java.util.Calendar; +import java.util.Comparator; +import java.util.Date; import java.util.List; +import java.util.SortedSet; +import java.util.TreeSet; -import org.hibernate.jdbc.Work; - +import com.google.common.collect.Sets; import com.raytheon.uf.edex.datadelivery.bandwidth.dao.SubscriptionRetrieval; +import com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.RetrievalStatus; /** * * DAO that handles {@link SubscriptionRetrieval} instances. @@ -37,7 +41,8 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.dao.SubscriptionRetrieval; * ------------ ---------- ----------- -------------------------- * Feb 13, 2013 1543 djohnson Initial creation * Feb 22, 2013 1543 djohnson Made public as YAJSW doesn't like Spring exceptions. - * 4/9/2013 1802 bphillip Changed to use new query method signatures in SessionManagedDao + * 4/9/2013 1802 bphillip Changed to use new query method signatures in SessionManagedDao + * Jun 03, 2013 2038 djohnson Add method to get subscription retrievals by provider, dataset, and status. * * * @@ -48,17 +53,21 @@ public class SubscriptionRetrievalDao extends BaseBandwidthAllocationDao implements ISubscriptionRetrievalDao { - private static final String GET_SUBSCRIPTIONRETRIEVAL_BY_PROVIDER_AND_DATASET_BASE = "from SubscriptionRetrieval sr where " - + " sr.bandwidthSubscription.id in (" - + " select sub.id from BandwidthSubscription sub where " - + " sub.provider = :provider and " - + " sub.dataSetName = :dataSetName"; + private static final String GET_SUBSCRIPTIONRETRIEVAL_BY_PROVIDER_AND_DATASET = "from SubscriptionRetrieval sr where " + + "sr.bandwidthSubscription.provider = :provider and " + + "sr.bandwidthSubscription.dataSetName = :dataSetName"; - private static final String GET_SUBSCRIPTIONRETRIEVAL_BY_PROVIDER_AND_DATASET = GET_SUBSCRIPTIONRETRIEVAL_BY_PROVIDER_AND_DATASET_BASE - + ")"; + private static final String GET_SUBSCRIPTIONRETRIEVAL_BY_PROVIDER_AND_DATASET_AND_BASEREFERENCETIME = GET_SUBSCRIPTIONRETRIEVAL_BY_PROVIDER_AND_DATASET + + " and sr.bandwidthSubscription.baseReferenceTime = :baseReferenceTime)"; - private static final String GET_SUBSCRIPTIONRETRIEVAL_BY_PROVIDER_AND_DATASET_AND_BASEREFERENCETIME = GET_SUBSCRIPTIONRETRIEVAL_BY_PROVIDER_AND_DATASET_BASE - + " and sub.baseReferenceTime = :baseReferenceTime)"; + private static final String GET_SUBSCRIPTIONRETRIEVAL_BY_PROVIDER_AND_DATASET_AND_STATUS = GET_SUBSCRIPTIONRETRIEVAL_BY_PROVIDER_AND_DATASET + + " and sr.status = :status order by sr.startTime"; + + private static final String GET_SUBSCRIPTIONRETRIEVAL_BY_PROVIDER_DATASET_STATUS_AND_DATES = GET_SUBSCRIPTIONRETRIEVAL_BY_PROVIDER_AND_DATASET + + " and sr.status = :status" + + " and sr.startTime >= :startDate" + + " and sr.startTime <= :endDate" + + " order by sr.startTime "; /** * {@inheritDoc} @@ -87,15 +96,66 @@ public class SubscriptionRetrievalDao extends public List getByProviderDataSet(String provider, String dataSetName) { return query(GET_SUBSCRIPTIONRETRIEVAL_BY_PROVIDER_AND_DATASET, - "provider", provider, "dataSetName", dataSetName); + "provider", provider, "dataSetName", dataSetName); } /** * {@inheritDoc} */ @Override - // TODO: Remove the requirement of this method - public void doWork(Work work) { - template.getSessionFactory().getCurrentSession().doWork(work); + public SortedSet getByProviderDataSetAndStatus( + String provider, String dataSetName, RetrievalStatus status) { + + final List results = query( + GET_SUBSCRIPTIONRETRIEVAL_BY_PROVIDER_AND_DATASET_AND_STATUS, + "provider", provider, "dataSetName", dataSetName, "status", + status); + + return orderByStart(results); } + + /** + * {@inheritDoc} + */ + @Override + public SortedSet getByProviderDataSetStatusAndDateRange( + String provider, String dataSetName, RetrievalStatus status, + Date earliestDate, Date latestDate) { + + final Calendar startDate = Calendar.getInstance(); + startDate.setTime(earliestDate); + + final Calendar endDate = Calendar.getInstance(); + endDate.setTime(latestDate); + + final List results = query( + GET_SUBSCRIPTIONRETRIEVAL_BY_PROVIDER_DATASET_STATUS_AND_DATES, + "provider", provider, "dataSetName", dataSetName, "status", + status, "startDate", startDate, "endDate", endDate); + + return orderByStart(results); + } + + /** + * Returns a {@link SortedSet} that orders the subscription retrievals by + * start date. + * + * @param results + * the results + * @return the set + */ + private SortedSet orderByStart( + List results) { + final TreeSet treeSet = Sets + .newTreeSet(new Comparator() { + @Override + public int compare(SubscriptionRetrieval o1, + SubscriptionRetrieval o2) { + return o1.getStartTime().compareTo(o2.getStartTime()); + } + }); + treeSet.addAll(results); + return treeSet; + } + } diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/retrieval/SubscriptionRetrievalAgent.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/retrieval/SubscriptionRetrievalAgent.java index 5a2ffbdb22..83c7ca7b12 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/retrieval/SubscriptionRetrievalAgent.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/retrieval/SubscriptionRetrievalAgent.java @@ -10,6 +10,7 @@ import java.util.List; import com.google.common.annotations.VisibleForTesting; import com.raytheon.uf.common.datadelivery.registry.Network; import com.raytheon.uf.common.datadelivery.registry.Provider; +import com.raytheon.uf.common.datadelivery.registry.ProviderType; import com.raytheon.uf.common.datadelivery.registry.Subscription; import com.raytheon.uf.common.datadelivery.registry.SubscriptionBundle; import com.raytheon.uf.common.datadelivery.registry.handlers.DataDeliveryHandlers; @@ -186,6 +187,9 @@ public class SubscriptionRetrievalAgent extends timer.start(); final int numberOfRetrievals = retrievals.size(); + final ProviderType providerType = bundle.getProvider() + .getProviderType(bundle.getDataType()); + final String plugin = providerType.getPlugin(); for (int i = 0; i < numberOfRetrievals; i++) { Retrieval retrieval = retrievals.get(i); RetrievalRequestRecord rec = new RetrievalRequestRecord( @@ -196,7 +200,7 @@ public class SubscriptionRetrievalAgent extends rec.setInsertTime(insertTime); rec.setNetwork(retrieval.getNetwork()); rec.setProvider(provider); - rec.setPlugin(retrieval.getProviderType().getPlugin()); + rec.setPlugin(plugin); rec.setSubscriptionType(retrieval.getSubscriptionType()); try { diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/BandwidthDaoUtil.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/BandwidthDaoUtil.java index c20bf293af..4c51ecc5a7 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/BandwidthDaoUtil.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/BandwidthDaoUtil.java @@ -19,6 +19,7 @@ **/ package com.raytheon.uf.edex.datadelivery.bandwidth.util; +import java.util.Arrays; import java.util.Calendar; import java.util.Date; import java.util.Iterator; @@ -26,7 +27,7 @@ import java.util.List; import java.util.SortedSet; import java.util.TreeSet; -import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Sets; import com.raytheon.uf.common.datadelivery.registry.AdhocSubscription; import com.raytheon.uf.common.datadelivery.registry.Subscription; import com.raytheon.uf.common.datadelivery.registry.Time; @@ -55,6 +56,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.RetrievalStatus; * Oct 24, 2012 1286 djohnson Extract methods from {@link BandwidthUtil}. * Dec 11, 2012 1286 djohnson FULFILLED allocations are not in the retrieval plan either. * Feb 14, 2013 1595 djohnson Fix not using calendar copies, and backwards max/min operations. + * Jun 03, 2013 2038 djohnson Add ability to schedule down to minute granularity. * Jun 04, 2013 223 mpduff Refactor changes. * * @@ -96,24 +98,8 @@ public class BandwidthDaoUtil { */ public SortedSet getRetrievalTimes(Subscription subscription, SortedSet cycles) { - - // TODO: Will need to address this calculation where obs are considered, - // not sure how the Time Object will be used to communicate all the - // times that an observation subscription should be fulfilled. For now, - // assume grid and move on. - SortedSet subscriptionTimes = new TreeSet(); - - // TODO: Can't schedule subscriptions without cycles, yet... - if (cycles.size() == 0) { - return subscriptionTimes; - } - - RetrievalPlan plan = retrievalManager.getPlan(subscription.getRoute()); - if (plan == null) { - return subscriptionTimes; - } - - return getRetrievalTimes(subscription, cycles, plan, subscriptionTimes); + return getRetrievalTimes(subscription, cycles, + Sets.newTreeSet(Arrays.asList(0))); } /** @@ -121,15 +107,47 @@ public class BandwidthDaoUtil { * the current retrieval plan for the specified subscription. * * @param subscription - * @param cycles - * @param plan - * @param subscriptionTimes + * @param retrievalInterval + * the retrieval interval + * @return the retrieval times + */ + public SortedSet getRetrievalTimes(Subscription subscription, + int retrievalInterval) { + // Add all hours of the days + final SortedSet hours = Sets.newTreeSet(); + for (int i = 0; i < TimeUtil.HOURS_PER_DAY; i++) { + hours.add(i); + } + + // Add every minute of the hour that is a multiple of the retrieval + // interval + final SortedSet minutes = Sets.newTreeSet(); + for (int i = 0; i < TimeUtil.MINUTES_PER_HOUR; i += retrievalInterval) { + minutes.add(i); + } + + return getRetrievalTimes(subscription, hours, minutes); + } + + /** + * Calculate all the retrieval times for a subscription that should be in + * the current retrieval plan for the specified subscription. + * + * @param subscription + * @param hours + * @param minutes * @return */ - @VisibleForTesting - SortedSet getRetrievalTimes(Subscription subscription, - SortedSet cycles, RetrievalPlan plan, - SortedSet subscriptionTimes) { + private SortedSet getRetrievalTimes(Subscription subscription, + SortedSet hours, SortedSet minutes) { + + SortedSet subscriptionTimes = new TreeSet(); + + RetrievalPlan plan = retrievalManager.getPlan(subscription.getRoute()); + if (plan == null) { + return subscriptionTimes; + } + Calendar planEnd = plan.getPlanEnd(); Calendar planStart = plan.getPlanStart(); @@ -200,24 +218,24 @@ public class BandwidthDaoUtil { outerloop: while (!subscriptionStartDate.after(subscriptionEndDate)) { - for (Integer cycle : cycles) { - // TODO: VERY grid specific. Should be transitioned to minutes? - // to support obs - // or rapidly (less than an hour) updated datasets.. + for (Integer cycle : hours) { subscriptionStartDate.set(Calendar.HOUR_OF_DAY, cycle); - if (subscriptionStartDate.after(subscriptionEndDate)) { - break outerloop; - } else { - Calendar time = TimeUtil.newCalendar(); - time.setTimeInMillis(subscriptionStartDate - .getTimeInMillis()); - subscriptionTimes.add(time); + for (Integer minute : minutes) { + subscriptionStartDate.set(Calendar.MINUTE, minute); + if (subscriptionStartDate.after(subscriptionEndDate)) { + break outerloop; + } else { + Calendar time = TimeUtil.newCalendar(); + time.setTimeInMillis(subscriptionStartDate + .getTimeInMillis()); + subscriptionTimes.add(time); + } } } // Start the next day.. subscriptionStartDate.add(Calendar.DAY_OF_YEAR, 1); - subscriptionStartDate.set(Calendar.HOUR_OF_DAY, cycles.first()); + subscriptionStartDate.set(Calendar.HOUR_OF_DAY, hours.first()); } // Now walk the subscription times and throw away anything outside the @@ -313,7 +331,7 @@ public class BandwidthDaoUtil { if (statusHandler.isPriorityEnabled(Priority.DEBUG)) { statusHandler .debug(String - .format("There wasn't applicable most recent dataset metadata to use for the adhoc subscription [%].", + .format("There wasn't applicable most recent dataset metadata to use for the adhoc subscription [%s].", adhoc.getName())); } } else { diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/ProviderDataTypeAvailabilityCalculator.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/ProviderDataTypeAvailabilityCalculator.java new file mode 100644 index 0000000000..ea7562b3dd --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/ProviderDataTypeAvailabilityCalculator.java @@ -0,0 +1,128 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.edex.datadelivery.bandwidth.util; + +import java.util.concurrent.ExecutionException; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.eventbus.Subscribe; +import com.raytheon.uf.common.datadelivery.registry.DataDeliveryRegistryObjectTypes; +import com.raytheon.uf.common.datadelivery.registry.DataType; +import com.raytheon.uf.common.datadelivery.registry.Provider; +import com.raytheon.uf.common.datadelivery.registry.ProviderType; +import com.raytheon.uf.common.datadelivery.registry.Subscription; +import com.raytheon.uf.common.datadelivery.registry.handlers.IProviderHandler; +import com.raytheon.uf.common.event.EventBus; +import com.raytheon.uf.common.registry.event.RegistryEvent; +import com.raytheon.uf.common.registry.handler.RegistryHandlerException; + +/** + * Uses provider configured values for the availability delay. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Jun 05, 2013 2038       djohnson     Initial creation
+ * 
+ *
+ * + * @author djohnson + * @version 1.0 + */ + +public class ProviderDataTypeAvailabilityCalculator implements + IDataSetAvailablityCalculator { + + private static final Provider PROVIDER_NOT_FOUND = new Provider(); + + private final LoadingCache providerCache; + + /** + * Constructor. + */ + public ProviderDataTypeAvailabilityCalculator( + final IProviderHandler providerHandler) { + + // TODO: This should probably be moved inside the registry handler + // itself + this.providerCache = CacheBuilder.newBuilder().build( + new CacheLoader() { + @Override + public Provider load(String key) + throws RegistryHandlerException { + Provider provider = providerHandler + .getByName(key); + if (provider == null) { + provider = PROVIDER_NOT_FOUND; + } + return provider; + } + }); + + EventBus.register(this); + } + + /** + * {@inheritDoc} + */ + @Override + public int getDataSetAvailablityDelay(Subscription subscription) { + final String providerName = subscription.getProvider(); + final DataType dataType = subscription.getDataSetType(); + + try { + Provider provider = this.providerCache.get(providerName); + + if (provider == PROVIDER_NOT_FOUND) { + throw new IllegalArgumentException( + "No availability delay registered for provider " + + providerName + " for data type " + dataType); + } + + final ProviderType providerType = provider + .getProviderType(dataType); + + if (providerType == null) { + throw new IllegalArgumentException( + "No availability delay registered for provider " + + providerName + " for data type " + dataType); + } + return providerType.getAvailabilityDelay(); + } catch (ExecutionException e) { + throw new IllegalStateException( + "Exception querying for the provider!", e); + } + } + + @Subscribe + public void registryEventListener(RegistryEvent re) { + final String objectType = re.getObjectType(); + + // If a provider event happens then expire the entire cache + if (DataDeliveryRegistryObjectTypes.PROVIDER.equals(objectType)) { + providerCache.invalidateAll(); + } + } +} diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/crawler/FileCommunicationStrategy.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/crawler/FileCommunicationStrategy.java index 533b4de6f6..4780298667 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/crawler/FileCommunicationStrategy.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/crawler/FileCommunicationStrategy.java @@ -230,7 +230,8 @@ class FileCommunicationStrategy implements CommunicationStrategy { coll.setFirstDate(pc.getFirstDateFormatted()); // TODO: figure a default data type and projection // strategy other than just the first one. - coll.setDataType(provider.getProviderType().get(0)); + coll.setDataType(provider.getProviderType().get(0) + .getDataType()); coll.setProjection(provider.getProjection().get(0).getType()); coll.setPeriodicity(pc.getPeriodicity()); coll.setUrlKey(pc.getUrlKey()); diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/MADIS-harvester.xml b/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/MADIS-harvester.xml index 5dce45db89..4ea1a93ef8 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/MADIS-harvester.xml +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/MADIS-harvester.xml @@ -10,7 +10,7 @@ !!!!!PLACE YOUR URL HERE!!!!!!--> http://your.url.here:8085 - Point + MadisLatLon MADIS Test LatLon Coverage diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/NOMADS-harvester.xml b/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/NOMADS-harvester.xml index d0baec43bf..f6a2be6687 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/NOMADS-harvester.xml +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/NOMADS-harvester.xml @@ -3,7 +3,7 @@ - Grid + NomadsLatLon Test LatLonGrid Coverage for NOMADS @@ -56,7 +56,7 @@ -1 2 - + narre narre day @@ -64,7 +64,7 @@ 20121108 yyyyMMdd - + wave_nahParameterLookup wave/nah nah @@ -73,7 +73,7 @@ 20121108 yyyyMMdd - + wave_akwParameterLookup wave/akw akw @@ -82,7 +82,7 @@ 20121108 yyyyMMdd - + gfsParameterLookup gfsLevelLookup gfs @@ -92,7 +92,7 @@ 20121108 yyyyMMdd - + wave_wnaParameterLookup wave/wna wna @@ -101,7 +101,7 @@ 20121108 yyyyMMdd - + wave/nfcens day @@ -109,7 +109,7 @@ 20121108 yyyyMMdd - + wave_mww3ParameterLookup wave/mww3 @@ -118,7 +118,7 @@ 20121108 yyyyMMdd - + wave_nphParameterLookup wave/nph nph @@ -127,7 +127,7 @@ 20121108 yyyyMMdd - + wave_nww3ParameterLookup wave/nww3 nww3 @@ -136,7 +136,7 @@ 20121108 yyyyMMdd - + gfs_hdParameterLookup gfs_hdLevelLookup gfs_hd @@ -146,7 +146,7 @@ 20121108 yyyyMMdd - + wave/glw glw day @@ -154,7 +154,7 @@ 20121108 yyyyMMdd - + wave_enpParameterLookup wave/enp enp @@ -163,7 +163,7 @@ 20121108 yyyyMMdd - + gfs_2p5ParameterLookup gfs_2p5LevelLookup gfs_2p5 @@ -173,7 +173,7 @@ 20121108 yyyyMMdd - + hireswParameterLookup hireswLevelLookup hiresw @@ -183,7 +183,7 @@ 20121108 yyyyMMdd - + cmcensParameterLookup cmcensLevelLookup cmcens @@ -193,7 +193,7 @@ 20121108 yyyyMMdd - + namParameterLookup namLevelLookup nam @@ -203,7 +203,7 @@ 20121108 yyyyMMdd - + srefParameterLookup srefLevelLookup sref @@ -213,7 +213,7 @@ 20121108 yyyyMMdd - + sref_bcParameterLookup sref_bcLevelLookup sref_bc @@ -223,7 +223,7 @@ 20121108 yyyyMMdd - + ofs_hiresParameterLookup ofs_hiresLevelLookup ofs/hires @@ -233,7 +233,7 @@ 20121108 yyyyMMdd - + ofsParameterLookup ofsLevelLookup ofs @@ -243,7 +243,7 @@ 20121108 yyyyMMdd - + akrtma akrtma day @@ -251,7 +251,7 @@ 20121108 yyyyMMdd - + fensParameterLookup fensLevelLookup fens @@ -261,7 +261,7 @@ 20121108 yyyyMMdd - + rtofsParameterLookup rtofsLevelLookup rtofs @@ -271,7 +271,7 @@ 20121108 yyyyMMdd - + iceParameterLookup ice ice @@ -280,7 +280,7 @@ 20121108 yyyyMMdd - + rtma2p5 rtma2p5 day @@ -288,7 +288,7 @@ 20121108 yyyyMMdd - + gurtmaParameterLookup gurtmaLevelLookup gurtma @@ -298,7 +298,7 @@ 20121108 yyyyMMdd - + naefs_bcParameterLookup naefs_bcLevelLookup naefs_bc @@ -308,7 +308,7 @@ 20121108 yyyyMMdd - + rapParameterLookup rapLevelLookup rap @@ -318,7 +318,7 @@ 20121108 yyyyMMdd - + naefs_ndgdParameterLookup naefs_ndgdLevelLookup naefs_ndgd @@ -328,7 +328,7 @@ 20121108 yyyyMMdd - + rtmaParameterLookup rtmaLevelLookup rtma @@ -338,7 +338,7 @@ 20121108 yyyyMMdd - + aqmParameterLookup aqmLevelLookup aqm @@ -348,7 +348,7 @@ 20121108 yyyyMMdd - + ncom ncom day @@ -356,7 +356,7 @@ 20121108 yyyyMMdd - + estofs day @@ -364,7 +364,7 @@ 20121108 yyyyMMdd - + fnlParameterLookup fnl fnl @@ -373,7 +373,7 @@ 20121108 yyyyMMdd - + hirtma hirtma day @@ -381,7 +381,7 @@ 20121108 yyyyMMdd - + prrtmaParameterLookup prrtmaLevelLookup prrtma @@ -391,7 +391,7 @@ 20121108 yyyyMMdd - + gens_bcParameterLookup gens_bcLevelLookup gens_bc @@ -401,7 +401,7 @@ 20121108 yyyyMMdd - + gensParameterLookup gensLevelLookup gens @@ -411,7 +411,7 @@ 20121108 yyyyMMdd - + gens_ndgdParameterLookup gens_ndgdLevelLookup gens_ndgd diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/OGC-harvester.xml.sample b/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/OGC-harvester.xml.sample index 4d021559ed..329561f2cd 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/OGC-harvester.xml.sample +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/OGC-harvester.xml.sample @@ -9,7 +9,7 @@ http://your.url.here:8085 - Point + MetarLatLon METAR Test LatLon Coverage diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/RetrievalGenerationHandler.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/RetrievalGenerationHandler.java index ea4e2d3c98..3501ff1145 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/RetrievalGenerationHandler.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/RetrievalGenerationHandler.java @@ -25,6 +25,7 @@ import java.util.Calendar; import java.util.Date; import java.util.List; +import com.raytheon.uf.common.datadelivery.registry.ProviderType; import com.raytheon.uf.common.datadelivery.registry.SubscriptionBundle; import com.raytheon.uf.common.datadelivery.retrieval.xml.Retrieval; import com.raytheon.uf.common.serialization.SerializationUtil; @@ -85,9 +86,6 @@ public class RetrievalGenerationHandler implements IGenerateRetrieval { List retrievals = rg.buildRetrieval(bundle); if (!CollectionUtil.isNullOrEmpty(retrievals)) { - // need to track in case something fails early - int numFinished = 0; - String owner = bundle.getSubscription().getOwner(); String provider = bundle.getSubscription().getProvider(); int priority = 3; @@ -101,12 +99,15 @@ public class RetrievalGenerationHandler implements IGenerateRetrieval { retrievals.size()); long cumultTime1 = 0; int index = 0; + final ProviderType providerType = bundle.getProvider() + .getProviderType(bundle.getDataType()); + final String plugin = providerType.getPlugin(); for (Retrieval retrieval : retrievals) { RetrievalRequestRecord rec = new RetrievalRequestRecord( subscriptionName, index++, -1L); rec.setOwner(owner); - rec.setPlugin(retrieval.getProviderType().getPlugin()); + rec.setPlugin(plugin); rec.setProvider(provider); rec.setSubscriptionType(retrieval.getSubscriptionType()); rec.setNetwork(retrieval.getNetwork()); @@ -125,7 +126,6 @@ public class RetrievalGenerationHandler implements IGenerateRetrieval { + subscriptionName + " Failed to serialize request [" + retrieval + "]", e); - numFinished++; rec.setRetrieval(new byte[0]); rec.setState(State.FAILED); } diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPRetrievalGenerator.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPRetrievalGenerator.java index fd2e8b9509..cb95ef825f 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPRetrievalGenerator.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPRetrievalGenerator.java @@ -36,8 +36,9 @@ import com.raytheon.uf.common.datadelivery.registry.GriddedDataSetMetaData; import com.raytheon.uf.common.datadelivery.registry.Levels; import com.raytheon.uf.common.datadelivery.registry.OpenDapGriddedDataSet; import com.raytheon.uf.common.datadelivery.registry.Parameter; -import com.raytheon.uf.common.datadelivery.registry.Provider.ProviderType; +import com.raytheon.uf.common.datadelivery.registry.Provider; import com.raytheon.uf.common.datadelivery.registry.Provider.ServiceType; +import com.raytheon.uf.common.datadelivery.registry.ProviderType; import com.raytheon.uf.common.datadelivery.registry.Subscription; import com.raytheon.uf.common.datadelivery.registry.SubscriptionBundle; import com.raytheon.uf.common.datadelivery.registry.Time; @@ -402,11 +403,7 @@ class OpenDAPRetrievalGenerator extends RetrievalGenerator { // Coverage and type processing Coverage cov = sub.getCoverage(); - ProviderType pt = null; - if (cov instanceof GriddedCoverage){ - pt = ProviderType.GRID; - retrieval.setProviderType(pt); - } else { + if (!(cov instanceof GriddedCoverage)) { throw new UnsupportedOperationException( "OPENDAP retrieval does not yet support coverages other than Gridded. "); } @@ -420,7 +417,18 @@ class OpenDAPRetrievalGenerator extends RetrievalGenerator { att.setParameter(lparam); att.setEnsemble(ensemble); att.setSubName(retrieval.getSubscriptionName()); - att.setPlugin(pt.getPlugin()); + Provider provider; + try { + provider = DataDeliveryHandlers.getProviderHandler().getByName( + sub.getProvider()); + } catch (RegistryHandlerException e) { + throw new IllegalArgumentException( + "Error looking up the provider!", e); + } + // Look up the provider's configured plugin for this data type + ProviderType providerType = provider.getProviderType(sub + .getDataSetType()); + att.setPlugin(providerType.getPlugin()); att.setProvider(sub.getProvider()); retrieval.addAttribute(att); diff --git a/tests/integration/com/raytheon/uf/edex/datadelivery/bandwidth/BandwidthManagerIntTest.java b/tests/integration/com/raytheon/uf/edex/datadelivery/bandwidth/BandwidthManagerIntTest.java index ac0834bf92..d5a93058fa 100644 --- a/tests/integration/com/raytheon/uf/edex/datadelivery/bandwidth/BandwidthManagerIntTest.java +++ b/tests/integration/com/raytheon/uf/edex/datadelivery/bandwidth/BandwidthManagerIntTest.java @@ -19,9 +19,11 @@ **/ package com.raytheon.uf.edex.datadelivery.bandwidth; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.sameInstance; @@ -48,11 +50,15 @@ import java.util.concurrent.CountDownLatch; import org.junit.Test; import com.raytheon.uf.common.datadelivery.registry.DataDeliveryRegistryObjectTypes; +import com.raytheon.uf.common.datadelivery.registry.DataType; import com.raytheon.uf.common.datadelivery.registry.GriddedDataSetMetaData; import com.raytheon.uf.common.datadelivery.registry.Network; import com.raytheon.uf.common.datadelivery.registry.OpenDapGriddedDataSetMetaData; import com.raytheon.uf.common.datadelivery.registry.OpenDapGriddedDataSetMetaDataFixture; import com.raytheon.uf.common.datadelivery.registry.ParameterFixture; +import com.raytheon.uf.common.datadelivery.registry.PointDataSetMetaData; +import com.raytheon.uf.common.datadelivery.registry.PointDataSetMetaDataFixture; +import com.raytheon.uf.common.datadelivery.registry.PointTime; import com.raytheon.uf.common.datadelivery.registry.SiteSubscription; import com.raytheon.uf.common.datadelivery.registry.SiteSubscriptionFixture; import com.raytheon.uf.common.datadelivery.registry.Subscription; @@ -98,6 +104,7 @@ import com.raytheon.uf.edex.datadelivery.retrieval.RetrievalManagerNotifyEvent; * Mar 11, 2013 1645 djohnson Test configuration file modifications. * Mar 28, 2013 1841 djohnson Subscription is now UserSubscription. * Apr 29, 2013 1910 djohnson Always shutdown bandwidth managers in tests. + * Jun 03, 2013 2038 djohnson Add support for point data based subscriptions. * * * @@ -120,6 +127,29 @@ public class BandwidthManagerIntTest extends AbstractWfoBandwidthManagerIntTest 1, 2, 3)); } + @Test + public void testAddingPointSubscriptionAllocatesAccordingToInterval() + throws SerializationException { + final int retrievalInterval = 10; + Subscription subscription = getPointDataSubscription(retrievalInterval); + + bandwidthManager.subscriptionUpdated(subscription); + + // 6 per hour, one every 10 minutes + final int expectedRetrievalsPerDay = TimeUtil.HOURS_PER_DAY + * (TimeUtil.MINUTES_PER_HOUR / retrievalInterval); + final int planDays = retrievalManager.getPlan(subscription.getRoute()) + .getPlanDays(); + // The number of retrievals per hour, per day, plus 1 because the last + // retrieval starts at the very last time in the retrieval plan + final int expectedNumberOfRetrievals = (planDays * expectedRetrievalsPerDay) + 1; + + assertEquals("Incorrect number of bandwidth allocations made!", + expectedNumberOfRetrievals, bandwidthDao + .getBandwidthAllocations(subscription.getRoute()) + .size()); + } + @Test public void testDataSetMetaDataUpdateSetsSubscriptionRetrievalsToReady() throws SerializationException, ParseException { @@ -128,7 +158,7 @@ public class BandwidthManagerIntTest extends AbstractWfoBandwidthManagerIntTest OpenDapGriddedDataSetMetaData metadata = OpenDapGriddedDataSetMetaDataFixture.INSTANCE .get(); - bandwidthManager.updateDataSetMetaData(metadata); + bandwidthManager.updateGriddedDataSetMetaData(metadata); Calendar cal = TimeUtil.newCalendar(); cal.setTime(metadata.getDate()); @@ -142,6 +172,65 @@ public class BandwidthManagerIntTest extends AbstractWfoBandwidthManagerIntTest .next().getStatus()); } + @Test + public void testPointDataSetMetaDataUpdateSetsSubscriptionRetrievalsToReady() + throws SerializationException, ParseException { + + final int retrievalInterval = 10; + final Subscription subscription = getPointDataSubscription(retrievalInterval); + final String providerName = subscription.getProvider(); + final String dataSetName = subscription.getDataSetName(); + + bandwidthManager.subscriptionUpdated(subscription); + + final List subscriptionRetrievals = bandwidthDao + .getSubscriptionRetrievals(providerName, + dataSetName); + + // We're going to send in a point data update with a time span that + // bridges these two retrievals + final SubscriptionRetrieval thirdRetrieval = subscriptionRetrievals + .get(2); + final SubscriptionRetrieval fourthRetrieval = subscriptionRetrievals + .get(3); + + // The start time for this point time is in between the third + // retrieval's start and end times, minus the retrievalInterval which + // becomes the latency + final long startTimeForPointTimeInMillis = ((thirdRetrieval + .getStartTime().getTimeInMillis() + thirdRetrieval.getEndTime() + .getTimeInMillis()) / 2) + - (retrievalInterval * TimeUtil.MILLIS_PER_MINUTE); + + // The end time for this point time is in between the fourth retrieval's + // start and end times, minus the retrievalInterval which becomes the + // latency + final long endTimeForPointTimeInMillis = ((fourthRetrieval + .getStartTime().getTimeInMillis() + fourthRetrieval + .getEndTime().getTimeInMillis()) / 2) + - (retrievalInterval * TimeUtil.MILLIS_PER_MINUTE); + + PointTime time = new PointTime(); + time.setTimes(Arrays. asList(new Date( + startTimeForPointTimeInMillis), new Date( + endTimeForPointTimeInMillis))); + + PointDataSetMetaData metadata = PointDataSetMetaDataFixture.INSTANCE + .get(); + metadata.setDataSetName(dataSetName); + metadata.setProviderName(providerName); + metadata.setTime(time); + + // Send in the new times to the bandwidth manager + bandwidthManager.updatePointDataSetMetaData(metadata); + + final SortedSet readyRetrievals = bandwidthDao + .getSubscriptionRetrievals(providerName, + dataSetName, RetrievalStatus.READY); + assertThat(readyRetrievals, hasSize(2)); + assertThat(readyRetrievals, contains(thirdRetrieval, fourthRetrieval)); + } + @Test public void testDataSetMetaDataUpdateSetsCorrectTimeOnSubscription() throws SerializationException, ParseException { @@ -158,7 +247,7 @@ public class BandwidthManagerIntTest extends AbstractWfoBandwidthManagerIntTest metadata.setDate(oneDayLater); // Send in the update - bandwidthManager.updateDataSetMetaData(metadata); + bandwidthManager.updateGriddedDataSetMetaData(metadata); Calendar cal = TimeUtil.newCalendar(); cal.setTime(metadata.getDate()); @@ -193,7 +282,7 @@ public class BandwidthManagerIntTest extends AbstractWfoBandwidthManagerIntTest dsmdTime.setStartDate(new Date(dsmdTime.getStartDate().getTime() + TimeUtil.MILLIS_PER_DAY)); - bandwidthManager.updateDataSetMetaData(update); + bandwidthManager.updateGriddedDataSetMetaData(update); List retrievals = bandwidthDao .getSubscriptionRetrievals(subscription.getProvider(), @@ -233,7 +322,7 @@ public class BandwidthManagerIntTest extends AbstractWfoBandwidthManagerIntTest RetrievalPlanTest.resizePlan(plan, TimeUtil.currentTimeMillis(), plan .getPlanEnd().getTimeInMillis()); - bandwidthManager.updateDataSetMetaData(update); + bandwidthManager.updateGriddedDataSetMetaData(update); List retrievals = bandwidthDao .getSubscriptionRetrievals(subscription.getProvider(), @@ -267,7 +356,7 @@ public class BandwidthManagerIntTest extends AbstractWfoBandwidthManagerIntTest RetrievalPlanTest.resizePlan(plan, TimeUtil.currentTimeMillis(), plan .getPlanEnd().getTimeInMillis()); - bandwidthManager.updateDataSetMetaData(update); + bandwidthManager.updateGriddedDataSetMetaData(update); List retrievals = bandwidthDao .getSubscriptionRetrievals(subscription.getProvider(), @@ -668,7 +757,7 @@ public class BandwidthManagerIntTest extends AbstractWfoBandwidthManagerIntTest // working at once waitForAllThreadsReadyLatch.countDown(); waitForAllThreadsReadyLatch.await(); - proposed.updateDataSetMetaData(OpenDapGriddedDataSetMetaDataFixture.INSTANCE + proposed.updateGriddedDataSetMetaData(OpenDapGriddedDataSetMetaDataFixture.INSTANCE .get(current)); } catch (Exception e) { queue.offer(e); @@ -983,11 +1072,7 @@ public class BandwidthManagerIntTest extends AbstractWfoBandwidthManagerIntTest List cycles) throws SerializationException { Subscription subscription = SiteSubscriptionFixture.INSTANCE.get(); subscription.getTime().setCycleTimes(cycles); - try { - bandwidthManager.subscriptionUpdated(subscription); - } catch (Throwable t) { - t.printStackTrace(); - } + bandwidthManager.subscriptionUpdated(subscription); assertEquals("Incorrect number of bandwidth allocations made!", retrievalManager.getPlan(subscription.getRoute()).getPlanDays() @@ -1026,4 +1111,22 @@ public class BandwidthManagerIntTest extends AbstractWfoBandwidthManagerIntTest return Network.OPSNET; } + /** + * Get a point data subscription with the given retrieval interval. + * + * @param retrievalInterval + * the retrieval interval + * @return + */ + protected Subscription getPointDataSubscription(int retrievalInterval) { + final PointTime pointTime = new PointTime(); + pointTime.setInterval(retrievalInterval); + + Subscription subscription = SiteSubscriptionFixture.INSTANCE.get(); + subscription.setTime(pointTime); + subscription.setDataSetType(DataType.POINT); + subscription.setLatencyInMinutes(retrievalInterval); + return subscription; + } + } diff --git a/tests/integration/com/raytheon/uf/edex/registry/ebxml/dao/AbstractRegistryTest.java b/tests/integration/com/raytheon/uf/edex/registry/ebxml/dao/AbstractRegistryTest.java index 8e9e97d25c..658a4c8848 100644 --- a/tests/integration/com/raytheon/uf/edex/registry/ebxml/dao/AbstractRegistryTest.java +++ b/tests/integration/com/raytheon/uf/edex/registry/ebxml/dao/AbstractRegistryTest.java @@ -60,6 +60,7 @@ import org.springframework.transaction.annotation.Transactional; import com.google.common.collect.Lists; import com.raytheon.uf.common.registry.ebxml.RegistryUtil; import com.raytheon.uf.common.util.SpringFiles; +import com.raytheon.uf.common.util.TestUtil; import com.raytheon.uf.edex.registry.ebxml.services.query.QueryConstants; import com.raytheon.uf.edex.registry.ebxml.services.query.QueryManagerImpl.RETURN_TYPE; import com.raytheon.uf.edex.registry.ebxml.util.EbxmlObjectUtil; @@ -76,6 +77,7 @@ import com.raytheon.uf.edex.registry.ebxml.util.EbxmlObjectUtil; * Apr 15, 2013 1914 djohnson Initial creation * Apr 18, 2013 1693 djohnson Consolidate reusable methods. * Apr 23, 2013 1910 djohnson Allow sub-classes to pass callables and monitor for fault exceptions. + * Jun 05, 2013 2038 djohnson Use TestUtil constant for transactionManager. * * * @@ -94,7 +96,7 @@ import com.raytheon.uf.edex.registry.ebxml.util.EbxmlObjectUtil; SpringFiles.UNIT_TEST_DB_BEANS_XML, SpringFiles.UNIT_TEST_EBXML_BEANS_XML, SpringFiles.UNIT_TEST_LOCALIZATION_BEANS_XML }) -@TransactionConfiguration(transactionManager = "metadataTxManager", defaultRollback = true) +@TransactionConfiguration(transactionManager = TestUtil.METADATA_TX_MANAGER, defaultRollback = true) @Transactional @Ignore public class AbstractRegistryTest { diff --git a/tests/resources/bandwidth/bandwidth-datadelivery-integrationtest-impl.xml b/tests/resources/bandwidth/bandwidth-datadelivery-integrationtest-impl.xml index 5a5068866c..19370ba981 100644 --- a/tests/resources/bandwidth/bandwidth-datadelivery-integrationtest-impl.xml +++ b/tests/resources/bandwidth/bandwidth-datadelivery-integrationtest-impl.xml @@ -31,4 +31,11 @@ factory-method="emptyMap"> + + + + + \ No newline at end of file diff --git a/tests/unit/com/raytheon/uf/common/datadelivery/registry/BaseSubscriptionFixture.java b/tests/unit/com/raytheon/uf/common/datadelivery/registry/BaseSubscriptionFixture.java index a869191b26..ad61b2ec5f 100644 --- a/tests/unit/com/raytheon/uf/common/datadelivery/registry/BaseSubscriptionFixture.java +++ b/tests/unit/com/raytheon/uf/common/datadelivery/registry/BaseSubscriptionFixture.java @@ -68,8 +68,7 @@ public abstract class BaseSubscriptionFixture extends .setDataSetName(OpenDapGriddedDataSetMetaDataFixture.INSTANCE .get(seedValue).getDataSetName()); subscription.setDataSetSize(seedValue); - subscription.setDataSetType(AbstractFixture.randomEnum(DataType.class, - random)); + subscription.setDataSetType(DataType.GRID); subscription.setDeleted(random.nextBoolean()); subscription.setDescription("description" + random.nextInt()); subscription.setFullDataSet(random.nextBoolean()); diff --git a/tests/unit/com/raytheon/uf/common/datadelivery/registry/PointDataSetMetaDataFixture.java b/tests/unit/com/raytheon/uf/common/datadelivery/registry/PointDataSetMetaDataFixture.java new file mode 100644 index 0000000000..5e96d11f93 --- /dev/null +++ b/tests/unit/com/raytheon/uf/common/datadelivery/registry/PointDataSetMetaDataFixture.java @@ -0,0 +1,82 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.common.datadelivery.registry; + +import java.text.ParseException; +import java.util.Random; + +import com.raytheon.uf.common.time.util.ImmutableDate; +import com.raytheon.uf.common.util.AbstractFixture; + +/** + * {@link AbstractFixture} implementation for {@link PointDataSetMetaData} + * objects. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Jun 05, 2013 2038      djohnson     Initial creation
+ * 
+ *
+ * + * @author djohnson + * @version 1.0 + */ + +public class PointDataSetMetaDataFixture extends + AbstractFixture { + + public static final PointDataSetMetaDataFixture INSTANCE = new PointDataSetMetaDataFixture(); + + /** + * Disabled constructor. + */ + private PointDataSetMetaDataFixture() { + } + + /** + * {@inheritDoc} + */ + @Override + public PointDataSetMetaData getInstance(long seedValue, Random random) { + final PointTime time = PointTimeFixture.INSTANCE.get(seedValue); + final PointDataSet dataSet = WFSPointDataSetFixture.INSTANCE + .get(seedValue); + + PointDataSetMetaData obj = new PointDataSetMetaData(); + obj.setDataSetDescription("description" + seedValue); + obj.setDataSetName(dataSet.getDataSetName()); + try { + obj.setDate(new ImmutableDate(TimeFixture.INSTANCE.get(seedValue) + .getStartDate())); + } catch (ParseException e) { + throw new RuntimeException(e); + } + obj.setProviderName(dataSet.getProviderName()); + obj.setTime(time); + obj.setUrl("http://" + seedValue); + + return obj; + } + +} diff --git a/tests/unit/com/raytheon/uf/common/datadelivery/registry/PointTimeFixture.java b/tests/unit/com/raytheon/uf/common/datadelivery/registry/PointTimeFixture.java new file mode 100644 index 0000000000..38bad685f5 --- /dev/null +++ b/tests/unit/com/raytheon/uf/common/datadelivery/registry/PointTimeFixture.java @@ -0,0 +1,63 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.common.datadelivery.registry; + +import java.util.Random; + +import com.raytheon.uf.common.util.AbstractFixture; + +/** + * Fixture for {@link PointTime} objects. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Jun 05, 2013 2038      djohnson     Initial creation
+ * 
+ *
+ * + * @author djohnson + * @version 1.0 + */ + +public class PointTimeFixture extends AbstractFixture { + + public static final PointTimeFixture INSTANCE = new PointTimeFixture(); + + /** + * Disabled. + */ + private PointTimeFixture() { + } + + /** + * {@inheritDoc} + */ + @Override + public PointTime getInstance(long seedValue, Random random) { + PointTime time = new PointTime(); + time.setInterval((int) seedValue + 1); + + return time; + } +} diff --git a/tests/unit/com/raytheon/uf/common/datadelivery/registry/ProviderFixture.java b/tests/unit/com/raytheon/uf/common/datadelivery/registry/ProviderFixture.java index 2cb4e0ff7a..f7d0b9ff36 100644 --- a/tests/unit/com/raytheon/uf/common/datadelivery/registry/ProviderFixture.java +++ b/tests/unit/com/raytheon/uf/common/datadelivery/registry/ProviderFixture.java @@ -19,11 +19,9 @@ **/ package com.raytheon.uf.common.datadelivery.registry; -import java.util.ArrayList; import java.util.Arrays; import java.util.Random; -import com.raytheon.uf.common.datadelivery.registry.Provider.ProviderType; import com.raytheon.uf.common.datadelivery.registry.Provider.ServiceType; import com.raytheon.uf.common.util.AbstractFixture; @@ -68,8 +66,8 @@ public class ProviderFixture extends AbstractFixture { // TODO: ProjectionFixture // provider.setProjection(ProjectionFixture.INSTANCE.get(seedValue)); provider.setServiceType(ServiceType.OPENDAP); - provider.setProviderType(new ArrayList(Arrays - .asList(AbstractFixture.randomEnum(ProviderType.class, random)))); + provider.setProviderType(Arrays. asList(new ProviderType( + DataType.GRID, "grid", 100))); return provider; } diff --git a/tests/unit/com/raytheon/uf/common/datadelivery/registry/WFSPointDataSetFixture.java b/tests/unit/com/raytheon/uf/common/datadelivery/registry/WFSPointDataSetFixture.java new file mode 100644 index 0000000000..686815b5bb --- /dev/null +++ b/tests/unit/com/raytheon/uf/common/datadelivery/registry/WFSPointDataSetFixture.java @@ -0,0 +1,70 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.common.datadelivery.registry; + +import java.util.Collections; +import java.util.Random; + +import com.raytheon.uf.common.util.AbstractFixture; + +/** + * {@link AbstractFixture} implementation for {@link WFSPointDataSet} objects. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Jun 05, 2013 2038      djohnson     Initial creation
+ * 
+ *
+ * + * @author djohnson + * @version 1.0 + */ + +public class WFSPointDataSetFixture extends AbstractFixture { + + public static final WFSPointDataSetFixture INSTANCE = new WFSPointDataSetFixture(); + + /** + * Disabled constructor. + */ + private WFSPointDataSetFixture() { + } + + /** + * {@inheritDoc} + */ + @Override + public WFSPointDataSet getInstance(long seedValue, Random random) { + WFSPointDataSet obj = new WFSPointDataSet(); + obj.setCollectionName("collectionName-" + seedValue); + obj.setDataSetName("dataSetName" + seedValue); + obj.setDataSetType(DataType.POINT); + obj.setTime(PointTimeFixture.INSTANCE.get(seedValue)); + obj.setParameters(Collections. emptyMap()); + obj.setProviderName(ProviderFixture.INSTANCE.get(seedValue).getName()); + + return obj; + } + +} diff --git a/tests/unit/com/raytheon/uf/common/util/TestUtil.java b/tests/unit/com/raytheon/uf/common/util/TestUtil.java index cbf7a313e2..10b976d5f6 100644 --- a/tests/unit/com/raytheon/uf/common/util/TestUtil.java +++ b/tests/unit/com/raytheon/uf/common/util/TestUtil.java @@ -59,6 +59,7 @@ import com.raytheon.uf.common.time.util.TimeUtil; * Jul 24, 2012 955 djohnson Initial creation * Sep 07, 2012 1102 djohnson Add test for more specification defined properties of hashCode/equals. * Oct 23, 2012 1286 djohnson setupTestClassDir() takes any class. + * Jun 05, 2013 2038 djohnson Add constant for metadata tx manager. * * * @@ -67,6 +68,9 @@ import com.raytheon.uf.common.time.util.TimeUtil; */ @Ignore public final class TestUtil { + + public static final String METADATA_TX_MANAGER = "metadataTxManager"; + public static final Pattern COMMA_PATTERN = Pattern.compile(","); private static final Pattern NEW_LINE_PATTERN = Pattern.compile("\n"); diff --git a/tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/AbstractBandwidthDaoTest.java b/tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/AbstractBandwidthDaoTest.java index 9e543b75d3..768f771b1e 100644 --- a/tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/AbstractBandwidthDaoTest.java +++ b/tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/AbstractBandwidthDaoTest.java @@ -19,24 +19,32 @@ **/ package com.raytheon.uf.edex.datadelivery.bandwidth; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import java.util.Arrays; import java.util.Calendar; +import java.util.Date; +import java.util.Iterator; import java.util.List; +import java.util.SortedSet; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; +import com.google.common.collect.Lists; import com.raytheon.uf.common.datadelivery.registry.Network; import com.raytheon.uf.common.datadelivery.registry.OpenDapGriddedDataSetMetaData; import com.raytheon.uf.common.datadelivery.registry.OpenDapGriddedDataSetMetaDataFixture; -import com.raytheon.uf.common.datadelivery.registry.Subscription; import com.raytheon.uf.common.datadelivery.registry.SiteSubscriptionFixture; +import com.raytheon.uf.common.datadelivery.registry.Subscription; import com.raytheon.uf.common.serialization.SerializationException; import com.raytheon.uf.common.time.util.ImmutableDate; import com.raytheon.uf.common.time.util.TimeUtil; @@ -44,8 +52,8 @@ import com.raytheon.uf.common.util.TestUtil; import com.raytheon.uf.edex.datadelivery.bandwidth.dao.BandwidthAllocation; import com.raytheon.uf.edex.datadelivery.bandwidth.dao.BandwidthAllocationFixture; import com.raytheon.uf.edex.datadelivery.bandwidth.dao.BandwidthDataSetUpdate; -import com.raytheon.uf.edex.datadelivery.bandwidth.dao.IBandwidthDao; import com.raytheon.uf.edex.datadelivery.bandwidth.dao.BandwidthSubscription; +import com.raytheon.uf.edex.datadelivery.bandwidth.dao.IBandwidthDao; import com.raytheon.uf.edex.datadelivery.bandwidth.dao.SubscriptionRetrieval; import com.raytheon.uf.edex.datadelivery.bandwidth.dao.SubscriptionRetrievalFixture; import com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.RetrievalStatus; @@ -61,6 +69,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.util.BandwidthUtil; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Nov 12, 2012 1286 djohnson Initial creation + * Jun 03, 2013 2038 djohnson Add test getting retrievals by dataset, provider, and status. * * * @@ -172,11 +181,13 @@ public abstract class AbstractBandwidthDaoTest { .get(1); OpenDapGriddedDataSetMetaData metaData2 = OpenDapGriddedDataSetMetaDataFixture.INSTANCE .get(2); - BandwidthDataSetUpdate metaDataDao = dao.newBandwidthDataSetUpdate(metaData); + BandwidthDataSetUpdate metaDataDao = dao + .newBandwidthDataSetUpdate(metaData); dao.newBandwidthDataSetUpdate(metaData2); - final List results = dao.getBandwidthDataSetUpdate( - metaData.getProviderName(), metaData.getDataSetName()); + final List results = dao + .getBandwidthDataSetUpdate(metaData.getProviderName(), + metaData.getDataSetName()); assertEquals(1, results.size()); final BandwidthDataSetUpdate result = results.iterator().next(); assertEquals(metaData.getDataSetName(), result.getDataSetName()); @@ -191,15 +202,17 @@ public abstract class AbstractBandwidthDaoTest { .get(1); metaData2.setDate(new ImmutableDate(metaData.getDate().getTime() + TimeUtil.MILLIS_PER_YEAR)); - BandwidthDataSetUpdate metaDataDao = dao.newBandwidthDataSetUpdate(metaData); + BandwidthDataSetUpdate metaDataDao = dao + .newBandwidthDataSetUpdate(metaData); dao.newBandwidthDataSetUpdate(metaData2); final ImmutableDate date1 = metaData.getDate(); Calendar cal = Calendar.getInstance(); cal.setTime(date1); - final List results = dao.getBandwidthDataSetUpdate( - metaData.getProviderName(), metaData.getDataSetName(), cal); + final List results = dao + .getBandwidthDataSetUpdate(metaData.getProviderName(), + metaData.getDataSetName(), cal); assertEquals(1, results.size()); final BandwidthDataSetUpdate result = results.iterator().next(); assertEquals(metaData.getDataSetName(), result.getDataSetName()); @@ -312,7 +325,8 @@ public abstract class AbstractBandwidthDaoTest { assertFalse("The two objects should not have the same id!", entity1.getId() == entity2.getId()); - final BandwidthSubscription result = dao.getBandwidthSubscription(entity2.getId()); + final BandwidthSubscription result = dao + .getBandwidthSubscription(entity2.getId()); assertEquals("Should have returned the entity with the correct id!", entity2.getId(), result.getId()); assertNotSame(entity2, result); @@ -323,7 +337,8 @@ public abstract class AbstractBandwidthDaoTest { throws SerializationException { final Calendar now = BandwidthUtil.now(); // Identical except for their base reference times and ids - dao.newBandwidthSubscription(SiteSubscriptionFixture.INSTANCE.get(), now); + dao.newBandwidthSubscription(SiteSubscriptionFixture.INSTANCE.get(), + now); final Calendar later = BandwidthUtil.now(); later.add(Calendar.HOUR, 1); @@ -407,9 +422,12 @@ public abstract class AbstractBandwidthDaoTest { .get(2); // Still have to persist the actual subscription daos - final BandwidthSubscription subDao1 = entity1.getBandwidthSubscription(); - final BandwidthSubscription subDao2 = entity2.getBandwidthSubscription(); - final BandwidthSubscription subDao3 = entity3.getBandwidthSubscription(); + final BandwidthSubscription subDao1 = entity1 + .getBandwidthSubscription(); + final BandwidthSubscription subDao2 = entity2 + .getBandwidthSubscription(); + final BandwidthSubscription subDao3 = entity3 + .getBandwidthSubscription(); // Give each a unique time final Calendar one = BandwidthUtil.now(); @@ -475,12 +493,15 @@ public abstract class AbstractBandwidthDaoTest { three.add(Calendar.HOUR, 1); // Three entities all the same except for base reference time - dao.newBandwidthSubscription(SiteSubscriptionFixture.INSTANCE.get(), one); - dao.newBandwidthSubscription(SiteSubscriptionFixture.INSTANCE.get(), two); + dao.newBandwidthSubscription(SiteSubscriptionFixture.INSTANCE.get(), + one); + dao.newBandwidthSubscription(SiteSubscriptionFixture.INSTANCE.get(), + two); BandwidthSubscription entity3 = dao.newBandwidthSubscription( SiteSubscriptionFixture.INSTANCE.get(), three); // One with same base reference time but different provider/dataset - dao.newBandwidthSubscription(SiteSubscriptionFixture.INSTANCE.get(2), three); + dao.newBandwidthSubscription(SiteSubscriptionFixture.INSTANCE.get(2), + three); List results = dao.getBandwidthSubscriptions( entity3.getProvider(), entity3.getDataSetName(), three); @@ -508,9 +529,12 @@ public abstract class AbstractBandwidthDaoTest { .get(2); // Still have to persist the actual subscription daos - final BandwidthSubscription subDao1 = entity1.getBandwidthSubscription(); - final BandwidthSubscription subDao2 = entity2.getBandwidthSubscription(); - final BandwidthSubscription subDao3 = entity3.getBandwidthSubscription(); + final BandwidthSubscription subDao1 = entity1 + .getBandwidthSubscription(); + final BandwidthSubscription subDao2 = entity2 + .getBandwidthSubscription(); + final BandwidthSubscription subDao3 = entity3 + .getBandwidthSubscription(); // This persists the subscription dao objects and sets them on the // retrievals @@ -547,9 +571,12 @@ public abstract class AbstractBandwidthDaoTest { .get(2); // Still have to persist the actual subscription daos - final BandwidthSubscription subDao1 = entity1.getBandwidthSubscription(); - final BandwidthSubscription subDao2 = entity2.getBandwidthSubscription(); - final BandwidthSubscription subDao3 = entity3.getBandwidthSubscription(); + final BandwidthSubscription subDao1 = entity1 + .getBandwidthSubscription(); + final BandwidthSubscription subDao2 = entity2 + .getBandwidthSubscription(); + final BandwidthSubscription subDao3 = entity3 + .getBandwidthSubscription(); // This persists the subscription dao objects and sets them on the // retrievals @@ -575,17 +602,20 @@ public abstract class AbstractBandwidthDaoTest { @Test public void testRemoveSubscriptionDao() throws SerializationException { final Calendar now = BandwidthUtil.now(); - dao.newBandwidthSubscription(SiteSubscriptionFixture.INSTANCE.get(1), now); + dao.newBandwidthSubscription(SiteSubscriptionFixture.INSTANCE.get(1), + now); final BandwidthSubscription entity2 = dao.newBandwidthSubscription( SiteSubscriptionFixture.INSTANCE.get(2), now); - dao.newBandwidthSubscription(SiteSubscriptionFixture.INSTANCE.get(3), now); + dao.newBandwidthSubscription(SiteSubscriptionFixture.INSTANCE.get(3), + now); assertEquals("Incorrect number of entities found!", 3, dao .getBandwidthSubscriptions().size()); dao.remove(entity2); - final List subscriptions = dao.getBandwidthSubscriptions(); + final List subscriptions = dao + .getBandwidthSubscriptions(); assertEquals("Incorrect number of entities found!", 2, subscriptions.size()); for (BandwidthSubscription subscription : subscriptions) { @@ -624,8 +654,7 @@ public abstract class AbstractBandwidthDaoTest { public void testUpdateBandwidthAllocation() { final long estimatedSize = 25L; - BandwidthAllocation entity = BandwidthAllocationFixture.INSTANCE - .get(); + BandwidthAllocation entity = BandwidthAllocationFixture.INSTANCE.get(); entity.setAgentType("someAgentType"); dao.store(entity); entity.setEstimatedSize(estimatedSize); @@ -647,7 +676,8 @@ public abstract class AbstractBandwidthDaoTest { dao.update(entity); assertEquals("Expected the entity to have been updated!", 25L, dao - .getBandwidthSubscriptions().iterator().next().getEstimatedSize()); + .getBandwidthSubscriptions().iterator().next() + .getEstimatedSize()); } @Test @@ -664,4 +694,124 @@ public abstract class AbstractBandwidthDaoTest { assertEquals("Expected the entity to have been updated!", 25L, dao .getSubscriptionRetrieval(entity.getId()).getEstimatedSize()); } + + @Test + public void testGetSubscriptionRetrievalsByProviderDataSetAndStatus() { + + final int numberOfScheduledEntities = 2; + final int numberOfReadyEntities = 3; + List entities = Lists + .newArrayListWithCapacity(numberOfScheduledEntities + + numberOfReadyEntities); + + // Create some scheduled entities + entities.addAll(getEntitiesInState(numberOfScheduledEntities, + RetrievalStatus.SCHEDULED)); + + // Create some ready entities + entities.addAll(getEntitiesInState(numberOfReadyEntities, + RetrievalStatus.READY)); + + for (int i = 0; i < entities.size(); i++) { + final SubscriptionRetrieval entity = entities.get(i); + // Give each one a unique start time + entity.getStartTime().add(Calendar.MINUTE, i); + dao.store(entity.getBandwidthSubscription()); + } + dao.store(entities); + + BandwidthSubscription bandwidthSubscription = entities.iterator() + .next().getBandwidthSubscription(); + final int actualNumberOfScheduledStatus = dao + .getSubscriptionRetrievals(bandwidthSubscription.getProvider(), + bandwidthSubscription.getDataSetName(), + RetrievalStatus.SCHEDULED).size(); + assertThat(actualNumberOfScheduledStatus, + is(equalTo(numberOfScheduledEntities))); + + final int actualNumberOfReadyStatus = dao.getSubscriptionRetrievals( + bandwidthSubscription.getProvider(), + bandwidthSubscription.getDataSetName(), RetrievalStatus.READY) + .size(); + assertThat(actualNumberOfReadyStatus, + is(equalTo(numberOfReadyEntities))); + } + + @Test + public void testGetSubscriptionRetrievalsByProviderDataSetStatusAndDates() { + + final int numberOfScheduledEntities = 2; + final int numberOfReadyEntities = 10; + List entities = Lists + .newArrayListWithCapacity(numberOfScheduledEntities + + numberOfReadyEntities); + + // Create some scheduled entities + entities.addAll(getEntitiesInState(numberOfScheduledEntities, + RetrievalStatus.SCHEDULED)); + + // Create some ready entities + List readyEntities = getEntitiesInState( + numberOfReadyEntities, RetrievalStatus.READY); + entities.addAll(readyEntities); + + // Persist the bandwidth subscriptions and create some unique times + for (int i = 0; i < entities.size(); i++) { + final SubscriptionRetrieval entity = entities.get(i); + + // Give each one a unique start time + final Calendar startTime = entity.getStartTime(); + startTime.add(Calendar.HOUR, i); + + // ... and end time + Calendar endTime = BandwidthUtil.copy(startTime); + endTime.add(Calendar.MINUTE, 5); + entity.setEndTime(endTime); + + dao.store(entity.getBandwidthSubscription()); + } + dao.store(entities); + + BandwidthSubscription bandwidthSubscription = entities.iterator() + .next().getBandwidthSubscription(); + // These are the entities we expect to get (two items) + List expectToGet = readyEntities.subList(3, 5); + final Iterator iter = expectToGet.iterator(); + + // Use the start time of the first retrieval and the end time of the + // second retrieval + final Date startTime = iter.next().getStartTime().getTime(); + final Date endTime = iter.next().getEndTime().getTime(); + + final SortedSet actualReceived = dao.getSubscriptionRetrievals(bandwidthSubscription.getProvider(), + bandwidthSubscription.getDataSetName(), RetrievalStatus.READY, + startTime, endTime); + + // Verify the correct number of retrievals were returned + assertThat(actualReceived, hasSize(expectToGet.size())); + + // Verify the two SubscriptionRetrievals are correct + Iterator actualIter = actualReceived.iterator(); + assertThat(actualIter.next(), is(equalTo(expectToGet.get(0)))); + assertThat(actualIter.next(), is(equalTo(expectToGet.get(1)))); + } + + /** + * Get the specified number of entities in the specified state. + * + * @param numberOfEntities + * @param state + * @return the entities + */ + protected static List getEntitiesInState( + int numberOfEntities, RetrievalStatus state) { + List entities = Lists.newArrayList(); + for (int i = 0; i < numberOfEntities; i++) { + SubscriptionRetrieval entity = SubscriptionRetrievalFixture.INSTANCE + .get(); + entity.setStatus(state); + entities.add(entity); + } + return entities; + } } diff --git a/tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/HibernateBandwidthDaoTest.java b/tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/HibernateBandwidthDaoTest.java index d4d793f940..9855374dfb 100644 --- a/tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/HibernateBandwidthDaoTest.java +++ b/tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/HibernateBandwidthDaoTest.java @@ -25,8 +25,11 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.annotation.DirtiesContext.ClassMode; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.test.context.transaction.TransactionConfiguration; +import org.springframework.transaction.annotation.Transactional; import com.raytheon.uf.common.util.SpringFiles; +import com.raytheon.uf.common.util.TestUtil; import com.raytheon.uf.edex.datadelivery.bandwidth.dao.IBandwidthDao; import com.raytheon.uf.edex.datadelivery.bandwidth.hibernate.HibernateBandwidthDao; @@ -41,6 +44,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.hibernate.HibernateBandwidthD * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Nov 13, 2012 1286 djohnson Initial creation + * Jun 03, 2013 2038 djohnson Use transactional semantics. * * * @@ -51,6 +55,8 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.hibernate.HibernateBandwidthD @ContextConfiguration(locations = { SpringFiles.UNIT_TEST_DB_BEANS_XML, SpringFiles.BANDWIDTH_DATADELIVERY_DAOS_XML, SpringFiles.RETRIEVAL_DATADELIVERY_DAOS_XML }) +@TransactionConfiguration(transactionManager = TestUtil.METADATA_TX_MANAGER, defaultRollback = true) +@Transactional @DirtiesContext(classMode = ClassMode.AFTER_EACH_TEST_METHOD) public class HibernateBandwidthDaoTest extends AbstractBandwidthDaoTest { diff --git a/tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/util/BandwidthDaoUtilTest.java b/tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/util/BandwidthDaoUtilTest.java index 303ef39478..ac05d0c3a0 100644 --- a/tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/util/BandwidthDaoUtilTest.java +++ b/tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/util/BandwidthDaoUtilTest.java @@ -19,7 +19,11 @@ **/ package com.raytheon.uf.edex.datadelivery.bandwidth.util; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -31,6 +35,7 @@ import java.util.Calendar; import java.util.Collection; import java.util.Date; import java.util.List; +import java.util.Map; import java.util.SortedSet; import java.util.TreeSet; @@ -38,6 +43,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import com.google.common.collect.Maps; import com.raytheon.uf.common.datadelivery.registry.Network; import com.raytheon.uf.common.datadelivery.registry.Subscription; import com.raytheon.uf.common.datadelivery.registry.SubscriptionBuilder; @@ -70,6 +76,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.RetrievalStatus; * Oct 24, 2012 1286 djohnson Initial creation * Feb 07, 2013 1543 djohnson Remove unnecessary test setup methods. * Feb 14, 2013 1595 djohnson Fix retrieval plan/subscription time intersections. + * Jun 05, 2013 2038 djohnson Use public API for getting retrieval times. * * * @@ -110,6 +117,11 @@ public class BandwidthDaoUtilTest { map = BandwidthMap.load(lf.getFile()); plan = new RetrievalPlan(Network.OPSNET, map, mockDao); + + Map retrievalPlans = Maps + .newEnumMap(Network.class); + retrievalPlans.put(Network.OPSNET, plan); + when(retrievalManager.getRetrievalPlans()).thenReturn(retrievalPlans); } @After @@ -130,10 +142,8 @@ public class BandwidthDaoUtilTest { TreeSet cycles = new TreeSet(subscription.getTime() .getCycleTimes()); - SortedSet subscriptionTimes = new TreeSet(); - - subscriptionTimes = bandwidthDaoUtil.getRetrievalTimes(subscription, - cycles, plan, subscriptionTimes); + SortedSet subscriptionTimes = bandwidthDaoUtil + .getRetrievalTimes(subscription, cycles); final List daysOfTheYear = Arrays.asList(3, 4); verifySubscriptionTimesContainsCyclesForSpecifiedDays(daysOfTheYear, @@ -154,10 +164,8 @@ public class BandwidthDaoUtilTest { TreeSet cycles = new TreeSet(subscription.getTime() .getCycleTimes()); - SortedSet subscriptionTimes = new TreeSet(); - - subscriptionTimes = bandwidthDaoUtil.getRetrievalTimes(subscription, - cycles, plan, subscriptionTimes); + SortedSet subscriptionTimes = bandwidthDaoUtil + .getRetrievalTimes(subscription, cycles); final List daysOfTheYear = Arrays.asList(4); verifySubscriptionTimesContainsCyclesForSpecifiedDays(daysOfTheYear, @@ -181,10 +189,8 @@ public class BandwidthDaoUtilTest { TreeSet cycles = new TreeSet(subscription.getTime() .getCycleTimes()); - SortedSet subscriptionTimes = new TreeSet(); - - subscriptionTimes = bandwidthDaoUtil.getRetrievalTimes(subscription, - cycles, plan, subscriptionTimes); + SortedSet subscriptionTimes = bandwidthDaoUtil + .getRetrievalTimes(subscription, cycles); final List daysOfTheYear = Arrays.asList(3); verifySubscriptionTimesContainsCyclesForSpecifiedDays(daysOfTheYear, @@ -217,6 +223,47 @@ public class BandwidthDaoUtilTest { verify(retrievalManager, never()).remove(alloc2); } + @Test + public void testGetRetrievalTimesReturnsEachIntervalMinuteOfEachHourInPlanWindow() { + Subscription subscription = new SubscriptionBuilder() + .withActivePeriodStart(plan.getPlanStart().getTime()) + .withActivePeriodEnd(plan.getPlanEnd().getTime()) + .withSubscriptionStart(TimeUtil.newImmutableDate()).build(); + + // A 30 minute interval should provide 0 and 30 minutes of every hour + // Make sure the subscription is "active" within the plan period + final int interval = 30; + SortedSet subscriptionTimes = bandwidthDaoUtil + .getRetrievalTimes(subscription, interval); + + // Expected size is two per hour (0 and 30 minutes), for every hour, + // over the retrieval plan days (2), plus 1 because the retrieval plan + // ends on a 0 minute time + final int expectedSize = TimeUtil.HOURS_PER_DAY * plan.getPlanDays() + * 2 + 1; + assertThat(subscriptionTimes, hasSize(expectedSize)); + + // Make sure we have the expected number of 0 and 30 minute scheduled + // times + int numberOfZeroMinuteTimes = 0; + int numberOfThirtyMinuteTimes = 0; + for (Calendar subscriptionTime : subscriptionTimes) { + final int minuteField = subscriptionTime.get(Calendar.MINUTE); + if (minuteField == 0) { + numberOfZeroMinuteTimes++; + } else if (minuteField == 30) { + numberOfThirtyMinuteTimes++; + } + } + + final int halfTheTimes = subscriptionTimes.size() / 2; + assertThat(numberOfZeroMinuteTimes, is(equalTo(halfTheTimes + 1))); + assertThat(numberOfThirtyMinuteTimes, is(equalTo(halfTheTimes))); + + // Would be nice to verify the days and hours, but the cycle based tests already + // do that and the code was reused, maybe add it later + } + /** * Verifies the subscription times contains the cycles for the specified * days. diff --git a/tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/util/ProviderDataTypeAvailabilityCalculatorTest.java b/tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/util/ProviderDataTypeAvailabilityCalculatorTest.java new file mode 100644 index 0000000000..56dad4aa99 --- /dev/null +++ b/tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/util/ProviderDataTypeAvailabilityCalculatorTest.java @@ -0,0 +1,111 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.edex.datadelivery.bandwidth.util; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.Arrays; + +import org.junit.Before; +import org.junit.Test; + +import com.raytheon.uf.common.datadelivery.registry.DataType; +import com.raytheon.uf.common.datadelivery.registry.Provider; +import com.raytheon.uf.common.datadelivery.registry.ProviderType; +import com.raytheon.uf.common.datadelivery.registry.SiteSubscription; +import com.raytheon.uf.common.datadelivery.registry.Subscription; +import com.raytheon.uf.common.datadelivery.registry.handlers.IProviderHandler; +import com.raytheon.uf.common.registry.handler.RegistryHandlerException; +import com.raytheon.uf.common.util.registry.RegistryException; + +/** + * Test {@link ProviderDataTypeAvailabilityCalculator}. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Jun 05, 2013 2038       djohnson     Initial creation
+ * 
+ *
+ * + * @author djohnson + * @version 1.0 + */ + +public class ProviderDataTypeAvailabilityCalculatorTest { + + private static final int ONE_HUNDRED = 100; + + private static final String PROVIDER_NAME = "someProviderName"; + + private static final Provider provider = new Provider(); + static { + provider.setName(PROVIDER_NAME); + provider.setProviderType(Arrays. asList(new ProviderType( + DataType.GRID, "grid", ONE_HUNDRED))); + } + + private final IProviderHandler providerHandler = mock(IProviderHandler.class); + + private final ProviderDataTypeAvailabilityCalculator availabilityCalculator = new ProviderDataTypeAvailabilityCalculator( + providerHandler); + + @Before + public void setUp() throws RegistryException, RegistryHandlerException { + when(providerHandler.getByName(PROVIDER_NAME)).thenReturn(provider); + } + + @Test + public void returnsConfiguredAvailabilityWhenRegistered() { + Subscription subscription = new SiteSubscription(); + subscription.setProvider(PROVIDER_NAME); + subscription.setDataSetType(DataType.GRID); + + assertThat( + availabilityCalculator.getDataSetAvailablityDelay(subscription), + is(equalTo(ONE_HUNDRED))); + } + + @Test(expected = IllegalArgumentException.class) + public void throwsIllegalArgumentExceptionWhenProviderNotRegistered() { + Subscription subscription = new SiteSubscription(); + subscription.setProvider("someOtherProviderName"); + subscription.setDataSetType(DataType.GRID); + + availabilityCalculator.getDataSetAvailablityDelay(subscription); + } + + @Test(expected = IllegalArgumentException.class) + public void throwsIllegalArgumentExceptionWhenDataTypeNotRegistered() { + Subscription subscription = new SiteSubscription(); + subscription.setProvider(PROVIDER_NAME); + subscription.setDataSetType(DataType.POINT); + + availabilityCalculator.getDataSetAvailablityDelay(subscription); + } + +} diff --git a/tests/unit/com/raytheon/uf/edex/datadelivery/retrieval/opendap/MockOpenDapRetrievalGenerator.java b/tests/unit/com/raytheon/uf/edex/datadelivery/retrieval/opendap/MockOpenDapRetrievalGenerator.java index b03c868311..fcb50c52c2 100644 --- a/tests/unit/com/raytheon/uf/edex/datadelivery/retrieval/opendap/MockOpenDapRetrievalGenerator.java +++ b/tests/unit/com/raytheon/uf/edex/datadelivery/retrieval/opendap/MockOpenDapRetrievalGenerator.java @@ -71,8 +71,8 @@ public class MockOpenDapRetrievalGenerator extends OpenDAPRetrievalGenerator { retrieval.setConnection(bundle.getConnection()); retrieval.setNetwork(subscription.getRoute()); retrieval.setOwner(subscription.getOwner()); - retrieval.setProviderType(bundle.getProvider().getProviderType() - .iterator().next()); + retrieval.setDataType(bundle.getProvider().getProviderType() + .iterator().next().getDataType()); retrieval.setServiceType(this.getServiceType()); retrieval.setSubscriptionName(subscription.getName()); retrieval.setSubscriptionType(getSubscriptionType(subscription)); diff --git a/tests/unit/com/raytheon/uf/viz/datadelivery/subscription/AbstractSubscriptionServiceTest.java b/tests/unit/com/raytheon/uf/viz/datadelivery/subscription/AbstractSubscriptionServiceTest.java index 555aa5308d..8120bde00d 100644 --- a/tests/unit/com/raytheon/uf/viz/datadelivery/subscription/AbstractSubscriptionServiceTest.java +++ b/tests/unit/com/raytheon/uf/viz/datadelivery/subscription/AbstractSubscriptionServiceTest.java @@ -47,8 +47,8 @@ import com.raytheon.uf.common.datadelivery.bandwidth.IBandwidthService; import com.raytheon.uf.common.datadelivery.bandwidth.IProposeScheduleResponse; import com.raytheon.uf.common.datadelivery.registry.AdhocSubscription; import com.raytheon.uf.common.datadelivery.registry.AdhocSubscriptionFixture; -import com.raytheon.uf.common.datadelivery.registry.Subscription; import com.raytheon.uf.common.datadelivery.registry.SiteSubscriptionFixture; +import com.raytheon.uf.common.datadelivery.registry.Subscription; import com.raytheon.uf.common.datadelivery.registry.handlers.DataDeliveryHandlers; import com.raytheon.uf.common.datadelivery.registry.handlers.ISubscriptionHandler; import com.raytheon.uf.common.datadelivery.service.ISubscriptionNotificationService; @@ -57,7 +57,6 @@ import com.raytheon.uf.common.datadelivery.service.subscription.ISubscriptionOve import com.raytheon.uf.common.localization.PathManagerFactoryTest; import com.raytheon.uf.common.registry.handler.RegistryHandlerException; import com.raytheon.uf.common.registry.handler.RegistryObjectHandlersUtil; -import com.raytheon.uf.common.util.FileUtil; import com.raytheon.uf.viz.datadelivery.subscription.ISubscriptionService.ISubscriptionServiceResult; import com.raytheon.uf.viz.datadelivery.subscription.SubscriptionService.ForceApplyPromptResponse; import com.raytheon.uf.viz.datadelivery.subscription.SubscriptionService.IDisplayForceApplyPrompt; @@ -350,26 +349,8 @@ public abstract class AbstractSubscriptionServiceTest { @Test public void testOverlappingSubscriptionsNotifiesUser() throws RegistryHandlerException { - final ISubscriptionHandler subscriptionHandler = DataDeliveryHandlers - .getSubscriptionHandler(); - - // Store a duplicate subscription - Subscription duplicateSub = sub1.copy("duplicateSub"); - subscriptionHandler.store(duplicateSub); - - final ISubscriptionOverlapResponse response = mock(ISubscriptionOverlapResponse.class); - when(subscriptionOverlapService.isOverlapping(duplicateSub, sub1)) - .thenReturn(response); - when(response.isOverlapping()).thenReturn(true); - - performServiceInteraction(); - - verify(mockDisplay).displayMessage( - mockPromptDisplayText, - ISubscriptionOverlapService.OVERLAPPING_SUBSCRIPTIONS - + FileUtil.EOL + duplicateSub.getName()); + // Not valid for adhocs } - /** * Verifies that the only interactions with the subscription handler are to * check for duplicate/overlapping subscriptions. diff --git a/tests/unit/com/raytheon/uf/viz/datadelivery/subscription/SubscriptionServiceStoreAdhocTest.java b/tests/unit/com/raytheon/uf/viz/datadelivery/subscription/SubscriptionServiceStoreAdhocTest.java index 1ee8a1911c..63a9353e6e 100644 --- a/tests/unit/com/raytheon/uf/viz/datadelivery/subscription/SubscriptionServiceStoreAdhocTest.java +++ b/tests/unit/com/raytheon/uf/viz/datadelivery/subscription/SubscriptionServiceStoreAdhocTest.java @@ -21,12 +21,15 @@ package com.raytheon.uf.viz.datadelivery.subscription; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; import java.util.Arrays; import java.util.Collections; import com.raytheon.uf.common.datadelivery.registry.Subscription; +import com.raytheon.uf.common.datadelivery.registry.handlers.DataDeliveryHandlers; +import com.raytheon.uf.common.datadelivery.registry.handlers.ISubscriptionHandler; import com.raytheon.uf.common.registry.handler.RegistryHandlerException; import com.raytheon.uf.common.util.CollectionUtil; import com.raytheon.uf.viz.datadelivery.subscription.ISubscriptionService.ISubscriptionServiceResult; @@ -147,6 +150,15 @@ public class SubscriptionServiceStoreAdhocTest extends // Adhoc subscriptions no longer interact with the registry } + @Override + protected void verifyOnlyCheckingForDuplicateSubscriptions() + throws RegistryHandlerException { + // Adhocs don't check for duplicates + final ISubscriptionHandler subscriptionHandler = DataDeliveryHandlers + .getSubscriptionHandler(); + verifyNoMoreInteractions(subscriptionHandler); + } + /** * {@inheritDoc} */ diff --git a/tests/unit/com/raytheon/uf/viz/datadelivery/subscription/presenter/CreateSubscriptionPresenterTest.java b/tests/unit/com/raytheon/uf/viz/datadelivery/subscription/presenter/CreateSubscriptionPresenterTest.java index 69d5a52f20..68b7621671 100644 --- a/tests/unit/com/raytheon/uf/viz/datadelivery/subscription/presenter/CreateSubscriptionPresenterTest.java +++ b/tests/unit/com/raytheon/uf/viz/datadelivery/subscription/presenter/CreateSubscriptionPresenterTest.java @@ -40,12 +40,13 @@ import org.junit.Before; import org.junit.Test; import com.google.common.collect.Sets; +import com.raytheon.uf.common.datadelivery.registry.DataType; import com.raytheon.uf.common.datadelivery.registry.GriddedDataSet; import com.raytheon.uf.common.datadelivery.registry.OpenDapGriddedDataSet; +import com.raytheon.uf.common.datadelivery.registry.SiteSubscription; +import com.raytheon.uf.common.datadelivery.registry.SiteSubscriptionFixture; import com.raytheon.uf.common.datadelivery.registry.Subscription; import com.raytheon.uf.common.datadelivery.registry.SubscriptionBuilder; -import com.raytheon.uf.common.datadelivery.registry.SiteSubscriptionFixture; -import com.raytheon.uf.common.datadelivery.registry.SiteSubscription; import com.raytheon.uf.common.registry.handler.RegistryHandlerException; import com.raytheon.uf.common.time.CalendarBuilder; import com.raytheon.uf.common.time.util.TimeUtil; @@ -91,6 +92,7 @@ public class CreateSubscriptionPresenterTest { dataSet.setDataSetName("name"); dataSet.setProviderName("provider"); dataSet.setForecastHours(Sets.newHashSet(0, 3, 6)); + dataSet.setDataSetType(DataType.GRID); dataSet.setCycles(Sets.newHashSet(0, 6, 12)); }