xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge "Issue #2636 - Fix data set availability offset. Now uses average offset of last 10 data set matches" into omaha_14.2.1

Browse source 
Former-commit-id: b53137d1af67ff95a91b4bf4d2aa78ef4c43b80a
This commit is contained in:
Richard Peter 2014-01-09 16:58:31 -06:00 • committed by Gerrit Code Review
commit 36ed029f0a
18 changed files with 476 additions and 420 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/DataSetMetaData.java
View file

@ -41,13 +41,15 @@ import com.raytheon.uf.common.time.util.ImmutableDate;
* Oct 16, 2012 0726 djohnson Override {@link #toString()}.
* Nov 19, 2012 1166 djohnson Clean up JAXB representation of registry objects.
* Sept, 30 2013 1797 dhladky Made generic based on Time
* Dec 20, 2013 2636 mpduff Add a dataset availability offset
* </pre>
*
* @author dhladky
* @version 1.0
*/
@XmlAccessorType(XmlAccessType.NONE)
@XmlSeeAlso({ GriddedDataSetMetaData.class, OpenDapGriddedDataSetMetaData.class, PointDataSetMetaData.class })
@XmlSeeAlso({ GriddedDataSetMetaData.class,
OpenDapGriddedDataSetMetaData.class, PointDataSetMetaData.class })
@RegistryObject({ "url" })
public abstract class DataSetMetaData<T extends Time> {
public static final String DATE_SLOT = "date";
@ -55,13 +57,12 @@ public abstract class DataSetMetaData<T extends Time> {
public static final String DATA_SET_NAME_SLOT = "dataSetName";
public static final String PROVIDER_NAME_SLOT = "providerName";
/**
* Compares the two instances of {@link DataSetMetaData} by their applicable
* date fields.
*/
public static Comparator<? super DataSetMetaData<?>> DATE_COMPARATOR = new Comparator<DataSetMetaData<?>>()
{
public static Comparator<? super DataSetMetaData<?>> DATE_COMPARATOR = new Comparator<DataSetMetaData<?>>() {
@Override
public int compare(DataSetMetaData<?> o1, DataSetMetaData<?> o2) {
@ -109,6 +110,11 @@ public abstract class DataSetMetaData<T extends Time> {
@DynamicSerializeElement
private ImmutableDate date;
@XmlElement
@DynamicSerializeElement
@SlotAttribute
protected int availabilityOffset;
public DataSetMetaData() {
}
@ -159,7 +165,6 @@ public abstract class DataSetMetaData<T extends Time> {
public void setProviderName(String providerName) {
this.providerName = providerName;
}
/**
* Get the date this object starts on. In the gridded world, this would
@ -182,6 +187,21 @@ public abstract class DataSetMetaData<T extends Time> {
this.date = date;
}
/**
* @return the availabilityOffset
*/
public int getAvailabilityOffset() {
return availabilityOffset;
}
/**
* @param availabilityOffset
* the availabilityOffset to set
*/
public void setAvailabilityOffset(int availabilityOffset) {
this.availabilityOffset = availabilityOffset;
}
@Override
public boolean equals(Object obj) {
if (obj instanceof DataSetMetaData) {

45
edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/GriddedDataSetMetaData.java
View file

@ -27,6 +27,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* Aug 20, 2012 0743 djohnson Store cycle in a slot.
* Nov 19, 2012 1166 djohnson Clean up JAXB representation of registry objects.
* Sept 30, 2013 1797 dhladky Generics
* Dec 20, 2013 2636 mpduff Add equals/hashcode.
*
* </pre>
*
@ -85,4 +86,48 @@ public abstract class GriddedDataSetMetaData extends
return cycle;
}
/*
* (non-Javadoc)
*
* @see java.lang.Object#hashCode()
*/
@Override
public int hashCode() {
final int prime = 31;
int result = super.hashCode();
result = prime * result + cycle;
result = prime * result
+ ((levelTypes == null) ? 0 : levelTypes.hashCode());
return result;
}
/*
* (non-Javadoc)
*
* @see java.lang.Object#equals(java.lang.Object)
*/
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (!super.equals(obj)) {
return false;
}
if (!(obj instanceof GriddedDataSetMetaData)) {
return false;
}
GriddedDataSetMetaData other = (GriddedDataSetMetaData) obj;
if (cycle != other.cycle) {
return false;
}
if (levelTypes == null) {
if (other.levelTypes != null) {
return false;
}
} else if (!levelTypes.equals(other.levelTypes)) {
return false;
}
return true;
}
}

6
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/res/spring/bandwidth-datadelivery-edex-impl.xml
View file

@ -50,10 +50,4 @@
<constructor-arg ref="SubscriptionHandler" />
<constructor-arg ref="subscriptionNotificationService" />
</bean>
<bean id="dataSetAvailabilityCalculator"
class="com.raytheon.uf.edex.datadelivery.bandwidth.util.ProviderDataTypeAvailabilityCalculator">
<constructor-arg ref="ProviderHandler" />
</bean>
</beans>

1
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/res/spring/bandwidth-datadelivery.xml
View file

@ -49,7 +49,6 @@
<bean id="bandwidthUtil"
class="com.raytheon.uf.edex.datadelivery.bandwidth.util.BandwidthUtil"
factory-method="getInstance">
<property name="dataSetAvailabilityCalculator" ref="dataSetAvailabilityCalculator" />
<property name="subscriptionLatencyCalculator">
<bean
class="com.raytheon.uf.edex.datadelivery.bandwidth.util.SubscriptionValueLatencyCalculator" />

4
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/IBandwidthManager.java
View file

@ -42,7 +42,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.interfaces.ISubscriptionAggre
* Oct 30, 2012 1286 djohnson Initial creation
* Jul 10, 2013 2106 djohnson Remove EDEX instance specific methods.
* 10/23/2013 2385 bphillip Change schedule method to scheduleAdhoc
*
* Jan 06, 2014 2636 mpduff Update javadoc
* </pre>
*
* @author djohnson
@ -52,7 +52,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.interfaces.ISubscriptionAggre
public interface IBandwidthManager<T extends Time, C extends Coverage> {
/**
* Schedule all cycles of a Subscription.
* Schedule all retrievals of a Subscription.
*
* @param subscription
* @return A list of bandwidth allocations that are not scheduled

15
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/processing/SimpleSubscriptionAggregator.java
View file

@ -4,6 +4,7 @@ import java.util.ArrayList;
import java.util.List;
import com.raytheon.uf.common.datadelivery.registry.Subscription;
import com.raytheon.uf.common.registry.handler.RegistryHandlerException;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
@ -32,7 +33,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.util.BandwidthUtil;
* Nov 20, 2012 1286 djohnson Change some logging to debug.
* Jun 13, 2013 2095 djohnson No need to query the database, we are only receiving new bandwidth subscriptions.
* Jul 11, 2013 2106 djohnson aggregate() signature changed.
*
* Jan 06, 2014 2636 mpduff Changed how data set offset is set.
* </pre>
*
* @author jspinks
@ -80,8 +81,16 @@ public class SimpleSubscriptionAggregator implements ISubscriptionAggregator {
subscriptionRetrieval.setSubscriptionLatency(BandwidthUtil
.getSubscriptionLatency(sub));
subscriptionRetrieval.setDataSetAvailablityDelay(BandwidthUtil
.getDataSetAvailablityDelay(sub));
int offset = 0;
try {
offset = BandwidthUtil.getDataSetAvailablityOffset(sub,
subDao.getBaseReferenceTime());
} catch (RegistryHandlerException e) {
statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(),
e);
}
subscriptionRetrieval.setDataSetAvailablityDelay(offset);
subscriptionRetrievals.add(subscriptionRetrieval);

122
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/AveragingAvailablityCalculator.java
View file

@ -3,11 +3,20 @@
*/
package com.raytheon.uf.edex.datadelivery.bandwidth.util;
import java.util.Calendar;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData;
import com.raytheon.uf.common.datadelivery.registry.GriddedDataSetMetaData;
import com.raytheon.uf.common.datadelivery.registry.Subscription;
import com.raytheon.uf.edex.datadelivery.bandwidth.dao.BandwidthDataSetUpdate;
import com.raytheon.uf.edex.datadelivery.bandwidth.dao.IBandwidthDao;
import com.raytheon.uf.common.datadelivery.registry.handlers.IDataSetMetaDataHandler;
import com.raytheon.uf.common.registry.handler.RegistryHandlerException;
import com.raytheon.uf.common.util.CollectionUtil;
/**
*
@ -20,67 +29,102 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.dao.IBandwidthDao;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Nov 9, 2012 1286 djohnson Add SW history.
* Jan 06, 2014 2636 mpduff Changed how offset is determined.
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public class AveragingAvailablityCalculator implements
IDataSetAvailablityCalculator {
public class AveragingAvailablityCalculator {
LoadingCache<NameProviderKey, List<DataSetMetaData>> cache = CacheBuilder
.newBuilder().maximumSize(1000)
.expireAfterWrite(30, TimeUnit.MINUTES)
.build(new CacheLoader<NameProviderKey, List<DataSetMetaData>>() {
@Override
public List<DataSetMetaData> load(NameProviderKey key)
throws RegistryHandlerException {
return handler.getByDataSet(key.getName(),
key.getProvider());
}
});
private static final int DATASET_AVAILABLE_DEFAULT = 60;
private final IBandwidthDao bandwidthDao;
private final IDataSetMetaDataHandler handler;
/**
* Constructor.
*
* @param dao
* dao
* @param handler
* The DataSetMetaDataHandler
*/
public AveragingAvailablityCalculator(IBandwidthDao dao) {
this.bandwidthDao = dao;
public AveragingAvailablityCalculator(IDataSetMetaDataHandler handler) {
this.handler = handler;
}
/**
* Retrieve the DataSet availability latency for a particular dataset. This
* time encompasses the number of minutes after the dataset base time that
* data is typically available.
* Get the average dataset offset for the provided subscription
*
* @param providerName
* The provider name for the dataset.
* @param subscription
* The subscription
*
* @param dataSetName
* The dataset name.
* @param referenceTime
* The base reference time
*
* @return The number of minutes of latency to expect.
* @throws RegistryHandlerException
*/
@Override
public int getDataSetAvailablityDelay(Subscription subscription) {
// Use a default of 60 minutes..
int delay = DATASET_AVAILABLE_DEFAULT;
List<BandwidthDataSetUpdate> recs = bandwidthDao
.getBandwidthDataSetUpdate(subscription.getProvider(),
subscription.getDataSetName());
public int getDataSetAvailablityOffset(Subscription subscription,
Calendar referenceTime) throws RegistryHandlerException {
int offset = 0;
NameProviderKey key = new NameProviderKey(
subscription.getDataSetName(), subscription.getProvider());
List<DataSetMetaData> records = null;
long totalLatency = 0;
int recordcount = 0;
// Average out a maximum of twenty records
for (BandwidthDataSetUpdate rec : recs) {
long diff = (rec.getUpdateTime().getTimeInMillis() - rec
.getDataSetBaseTime().getTimeInMillis()) / 60000;
// Make sure some funky dates don't mess with the average..
totalLatency += Math.max(0, diff);
if (recordcount++ > 20) {
break;
try {
records = cache.get(key);
if (!CollectionUtil.isNullOrEmpty(records)) {
DataSetMetaData md = records.get(0);
if (md instanceof GriddedDataSetMetaData) {
offset = getOffsetForGrid(records, referenceTime);
}
// No availability delay for point data.
}
} catch (ExecutionException e) {
throw new RegistryHandlerException(e);
}
return offset;
}
/**
* Get the availability offset for gridded data.
*
* @param records
* List of DataSetMetaData records
* @param referenceTime
* The data's base reference time
* @return The offset in minutes
*/
private int getOffsetForGrid(List<DataSetMetaData> records,
Calendar referenceTime) {
int cycle = referenceTime.get(Calendar.HOUR_OF_DAY);
int count = 0;
int total = 0;
for (DataSetMetaData md : records) {
GriddedDataSetMetaData gmd = (GriddedDataSetMetaData) md;
if (gmd.getCycle() == cycle) {
total += gmd.getAvailabilityOffset();
count++;
if (count == 10) {
break;
}
}
}
// Make sure we did some kind of calculation.
if (totalLatency > 0) {
delay = (int) (totalLatency / recordcount);
if (count > 0) {
return total / count;
}
return delay;
return 0;
}
}

23
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/BandwidthDaoUtil.java
View file

@ -76,7 +76,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.RetrievalStatus;
* Nov 20, 2013 2448 bgonzale Fix for subscription start time set to first cycle time.
* Fix for subscription end time set to end of day.
* Dec 02, 2013 2545 mpduff Fix for delay starting retrievals, execute adhoc upon subscribing.
*
* Dec 20, 2013 2636 mpduff Fix dataset offset.
* </pre>
*
* @author djohnson
@ -297,18 +297,27 @@ public class BandwidthDaoUtil<T extends Time, C extends Coverage> {
// Now walk the subscription times and throw away anything outside the
// plan hours, taking into account the availability delay...
int availabilityDelay = BandwidthUtil
.getDataSetAvailablityDelay(subscription);
int availabilityOffset = 0;
Iterator<Calendar> itr = subscriptionTimes.iterator();
while (itr.hasNext()) {
availabilityOffset = 0;
Calendar time = itr.next();
Calendar withAvailabilityDelay = TimeUtil.newCalendar(time);
withAvailabilityDelay.add(Calendar.MINUTE, availabilityDelay);
try {
availabilityOffset = BandwidthUtil.getDataSetAvailablityOffset(
subscription, time);
} catch (RegistryHandlerException e) {
// Error occurred querying the registry. Log and continue on
statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(),
e);
}
Calendar withAvailabilityOffset = TimeUtil.newCalendar(time);
withAvailabilityOffset.add(Calendar.MINUTE, availabilityOffset);
// We allow base reference times that are still possible to retrieve
// within the availability window to be included
if (withAvailabilityDelay.before(planStart) || time.after(planEnd)) {
if (withAvailabilityOffset.before(planStart) || time.after(planEnd)) {
itr.remove();
}
}

38
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/BandwidthUtil.java
View file

@ -6,6 +6,8 @@ import java.util.Date;
import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData;
import com.raytheon.uf.common.datadelivery.registry.DataType;
import com.raytheon.uf.common.datadelivery.registry.Subscription;
import com.raytheon.uf.common.datadelivery.registry.handlers.DataDeliveryHandlers;
import com.raytheon.uf.common.registry.handler.RegistryHandlerException;
import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.edex.datadelivery.bandwidth.dao.BandwidthDataSetUpdate;
@ -29,6 +31,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.dao.BandwidthSubscription;
* Jun 25, 2013 2106 djohnson CheapClone was cheap in ease, not performance.
* Jul 11, 2013 2106 djohnson Use SubscriptionPriority enum.
* Oct 30, 2013 2448 dhladky Moved methods to TimeUtil.
* Dec 20, 2013 2636 mpduff Changed dataset delay to offset.
*
* </pre>
*
@ -49,15 +52,15 @@ public class BandwidthUtil {
// algorithms can be injected with Spring..
private static final BandwidthUtil instance = new BandwidthUtil();
// An implementation of the data set availability delay calculator
// injected with Spring.
private IDataSetAvailablityCalculator dataSetAvailabilityCalculator;
private final AveragingAvailablityCalculator dataSetAvailabilityCalculator;
private ISubscriptionLatencyCalculator subscriptionLatencyCalculator;
private ISubscriptionRescheduleStrategy subscriptionRescheduleStrategy;
private BandwidthUtil() {
dataSetAvailabilityCalculator = new AveragingAvailablityCalculator(
DataDeliveryHandlers.getDataSetMetaDataHandler());
};
public static int getSubscriptionLatency(Subscription<?, ?> subscription) {
@ -71,7 +74,7 @@ public class BandwidthUtil {
public static Calendar max(Date lhs, Calendar rhs) {
return max(TimeUtil.newCalendar(lhs), rhs);
}
public static Calendar max(Calendar lhs, Calendar rhs) {
Calendar calendar = null;
if (lhs != null && rhs != null) {
@ -112,22 +115,21 @@ public class BandwidthUtil {
}
/**
* Calculate the number of minutes of delay between a Subscriptions base
* Calculate the number of minutes of offset between a Subscriptions base
* reference time and the time the data should be available.
*
* @param subscription
* The Subscription Object to obtain the availability for.
*
* @return The delay in minutes.
* @param referenceTime
* Data reference time
* @return The offset in minutes.
* @throws RegistryHandlerException
*/
public static int getDataSetAvailablityDelay(Subscription<?, ?> subscription) {
public static int getDataSetAvailablityOffset(
Subscription<?, ?> subscription, Calendar referenceTime)
throws RegistryHandlerException {
return instance.dataSetAvailabilityCalculator
.getDataSetAvailablityDelay(subscription);
}
public void setDataSetAvailabilityCalculator(
IDataSetAvailablityCalculator dataSetAvailabilityCalculator) {
this.dataSetAvailabilityCalculator = dataSetAvailabilityCalculator;
.getDataSetAvailablityOffset(subscription, referenceTime);
}
/**
@ -268,16 +270,16 @@ public class BandwidthUtil {
.subscriptionRequiresReschedule(subscription, old);
}
/**
* Sets up the activePeriod Start/End to plan Start/End calendar
*/
public static Calendar planToPeriodCompareCalendar(Calendar planCalendar, Calendar activePeriod) {
public static Calendar planToPeriodCompareCalendar(Calendar planCalendar,
Calendar activePeriod) {
Calendar cal = TimeUtil.newCalendar(planCalendar);
cal.set(Calendar.MONTH, activePeriod.get(Calendar.MONTH));
cal.set(Calendar.DAY_OF_MONTH, activePeriod.get(Calendar.DAY_OF_MONTH));
return cal;
}
}

37
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/IDataSetAvailablityCalculator.java
View file

@ -1,37 +0,0 @@
/**
*
*/
package com.raytheon.uf.edex.datadelivery.bandwidth.util;
import com.raytheon.uf.common.datadelivery.registry.Subscription;
/**
* Interface for a pluggable implementation for calculating a data sets
* availability.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Nov 9, 2012 1286 djohnson Added SW histor, rename to comply with AWIPS standards.
*
* </pre>
*
* @author jspinks
* @version 1.0
*/
public interface IDataSetAvailablityCalculator {
/**
* Calculate the number of minutes of delay between a Subscriptions base
* reference time and the time the data should be available.
*
* @param subscription
* The Subscription Object to obtain the availability for.
*
* @return The delay in minutes.
*/
int getDataSetAvailablityDelay(Subscription subscription);
}

117
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/NameProviderKey.java Normal file
View file

@ -0,0 +1,117 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.bandwidth.util;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
/**
* Object holding name and provider Strings to act as a hash key.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Dec 20, 2013 2636 mpduff Initial creation
*
* </pre>
*
* @author mpduff
* @version 1.0
*/
public class NameProviderKey {
private final String name;
private final String provider;
/**
* Constructor.
*
* @param name
* The name
* @param provider
* The provider
*/
public NameProviderKey(String name, String provider) {
this.name = name;
this.provider = provider;
}
/**
* @return the name
*/
public String getName() {
return name;
}
/**
* @return the provider
*/
public String getProvider() {
return provider;
}
/*
* (non-Javadoc)
*
* @see java.lang.Object#toString()
*/
@Override
public String toString() {
return this.name + " " + this.provider;
}
/*
* (non-Javadoc)
*
* @see java.lang.Object#hashCode()
*/
@Override
public int hashCode() {
HashCodeBuilder builder = new HashCodeBuilder();
builder.append(name);
builder.append(provider);
return builder.hashCode();
}
/*
* (non-Javadoc)
*
* @see java.lang.Object#equals(java.lang.Object)
*/
@Override
public boolean equals(Object obj) {
if (!(obj instanceof NameProviderKey)) {
return false;
}
NameProviderKey other = (NameProviderKey) obj;
EqualsBuilder builder = new EqualsBuilder();
builder.append(this.name, other.getName());
builder.append(this.provider, other.getProvider());
return builder.isEquals();
}
}

128
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/ProviderDataTypeAvailabilityCalculator.java
View file

@ -1,128 +0,0 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.bandwidth.util;
import java.util.concurrent.ExecutionException;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.eventbus.Subscribe;
import com.raytheon.uf.common.datadelivery.registry.DataDeliveryRegistryObjectTypes;
import com.raytheon.uf.common.datadelivery.registry.DataType;
import com.raytheon.uf.common.datadelivery.registry.Provider;
import com.raytheon.uf.common.datadelivery.registry.ProviderType;
import com.raytheon.uf.common.datadelivery.registry.Subscription;
import com.raytheon.uf.common.datadelivery.registry.handlers.IProviderHandler;
import com.raytheon.uf.common.event.EventBus;
import com.raytheon.uf.common.registry.event.RegistryEvent;
import com.raytheon.uf.common.registry.handler.RegistryHandlerException;
/**
* Uses provider configured values for the availability delay.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 05, 2013 2038 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public class ProviderDataTypeAvailabilityCalculator implements
IDataSetAvailablityCalculator {
private static final Provider PROVIDER_NOT_FOUND = new Provider();
private final LoadingCache<String, Provider> providerCache;
/**
* Constructor.
*/
public ProviderDataTypeAvailabilityCalculator(
final IProviderHandler providerHandler) {
// TODO: This should probably be moved inside the registry handler
// itself
this.providerCache = CacheBuilder.newBuilder().build(
new CacheLoader<String, Provider>() {
@Override
public Provider load(String key)
throws RegistryHandlerException {
Provider provider = providerHandler
.getByName(key);
if (provider == null) {
provider = PROVIDER_NOT_FOUND;
}
return provider;
}
});
EventBus.register(this);
}
/**
* {@inheritDoc}
*/
@Override
public int getDataSetAvailablityDelay(Subscription subscription) {
final String providerName = subscription.getProvider();
final DataType dataType = subscription.getDataSetType();
try {
Provider provider = this.providerCache.get(providerName);
if (provider == PROVIDER_NOT_FOUND) {
throw new IllegalArgumentException(
"No availability delay registered for provider "
+ providerName + " for data type " + dataType);
}
final ProviderType providerType = provider
.getProviderType(dataType);
if (providerType == null) {
throw new IllegalArgumentException(
"No availability delay registered for provider "
+ providerName + " for data type " + dataType);
}
return providerType.getAvailabilityDelay();
} catch (ExecutionException e) {
throw new IllegalStateException(
"Exception querying for the provider!", e);
}
}
@Subscribe
public void registryEventListener(RegistryEvent re) {
final String objectType = re.getObjectType();
// If a provider event happens then expire the entire cache
if (DataDeliveryRegistryObjectTypes.PROVIDER.equals(objectType)) {
providerCache.invalidateAll();
}
}
}

42
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/SimpleAvailablityCalculator.java
View file

@ -1,42 +0,0 @@
/**
*
*/
package com.raytheon.uf.edex.datadelivery.bandwidth.util;
import com.raytheon.uf.common.datadelivery.registry.Subscription;
/**
* @author jspinks
*
*/
public class SimpleAvailablityCalculator implements
IDataSetAvailablityCalculator {
private int delay;
public int getDelay() {
return delay;
}
public void setDelay(int delay) {
this.delay = delay;
}
/**
* Retrieve the DataSet availability latency for a particular dataset. This
* time encompasses the number of minutes after the dataset base time that
* data is typically available.
*
* @param providerName
* The provider name for the dataset.
*
* @param dataSetName
* The dataset name.
*
* @return The number of minutes of latency to expect.
*/
@Override
public int getDataSetAvailablityDelay(Subscription subscription) {
return delay;
}
}

4
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPMetaDataParser.java
View file

@ -94,7 +94,7 @@ import dods.dap.DAS;
* Jan 24, 2013 1527 dhladky Changed 0DEG to FRZ
* Sept 25, 2013 1797 dhladky separated time from gridded time
* Oct 10, 2013 1797 bgonzale Refactored registry Time objects.
* Dec 18, 2013 2636 mpduff Calculate a data availability delay for the dataset.
* Dec 18, 2013 2636 mpduff Calculate a data availability delay for the dataset and dataset meta data.
*
* </pre>
*
@ -267,6 +267,7 @@ class OpenDAPMetaDataParser extends MetaDataParser {
long now = TimeUtil.newGmtCalendar().getTimeInMillis();
long offset = (now - startMillis) / TimeUtil.MILLIS_PER_MINUTE;
dataSet.setAvailabilityOffset((int) offset);
gdsmd.setAvailabilityOffset((int) offset);
if (statusHandler.isPriorityEnabled(Priority.DEBUG)) {
statusHandler.debug("Dataset Name: "
@ -776,5 +777,4 @@ class OpenDAPMetaDataParser extends MetaDataParser {
return parsedMetadatas;
}
}

5
edexOsgi/com.raytheon.uf.edex.registry.ebxml/src/com/raytheon/uf/edex/registry/ebxml/dao/DbInit.java
View file

@ -90,6 +90,7 @@ import com.raytheon.uf.edex.registry.ebxml.init.RegistryInitializedListener;
* they can't fail the ebxml schema creation/population.
* Nov 01, 2013 2361 njensen Use EbxmlJaxbManager instead of SerializationUtil
* Nov 14, 2013 2552 bkowal EbxmlJaxbManager is now accessed via getInstance
* Dec 20, 2013 2636 mpduff Set initialized to true before postInitialized is called.
* </pre>
*
* @author bphillip
@ -346,9 +347,9 @@ public class DbInit extends com.raytheon.uf.edex.database.init.DbInit implements
statusHandler.fatal("Error initializing EBXML database!", e);
}
myself.postInitDb();
INITIALIZED = true;
myself.postInitDb();
}
}

138
tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/util/AveragingAvailabilityCalculatorTest.java Normal file
View file

@ -0,0 +1,138 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.bandwidth.util;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;
import org.junit.Before;
import org.junit.Test;
import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData;
import com.raytheon.uf.common.datadelivery.registry.GriddedDataSetMetaData;
import com.raytheon.uf.common.datadelivery.registry.OpenDapGriddedDataSetMetaData;
import com.raytheon.uf.common.datadelivery.registry.PointDataSetMetaData;
import com.raytheon.uf.common.datadelivery.registry.SiteSubscription;
import com.raytheon.uf.common.datadelivery.registry.Subscription;
import com.raytheon.uf.common.datadelivery.registry.handlers.IDataSetMetaDataHandler;
import com.raytheon.uf.common.registry.handler.RegistryHandlerException;
import com.raytheon.uf.common.time.util.ImmutableDate;
import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.common.util.registry.RegistryException;
/**
* Test dataset offset average calculations.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Dec 20, 2013 2636 mpduff Initial creation
*
* </pre>
*
* @author mpduff
* @version 1.0
*/
public class AveragingAvailabilityCalculatorTest {
private static final String NAME = "DataSetName";
private static final String NOMADS = "NOMADS";
private static final String MADIS = "MADIS";
private static final int[] offsets = new int[] { 5, 4, 6, 7, 9, 5, 4, 5, 6,
4, 3, 6, 2 };
private final IDataSetMetaDataHandler handler = mock(IDataSetMetaDataHandler.class);
private final AveragingAvailablityCalculator availabilityCalculator = new AveragingAvailablityCalculator(
handler);
@Before
public void setUp() throws RegistryException, RegistryHandlerException {
List<DataSetMetaData> gmdList = new ArrayList<DataSetMetaData>();
List<DataSetMetaData> pointList = new ArrayList<DataSetMetaData>();
Calendar cal = TimeUtil.newGmtCalendar();
cal.set(Calendar.MONTH, 2);
cal.set(Calendar.DATE, 20);
cal.set(Calendar.MINUTE, 0);
cal.set(Calendar.SECOND, 0);
cal.set(Calendar.MILLISECOND, 0);
// Create 20 DataSetMetaData objects, 10 each for grid and point
for (int i = 0; i < offsets.length; i++) {
cal.add(Calendar.DATE, -1);
DataSetMetaData gmd = new OpenDapGriddedDataSetMetaData();
gmd.setAvailabilityOffset(offsets[i]);
gmd.setDate(new ImmutableDate(cal.getTime()));
((GriddedDataSetMetaData) gmd).setCycle(cal
.get(Calendar.HOUR_OF_DAY));
gmdList.add(gmd);
DataSetMetaData pmd = new PointDataSetMetaData();
pmd.setAvailabilityOffset(offsets[i]);
pmd.setDate(new ImmutableDate(cal.getTime()));
pointList.add(pmd);
}
when(handler.getByDataSet(NAME, NOMADS)).thenReturn(gmdList);
when(handler.getByDataSet(NAME, MADIS)).thenReturn(pointList);
}
@Test
public void testGridCalculation() throws RegistryHandlerException {
Calendar refTime = TimeUtil.newGmtCalendar();
refTime.set(Calendar.MONTH, 2);
refTime.set(Calendar.DATE, 21);
Subscription subscription = new SiteSubscription();
subscription.setProvider(NOMADS);
subscription.setDataSetName(NAME);
int offset = availabilityCalculator.getDataSetAvailablityOffset(
subscription, refTime);
assertTrue("Average not correct.", offset == getAvg());
}
/**
* Get the average. The average calculator uses the last 10 values in the
* avg calculation.
*
* @return the average
*/
private int getAvg() {
int total = 0;
for (int i = 0; i < 10; i++) {
total += offsets[i];
}
return total / 10;
}
}

30
tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/util/BandwidthDaoUtilTest.java
View file

@ -82,6 +82,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.RetrievalStatus;
* Jun 05, 2013 2038 djohnson Use public API for getting retrieval times.
* Jun 25, 2013 2106 djohnson RetrievalPlan uses setters instead of constructor injection now.
* Sept 25, 2013 1797 dhladky separated time and gridded time
* Jan 07, 2014 2636 mpduff Removed dataset availability offset calculator (not used).
*
* </pre>
*
@ -106,11 +107,6 @@ public class BandwidthDaoUtilTest {
public void setUp() {
TimeUtilTest.freezeTime(TimeUtil.MILLIS_PER_DAY * 2);
SimpleAvailablityCalculator dataSetAvailabilityCalculator = new SimpleAvailablityCalculator();
dataSetAvailabilityCalculator.setDelay(0);
BandwidthUtil.getInstance().setDataSetAvailabilityCalculator(
dataSetAvailabilityCalculator);
PathManagerFactoryTest.initLocalization();
IPathManager pm = PathManagerFactory.getPathManager();
@ -156,11 +152,11 @@ public class BandwidthDaoUtilTest {
.withActivePeriodStart(plan.getPlanStart().getTime())
.withActivePeriodEnd(plan.getPlanEnd().getTime())
.withSubscriptionStart(TimeUtil.newImmutableDate()).build();
((GriddedTime)subscription.getTime()).setCycleTimes(
Arrays.asList(Integer.valueOf(9), Integer.valueOf(0)));
((GriddedTime) subscription.getTime()).setCycleTimes(Arrays.asList(
Integer.valueOf(9), Integer.valueOf(0)));
TreeSet<Integer> cycles = new TreeSet<Integer>(((GriddedTime)subscription.getTime())
.getCycleTimes());
TreeSet<Integer> cycles = new TreeSet<Integer>(
((GriddedTime) subscription.getTime()).getCycleTimes());
SortedSet<Calendar> subscriptionTimes = bandwidthDaoUtil
.getRetrievalTimes(subscription, cycles);
@ -178,11 +174,11 @@ public class BandwidthDaoUtilTest {
Subscription subscription = new SubscriptionBuilder()
.withSubscriptionStart(startsTwoDaysIntoPlan)
.withSubscriptionEnd(plan.getPlanEnd().getTime()).build();
((GriddedTime)subscription.getTime()).setCycleTimes(
Arrays.asList(Integer.valueOf(9), Integer.valueOf(0)));
((GriddedTime) subscription.getTime()).setCycleTimes(Arrays.asList(
Integer.valueOf(9), Integer.valueOf(0)));
TreeSet<Integer> cycles = new TreeSet<Integer>(((GriddedTime)subscription.getTime())
.getCycleTimes());
TreeSet<Integer> cycles = new TreeSet<Integer>(
((GriddedTime) subscription.getTime()).getCycleTimes());
SortedSet<Calendar> subscriptionTimes = bandwidthDaoUtil
.getRetrievalTimes(subscription, cycles);
@ -203,11 +199,11 @@ public class BandwidthDaoUtilTest {
Subscription subscription = new SubscriptionBuilder()
.withSubscriptionStart(plan.getPlanStart().getTime())
.withSubscriptionEnd(endsOneDayBeforePlan).build();
((GriddedTime)subscription.getTime()).setCycleTimes(
Arrays.asList(Integer.valueOf(9), Integer.valueOf(0)));
((GriddedTime) subscription.getTime()).setCycleTimes(Arrays.asList(
Integer.valueOf(9), Integer.valueOf(0)));
TreeSet<Integer> cycles = new TreeSet<Integer>(((GriddedTime)subscription.getTime())
.getCycleTimes());
TreeSet<Integer> cycles = new TreeSet<Integer>(
((GriddedTime) subscription.getTime()).getCycleTimes());
SortedSet<Calendar> subscriptionTimes = bandwidthDaoUtil
.getRetrievalTimes(subscription, cycles);

111
tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/util/ProviderDataTypeAvailabilityCalculatorTest.java
View file

@ -1,111 +0,0 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.bandwidth.util;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.util.Arrays;
import org.junit.Before;
import org.junit.Test;
import com.raytheon.uf.common.datadelivery.registry.DataType;
import com.raytheon.uf.common.datadelivery.registry.Provider;
import com.raytheon.uf.common.datadelivery.registry.ProviderType;
import com.raytheon.uf.common.datadelivery.registry.SiteSubscription;
import com.raytheon.uf.common.datadelivery.registry.Subscription;
import com.raytheon.uf.common.datadelivery.registry.handlers.IProviderHandler;
import com.raytheon.uf.common.registry.handler.RegistryHandlerException;
import com.raytheon.uf.common.util.registry.RegistryException;
/**
* Test {@link ProviderDataTypeAvailabilityCalculator}.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 05, 2013 2038 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public class ProviderDataTypeAvailabilityCalculatorTest {
private static final int ONE_HUNDRED = 100;
private static final String PROVIDER_NAME = "someProviderName";
private static final Provider provider = new Provider();
static {
provider.setName(PROVIDER_NAME);
provider.setProviderType(Arrays.<ProviderType> asList(new ProviderType(
DataType.GRID, "grid", ONE_HUNDRED)));
}
private final IProviderHandler providerHandler = mock(IProviderHandler.class);
private final ProviderDataTypeAvailabilityCalculator availabilityCalculator = new ProviderDataTypeAvailabilityCalculator(
providerHandler);
@Before
public void setUp() throws RegistryException, RegistryHandlerException {
when(providerHandler.getByName(PROVIDER_NAME)).thenReturn(provider);
}
@Test
public void returnsConfiguredAvailabilityWhenRegistered() {
Subscription subscription = new SiteSubscription();
subscription.setProvider(PROVIDER_NAME);
subscription.setDataSetType(DataType.GRID);
assertThat(
availabilityCalculator.getDataSetAvailablityDelay(subscription),
is(equalTo(ONE_HUNDRED)));
}
@Test(expected = IllegalArgumentException.class)
public void throwsIllegalArgumentExceptionWhenProviderNotRegistered() {
Subscription subscription = new SiteSubscription();
subscription.setProvider("someOtherProviderName");
subscription.setDataSetType(DataType.GRID);
availabilityCalculator.getDataSetAvailablityDelay(subscription);
}
@Test(expected = IllegalArgumentException.class)
public void throwsIllegalArgumentExceptionWhenDataTypeNotRegistered() {
Subscription subscription = new SiteSubscription();
subscription.setProvider(PROVIDER_NAME);
subscription.setDataSetType(DataType.POINT);
availabilityCalculator.getDataSetAvailablityDelay(subscription);
}
}