xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #2636 - Fix data set availability offset. Now uses average offset of last 10 data set matches

Browse source 
Former-commit-id: b841e795f2a18bf902ec373c9f79b191db177fde
This commit is contained in:
Mike Duff 2014-01-07 12:18:06 -06:00
parent 6febe84651
commit 183ca614df
18 changed files with 476 additions and 420 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/DataSetMetaData.java
View file

@ -41,13 +41,15 @@ import com.raytheon.uf.common.time.util.ImmutableDate;
* Oct 16, 2012 0726 djohnson Override {@link #toString()}.
* Nov 19, 2012 1166 djohnson Clean up JAXB representation of registry objects.
* Sept, 30 2013 1797 dhladky Made generic based on Time
* Dec 20, 2013 2636 mpduff Add a dataset availability offset
* </pre>
*
* @author dhladky
* @version 1.0
*/
@XmlAccessorType(XmlAccessType.NONE)
@XmlSeeAlso({ GriddedDataSetMetaData.class, OpenDapGriddedDataSetMetaData.class, PointDataSetMetaData.class })
@XmlSeeAlso({ GriddedDataSetMetaData.class,
OpenDapGriddedDataSetMetaData.class, PointDataSetMetaData.class })
@RegistryObject({ "url" })
public abstract class DataSetMetaData<T extends Time> {
public static final String DATE_SLOT = "date";
@ -55,13 +57,12 @@ public abstract class DataSetMetaData<T extends Time> {
public static final String DATA_SET_NAME_SLOT = "dataSetName";
public static final String PROVIDER_NAME_SLOT = "providerName";
/**
* Compares the two instances of {@link DataSetMetaData} by their applicable
* date fields.
*/
public static Comparator<? super DataSetMetaData<?>> DATE_COMPARATOR = new Comparator<DataSetMetaData<?>>()
{
public static Comparator<? super DataSetMetaData<?>> DATE_COMPARATOR = new Comparator<DataSetMetaData<?>>() {
@Override
public int compare(DataSetMetaData<?> o1, DataSetMetaData<?> o2) {
@ -109,6 +110,11 @@ public abstract class DataSetMetaData<T extends Time> {
@DynamicSerializeElement
private ImmutableDate date;
@XmlElement
@DynamicSerializeElement
@SlotAttribute
protected int availabilityOffset;
public DataSetMetaData() {
}
@ -159,7 +165,6 @@ public abstract class DataSetMetaData<T extends Time> {
public void setProviderName(String providerName) {
this.providerName = providerName;
}
/**
* Get the date this object starts on. In the gridded world, this would
@ -182,6 +187,21 @@ public abstract class DataSetMetaData<T extends Time> {
this.date = date;
}
/**
* @return the availabilityOffset
*/
public int getAvailabilityOffset() {
return availabilityOffset;
}
/**
* @param availabilityOffset
* the availabilityOffset to set
*/
public void setAvailabilityOffset(int availabilityOffset) {
this.availabilityOffset = availabilityOffset;
}
@Override
public boolean equals(Object obj) {
if (obj instanceof DataSetMetaData) {

45
edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/GriddedDataSetMetaData.java
View file

@ -27,6 +27,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* Aug 20, 2012 0743 djohnson Store cycle in a slot.
* Nov 19, 2012 1166 djohnson Clean up JAXB representation of registry objects.
* Sept 30, 2013 1797 dhladky Generics
* Dec 20, 2013 2636 mpduff Add equals/hashcode.
*
* </pre>
*
@ -85,4 +86,48 @@ public abstract class GriddedDataSetMetaData extends
return cycle;
}
/*
* (non-Javadoc)
*
* @see java.lang.Object#hashCode()
*/
@Override
public int hashCode() {
final int prime = 31;
int result = super.hashCode();
result = prime * result + cycle;
result = prime * result
+ ((levelTypes == null) ? 0 : levelTypes.hashCode());
return result;
}
/*
* (non-Javadoc)
*
* @see java.lang.Object#equals(java.lang.Object)
*/
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (!super.equals(obj)) {
return false;
}
if (!(obj instanceof GriddedDataSetMetaData)) {
return false;
}
GriddedDataSetMetaData other = (GriddedDataSetMetaData) obj;
if (cycle != other.cycle) {
return false;
}
if (levelTypes == null) {
if (other.levelTypes != null) {
return false;
}
} else if (!levelTypes.equals(other.levelTypes)) {
return false;
}
return true;
}
}

6
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/res/spring/bandwidth-datadelivery-edex-impl.xml
View file

@ -50,10 +50,4 @@
<constructor-arg ref="SubscriptionHandler" />
<constructor-arg ref="subscriptionNotificationService" />
</bean>
<bean id="dataSetAvailabilityCalculator"
class="com.raytheon.uf.edex.datadelivery.bandwidth.util.ProviderDataTypeAvailabilityCalculator">
<constructor-arg ref="ProviderHandler" />
</bean>
</beans>

1
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/res/spring/bandwidth-datadelivery.xml
View file

@ -49,7 +49,6 @@
<bean id="bandwidthUtil"
class="com.raytheon.uf.edex.datadelivery.bandwidth.util.BandwidthUtil"
factory-method="getInstance">
<property name="dataSetAvailabilityCalculator" ref="dataSetAvailabilityCalculator" />
<property name="subscriptionLatencyCalculator">
<bean
class="com.raytheon.uf.edex.datadelivery.bandwidth.util.SubscriptionValueLatencyCalculator" />

4
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/IBandwidthManager.java
View file

@ -42,7 +42,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.interfaces.ISubscriptionAggre
* Oct 30, 2012 1286 djohnson Initial creation
* Jul 10, 2013 2106 djohnson Remove EDEX instance specific methods.
* 10/23/2013 2385 bphillip Change schedule method to scheduleAdhoc
*
* Jan 06, 2014 2636 mpduff Update javadoc
* </pre>
*
* @author djohnson
@ -52,7 +52,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.interfaces.ISubscriptionAggre
public interface IBandwidthManager<T extends Time, C extends Coverage> {
/**
* Schedule all cycles of a Subscription.
* Schedule all retrievals of a Subscription.
*
* @param subscription
* @return A list of bandwidth allocations that are not scheduled

15
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/processing/SimpleSubscriptionAggregator.java
View file

@ -4,6 +4,7 @@ import java.util.ArrayList;
import java.util.List;
import com.raytheon.uf.common.datadelivery.registry.Subscription;
import com.raytheon.uf.common.registry.handler.RegistryHandlerException;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
@ -32,7 +33,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.util.BandwidthUtil;
* Nov 20, 2012 1286 djohnson Change some logging to debug.
* Jun 13, 2013 2095 djohnson No need to query the database, we are only receiving new bandwidth subscriptions.
* Jul 11, 2013 2106 djohnson aggregate() signature changed.
*
* Jan 06, 2014 2636 mpduff Changed how data set offset is set.
* </pre>
*
* @author jspinks
@ -80,8 +81,16 @@ public class SimpleSubscriptionAggregator implements ISubscriptionAggregator {
subscriptionRetrieval.setSubscriptionLatency(BandwidthUtil
.getSubscriptionLatency(sub));
subscriptionRetrieval.setDataSetAvailablityDelay(BandwidthUtil
.getDataSetAvailablityDelay(sub));
int offset = 0;
try {
offset = BandwidthUtil.getDataSetAvailablityOffset(sub,
subDao.getBaseReferenceTime());
} catch (RegistryHandlerException e) {
statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(),
e);
}
subscriptionRetrieval.setDataSetAvailablityDelay(offset);
subscriptionRetrievals.add(subscriptionRetrieval);

122
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/AveragingAvailablityCalculator.java
View file

@ -3,11 +3,20 @@
*/
package com.raytheon.uf.edex.datadelivery.bandwidth.util;
import java.util.Calendar;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData;
import com.raytheon.uf.common.datadelivery.registry.GriddedDataSetMetaData;
import com.raytheon.uf.common.datadelivery.registry.Subscription;
import com.raytheon.uf.edex.datadelivery.bandwidth.dao.BandwidthDataSetUpdate;
import com.raytheon.uf.edex.datadelivery.bandwidth.dao.IBandwidthDao;
import com.raytheon.uf.common.datadelivery.registry.handlers.IDataSetMetaDataHandler;
import com.raytheon.uf.common.registry.handler.RegistryHandlerException;
import com.raytheon.uf.common.util.CollectionUtil;
/**
*
@ -20,67 +29,102 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.dao.IBandwidthDao;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Nov 9, 2012 1286 djohnson Add SW history.
* Jan 06, 2014 2636 mpduff Changed how offset is determined.
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public class AveragingAvailablityCalculator implements
IDataSetAvailablityCalculator {
public class AveragingAvailablityCalculator {
LoadingCache<NameProviderKey, List<DataSetMetaData>> cache = CacheBuilder
.newBuilder().maximumSize(1000)
.expireAfterWrite(30, TimeUnit.MINUTES)
.build(new CacheLoader<NameProviderKey, List<DataSetMetaData>>() {
@Override
public List<DataSetMetaData> load(NameProviderKey key)
throws RegistryHandlerException {
return handler.getByDataSet(key.getName(),
key.getProvider());
}
});
private static final int DATASET_AVAILABLE_DEFAULT = 60;
private final IBandwidthDao bandwidthDao;
private final IDataSetMetaDataHandler handler;
/**
* Constructor.
*
* @param dao
* dao
* @param handler
* The DataSetMetaDataHandler
*/
public AveragingAvailablityCalculator(IBandwidthDao dao) {
this.bandwidthDao = dao;
public AveragingAvailablityCalculator(IDataSetMetaDataHandler handler) {
this.handler = handler;
}
/**
* Retrieve the DataSet availability latency for a particular dataset. This
* time encompasses the number of minutes after the dataset base time that
* data is typically available.
* Get the average dataset offset for the provided subscription
*
* @param providerName
* The provider name for the dataset.
* @param subscription
* The subscription
*
* @param dataSetName
* The dataset name.
* @param referenceTime
* The base reference time
*
* @return The number of minutes of latency to expect.
* @throws RegistryHandlerException
*/
@Override
public int getDataSetAvailablityDelay(Subscription subscription) {
// Use a default of 60 minutes..
int delay = DATASET_AVAILABLE_DEFAULT;
List<BandwidthDataSetUpdate> recs = bandwidthDao
.getBandwidthDataSetUpdate(subscription.getProvider(),
subscription.getDataSetName());
public int getDataSetAvailablityOffset(Subscription subscription,
Calendar referenceTime) throws RegistryHandlerException {
int offset = 0;
NameProviderKey key = new NameProviderKey(
subscription.getDataSetName(), subscription.getProvider());
List<DataSetMetaData> records = null;
long totalLatency = 0;
int recordcount = 0;
// Average out a maximum of twenty records
for (BandwidthDataSetUpdate rec : recs) {
long diff = (rec.getUpdateTime().getTimeInMillis() - rec
.getDataSetBaseTime().getTimeInMillis()) / 60000;
// Make sure some funky dates don't mess with the average..
totalLatency += Math.max(0, diff);
if (recordcount++ > 20) {
break;
try {
records = cache.get(key);
if (!CollectionUtil.isNullOrEmpty(records)) {
DataSetMetaData md = records.get(0);
if (md instanceof GriddedDataSetMetaData) {
offset = getOffsetForGrid(records, referenceTime);
}
// No availability delay for point data.
}
} catch (ExecutionException e) {
throw new RegistryHandlerException(e);
}
return offset;
}
/**
* Get the availability offset for gridded data.
*
* @param records
* List of DataSetMetaData records
* @param referenceTime
* The data's base reference time
* @return The offset in minutes
*/
private int getOffsetForGrid(List<DataSetMetaData> records,
Calendar referenceTime) {
int cycle = referenceTime.get(Calendar.HOUR_OF_DAY);
int count = 0;
int total = 0;
for (DataSetMetaData md : records) {
GriddedDataSetMetaData gmd = (GriddedDataSetMetaData) md;
if (gmd.getCycle() == cycle) {
total += gmd.getAvailabilityOffset();
count++;
if (count == 10) {
break;
}
}
}
// Make sure we did some kind of calculation.
if (totalLatency > 0) {
delay = (int) (totalLatency / recordcount);
if (count > 0) {
return total / count;
}
return delay;
return 0;
}
}

23
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/BandwidthDaoUtil.java
View file

@ -76,7 +76,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.RetrievalStatus;
* Nov 20, 2013 2448 bgonzale Fix for subscription start time set to first cycle time.
* Fix for subscription end time set to end of day.
* Dec 02, 2013 2545 mpduff Fix for delay starting retrievals, execute adhoc upon subscribing.
*
* Dec 20, 2013 2636 mpduff Fix dataset offset.
* </pre>
*
* @author djohnson
@ -297,18 +297,27 @@ public class BandwidthDaoUtil<T extends Time, C extends Coverage> {
// Now walk the subscription times and throw away anything outside the
// plan hours, taking into account the availability delay...
int availabilityDelay = BandwidthUtil
.getDataSetAvailablityDelay(subscription);
int availabilityOffset = 0;
Iterator<Calendar> itr = subscriptionTimes.iterator();
while (itr.hasNext()) {
availabilityOffset = 0;
Calendar time = itr.next();
Calendar withAvailabilityDelay = TimeUtil.newCalendar(time);
withAvailabilityDelay.add(Calendar.MINUTE, availabilityDelay);
try {
availabilityOffset = BandwidthUtil.getDataSetAvailablityOffset(
subscription, time);
} catch (RegistryHandlerException e) {
// Error occurred querying the registry. Log and continue on
statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(),
e);
}
Calendar withAvailabilityOffset = TimeUtil.newCalendar(time);
withAvailabilityOffset.add(Calendar.MINUTE, availabilityOffset);
// We allow base reference times that are still possible to retrieve
// within the availability window to be included
if (withAvailabilityDelay.before(planStart) || time.after(planEnd)) {
if (withAvailabilityOffset.before(planStart) || time.after(planEnd)) {
itr.remove();
}
}

38
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/BandwidthUtil.java
View file

@ -6,6 +6,8 @@ import java.util.Date;
import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData;
import com.raytheon.uf.common.datadelivery.registry.DataType;
import com.raytheon.uf.common.datadelivery.registry.Subscription;
import com.raytheon.uf.common.datadelivery.registry.handlers.DataDeliveryHandlers;
import com.raytheon.uf.common.registry.handler.RegistryHandlerException;
import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.edex.datadelivery.bandwidth.dao.BandwidthDataSetUpdate;
@ -29,6 +31,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.dao.BandwidthSubscription;
* Jun 25, 2013 2106 djohnson CheapClone was cheap in ease, not performance.
* Jul 11, 2013 2106 djohnson Use SubscriptionPriority enum.
* Oct 30, 2013 2448 dhladky Moved methods to TimeUtil.
* Dec 20, 2013 2636 mpduff Changed dataset delay to offset.
*
* </pre>
*
@ -49,15 +52,15 @@ public class BandwidthUtil {
// algorithms can be injected with Spring..
private static final BandwidthUtil instance = new BandwidthUtil();
// An implementation of the data set availability delay calculator
// injected with Spring.
private IDataSetAvailablityCalculator dataSetAvailabilityCalculator;
private final AveragingAvailablityCalculator dataSetAvailabilityCalculator;
private ISubscriptionLatencyCalculator subscriptionLatencyCalculator;
private ISubscriptionRescheduleStrategy subscriptionRescheduleStrategy;
private BandwidthUtil() {
dataSetAvailabilityCalculator = new AveragingAvailablityCalculator(
DataDeliveryHandlers.getDataSetMetaDataHandler());
};
public static int getSubscriptionLatency(Subscription<?, ?> subscription) {
@ -71,7 +74,7 @@ public class BandwidthUtil {
public static Calendar max(Date lhs, Calendar rhs) {
return max(TimeUtil.newCalendar(lhs), rhs);
}
public static Calendar max(Calendar lhs, Calendar rhs) {
Calendar calendar = null;
if (lhs != null && rhs != null) {
@ -112,22 +115,21 @@ public class BandwidthUtil {
}
/**
* Calculate the number of minutes of delay between a Subscriptions base
* Calculate the number of minutes of offset between a Subscriptions base
* reference time and the time the data should be available.
*
* @param subscription
* The Subscription Object to obtain the availability for.
*
* @return The delay in minutes.
* @param referenceTime
* Data reference time
* @return The offset in minutes.
* @throws RegistryHandlerException
*/
public static int getDataSetAvailablityDelay(Subscription<?, ?> subscription) {
public static int getDataSetAvailablityOffset(
Subscription<?, ?> subscription, Calendar referenceTime)
throws RegistryHandlerException {
return instance.dataSetAvailabilityCalculator
.getDataSetAvailablityDelay(subscription);
}
public void setDataSetAvailabilityCalculator(
IDataSetAvailablityCalculator dataSetAvailabilityCalculator) {
this.dataSetAvailabilityCalculator = dataSetAvailabilityCalculator;
.getDataSetAvailablityOffset(subscription, referenceTime);
}
/**
@ -268,16 +270,16 @@ public class BandwidthUtil {
.subscriptionRequiresReschedule(subscription, old);
}
/**
* Sets up the activePeriod Start/End to plan Start/End calendar
*/
public static Calendar planToPeriodCompareCalendar(Calendar planCalendar, Calendar activePeriod) {
public static Calendar planToPeriodCompareCalendar(Calendar planCalendar,
Calendar activePeriod) {
Calendar cal = TimeUtil.newCalendar(planCalendar);
cal.set(Calendar.MONTH, activePeriod.get(Calendar.MONTH));
cal.set(Calendar.DAY_OF_MONTH, activePeriod.get(Calendar.DAY_OF_MONTH));
return cal;
}
}

37
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/IDataSetAvailablityCalculator.java
View file

@ -1,37 +0,0 @@
/**
*
*/
package com.raytheon.uf.edex.datadelivery.bandwidth.util;
import com.raytheon.uf.common.datadelivery.registry.Subscription;
/**
* Interface for a pluggable implementation for calculating a data sets
* availability.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Nov 9, 2012 1286 djohnson Added SW histor, rename to comply with AWIPS standards.
*
* </pre>
*
* @author jspinks
* @version 1.0
*/
public interface IDataSetAvailablityCalculator {
/**
* Calculate the number of minutes of delay between a Subscriptions base
* reference time and the time the data should be available.
*
* @param subscription
* The Subscription Object to obtain the availability for.
*
* @return The delay in minutes.
*/
int getDataSetAvailablityDelay(Subscription subscription);
}

117
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/NameProviderKey.java Normal file
View file

@ -0,0 +1,117 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.bandwidth.util;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
/**
* Object holding name and provider Strings to act as a hash key.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Dec 20, 2013 2636 mpduff Initial creation
*
* </pre>
*
* @author mpduff
* @version 1.0
*/
public class NameProviderKey {
private final String name;
private final String provider;
/**
* Constructor.
*
* @param name
* The name
* @param provider
* The provider
*/
public NameProviderKey(String name, String provider) {
this.name = name;
this.provider = provider;
}
/**
* @return the name
*/
public String getName() {
return name;
}
/**
* @return the provider
*/
public String getProvider() {
return provider;
}
/*
* (non-Javadoc)
*
* @see java.lang.Object#toString()
*/
@Override
public String toString() {
return this.name + " " + this.provider;
}
/*
* (non-Javadoc)
*
* @see java.lang.Object#hashCode()
*/
@Override
public int hashCode() {
HashCodeBuilder builder = new HashCodeBuilder();
builder.append(name);
builder.append(provider);
return builder.hashCode();
}
/*
* (non-Javadoc)
*
* @see java.lang.Object#equals(java.lang.Object)
*/
@Override
public boolean equals(Object obj) {
if (!(obj instanceof NameProviderKey)) {
return false;
}
NameProviderKey other = (NameProviderKey) obj;
EqualsBuilder builder = new EqualsBuilder();
builder.append(this.name, other.getName());
builder.append(this.provider, other.getProvider());
return builder.isEquals();
}
}

128
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/ProviderDataTypeAvailabilityCalculator.java
View file

@ -1,128 +0,0 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.bandwidth.util;
import java.util.concurrent.ExecutionException;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.eventbus.Subscribe;
import com.raytheon.uf.common.datadelivery.registry.DataDeliveryRegistryObjectTypes;
import com.raytheon.uf.common.datadelivery.registry.DataType;
import com.raytheon.uf.common.datadelivery.registry.Provider;
import com.raytheon.uf.common.datadelivery.registry.ProviderType;
import com.raytheon.uf.common.datadelivery.registry.Subscription;
import com.raytheon.uf.common.datadelivery.registry.handlers.IProviderHandler;
import com.raytheon.uf.common.event.EventBus;
import com.raytheon.uf.common.registry.event.RegistryEvent;
import com.raytheon.uf.common.registry.handler.RegistryHandlerException;
/**
* Uses provider configured values for the availability delay.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 05, 2013 2038 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public class ProviderDataTypeAvailabilityCalculator implements
IDataSetAvailablityCalculator {
private static final Provider PROVIDER_NOT_FOUND = new Provider();
private final LoadingCache<String, Provider> providerCache;
/**
* Constructor.
*/
public ProviderDataTypeAvailabilityCalculator(
final IProviderHandler providerHandler) {
// TODO: This should probably be moved inside the registry handler
// itself
this.providerCache = CacheBuilder.newBuilder().build(
new CacheLoader<String, Provider>() {
@Override
public Provider load(String key)
throws RegistryHandlerException {
Provider provider = providerHandler
.getByName(key);
if (provider == null) {
provider = PROVIDER_NOT_FOUND;
}
return provider;
}
});
EventBus.register(this);
}
/**
* {@inheritDoc}
*/
@Override
public int getDataSetAvailablityDelay(Subscription subscription) {
final String providerName = subscription.getProvider();
final DataType dataType = subscription.getDataSetType();
try {
Provider provider = this.providerCache.get(providerName);
if (provider == PROVIDER_NOT_FOUND) {
throw new IllegalArgumentException(
"No availability delay registered for provider "
+ providerName + " for data type " + dataType);
}
final ProviderType providerType = provider
.getProviderType(dataType);
if (providerType == null) {
throw new IllegalArgumentException(
"No availability delay registered for provider "
+ providerName + " for data type " + dataType);
}
return providerType.getAvailabilityDelay();
} catch (ExecutionException e) {
throw new IllegalStateException(
"Exception querying for the provider!", e);
}
}
@Subscribe
public void registryEventListener(RegistryEvent re) {
final String objectType = re.getObjectType();
// If a provider event happens then expire the entire cache
if (DataDeliveryRegistryObjectTypes.PROVIDER.equals(objectType)) {
providerCache.invalidateAll();
}
}
}

42
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/SimpleAvailablityCalculator.java
View file

@ -1,42 +0,0 @@
/**
*
*/
package com.raytheon.uf.edex.datadelivery.bandwidth.util;
import com.raytheon.uf.common.datadelivery.registry.Subscription;
/**
* @author jspinks
*
*/
public class SimpleAvailablityCalculator implements
IDataSetAvailablityCalculator {
private int delay;
public int getDelay() {
return delay;
}
public void setDelay(int delay) {
this.delay = delay;
}
/**
* Retrieve the DataSet availability latency for a particular dataset. This
* time encompasses the number of minutes after the dataset base time that
* data is typically available.
*
* @param providerName
* The provider name for the dataset.
*
* @param dataSetName
* The dataset name.
*
* @return The number of minutes of latency to expect.
*/
@Override
public int getDataSetAvailablityDelay(Subscription subscription) {
return delay;
}
}

4
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/opendap/OpenDAPMetaDataParser.java
View file

@ -94,7 +94,7 @@ import dods.dap.DAS;
* Jan 24, 2013 1527 dhladky Changed 0DEG to FRZ
* Sept 25, 2013 1797 dhladky separated time from gridded time
* Oct 10, 2013 1797 bgonzale Refactored registry Time objects.
* Dec 18, 2013 2636 mpduff Calculate a data availability delay for the dataset.
* Dec 18, 2013 2636 mpduff Calculate a data availability delay for the dataset and dataset meta data.
*
* </pre>
*
@ -267,6 +267,7 @@ class OpenDAPMetaDataParser extends MetaDataParser {
long now = TimeUtil.newGmtCalendar().getTimeInMillis();
long offset = (now - startMillis) / TimeUtil.MILLIS_PER_MINUTE;
dataSet.setAvailabilityOffset((int) offset);
gdsmd.setAvailabilityOffset((int) offset);
if (statusHandler.isPriorityEnabled(Priority.DEBUG)) {
statusHandler.debug("Dataset Name: "
@ -776,5 +777,4 @@ class OpenDAPMetaDataParser extends MetaDataParser {
return parsedMetadatas;
}
}

5
edexOsgi/com.raytheon.uf.edex.registry.ebxml/src/com/raytheon/uf/edex/registry/ebxml/dao/DbInit.java
View file

@ -90,6 +90,7 @@ import com.raytheon.uf.edex.registry.ebxml.init.RegistryInitializedListener;
* they can't fail the ebxml schema creation/population.
* Nov 01, 2013 2361 njensen Use EbxmlJaxbManager instead of SerializationUtil
* Nov 14, 2013 2552 bkowal EbxmlJaxbManager is now accessed via getInstance
* Dec 20, 2013 2636 mpduff Set initialized to true before postInitialized is called.
* </pre>
*
* @author bphillip
@ -346,9 +347,9 @@ public class DbInit extends com.raytheon.uf.edex.database.init.DbInit implements
statusHandler.fatal("Error initializing EBXML database!", e);
}
myself.postInitDb();
INITIALIZED = true;
myself.postInitDb();
}
}

138
tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/util/AveragingAvailabilityCalculatorTest.java Normal file
View file

@ -0,0 +1,138 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.bandwidth.util;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;
import org.junit.Before;
import org.junit.Test;
import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData;
import com.raytheon.uf.common.datadelivery.registry.GriddedDataSetMetaData;
import com.raytheon.uf.common.datadelivery.registry.OpenDapGriddedDataSetMetaData;
import com.raytheon.uf.common.datadelivery.registry.PointDataSetMetaData;
import com.raytheon.uf.common.datadelivery.registry.SiteSubscription;
import com.raytheon.uf.common.datadelivery.registry.Subscription;
import com.raytheon.uf.common.datadelivery.registry.handlers.IDataSetMetaDataHandler;
import com.raytheon.uf.common.registry.handler.RegistryHandlerException;
import com.raytheon.uf.common.time.util.ImmutableDate;
import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.common.util.registry.RegistryException;
/**
* Test dataset offset average calculations.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Dec 20, 2013 2636 mpduff Initial creation
*
* </pre>
*
* @author mpduff
* @version 1.0
*/
public class AveragingAvailabilityCalculatorTest {
private static final String NAME = "DataSetName";
private static final String NOMADS = "NOMADS";
private static final String MADIS = "MADIS";
private static final int[] offsets = new int[] { 5, 4, 6, 7, 9, 5, 4, 5, 6,
4, 3, 6, 2 };
private final IDataSetMetaDataHandler handler = mock(IDataSetMetaDataHandler.class);
private final AveragingAvailablityCalculator availabilityCalculator = new AveragingAvailablityCalculator(
handler);
@Before
public void setUp() throws RegistryException, RegistryHandlerException {
List<DataSetMetaData> gmdList = new ArrayList<DataSetMetaData>();
List<DataSetMetaData> pointList = new ArrayList<DataSetMetaData>();
Calendar cal = TimeUtil.newGmtCalendar();
cal.set(Calendar.MONTH, 2);
cal.set(Calendar.DATE, 20);
cal.set(Calendar.MINUTE, 0);
cal.set(Calendar.SECOND, 0);
cal.set(Calendar.MILLISECOND, 0);
// Create 20 DataSetMetaData objects, 10 each for grid and point
for (int i = 0; i < offsets.length; i++) {
cal.add(Calendar.DATE, -1);
DataSetMetaData gmd = new OpenDapGriddedDataSetMetaData();
gmd.setAvailabilityOffset(offsets[i]);
gmd.setDate(new ImmutableDate(cal.getTime()));
((GriddedDataSetMetaData) gmd).setCycle(cal
.get(Calendar.HOUR_OF_DAY));
gmdList.add(gmd);
DataSetMetaData pmd = new PointDataSetMetaData();
pmd.setAvailabilityOffset(offsets[i]);
pmd.setDate(new ImmutableDate(cal.getTime()));
pointList.add(pmd);
}
when(handler.getByDataSet(NAME, NOMADS)).thenReturn(gmdList);
when(handler.getByDataSet(NAME, MADIS)).thenReturn(pointList);
}
@Test
public void testGridCalculation() throws RegistryHandlerException {
Calendar refTime = TimeUtil.newGmtCalendar();
refTime.set(Calendar.MONTH, 2);
refTime.set(Calendar.DATE, 21);
Subscription subscription = new SiteSubscription();
subscription.setProvider(NOMADS);
subscription.setDataSetName(NAME);
int offset = availabilityCalculator.getDataSetAvailablityOffset(
subscription, refTime);
assertTrue("Average not correct.", offset == getAvg());
}
/**
* Get the average. The average calculator uses the last 10 values in the
* avg calculation.
*
* @return the average
*/
private int getAvg() {
int total = 0;
for (int i = 0; i < 10; i++) {
total += offsets[i];
}
return total / 10;
}
}

30
tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/util/BandwidthDaoUtilTest.java
View file

@ -82,6 +82,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.RetrievalStatus;
* Jun 05, 2013 2038 djohnson Use public API for getting retrieval times.
* Jun 25, 2013 2106 djohnson RetrievalPlan uses setters instead of constructor injection now.
* Sept 25, 2013 1797 dhladky separated time and gridded time
* Jan 07, 2014 2636 mpduff Removed dataset availability offset calculator (not used).
*
* </pre>
*
@ -106,11 +107,6 @@ public class BandwidthDaoUtilTest {
public void setUp() {
TimeUtilTest.freezeTime(TimeUtil.MILLIS_PER_DAY * 2);
SimpleAvailablityCalculator dataSetAvailabilityCalculator = new SimpleAvailablityCalculator();
dataSetAvailabilityCalculator.setDelay(0);
BandwidthUtil.getInstance().setDataSetAvailabilityCalculator(
dataSetAvailabilityCalculator);
PathManagerFactoryTest.initLocalization();
IPathManager pm = PathManagerFactory.getPathManager();
@ -156,11 +152,11 @@ public class BandwidthDaoUtilTest {
.withActivePeriodStart(plan.getPlanStart().getTime())
.withActivePeriodEnd(plan.getPlanEnd().getTime())
.withSubscriptionStart(TimeUtil.newImmutableDate()).build();
((GriddedTime)subscription.getTime()).setCycleTimes(
Arrays.asList(Integer.valueOf(9), Integer.valueOf(0)));
((GriddedTime) subscription.getTime()).setCycleTimes(Arrays.asList(
Integer.valueOf(9), Integer.valueOf(0)));
TreeSet<Integer> cycles = new TreeSet<Integer>(((GriddedTime)subscription.getTime())
.getCycleTimes());
TreeSet<Integer> cycles = new TreeSet<Integer>(
((GriddedTime) subscription.getTime()).getCycleTimes());
SortedSet<Calendar> subscriptionTimes = bandwidthDaoUtil
.getRetrievalTimes(subscription, cycles);
@ -178,11 +174,11 @@ public class BandwidthDaoUtilTest {
Subscription subscription = new SubscriptionBuilder()
.withSubscriptionStart(startsTwoDaysIntoPlan)
.withSubscriptionEnd(plan.getPlanEnd().getTime()).build();
((GriddedTime)subscription.getTime()).setCycleTimes(
Arrays.asList(Integer.valueOf(9), Integer.valueOf(0)));
((GriddedTime) subscription.getTime()).setCycleTimes(Arrays.asList(
Integer.valueOf(9), Integer.valueOf(0)));
TreeSet<Integer> cycles = new TreeSet<Integer>(((GriddedTime)subscription.getTime())
.getCycleTimes());
TreeSet<Integer> cycles = new TreeSet<Integer>(
((GriddedTime) subscription.getTime()).getCycleTimes());
SortedSet<Calendar> subscriptionTimes = bandwidthDaoUtil
.getRetrievalTimes(subscription, cycles);
@ -203,11 +199,11 @@ public class BandwidthDaoUtilTest {
Subscription subscription = new SubscriptionBuilder()
.withSubscriptionStart(plan.getPlanStart().getTime())
.withSubscriptionEnd(endsOneDayBeforePlan).build();
((GriddedTime)subscription.getTime()).setCycleTimes(
Arrays.asList(Integer.valueOf(9), Integer.valueOf(0)));
((GriddedTime) subscription.getTime()).setCycleTimes(Arrays.asList(
Integer.valueOf(9), Integer.valueOf(0)));
TreeSet<Integer> cycles = new TreeSet<Integer>(((GriddedTime)subscription.getTime())
.getCycleTimes());
TreeSet<Integer> cycles = new TreeSet<Integer>(
((GriddedTime) subscription.getTime()).getCycleTimes());
SortedSet<Calendar> subscriptionTimes = bandwidthDaoUtil
.getRetrievalTimes(subscription, cycles);

111
tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/util/ProviderDataTypeAvailabilityCalculatorTest.java
View file

@ -1,111 +0,0 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.bandwidth.util;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.util.Arrays;
import org.junit.Before;
import org.junit.Test;
import com.raytheon.uf.common.datadelivery.registry.DataType;
import com.raytheon.uf.common.datadelivery.registry.Provider;
import com.raytheon.uf.common.datadelivery.registry.ProviderType;
import com.raytheon.uf.common.datadelivery.registry.SiteSubscription;
import com.raytheon.uf.common.datadelivery.registry.Subscription;
import com.raytheon.uf.common.datadelivery.registry.handlers.IProviderHandler;
import com.raytheon.uf.common.registry.handler.RegistryHandlerException;
import com.raytheon.uf.common.util.registry.RegistryException;
/**
* Test {@link ProviderDataTypeAvailabilityCalculator}.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 05, 2013 2038 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public class ProviderDataTypeAvailabilityCalculatorTest {
private static final int ONE_HUNDRED = 100;
private static final String PROVIDER_NAME = "someProviderName";
private static final Provider provider = new Provider();
static {
provider.setName(PROVIDER_NAME);
provider.setProviderType(Arrays.<ProviderType> asList(new ProviderType(
DataType.GRID, "grid", ONE_HUNDRED)));
}
private final IProviderHandler providerHandler = mock(IProviderHandler.class);
private final ProviderDataTypeAvailabilityCalculator availabilityCalculator = new ProviderDataTypeAvailabilityCalculator(
providerHandler);
@Before
public void setUp() throws RegistryException, RegistryHandlerException {
when(providerHandler.getByName(PROVIDER_NAME)).thenReturn(provider);
}
@Test
public void returnsConfiguredAvailabilityWhenRegistered() {
Subscription subscription = new SiteSubscription();
subscription.setProvider(PROVIDER_NAME);
subscription.setDataSetType(DataType.GRID);
assertThat(
availabilityCalculator.getDataSetAvailablityDelay(subscription),
is(equalTo(ONE_HUNDRED)));
}
@Test(expected = IllegalArgumentException.class)
public void throwsIllegalArgumentExceptionWhenProviderNotRegistered() {
Subscription subscription = new SiteSubscription();
subscription.setProvider("someOtherProviderName");
subscription.setDataSetType(DataType.GRID);
availabilityCalculator.getDataSetAvailablityDelay(subscription);
}
@Test(expected = IllegalArgumentException.class)
public void throwsIllegalArgumentExceptionWhenDataTypeNotRegistered() {
Subscription subscription = new SiteSubscription();
subscription.setProvider(PROVIDER_NAME);
subscription.setDataSetType(DataType.POINT);
availabilityCalculator.getDataSetAvailablityDelay(subscription);
}
}