xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master_14.3.1' (14.3.1-3) into omaha_14.3.1

Browse source 
Conflicts:
	edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/BandwidthManager.java
	edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/EdexBandwidthManager.java
	edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/BandwidthDaoUtil.java
	edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/res/spring/harvester-datadelivery.xml
	edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/util/RetrievalGeneratorUtilities.java

Former-commit-id: ccaaa1cfea [formerly 333bf8b36d [formerly 2e5ee5565d1847490e46c59866b14f77555e2a7e]]
Former-commit-id: 333bf8b36d
Former-commit-id: 0ac4de6ad0
This commit is contained in:
Dave Hladky 2014-05-05 12:15:14 -05:00
commit 7d244313a5
42 changed files with 575 additions and 785 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/common/ui/TableDataManager.java
View file

@ -31,6 +31,7 @@ package com.raytheon.uf.viz.datadelivery.common.ui;
* Jun 06, 2012 lvenable Initial creation * Jun 06, 2012 lvenable Initial creation
* Apr 10, 2013 1891 djohnson Declare variable as List. * Apr 10, 2013 1891 djohnson Declare variable as List.
* Feb 07, 2014 2453 mpduff Added getSize(). * Feb 07, 2014 2453 mpduff Added getSize().
* Apr 18, 2014 3012 dhladky Null check.
* *
* </pre> * </pre>
* *
@ -134,7 +135,11 @@ public class TableDataManager<T extends ITableData<T>> implements ISortTable {
if (index >= 0 && index < tableData.size()) { if (index >= 0 && index < tableData.size()) {
return tableData.get(index); return tableData.get(index);
} else { } else {
return tableData.get(0); if (!tableData.isEmpty()) {
return tableData.get(0);
} else {
return null;
}
} }
} }

4
cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/notification/NotificationTableComp.java
View file

@ -85,6 +85,7 @@ import com.raytheon.uf.viz.datadelivery.utils.NotificationHandler;
* Oct 15, 2013 2451 skorolev Get highlighted rows after message update. * Oct 15, 2013 2451 skorolev Get highlighted rows after message update.
* Nov 01, 2013 2431 skorolev Changed labels on the table. * Nov 01, 2013 2431 skorolev Changed labels on the table.
* Feb 07, 2014 2453 mpduff Refactored. * Feb 07, 2014 2453 mpduff Refactored.
* Apr 18, 2014 3012 dhladky Null check.
* </pre> * </pre>
* *
* @author lvenable * @author lvenable
@ -941,6 +942,9 @@ public class NotificationTableComp extends TableComp implements ITableFind {
for (int index : indices) { for (int index : indices) {
NotificationRowData rowData = filteredTableList NotificationRowData rowData = filteredTableList
.getDataRow(index); .getDataRow(index);
if (rowData == null) {
continue;
}
selectedRowIds.add(rowData.getId()); selectedRowIds.add(rowData.getId());
} }
} }

25
cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/SubscriptionManagerDlg.java
View file

@ -159,6 +159,7 @@ import com.raytheon.viz.ui.presenter.IDisplay;
* Mar 24, 2014 #2951 lvenable Added dispose checks for SWT widgets. * Mar 24, 2014 #2951 lvenable Added dispose checks for SWT widgets.
* Mar 31, 2014 2889 dhladky Added username for notification center tracking. * Mar 31, 2014 2889 dhladky Added username for notification center tracking.
* Apr 2, 2014 2974 dhladky DD ID added to list for dropdowns in DD. * Apr 2, 2014 2974 dhladky DD ID added to list for dropdowns in DD.
* Apr 18, 2014 3012 dhladky Null check.
* *
* </pre> * </pre>
* *
@ -1001,18 +1002,20 @@ public class SubscriptionManagerDlg extends CaveSWTDialog implements
SubscriptionManagerRowData removedItem = tableComp SubscriptionManagerRowData removedItem = tableComp
.getSubscriptionData().getDataRow(idx); .getSubscriptionData().getDataRow(idx);
Subscription sub = removedItem.getSubscription(); Subscription sub = removedItem.getSubscription();
if (sub instanceof SharedSubscription) { if (sub != null) {
sub.getOfficeIDs().remove(CURRENT_SITE); if (sub instanceof SharedSubscription) {
if (sub.getOfficeIDs().size() > 0) { sub.getOfficeIDs().remove(CURRENT_SITE);
subsToUpdate.add(sub); if (sub.getOfficeIDs().size() > 0) {
subsToUpdate.add(sub);
} else {
subsToDelete.add(sub);
}
} else { } else {
subsToDelete.add(sub); subsToDelete.add(removedItem.getSubscription());
} }
} else {
subsToDelete.add(removedItem.getSubscription());
}
deleteList.add(removedItem); deleteList.add(removedItem);
}
} }
String message = getMessage(subsToDelete, subsToUpdate); String message = getMessage(subsToDelete, subsToUpdate);
@ -1189,7 +1192,9 @@ public class SubscriptionManagerDlg extends CaveSWTDialog implements
int idx = selectionIndices[i]; int idx = selectionIndices[i];
SubscriptionManagerRowData rowData = tableComp SubscriptionManagerRowData rowData = tableComp
.getSubscriptionData().getDataRow(idx); .getSubscriptionData().getDataRow(idx);
if (rowData == null) {
continue;
}
Subscription sub = rowData.getSubscription(); Subscription sub = rowData.getSubscription();
if (activate) { if (activate) {
sub.activate(); sub.activate();

7
cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/SubscriptionTableComp.java
View file

@ -113,7 +113,7 @@ import com.raytheon.uf.viz.datadelivery.utils.DataDeliveryUtils.TABLE_TYPE;
* Feb 04, 2014 2722 mpduff Add last update time. * Feb 04, 2014 2722 mpduff Add last update time.
* Feb 11, 2014 2771 bgonzale Use Data Delivery ID instead of Site. * Feb 11, 2014 2771 bgonzale Use Data Delivery ID instead of Site.
* Mar 24, 2014 #2951 lvenable Added dispose checks for SWT widgets. * Mar 24, 2014 #2951 lvenable Added dispose checks for SWT widgets.
* * Apr 18, 2014 3012 dhladky Null check.
* @version 1.0 * @version 1.0
*/ */
@ -325,6 +325,9 @@ public class SubscriptionTableComp extends TableComp implements IGroupAction {
SubscriptionManagerRowData rowData = subManagerData SubscriptionManagerRowData rowData = subManagerData
.getDataRow(selectionIndices[i]); .getDataRow(selectionIndices[i]);
if (rowData == null) {
continue;
}
// get the subscription details to be displayed to the user // get the subscription details to be displayed to the user
printDetails.append(DataDeliveryUtils.formatDetails(rowData printDetails.append(DataDeliveryUtils.formatDetails(rowData
.getSubscription())); .getSubscription()));
@ -911,4 +914,4 @@ public class SubscriptionTableComp extends TableComp implements IGroupAction {
public long getLastUpdateTime() { public long getLastUpdateTime() {
return lastUpdateTime; return lastUpdateTime;
} }
} }

4
cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/approve/SubApprovalTableComp.java
View file

@ -74,6 +74,7 @@ import com.raytheon.uf.viz.datadelivery.utils.DataDeliveryUtils.TABLE_TYPE;
* denied pending messages. * denied pending messages.
* Apr 05, 2013 1841 djohnson Refresh entire table on receiving a notification of the correct type. * Apr 05, 2013 1841 djohnson Refresh entire table on receiving a notification of the correct type.
* Apr 10, 2013 1891 djohnson Move logic to get column display text to the column definition, fix sorting. * Apr 10, 2013 1891 djohnson Move logic to get column display text to the column definition, fix sorting.
* Apr 18, 2014 3012 dhladky Null check.
* </pre> * </pre>
* *
* @author lvenable * @author lvenable
@ -270,6 +271,9 @@ public class SubApprovalTableComp extends TableComp {
SubscriptionApprovalRowData rowData = pendingSubData.getDataRow(table SubscriptionApprovalRowData rowData = pendingSubData.getDataRow(table
.getSelectionIndex()); .getSelectionIndex());
if (rowData == null) {
return;
}
// Get the subscription object // Get the subscription object
InitialPendingSubscription pendingSub = rowData.getSubscription(); InitialPendingSubscription pendingSub = rowData.getSubscription();
diffDetails.append("Subscription Name: ") diffDetails.append("Subscription Name: ")

24
cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/approve/SubscriptionApprovalDlg.java
View file

@ -96,6 +96,7 @@ import com.raytheon.viz.ui.presenter.IDisplay;
* Sep 03, 2013 2315 mpduff Add subscription name to denied approval message. * Sep 03, 2013 2315 mpduff Add subscription name to denied approval message.
* Oct 23, 2013 2292 mpduff Move subscription overlap checks to edex. * Oct 23, 2013 2292 mpduff Move subscription overlap checks to edex.
* Mar 31, 2014 2889 dhladky Added username for notification center tracking. * Mar 31, 2014 2889 dhladky Added username for notification center tracking.
* Apr 18, 2014 3012 dhladky Null check.
* *
* </pre> * </pre>
* *
@ -368,13 +369,15 @@ public class SubscriptionApprovalDlg extends CaveSWTDialog implements
for (int idx : tableComp.getTable().getSelectionIndices()) { for (int idx : tableComp.getTable().getSelectionIndices()) {
SubscriptionApprovalRowData approvedItem = pendingSubData SubscriptionApprovalRowData approvedItem = pendingSubData
.getDataRow(idx); .getDataRow(idx);
if (site) { if (approvedItem != null) {
approveList.add(approvedItem); if (site) {
} else {
if (approvedItem.isOwner(user)) {
approveList.add(approvedItem); approveList.add(approvedItem);
} else { } else {
notApprovedSubList.add(approvedItem.getSubName()); if (approvedItem.isOwner(user)) {
approveList.add(approvedItem);
} else {
notApprovedSubList.add(approvedItem.getSubName());
}
} }
} }
} }
@ -461,12 +464,13 @@ public class SubscriptionApprovalDlg extends CaveSWTDialog implements
for (int idx : tableComp.getTable().getSelectionIndices()) { for (int idx : tableComp.getTable().getSelectionIndices()) {
SubscriptionApprovalRowData removedItem = pendingSubData SubscriptionApprovalRowData removedItem = pendingSubData
.getDataRow(idx); .getDataRow(idx);
if (removedItem != null) {
if (site) { if (site) {
deleteList.add(removedItem);
} else {
if (removedItem.isOwner(user)) {
deleteList.add(removedItem); deleteList.add(removedItem);
} else {
if (removedItem.isOwner(user)) {
deleteList.add(removedItem);
}
} }
} }
} }

32
edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/ifpAG/ASCIIGrid.java
View file

@ -25,6 +25,7 @@ import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.io.PrintWriter; import java.io.PrintWriter;
import java.text.DateFormat; import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
@ -34,6 +35,7 @@ import java.util.Scanner;
import com.raytheon.edex.plugin.gfe.config.IFPServerConfig; import com.raytheon.edex.plugin.gfe.config.IFPServerConfig;
import com.raytheon.edex.plugin.gfe.config.IFPServerConfigManager; import com.raytheon.edex.plugin.gfe.config.IFPServerConfigManager;
import com.raytheon.edex.plugin.gfe.exception.GfeConfigurationException; import com.raytheon.edex.plugin.gfe.exception.GfeConfigurationException;
import com.raytheon.edex.plugin.gfe.server.IFPServer;
import com.raytheon.uf.common.dataplugin.gfe.GridDataHistory; import com.raytheon.uf.common.dataplugin.gfe.GridDataHistory;
import com.raytheon.uf.common.dataplugin.gfe.GridDataHistory.OriginType; import com.raytheon.uf.common.dataplugin.gfe.GridDataHistory.OriginType;
import com.raytheon.uf.common.dataplugin.gfe.RemapGrid; import com.raytheon.uf.common.dataplugin.gfe.RemapGrid;
@ -46,6 +48,7 @@ import com.raytheon.uf.common.dataplugin.gfe.db.objects.GridParmInfo;
import com.raytheon.uf.common.dataplugin.gfe.db.objects.ParmID; import com.raytheon.uf.common.dataplugin.gfe.db.objects.ParmID;
import com.raytheon.uf.common.dataplugin.gfe.db.objects.TimeConstraints; import com.raytheon.uf.common.dataplugin.gfe.db.objects.TimeConstraints;
import com.raytheon.uf.common.dataplugin.gfe.discrete.DiscreteKey; import com.raytheon.uf.common.dataplugin.gfe.discrete.DiscreteKey;
import com.raytheon.uf.common.dataplugin.gfe.exception.GfeException;
import com.raytheon.uf.common.dataplugin.gfe.grid.Grid2DByte; import com.raytheon.uf.common.dataplugin.gfe.grid.Grid2DByte;
import com.raytheon.uf.common.dataplugin.gfe.grid.Grid2DFloat; import com.raytheon.uf.common.dataplugin.gfe.grid.Grid2DFloat;
import com.raytheon.uf.common.dataplugin.gfe.slice.DiscreteGridSlice; import com.raytheon.uf.common.dataplugin.gfe.slice.DiscreteGridSlice;
@ -72,6 +75,7 @@ import com.vividsolutions.jts.geom.Coordinate;
* Apr 13, 2011 #8393 dgilling Initial creation * Apr 13, 2011 #8393 dgilling Initial creation
* 02/19/13 #1637 randerso Added exception handling for Discrete and Weather * 02/19/13 #1637 randerso Added exception handling for Discrete and Weather
* 10/31/2013 #2508 randerso Change to use DiscreteGridSlice.getKeys() * 10/31/2013 #2508 randerso Change to use DiscreteGridSlice.getKeys()
* 04/22/2014 #3050 randerso Allow exceptions to propagate to caller from readASCIIGridData
* *
* </pre> * </pre>
* *
@ -353,12 +357,14 @@ public class ASCIIGrid {
} }
public String readASCIIGridData(File aGridData) public String readASCIIGridData(File aGridData)
throws FileNotFoundException { throws FileNotFoundException, GfeException, ParseException {
List<IGridSlice> gridSlices = new ArrayList<IGridSlice>(); List<IGridSlice> gridSlices = new ArrayList<IGridSlice>();
Scanner inputStream = new Scanner(aGridData, "US-ASCII");
while (true) { Scanner inputStream = null;
try { try {
inputStream = new Scanner(aGridData, "US-ASCII");
while (true) {
// read the ASCIIGRID keyword // read the ASCIIGRID keyword
// if we have an ASCIIGRID to read // if we have an ASCIIGRID to read
if (!inputStream.next().equals("ASCIIGRID")) { if (!inputStream.next().equals("ASCIIGRID")) {
@ -421,8 +427,12 @@ public class ASCIIGrid {
float yExtent = inputStream.nextFloat(); float yExtent = inputStream.nextFloat();
// make the GridLocation // make the GridLocation
IFPServerConfig config = IFPServerConfigManager IFPServer ifpServer = IFPServer.getActiveServer(dbSiteId);
.getServerConfig(dbSiteId); if (ifpServer == null) {
throw new GfeException("No active IFPServer for site: "
+ dbSiteId);
}
IFPServerConfig config = ifpServer.getConfig();
GridLocation baseGLoc = config.dbDomain(); GridLocation baseGLoc = config.dbDomain();
ProjectionData projData = config.getProjectionData(projId); ProjectionData projData = config.getProjectionData(projId);
GridLocation gLocation = new GridLocation(dbSiteId, projData, GridLocation gLocation = new GridLocation(dbSiteId, projData,
@ -600,14 +610,12 @@ public class ASCIIGrid {
if (!inputStream.hasNext()) { if (!inputStream.hasNext()) {
break; break;
} }
}
} catch (Exception e) { } finally {
statusHandler.handle(Priority.PROBLEM, if (inputStream != null) {
"Caught exception in readASCIIGridData()", e); inputStream.close();
break;
} }
} }
inputStream.close();
this.gridSlices = gridSlices; this.gridSlices = gridSlices;
return ""; return "";

19
edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/SaveASCIIGridsHandler.java
View file

@ -61,6 +61,9 @@ import com.raytheon.uf.common.status.UFStatus;
* Apr 21, 2011 dgilling Initial creation * Apr 21, 2011 dgilling Initial creation
* Apr 23, 2013 1949 rjpeter Removed extra lock table look up * Apr 23, 2013 1949 rjpeter Removed extra lock table look up
* Jun 13, 2013 #2044 randerso Refactored to use IFPServer * Jun 13, 2013 #2044 randerso Refactored to use IFPServer
* Apr 21, 2014 #3050 randerso Get the IFPServer instance based on the
* site in the ParmID
*
* </pre> * </pre>
* *
* @author dgilling * @author dgilling
@ -83,9 +86,6 @@ public class SaveASCIIGridsHandler extends BaseGfeRequestHandler implements
@Override @Override
public ServerResponse<String> handleRequest(SaveASCIIGridsRequest request) public ServerResponse<String> handleRequest(SaveASCIIGridsRequest request)
throws Exception { throws Exception {
IFPServer ifpServer = getIfpServer(request);
GridParmManager gridParmMgr = ifpServer.getGridParmMgr();
LockManager lockMgr = ifpServer.getLockMgr();
ServerResponse<String> sr = new ServerResponse<String>(); ServerResponse<String> sr = new ServerResponse<String>();
@ -97,10 +97,23 @@ public class SaveASCIIGridsHandler extends BaseGfeRequestHandler implements
sr.addMessage(msg); sr.addMessage(msg);
} }
String prevSiteID = null;
int ngrids = agrid.getGridSlices().size(); int ngrids = agrid.getGridSlices().size();
for (int i = 0; i < ngrids; i++) { for (int i = 0; i < ngrids; i++) {
ParmID pid = agrid.getGridSlices().get(i).getGridInfo().getParmID(); ParmID pid = agrid.getGridSlices().get(i).getGridInfo().getParmID();
// get the server for this site
String siteID = pid.getDbId().getSiteId();
IFPServer ifpServer = IFPServer.getActiveServer(siteID);
if (ifpServer == null && !siteID.equals(prevSiteID)) {
sr.addMessage("No active IFPServer for site: " + siteID);
continue;
}
prevSiteID = siteID;
GridParmManager gridParmMgr = ifpServer.getGridParmMgr();
LockManager lockMgr = ifpServer.getLockMgr();
// get a list of available databases, see if the grid is part of an // get a list of available databases, see if the grid is part of an
// existing database. // existing database.
ServerResponse<List<DatabaseID>> srDbInv = gridParmMgr ServerResponse<List<DatabaseID>> srDbInv = gridParmMgr

14
edexOsgi/com.raytheon.uf.common.datadelivery.harvester/src/com/raytheon/uf/common/datadelivery/harvester/HarvesterConfig.java
View file

@ -19,6 +19,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* 12 Sept, 2012 1038 dhladky Initial creation * 12 Sept, 2012 1038 dhladky Initial creation
* 1 May 2013 1959 dhladky remove backup registry references * 1 May 2013 1959 dhladky remove backup registry references
* 23 Oct, 2013 2361 njensen Remove ISerializableObject * 23 Oct, 2013 2361 njensen Remove ISerializableObject
* 15 Apr, 2014 3012 dhladky Added retention time for this provider in registry.
* *
* </pre> * </pre>
* *
@ -37,6 +38,11 @@ public class HarvesterConfig {
@XmlElement(name = "agent") @XmlElement(name = "agent")
@DynamicSerializeElement @DynamicSerializeElement
private Agent agent; private Agent agent;
/** default of 7 days **/
@XmlElement(name = "retention")
@DynamicSerializeElement
private String retention = "7";
public HarvesterConfig() { public HarvesterConfig() {
@ -58,4 +64,12 @@ public class HarvesterConfig {
this.provider = provider; this.provider = provider;
} }
public String getRetention() {
return retention;
}
public void setRetention(String retention) {
this.retention = retention;
}
} }

2
edexOsgi/com.raytheon.uf.common.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/NOMADS-harvester.xml
View file

@ -28,6 +28,8 @@
<url>http://nomads.ncep.noaa.gov:9090/dods/</url> <url>http://nomads.ncep.noaa.gov:9090/dods/</url>
</connection> </connection>
</provider> </provider>
<!-- default one week of DataSetMetaData retention -->
<retention>7</retention>
<agent xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="crawlAgent"> <agent xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="crawlAgent">
<crawlDir>/awips2/crawl</crawlDir> <crawlDir>/awips2/crawl</crawlDir>

2
edexOsgi/com.raytheon.uf.common.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/OGC-harvester.xml.sample
View file

@ -15,6 +15,8 @@
<description>METAR Test LatLon Coverage</description> <description>METAR Test LatLon Coverage</description>
</projection> </projection>
</provider> </provider>
<!-- retention time in days -->
<retention>1</retention>
<agent xsi:type="ogcAgent" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> <agent xsi:type="ogcAgent" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<dateFormat>HHddMMMyyyy</dateFormat> <dateFormat>HHddMMMyyyy</dateFormat>
<layer name="metar"> <layer name="metar">

12
edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/DataSetMetaData.java
View file

@ -43,7 +43,8 @@ import com.raytheon.uf.common.time.util.ImmutableDate;
* Nov 19, 2012 1166 djohnson Clean up JAXB representation of registry objects. * Nov 19, 2012 1166 djohnson Clean up JAXB representation of registry objects.
* Sept, 30 2013 1797 dhladky Made generic based on Time * Sept, 30 2013 1797 dhladky Made generic based on Time
* Dec 20, 2013 2636 mpduff Add a dataset availability offset * Dec 20, 2013 2636 mpduff Add a dataset availability offset
* jan 23, 2013 2584 dhladky Versions. * jan 23, 2013 2584 dhladky Versions.
* Apr 14, 2013 3012 dhladky Removed unused methods.
* </pre> * </pre>
* *
* @author dhladky * @author dhladky
@ -229,13 +230,4 @@ public abstract class DataSetMetaData<T extends Time> {
return url; return url;
} }
/**
* Accepts a {@link IDataSetMetaDataVisitor} which can perform arbitrary
* processing on this {@link DataSetMetaData} instance. Should be defined by
* each concrete class instance.
*
* @param visitor
* the visitor
*/
public abstract void accept(IDataSetMetaDataVisitor visitor);
} }

50
edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/IDataSetMetaDataVisitor.java
View file

@ -1,50 +0,0 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.datadelivery.registry;
/**
* Defines a type that can visit {@link DataSetMetaData} instances and perform
* some activity.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Sep 4, 2012 1102 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public interface IDataSetMetaDataVisitor {
/**
* Visits an {@link OpenDapGriddedDataSetMetaData} instance.
*
* @param metaData
* the metaData
*/
void visit(OpenDapGriddedDataSetMetaData metaData);
}

11
edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/OpenDapGriddedDataSetMetaData.java
View file

@ -36,7 +36,8 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* Sep 4, 2012 1102 djohnson Initial creation * Sep 4, 2012 1102 djohnson Initial creation
* jan 23, 2013 2584 dhladky Versions * jan 23, 2014 2584 dhladky Versions
* Apr 14, 2014 3012 dhladky Unneeded method removed.
* *
* </pre> * </pre>
* *
@ -49,12 +50,4 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
@DynamicSerialize @DynamicSerialize
public class OpenDapGriddedDataSetMetaData extends GriddedDataSetMetaData { public class OpenDapGriddedDataSetMetaData extends GriddedDataSetMetaData {
/**
* {@inheritDoc}
*/
@Override
public void accept(IDataSetMetaDataVisitor visitor) {
visitor.visit(this);
}
} }

8
edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/PointDataSetMetaData.java
View file

@ -34,6 +34,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* Aug 20, 2012 754 dhladky Initial creation * Aug 20, 2012 754 dhladky Initial creation
* Sept 30, 2013 1797 dhladky Generics * Sept 30, 2013 1797 dhladky Generics
* Apr 14, 2014 3012 dhladky Unneeded method removed.
* *
* </pre> * </pre>
* *
@ -50,10 +51,5 @@ public class PointDataSetMetaData extends DataSetMetaData<PointTime> {
public PointDataSetMetaData() { public PointDataSetMetaData() {
} }
@Override
public void accept(IDataSetMetaDataVisitor visitor) {
// TODO: not sure what this does?
}
} }

26
edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/xml/DataSetInformation.java
View file

@ -20,7 +20,6 @@ package com.raytheon.uf.common.datadelivery.retrieval.xml;
* further licensing information. * further licensing information.
**/ **/
import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlElement;
@ -28,6 +27,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
import com.raytheon.uf.common.time.util.TimeUtil;
/** /**
* Data Set Info XML Object. * Data Set Info XML Object.
@ -39,6 +39,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* Jan 14, 2014 dhladky Initial creation. * Jan 14, 2014 dhladky Initial creation.
* Apr 09, 2014 #3012 dhladky Fixed incorrect default calc.
* *
* </pre> * </pre>
* *
@ -54,25 +55,26 @@ public class DataSetInformation {
@XmlElement(name = "modelName", type = String.class) @XmlElement(name = "modelName", type = String.class)
@DynamicSerializeElement @DynamicSerializeElement
protected String modelName; protected String modelName;
@XmlElement(name = "multiplier", type = Double.class) @XmlElement(name = "multiplier", type = Double.class)
@DynamicSerializeElement @DynamicSerializeElement
protected Double multiplier; protected Double multiplier;
@XmlElement(name = "modelRunIncrement", type = Integer.class) @XmlElement(name = "modelRunIncrement", type = Integer.class)
@DynamicSerializeElement @DynamicSerializeElement
protected Integer modelRunIncrement; protected Integer modelRunIncrement;
@XmlElement(name = "defaultOffset", type = Integer.class) @XmlElement(name = "defaultOffset", type = Integer.class)
@DynamicSerializeElement @DynamicSerializeElement
protected Integer defaultOffset; protected Integer defaultOffset;
public DataSetInformation() { public DataSetInformation() {
} }
public DataSetInformation(String modelName, Double multiplier, int modelRunIncrement, int defaultOffset) { public DataSetInformation(String modelName, Double multiplier,
int modelRunIncrement, int defaultOffset) {
this.modelName = modelName; this.modelName = modelName;
this.multiplier = multiplier; this.multiplier = multiplier;
this.modelRunIncrement = modelRunIncrement; this.modelRunIncrement = modelRunIncrement;
@ -111,8 +113,12 @@ public class DataSetInformation {
this.defaultOffset = defaultOffset; this.defaultOffset = defaultOffset;
} }
/**
* The range int is in minutes, so we need multiplier * model running increment * minutes per hour
* @return
*/
public int getRange() { public int getRange() {
return (int) (getMultiplier() * getModelRunIncrement()); return (int) ((getMultiplier() * getModelRunIncrement()) * TimeUtil.MINUTES_PER_HOUR);
} }
} }

24
edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/dao/SessionManagedDao.java
View file

@ -21,6 +21,7 @@
package com.raytheon.uf.edex.database.dao; package com.raytheon.uf.edex.database.dao;
import java.io.Serializable; import java.io.Serializable;
import java.util.Calendar;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.Iterator; import java.util.Iterator;
@ -41,6 +42,7 @@ import org.springframework.transaction.annotation.Transactional;
import com.raytheon.uf.common.dataplugin.persist.IPersistableDataObject; import com.raytheon.uf.common.dataplugin.persist.IPersistableDataObject;
import com.raytheon.uf.common.status.IUFStatusHandler; import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus; import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.edex.database.DataAccessLayerException; import com.raytheon.uf.edex.database.DataAccessLayerException;
/** /**
@ -65,6 +67,7 @@ import com.raytheon.uf.edex.database.DataAccessLayerException;
* 12/9/2013 2613 bphillip Added flushAndClearSession method * 12/9/2013 2613 bphillip Added flushAndClearSession method
* Jan 17, 2014 2459 mpduff Added null check to prevent NPE. * Jan 17, 2014 2459 mpduff Added null check to prevent NPE.
* 2/13/2014 2769 bphillip Added read-only flag to query methods and loadById method * 2/13/2014 2769 bphillip Added read-only flag to query methods and loadById method
* 4/18/2014 3012 dhladky Diagnostic addition.
* *
* </pre> * </pre>
* *
@ -214,7 +217,26 @@ public abstract class SessionManagedDao<IDENTIFIER extends Serializable, ENTITY
*/ */
@Transactional(propagation = Propagation.REQUIRED, readOnly = true) @Transactional(propagation = Propagation.REQUIRED, readOnly = true)
public List<ENTITY> query(String queryString, Object... params) { public List<ENTITY> query(String queryString, Object... params) {
return executeHQLQuery(queryString, 0, params);
List<ENTITY> stuff = executeHQLQuery(queryString, 0, params);
// Used for diagnostics
if (statusHandler.isPriorityEnabled(Priority.DEBUG)) {
statusHandler.debug("Query: " + queryString);
for (Object o : params) {
if (o instanceof Calendar) {
statusHandler.debug("param: "
+ ((Calendar) o).getTime().toString());
} else {
statusHandler.debug("param: " + o.toString());
}
}
statusHandler.debug("return size: " + stuff.size() + "\n");
}
return stuff;
} }
@Transactional(propagation = Propagation.REQUIRED, readOnly = true) @Transactional(propagation = Propagation.REQUIRED, readOnly = true)

3
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/META-INF/MANIFEST.MF
View file

@ -34,7 +34,8 @@ Require-Bundle: com.raytheon.uf.common.status;bundle-version="1.12.1174",
com.raytheon.uf.common.datadelivery.service;bundle-version="1.0.0", com.raytheon.uf.common.datadelivery.service;bundle-version="1.0.0",
org.eclipse.jetty;bundle-version="7.6.14", org.eclipse.jetty;bundle-version="7.6.14",
com.raytheon.uf.common.serialization.comm;bundle-version="1.12.1174", com.raytheon.uf.common.serialization.comm;bundle-version="1.12.1174",
org.quartz;bundle-version="1.8.6" org.quartz;bundle-version="1.8.6",
com.raytheon.uf.edex.datadelivery.registry;bundle-version="1.0.0"
Export-Package: com.raytheon.uf.edex.datadelivery.bandwidth, Export-Package: com.raytheon.uf.edex.datadelivery.bandwidth,
com.raytheon.uf.edex.datadelivery.bandwidth.dao, com.raytheon.uf.edex.datadelivery.bandwidth.dao,
com.raytheon.uf.edex.datadelivery.bandwidth.hibernate, com.raytheon.uf.edex.datadelivery.bandwidth.hibernate,

59
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/BandwidthManager.java
View file

@ -148,6 +148,7 @@ import com.raytheon.uf.edex.registry.ebxml.util.RegistryIdUtil;
* Feb 11, 2014 2771 bgonzale Added handler for GET_DATADELIVERY_ID request. * Feb 11, 2014 2771 bgonzale Added handler for GET_DATADELIVERY_ID request.
* Feb 10, 2014 2636 mpduff Changed how retrieval plan is updated over time. * Feb 10, 2014 2636 mpduff Changed how retrieval plan is updated over time.
* Apr 02, 2014 2810 dhladky Priority sorting of subscriptions. * Apr 02, 2014 2810 dhladky Priority sorting of subscriptions.
* Apr 09, 2014 3012 dhladky Range the querries for metadata checks to subscriptions.
* Apr 22, 2014 2992 dhladky Ability to get list of all registry nodes containing data. * Apr 22, 2014 2992 dhladky Ability to get list of all registry nodes containing data.
* *
* </pre> * </pre>
@ -178,8 +179,14 @@ public abstract class BandwidthManager<T extends Time, C extends Coverage>
private final IBandwidthDbInit dbInit; private final IBandwidthDbInit dbInit;
/** used for min time range **/
public static final String MIN_RANGE_TIME = "min";
private final RegistryIdUtil idUtil; private final RegistryIdUtil idUtil;
/** used for max time range **/
public static final String MAX_RANGE_TIME = "max";
// Instance variable and not static, because there are multiple child // Instance variable and not static, because there are multiple child
// implementation classes which should each have a unique prefix // implementation classes which should each have a unique prefix
private final IPerformanceStatusHandler performanceHandler = PerformanceStatus private final IPerformanceStatusHandler performanceHandler = PerformanceStatus
@ -379,7 +386,9 @@ public abstract class BandwidthManager<T extends Time, C extends Coverage>
.getBaseReferenceTime(); .getBaseReferenceTime();
Calendar startTime = TimeUtil.newGmtCalendar(retrievalTime Calendar startTime = TimeUtil.newGmtCalendar(retrievalTime
.getTime()); .getTime());
startTime.add(Calendar.MINUTE,
retrieval.getDataSetAvailablityDelay());
int maxLatency = retrieval.getSubscriptionLatency(); int maxLatency = retrieval.getSubscriptionLatency();
retrieval.setStartTime(startTime); retrieval.setStartTime(startTime);
@ -458,8 +467,14 @@ public abstract class BandwidthManager<T extends Time, C extends Coverage>
@Override @Override
public List<BandwidthAllocation> schedule(Subscription<T, C> subscription) { public List<BandwidthAllocation> schedule(Subscription<T, C> subscription) {
List<BandwidthAllocation> unscheduled = null;
List<BandwidthAllocation> unscheduled = Collections.emptyList();
if (subscription instanceof RecurringSubscription) {
if (!((RecurringSubscription<T, C>) subscription).shouldSchedule()) {
return unscheduled;
}
}
final DataType dataSetType = subscription.getDataSetType(); final DataType dataSetType = subscription.getDataSetType();
switch (dataSetType) { switch (dataSetType) {
case GRID: case GRID:
@ -478,7 +493,7 @@ public abstract class BandwidthManager<T extends Time, C extends Coverage>
return unscheduled; return unscheduled;
} }
/** /**
* Update the retrieval plan for this subscription. * Update the retrieval plan for this subscription.
* *
@ -736,7 +751,10 @@ public abstract class BandwidthManager<T extends Time, C extends Coverage>
Subscription<T, C> subscription) { Subscription<T, C> subscription) {
List<BandwidthAllocation> unscheduled = schedule(subscription, List<BandwidthAllocation> unscheduled = schedule(subscription,
((PointTime) subscription.getTime()).getInterval()); ((PointTime) subscription.getTime()).getInterval());
unscheduled.addAll(getMostRecent(subscription, false)); // add an adhoc if one exists and isn't in startup mode
if (EDEXUtil.isRunning()) {
unscheduled.addAll(getMostRecent(subscription, false));
}
return unscheduled; return unscheduled;
} }
@ -760,10 +778,20 @@ public abstract class BandwidthManager<T extends Time, C extends Coverage>
if (subscribedToCycles) { if (subscribedToCycles) {
unscheduled = schedule(subscription, Sets.newTreeSet(cycles)); unscheduled = schedule(subscription, Sets.newTreeSet(cycles));
} }
// add an adhoc if one exists and isn't in startup mode
if (EDEXUtil.isRunning()) {
unscheduled.addAll(getMostRecent(subscription, true));
}
return unscheduled; return unscheduled;
} }
/**
* Schedule the most recent dataset update if one exists.
* @param subscription
* @param useMostRecentDataSetUpdate
* @return
*/
private List<BandwidthAllocation> getMostRecent( private List<BandwidthAllocation> getMostRecent(
Subscription<T, C> subscription, boolean useMostRecentDataSetUpdate) { Subscription<T, C> subscription, boolean useMostRecentDataSetUpdate) {
List<BandwidthAllocation> unscheduled = Collections.emptyList(); List<BandwidthAllocation> unscheduled = Collections.emptyList();
@ -1717,4 +1745,27 @@ public abstract class BandwidthManager<T extends Time, C extends Coverage>
return dataSetMetaDataTime; return dataSetMetaDataTime;
} }
/**
* Sets a range based on the baseReferenceTime hour.
* @param baseReferenceTime
* @return
*/
public static Map<String, Date> getBaseReferenceTimeDateRange(Calendar baseReferenceTime) {
Map<String, Date> dates = new HashMap<String, Date>(2);
// Set min range to baseReferenceTime hour "00" minutes, "00" seconds
// Set max range to baseReferenceTime hour "59" minutes, "59" seconds
Calendar min = TimeUtil.newGmtCalendar(baseReferenceTime.getTime());
min.set(Calendar.MINUTE, 0);
min.set(Calendar.SECOND, 0);
Calendar max = TimeUtil.newGmtCalendar(baseReferenceTime.getTime());
max.set(Calendar.MINUTE, 59);
max.set(Calendar.SECOND, 59);
dates.put(MIN_RANGE_TIME, min.getTime());
dates.put(MAX_RANGE_TIME, max.getTime());
return dates;
}
} }

46
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/EdexBandwidthManager.java
View file

@ -128,6 +128,7 @@ import com.raytheon.uf.edex.registry.ebxml.util.RegistryIdUtil;
* Feb 10, 2014 2636 mpduff Pass Network map to be scheduled. * Feb 10, 2014 2636 mpduff Pass Network map to be scheduled.
* Feb 21, 2014, 2636 dhladky Try catch to keep MaintTask from dying. * Feb 21, 2014, 2636 dhladky Try catch to keep MaintTask from dying.
* Mar 31, 2014 2889 dhladky Added username for notification center tracking. * Mar 31, 2014 2889 dhladky Added username for notification center tracking.
* Apr 09, 2014 3012 dhladky Range the queries for metadata checks, adhoc firing prevention.
* Apr 22, 2014 2992 dhladky Added IdUtil for siteList * Apr 22, 2014 2992 dhladky Added IdUtil for siteList
* </pre> * </pre>
* *
@ -295,7 +296,6 @@ public abstract class EdexBandwidthManager<T extends Time, C extends Coverage>
* @param subscription * @param subscription
* The completed subscription. * The completed subscription.
*/ */
@SuppressWarnings("unchecked")
@Subscribe @Subscribe
public void subscriptionFulfilled( public void subscriptionFulfilled(
SubscriptionRetrievalFulfilled subscriptionRetrievalFulfilled) { SubscriptionRetrievalFulfilled subscriptionRetrievalFulfilled) {
@ -385,6 +385,7 @@ public abstract class EdexBandwidthManager<T extends Time, C extends Coverage>
Subscription<T, C> sub = getRegistryObjectById(subscriptionHandler, Subscription<T, C> sub = getRegistryObjectById(subscriptionHandler,
re.getId()); re.getId());
sendSubscriptionNotificationEvent(re, sub); sendSubscriptionNotificationEvent(re, sub);
} }
} }
@ -450,9 +451,9 @@ public abstract class EdexBandwidthManager<T extends Time, C extends Coverage>
+ dsmd.getDataSetName() + "] to [rap_f]"); + dsmd.getDataSetName() + "] to [rap_f]");
dsmd.setDataSetName("rap_f"); dsmd.setDataSetName("rap_f");
} }
// TODO: End of hack..
BandwidthEventBus.publish(dsmd); BandwidthEventBus.publish(dsmd);
} else { } else {
statusHandler.error("No DataSetMetaData found for id [" + id + "]"); statusHandler.error("No DataSetMetaData found for id [" + id + "]");
} }
@ -492,6 +493,7 @@ public abstract class EdexBandwidthManager<T extends Time, C extends Coverage>
@Subscribe @Subscribe
public void updateGriddedDataSetMetaData( public void updateGriddedDataSetMetaData(
GriddedDataSetMetaData dataSetMetaData) throws ParseException { GriddedDataSetMetaData dataSetMetaData) throws ParseException {
// Daily/Hourly/Monthly datasets // Daily/Hourly/Monthly datasets
if (dataSetMetaData.getCycle() == GriddedDataSetMetaData.NO_CYCLE) { if (dataSetMetaData.getCycle() == GriddedDataSetMetaData.NO_CYCLE) {
updateDataSetMetaDataWithoutCycle((DataSetMetaData<T>) dataSetMetaData); updateDataSetMetaDataWithoutCycle((DataSetMetaData<T>) dataSetMetaData);
@ -500,6 +502,7 @@ public abstract class EdexBandwidthManager<T extends Time, C extends Coverage>
else { else {
updateDataSetMetaDataWithCycle((DataSetMetaData<T>) dataSetMetaData); updateDataSetMetaDataWithCycle((DataSetMetaData<T>) dataSetMetaData);
} }
} }
/** /**
@ -667,19 +670,20 @@ public abstract class EdexBandwidthManager<T extends Time, C extends Coverage>
DataSetMetaData<T> dataSetMetaData) throws ParseException { DataSetMetaData<T> dataSetMetaData) throws ParseException {
BandwidthDataSetUpdate dataset = bandwidthDao BandwidthDataSetUpdate dataset = bandwidthDao
.newBandwidthDataSetUpdate(dataSetMetaData); .newBandwidthDataSetUpdate(dataSetMetaData);
// Looking for active subscriptions to the dataset. // Range the query for subscriptions within the baseReferenceTime hour.
List<SubscriptionRetrieval> subscriptions = bandwidthDao // SOME models, RAP and RTMA, come not exactly on the hour. This causes these
.getSubscriptionRetrievals(dataset.getProviderName(), // subscriptions to be missed because baseReferenceTimes are on the hour.
dataset.getDataSetName(), dataset.getDataSetBaseTime()); Map<String, Date> timeRange = getBaseReferenceTimeDateRange(dataset.getDataSetBaseTime());
final SortedSet<SubscriptionRetrieval> subscriptions = bandwidthDao
.getSubscriptionRetrievals(dataset.getProviderName(), dataset.getDataSetName(),
RetrievalStatus.SCHEDULED, timeRange.get(MIN_RANGE_TIME), timeRange.get(MAX_RANGE_TIME));
if (!subscriptions.isEmpty()) { if (!subscriptions.isEmpty()) {
// Loop through the scheduled SubscriptionRetrievals and mark // Loop through the scheduled SubscriptionRetrievals and mark
// the scheduled retrievals as ready for retrieval // the scheduled retrievals as ready for retrieval
for (SubscriptionRetrieval retrieval : subscriptions) { for (SubscriptionRetrieval retrieval : subscriptions) {
// TODO: Evaluate the state changes for receiving multiple
// dataset update messages. This seems to be happening
// quite a bit.
if (RetrievalStatus.SCHEDULED.equals(retrieval.getStatus())) { if (RetrievalStatus.SCHEDULED.equals(retrieval.getStatus())) {
// Need to update the Subscription Object in the // Need to update the Subscription Object in the
@ -723,14 +727,12 @@ public abstract class EdexBandwidthManager<T extends Time, C extends Coverage>
} }
} else { } else {
if (statusHandler.isPriorityEnabled(Priority.DEBUG)) { statusHandler
statusHandler .debug("No Subscriptions scheduled for BandwidthDataSetUpdate ["
.debug("No Subscriptions scheduled for BandwidthDataSetUpdate [" + dataset.getIdentifier()
+ dataset.getIdentifier() + "] base time ["
+ "] base time [" + BandwidthUtil.format(dataset.getDataSetBaseTime())
+ BandwidthUtil.format(dataset + "]");
.getDataSetBaseTime()) + "]");
}
} }
} }
@ -867,7 +869,8 @@ public abstract class EdexBandwidthManager<T extends Time, C extends Coverage>
+ plan.getPlanEnd().getTime()); + plan.getPlanEnd().getTime());
statusHandler.info("MaintenanceTask: Update schedule"); statusHandler.info("MaintenanceTask: Update schedule");
} }
// Find DEFERRED Allocations and load them into the plan... // Find DEFERRED Allocations and load them into the
// plan...
List<BandwidthAllocation> deferred = bandwidthDao List<BandwidthAllocation> deferred = bandwidthDao
.getDeferred(plan.getNetwork(), plan.getPlanEnd()); .getDeferred(plan.getNetwork(), plan.getPlanEnd());
if (!deferred.isEmpty()) { if (!deferred.isEmpty()) {
@ -886,8 +889,11 @@ public abstract class EdexBandwidthManager<T extends Time, C extends Coverage>
+ " Subscriptions processed."); + " Subscriptions processed.");
} catch (Throwable t) { } catch (Throwable t) {
statusHandler.error("MaintenanceTask: Subscription update scheduling has failed", t); statusHandler
.error("MaintenanceTask: Subscription update scheduling has failed",
t);
} }
} }
} }
} }

6
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/hibernate/HibernateBandwidthInitializer.java
View file

@ -39,6 +39,7 @@ import com.raytheon.uf.edex.registry.ebxml.util.RegistryIdUtil;
* Feb 06, 2014 2636 bgonzale Use scheduling initialization method after registry init. * Feb 06, 2014 2636 bgonzale Use scheduling initialization method after registry init.
* Feb 11, 2014 2771 bgonzale Use Data Delivery ID instead of Site. * Feb 11, 2014 2771 bgonzale Use Data Delivery ID instead of Site.
* Feb 14, 2014 2636 mpduff Clean up logging * Feb 14, 2014 2636 mpduff Clean up logging
* Apr 09, 2014 3012 dhladky Adhoc firing prevention.
* </pre> * </pre>
* *
* @author djohnson * @author djohnson
@ -102,19 +103,22 @@ public class HibernateBandwidthInitializer implements BandwidthInitializer {
@Override @Override
public void executeAfterRegistryInit() { public void executeAfterRegistryInit() {
try { try {
Map<Network, List<Subscription>> subMap = findSubscriptionsStrategy Map<Network, List<Subscription>> subMap = findSubscriptionsStrategy
.findSubscriptionsToSchedule(); .findSubscriptionsToSchedule();
List<String> unscheduled = instance.initializeScheduling(subMap); List<String> unscheduled = instance.initializeScheduling(subMap);
if (!unscheduled.isEmpty()) { if (!unscheduled.isEmpty()) {
StringBuilder sb = new StringBuilder("The following subscriptions could not be scheduled at startup: "); StringBuilder sb = new StringBuilder(
"The following subscriptions could not be scheduled at startup: ");
sb.append(StringUtil.NEWLINE); sb.append(StringUtil.NEWLINE);
for (String subscription : unscheduled) { for (String subscription : unscheduled) {
sb.append(subscription).append(" "); sb.append(subscription).append(" ");
} }
statusHandler.handle(Priority.INFO, sb.toString()); statusHandler.handle(Priority.INFO, sb.toString());
} }
} catch (Exception e) { } catch (Exception e) {
statusHandler.error( statusHandler.error(
"Failed to query for subscriptions to schedule", e); "Failed to query for subscriptions to schedule", e);

4
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/processing/SimpleSubscriptionAggregator.java
View file

@ -33,7 +33,6 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.dao.SubscriptionRetrieval;
import com.raytheon.uf.edex.datadelivery.bandwidth.interfaces.ISubscriptionAggregator; import com.raytheon.uf.edex.datadelivery.bandwidth.interfaces.ISubscriptionAggregator;
import com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.RetrievalAgent; import com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.RetrievalAgent;
import com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.RetrievalStatus; import com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.RetrievalStatus;
import com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.SubscriptionRetrievalAgent;
import com.raytheon.uf.edex.datadelivery.bandwidth.util.BandwidthUtil; import com.raytheon.uf.edex.datadelivery.bandwidth.util.BandwidthUtil;
/** /**
@ -55,6 +54,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.util.BandwidthUtil;
* Jul 11, 2013 2106 djohnson aggregate() signature changed. * Jul 11, 2013 2106 djohnson aggregate() signature changed.
* Jan 06, 2014 2636 mpduff Changed how data set offset is set. * Jan 06, 2014 2636 mpduff Changed how data set offset is set.
* Jan 30, 2014 2686 dhladky refactor of retrieval. * Jan 30, 2014 2686 dhladky refactor of retrieval.
* Apr 15, 2014 3012 dhladky help with confusing nomenclature.
* </pre> * </pre>
* *
* @author jspinks * @author jspinks
@ -87,6 +87,8 @@ public class SimpleSubscriptionAggregator implements ISubscriptionAggregator {
// (i.e. has SubscriptionRetrievals associated with it) if // (i.e. has SubscriptionRetrievals associated with it) if
// not, create a SubscriptionRetrieval for the subscription // not, create a SubscriptionRetrieval for the subscription
// Do NOT confuse this with an actual SubscriptionRetrieval.
// This SubscriptionRetrieval object is a BandwidthAllocation object
SubscriptionRetrieval subscriptionRetrieval = new SubscriptionRetrieval(); SubscriptionRetrieval subscriptionRetrieval = new SubscriptionRetrieval();
// Link this SubscriptionRetrieval with the subscription. // Link this SubscriptionRetrieval with the subscription.
subscriptionRetrieval.setBandwidthSubscription(subDao); subscriptionRetrieval.setBandwidthSubscription(subDao);

89
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/BandwidthDaoUtil.java
View file

@ -84,6 +84,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.RetrievalStatus;
* active period. * active period.
* Jan 29, 2014 2636 mpduff Scheduling refactor. * Jan 29, 2014 2636 mpduff Scheduling refactor.
* Feb 11, 2014 2636 mpduff Change how retrieval times are calculated. * Feb 11, 2014 2636 mpduff Change how retrieval times are calculated.
* Apr 15, 2014 3012 dhladky Fixed improper offsets.
* Apr 21, 2014 2887 dhladky Missed start/end in previous call, needs shouldScheduleForTime(); * Apr 21, 2014 2887 dhladky Missed start/end in previous call, needs shouldScheduleForTime();
* </pre> * </pre>
* *
@ -180,6 +181,7 @@ public class BandwidthDaoUtil<T extends Time, C extends Coverage> {
// based on plan start and subscription start. // based on plan start and subscription start.
Calendar subscriptionCalculatedStart = subscription Calendar subscriptionCalculatedStart = subscription
.calculateStart(planStart); .calculateStart(planStart);
// end time when when subscription is last valid for scheduling based on // end time when when subscription is last valid for scheduling based on
// plan end and subscription end. // plan end and subscription end.
Calendar subscriptionCalculatedEnd = subscription.calculateEnd(planEnd); Calendar subscriptionCalculatedEnd = subscription.calculateEnd(planEnd);
@ -201,55 +203,54 @@ public class BandwidthDaoUtil<T extends Time, C extends Coverage> {
Calendar start = TimeUtil.newGmtCalendar(subscriptionCalculatedStart Calendar start = TimeUtil.newGmtCalendar(subscriptionCalculatedStart
.getTime()); .getTime());
int availabilityOffset = 0;
try {
availabilityOffset = BandwidthUtil.getDataSetAvailablityOffset(
subscription, start);
} catch (RegistryHandlerException e) {
// Error occurred querying the registry. Log and continue on
statusHandler
.handle(Priority.PROBLEM,
"Unable to retrieve data availability offset, using 0 for the offset.",
e);
}
while (!start.after(subscriptionCalculatedEnd)) { while (!start.after(subscriptionCalculatedEnd)) {
if (!hours.isEmpty()) { for (Integer cycle : hours) {
for (Integer cycle : hours) { start.set(Calendar.HOUR_OF_DAY, cycle);
start.set(Calendar.HOUR_OF_DAY, cycle);
for (Integer minute : minutes) {
start.set(Calendar.MINUTE, minute);
Calendar retrievalTime = TimeUtil.newGmtCalendar();
retrievalTime.setTimeInMillis(start.getTimeInMillis());
retrievalTime.add(Calendar.MINUTE, availabilityOffset);
if (retrievalTime.after(planStart) // calculate the offset, every hour
&& retrievalTime.before(planEnd)) { int availabilityOffset = 0;
// Check for nonsense try {
/* availabilityOffset = BandwidthUtil
* Fine grain check by hour and minute, for .getDataSetAvailablityOffset(subscription, start);
* subscription(start/end), activePeriod(start/end) } catch (RegistryHandlerException e) {
*/ // Error occurred querying the registry. Log and continue on
statusHandler
// TODO: IMPORTANT NOTE: WHEN 14.2.1 MERGES IN. THIS NEEDS .handle(Priority.PROBLEM,
// TO CHECK AGAINST THE OFFSET BASE REFTIME, THE BASE REFTIME "Unable to retrieve data availability offset, using 0 for the offset.",
// WILL BE WHAT IS ADDED IF THE CHECK IS TRUE. DO NOT BLINDLY e);
// MERGE 14.2.1's CODE OVER THIS.
if (!subscription
.shouldScheduleForTime(retrievalTime)) {
// don't schedule this retrieval time,
// outside subscription window
continue;
}
subscriptionTimes.add(retrievalTime);
}
}
} }
// Start the next day.. for (Integer minute : minutes) {
start.add(Calendar.DAY_OF_YEAR, 1);
start.set(Calendar.HOUR_OF_DAY, hours.first()); start.set(Calendar.MINUTE, minute);
Calendar baseRefTime = TimeUtil.newGmtCalendar();
baseRefTime.setTimeInMillis(start.getTimeInMillis());
// add the offset and check if it falls within window
Calendar offsetBaseRefTime = TimeUtil
.newGmtCalendar(baseRefTime.getTime());
offsetBaseRefTime.add(Calendar.MINUTE, availabilityOffset);
if (offsetBaseRefTime.after(planStart)
&& offsetBaseRefTime.before(planEnd)) {
/*
* Fine grain check by hour and minute, for
* subscription(start/end), activePeriod(start/end)
*/
if (!subscription.shouldScheduleForTime(baseRefTime)) {
// don't schedule this retrieval time,
// outside subscription window
continue;
}
subscriptionTimes.add(baseRefTime);
}
}
} }
// Start the next day..
start.add(Calendar.DAY_OF_YEAR, 1);
start.set(Calendar.HOUR_OF_DAY, hours.first());
} }
return subscriptionTimes; return subscriptionTimes;

3
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/BandwidthUtil.java
View file

@ -33,6 +33,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.dao.BandwidthSubscription;
* Oct 30, 2013 2448 dhladky Moved methods to TimeUtil. * Oct 30, 2013 2448 dhladky Moved methods to TimeUtil.
* Dec 20, 2013 2636 mpduff Changed dataset delay to offset. * Dec 20, 2013 2636 mpduff Changed dataset delay to offset.
* Jan 08, 2014 2615 bgonzale Moved Calendar min and max methods to TimeUtil. * Jan 08, 2014 2615 bgonzale Moved Calendar min and max methods to TimeUtil.
* Apr 09, 2014 3012 dhladky GMT Calendar use.
* *
* </pre> * </pre>
* *
@ -191,7 +192,7 @@ public class BandwidthUtil {
dao.setDataSetName(dataSetMetaData.getDataSetName()); dao.setDataSetName(dataSetMetaData.getDataSetName());
dao.setProviderName(dataSetMetaData.getProviderName()); dao.setProviderName(dataSetMetaData.getProviderName());
dao.setUpdateTime(BandwidthUtil.now()); dao.setUpdateTime(BandwidthUtil.now());
dao.setDataSetBaseTime(TimeUtil.newCalendar(dataSetMetaData.getDate())); dao.setDataSetBaseTime(TimeUtil.newGmtCalendar(dataSetMetaData.getDate()));
dao.setUrl(dataSetMetaData.getUrl()); dao.setUrl(dataSetMetaData.getUrl());
return dao; return dao;

69
edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/res/spring/harvester-datadelivery.xml
View file

@ -10,43 +10,25 @@
<constructor-arg ref="ProviderHandler" /> <constructor-arg ref="ProviderHandler" />
</bean> </bean>
<bean id="DataSetMetaDataPurgeLauncher" class="com.raytheon.uf.edex.datadelivery.harvester.purge.DataSetMetaDataPurgeLauncher" <!-- Start of DataSetMetaData purge configuration -->
factory-method="getInstance" depends-on="DbInit"> <bean id="DataSetMetaDataPurgeLauncher" class="com.raytheon.uf.edex.datadelivery.harvester.purge.DataSetMetaDataPurgeLauncher" depends-on="registryInit">
<constructor-arg ref="registryObjectDao" />
</bean> </bean>
<camelContext id="MetaData-context" <camelContext id="DataSetMetaDataPurge-context"
xmlns="http://camel.apache.org/schema/spring" xmlns="http://camel.apache.org/schema/spring"
errorHandlerRef="errorHandler"> errorHandlerRef="errorHandler">
<endpoint id="metaDataCron" uri="quartz://datadelivery/harvester/?cron=${metadata-process.cron}"/> <endpoint id="dataSetMetaDataPurgeCron" uri="quartz://datadelivery/metaDataPurge/?cron=${metadata-purge.cron}"/>
<endpoint id="harvesterProcessWorkEndpoint" uri="jms-generic:queue:metaDataProcessWork?concurrentConsumers=${metadata-process.threads}&amp;threadName=harvester"/> <endpoint id="dataSetMetaDataWorkEndpoint" uri="jms-generic:queue:dataSetMetaDataPurgeWork?concurrentConsumers=1&amp;threadName=dataSetMetaDataPurge"/>
<endpoint id="datasetMetaDataPurgeCron" uri="quartz://datadelivery/metaDataPurge/?cron=${metadata-purge.cron}"/>
<route id="metaDataProcess">
<from uri="metaDataCron" />
<to uri="jms-generic:queue:metaDataProcessWork" />
</route>
<route id="metaDataProcessWork">
<from uri="harvesterProcessWorkEndpoint" />
<doTry>
<pipeline>
<bean ref="MetaDataProcessor" method="metaDataCheck" />
</pipeline>
<doCatch>
<exception>java.lang.Throwable</exception>
<to uri="log:metaDataProcess?level=ERROR"/>
</doCatch>
</doTry>
</route>
<route id="metaDataPurge"> <route id="metaDataPurge">
<from uri="datasetMetaDataPurgeCron" /> <from uri="dataSetMetaDataPurgeCron" />
<to uri="jms-generic:queue:metaDataPurgeWork" /> <to uri="jms-generic:queue:dataSetMetaDataPurgeWork" />
</route> </route>
<route id="metaDataPurgeWork"> <route id="dataSetMetaDataPurgeWork">
<from uri="jms-generic:queue:metaDataPurgeWork" /> <from uri="dataSetMetaDataWorkEndpoint" />
<doTry> <doTry>
<pipeline> <pipeline>
<bean ref="DataSetMetaDataPurgeLauncher" method="runPurge" /> <bean ref="DataSetMetaDataPurgeLauncher" method="runPurge" />
@ -59,4 +41,31 @@
</route> </route>
</camelContext> </camelContext>
<camelContext id="MetaData-context"
xmlns="http://camel.apache.org/schema/spring"
errorHandlerRef="errorHandler">
<endpoint id="metaDataCron" uri="quartz://datadelivery/harvester/?cron=${metadata-process.cron}"/>
<endpoint id="harvesterProcessWorkEndpoint" uri="jms-generic:queue:metaDataProcessWork?concurrentConsumers=${metadata-process.threads}&amp;threadName=harvester"/>
<route id="metaDataProcess">
<from uri="metaDataCron" />
<to uri="jms-generic:queue:metaDataProcessWork" />
</route>
<route id="metaDataProcessWork">
<from uri="harvesterProcessWorkEndpoint" />
<doTry>
<pipeline>
<bean ref="MetaDataProcessor" method="metaDataCheck" />
</pipeline>
<doCatch>
<exception>java.lang.Throwable</exception>
<to uri="log:metaDataProcess?level=ERROR"/>
</doCatch>
</doTry>
</route>
</camelContext>
</beans> </beans>

2
edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/resources/com.raytheon.uf.edex.datadelivery.harvester.properties
View file

@ -2,6 +2,6 @@
metadata-process.cron=0+*+*+*+*+? metadata-process.cron=0+*+*+*+*+?
# The cron pattern for how often the DataSetMetaDataPurgeLauncher will run, e.g. # The cron pattern for how often the DataSetMetaDataPurgeLauncher will run, e.g.
# how often to check for purgeable dataset metadata instances # how often to check for purgeable dataset metadata instances
metadata-purge.cron=0+0+3+*+*+? metadata-purge.cron=0+0+8+*+*+?
# how many meta data process threads to use to process metadata # how many meta data process threads to use to process metadata
metadata-process.threads=2 metadata-process.threads=2

24
edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/DataSetMetaDataPurgeLauncher.java
View file

@ -24,6 +24,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import com.raytheon.uf.common.status.IUFStatusHandler; import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus; import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority; import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.edex.registry.ebxml.dao.RegistryObjectDao;
/** /**
* Container class to hold the {@link IDataSetMetaDataPurgeTask} instance. It * Container class to hold the {@link IDataSetMetaDataPurgeTask} instance. It
@ -36,28 +37,32 @@ import com.raytheon.uf.common.status.UFStatus.Priority;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* Sep 4, 2012 1102 djohnson Initial creation * Sep 4, 2012 1102 djohnson Initial creation
* Apr 12,2014 3012 dhladky Purge never worked, fixed to make work.
* *
* </pre> * </pre>
* *
* @author djohnson * @author djohnson
* @version 1.0 * @version 1.0
*/ */
public final class DataSetMetaDataPurgeLauncher { public class DataSetMetaDataPurgeLauncher {
private static final IUFStatusHandler statusHandler = UFStatus private static final IUFStatusHandler statusHandler = UFStatus
.getHandler(DataSetMetaDataPurgeLauncher.class); .getHandler(DataSetMetaDataPurgeLauncher.class);
private static final DataSetMetaDataPurgeLauncher INSTANCE = new DataSetMetaDataPurgeLauncher(); private IDataSetMetaDataPurgeTask PURGE_TASK = null;
private static final IDataSetMetaDataPurgeTask PURGE_TASK = new DataSetMetaDataPurgeTaskImpl();
private static final AtomicBoolean running = new AtomicBoolean(); private static final AtomicBoolean running = new AtomicBoolean();
/** /**
* Disabled constructor. * Public constructor.
*/ */
private DataSetMetaDataPurgeLauncher() { public DataSetMetaDataPurgeLauncher(RegistryObjectDao rdo) {
PURGE_TASK = new DataSetMetaDataPurgeTaskImpl(rdo);
} }
/**
* Try purging datasets.
*/
public void runPurge() { public void runPurge() {
try { try {
if (running.compareAndSet(false, true)) { if (running.compareAndSet(false, true)) {
@ -70,11 +75,4 @@ public final class DataSetMetaDataPurgeLauncher {
running.set(false); running.set(false);
} }
} }
/**
* @return the instance
*/
public static DataSetMetaDataPurgeLauncher getInstance() {
return INSTANCE;
}
} }

258
edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/DataSetMetaDataPurgeTaskImpl.java
View file

@ -22,33 +22,29 @@ package com.raytheon.uf.edex.datadelivery.harvester.purge;
import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkNotNull;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Calendar;
import java.util.Collections; import java.util.Collections;
import java.util.Iterator; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.SortedSet; import java.util.Map;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Multimap;
import com.google.common.collect.Ordering;
import com.google.common.collect.TreeMultimap;
import com.raytheon.uf.common.datadelivery.harvester.Agent; import com.raytheon.uf.common.datadelivery.harvester.Agent;
import com.raytheon.uf.common.datadelivery.harvester.CrawlAgent; import com.raytheon.uf.common.datadelivery.harvester.CrawlAgent;
import com.raytheon.uf.common.datadelivery.harvester.HarvesterConfig; import com.raytheon.uf.common.datadelivery.harvester.HarvesterConfig;
import com.raytheon.uf.common.datadelivery.harvester.HarvesterConfigurationManager;
import com.raytheon.uf.common.datadelivery.registry.DataDeliveryRegistryObjectTypes;
import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData; import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData;
import com.raytheon.uf.common.datadelivery.registry.IDataSetMetaDataVisitor;
import com.raytheon.uf.common.datadelivery.registry.OpenDapGriddedDataSetMetaData;
import com.raytheon.uf.common.datadelivery.registry.handlers.DataDeliveryHandlers; import com.raytheon.uf.common.datadelivery.registry.handlers.DataDeliveryHandlers;
import com.raytheon.uf.common.localization.LocalizationFile; import com.raytheon.uf.common.localization.LocalizationFile;
import com.raytheon.uf.common.registry.ebxml.RegistryUtil;
import com.raytheon.uf.common.registry.handler.RegistryHandlerException; import com.raytheon.uf.common.registry.handler.RegistryHandlerException;
import com.raytheon.uf.common.serialization.SerializationUtil;
import com.raytheon.uf.common.status.IUFStatusHandler; import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus; import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority; import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.time.util.ITimer; import com.raytheon.uf.common.time.util.ITimer;
import com.raytheon.uf.common.time.util.TimeUtil; import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.edex.datadelivery.harvester.crawler.CrawlLauncher; import com.raytheon.uf.edex.datadelivery.harvester.crawler.CrawlLauncher;
import com.raytheon.uf.edex.registry.ebxml.dao.RegistryObjectDao;
/** /**
* Purges {@link DataSetMetaData} instances that are no longer accessible on * Purges {@link DataSetMetaData} instances that are no longer accessible on
@ -66,50 +62,22 @@ import com.raytheon.uf.edex.datadelivery.harvester.crawler.CrawlLauncher;
* Oct 05, 2012 1241 djohnson Replace RegistryManager calls with registry handler calls. * Oct 05, 2012 1241 djohnson Replace RegistryManager calls with registry handler calls.
* Dec 12, 2012 1410 dhladky multi provider configurations. * Dec 12, 2012 1410 dhladky multi provider configurations.
* Sept 30, 2013 1797 dhladky Generics * Sept 30, 2013 1797 dhladky Generics
* Apr 12,2014 3012 dhladky Purge never worked, fixed to make work.
* *
* </pre> * </pre>
* *
* @author djohnson * @author djohnson
* @version 1.0 * @version 1.0
*/ */
class DataSetMetaDataPurgeTaskImpl implements IDataSetMetaDataPurgeTask, public class DataSetMetaDataPurgeTaskImpl implements IDataSetMetaDataPurgeTask {
IDataSetMetaDataVisitor {
/**
* Maintains state for an instance of the purge task.
*/
private static class State {
/**
* This boolean flag is used to mark whether or not the DataSetMetaData
* group should be continued, it will be set to false when the purge has
* found a DataSetMetaData instance that should NOT be purged
*/
private boolean continueWithDataSet = true;
/**
* The harvester configurations instance at the time the purge started.
*/
private List<HarvesterConfig> harvesterConfigs;
}
private static final IUFStatusHandler statusHandler = UFStatus private static final IUFStatusHandler statusHandler = UFStatus
.getHandler(DataSetMetaDataPurgeTaskImpl.class); .getHandler(DataSetMetaDataPurgeTaskImpl.class);
private static final IOpenDapGriddedPurge GRIDDED_OPENDAP = new OpenDapGriddedPurgeImpl();
/**
* This is the unique identifying key for this metadata's dataset in the
* map.
*
* @param metaData
* the metaDat
* @return the key
*/
@VisibleForTesting
static String getDatasetMetaDataMapKey(DataSetMetaData<?> metaData) {
return metaData.getDataSetName() + metaData.getProviderName();
}
/** Data access object for registry objects */
private RegistryObjectDao rdo;
/** /**
* Purges a {@link DataSetMetaData} instance. * Purges a {@link DataSetMetaData} instance.
* *
@ -133,83 +101,40 @@ class DataSetMetaDataPurgeTaskImpl implements IDataSetMetaDataPurgeTask,
} }
} }
private final IOpenDapGriddedPurge openDapGriddedPurge;
// Used to maintain state on a per-thread basis, in case two purges somehow
// overrun each other
private final ThreadLocal<State> threadState = new ThreadLocal<State>();
/** /**
* Default Constructor. * Default Constructor.
*/ */
DataSetMetaDataPurgeTaskImpl() { public DataSetMetaDataPurgeTaskImpl(RegistryObjectDao rdo) {
this(GRIDDED_OPENDAP); this.rdo = rdo;
} }
/** /**
* Constructor accepting specific purge strategies. * Gets the entire list of DSMD ids from the registry.
*
* @param openDapGriddedPurge
* openDapGriddedPurge
*
*/
@VisibleForTesting
DataSetMetaDataPurgeTaskImpl(IOpenDapGriddedPurge openDapGriddedPurge) {
this.openDapGriddedPurge = openDapGriddedPurge;
}
/**
* Clears the state for a running instance.
*/
private void clearState() {
threadState.set(null);
}
/**
* Returns all {@link DataSetMetaData} instances that are to be checked for
* validity.
*
* @return the {@link DataSetMetaData} instances
*/
@SuppressWarnings("rawtypes")
@VisibleForTesting
List<DataSetMetaData> getDataSetMetaDatas() {
try {
return DataDeliveryHandlers.getDataSetMetaDataHandler().getAll();
} catch (RegistryHandlerException e) {
statusHandler.handle(Priority.PROBLEM,
"Unable to retrieve DataSetMetaData instances!", e);
return Collections.emptyList();
}
}
/**
* Creates a map from the DataSetMetaData key as defined by
* {@link #getDatasetMetaDataMapKey(DataSetMetaData)} to the
* {@link SortedSet} of instances.
* *
* @return the map * @return the map
*/ */
@VisibleForTesting @VisibleForTesting
Multimap<String, DataSetMetaData<?>> getDataSetNameKeyedInstanceMap() { List<String> getDataSetMetaDataIds() {
Multimap<String, DataSetMetaData<?>> map = TreeMultimap.create( ArrayList<String> ids = null;
Ordering.<String> natural(), DataSetMetaData.DATE_COMPARATOR); try {
// Gets the list of all available lids for current DataSetMetaData objects
for (DataSetMetaData<?> metaData : getDataSetMetaDatas()) { ids = (ArrayList<String>) rdo.getRegistryObjectIdsOfType(DataDeliveryRegistryObjectTypes.DATASETMETADATA);
String key = getDatasetMetaDataMapKey(metaData); } catch (Exception e) {
map.put(key, metaData); statusHandler.handle(Priority.PROBLEM,
"Unable to retrieve DataSetMetaData ids!", e);
return Collections.emptyList();
} }
return map; return ids;
} }
/** /**
* Returns the HarvesterConfig Array from localization. * Returns the Retention times by Provider name.
* *
* @return the {@link HarvesterConfig} * @return the {@link HarvesterConfig}
*/ */
@VisibleForTesting @VisibleForTesting
List<HarvesterConfig> getHarvesterConfigs() { static Map<String, String> getHarvesterConfigs() {
// first get the Localization directory and find all harvester // first get the Localization directory and find all harvester
// configs // configs
@ -220,8 +145,7 @@ class DataSetMetaDataPurgeTaskImpl implements IDataSetMetaDataPurgeTask,
HarvesterConfig hc = null; HarvesterConfig hc = null;
try { try {
hc = SerializationUtil.jaxbUnmarshalFromXmlFile( hc = HarvesterConfigurationManager.getHarvesterFile(lf.getFile());
HarvesterConfig.class, lf.getFile());
} catch (Exception se) { } catch (Exception se) {
statusHandler.handle(Priority.PROBLEM, statusHandler.handle(Priority.PROBLEM,
se.getLocalizedMessage(), se); se.getLocalizedMessage(), se);
@ -238,102 +162,78 @@ class DataSetMetaDataPurgeTaskImpl implements IDataSetMetaDataPurgeTask,
} }
} }
} }
Map<String, String> configMap = null;
return configs; if (!configs.isEmpty()) {
} configMap = new HashMap<String, String>(
configs.size());
/** for (HarvesterConfig config : configs) {
* This method consolidates the logic of applying a purge strategy for a configMap.put(config.getProvider().getName(), config.getRetention());
* specific data type and service (e.g. OpenDAP for Gridded data) on a
* specific {@link DataSetMetaData} of that type. The generics ensure strict
* adherence to the data type mappings.
*
* @param <T>
* the type that extends DataSetMetaData
* @param metaData
* the metadata instance
* @param purge
* the purge strategy
*/
private <T extends DataSetMetaData<?>> void handleVisit(T metaData,
IServiceDataSetMetaDataPurge<T> purge) {
State state = threadState.get();
List<HarvesterConfig> harvesterConfigs = state.harvesterConfigs;
for (HarvesterConfig config : harvesterConfigs) {
if (purge.isTimeToPurge(metaData, config)) {
purgeMetaData(metaData);
} else {
// Found a non-purgeable metadata instance
state.continueWithDataSet = false;
if (statusHandler.isPriorityEnabled(Priority.DEBUG)) {
final String id = RegistryUtil
.getRegistryObjectKey(metaData);
statusHandler
.debug(String
.format("Provider: "
+ config.getProvider().getName()
+ " : DataSetMetaData with id [%s] does not require purging.",
id));
}
} }
} else {
return Collections.emptyMap();
} }
return configMap;
} }
/**
* Initializes the state for a running instance.
*
* @return the State instance
*/
@VisibleForTesting
State initializeState() {
State state = new State();
state.harvesterConfigs = getHarvesterConfigs();
threadState.set(state);
return state;
}
/** /**
* {@inheritDoc} * {@inheritDoc}
*/ */
@Override @Override
public void run() { public void run() {
ITimer timer = TimeUtil.getTimer(); ITimer timer = TimeUtil.getTimer();
timer.start(); timer.start();
Multimap<String, DataSetMetaData<?>> dataSetKeyedMap = getDataSetNameKeyedInstanceMap(); List<String> idList = getDataSetMetaDataIds();
Map<String, String> configMap = getHarvesterConfigs();
int deletes = 0;
for (String id : idList) {
try { try {
State state = initializeState();
for (String key : dataSetKeyedMap.keySet()) { DataSetMetaData<?> metaData = DataDeliveryHandlers
Collection<DataSetMetaData<?>> metaDatas = dataSetKeyedMap .getDataSetMetaDataHandler().getById(id);
.get(key); Integer retention = Integer.valueOf(configMap.get(metaData.getProviderName()));
Iterator<DataSetMetaData<?>> iter = metaDatas.iterator();
state.continueWithDataSet = true; if (retention != null) {
while (iter.hasNext() && state.continueWithDataSet) {
DataSetMetaData<?> metaData = iter.next(); if (retention == -1) {
metaData.accept(this); // no purging for this DSMD type
continue;
} else {
// retention is in days
retention = retention * (-1);
// we are subtracting from current
Calendar thresholdTime = TimeUtil.newGmtCalendar();
thresholdTime.add(Calendar.DAY_OF_YEAR, retention);
if (thresholdTime.getTimeInMillis() >= metaData
.getDate().getTime()) {
purgeMetaData(metaData);
deletes++;
}
}
} else {
statusHandler
.warn("No retention time set for this DataSetMetaData provider! "
+ id
+ "Provider: "
+ metaData.getProviderName());
} }
} catch (Exception e) {
statusHandler.error("DataSetMetaData purge failed! " + id, e);
} }
} finally {
clearState();
} }
timer.stop(); timer.stop();
statusHandler.info(String.format( statusHandler.info(String.format(
"DataSetMetaData purge completed in %s ms.", "DataSetMetaData purge completed in %s ms.",
timer.getElapsedTime())); timer.getElapsedTime()+" deleted: "+deletes));
}
/**
* {@inheritDoc}
*/
@Override
public void visit(OpenDapGriddedDataSetMetaData metaData) {
handleVisit(metaData, openDapGriddedPurge);
} }
} }

4
edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/IDataSetMetaDataPurgeTask.java
View file

@ -16,7 +16,9 @@
* *
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information. * further licensing information.
**/ */
package com.raytheon.uf.edex.datadelivery.harvester.purge; package com.raytheon.uf.edex.datadelivery.harvester.purge;
import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData; import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData;

44
edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/IOpenDapGriddedPurge.java
View file

@ -1,44 +0,0 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.harvester.purge;
import com.raytheon.uf.common.datadelivery.registry.OpenDapGriddedDataSetMetaData;
/**
* Marker interface for the gridded OpenDAP purge strategy. Intentionally
* package-level as it's an internal implementation detail that should only
* reside and be accessible within this package.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Sep 4, 2012 1102 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
interface IOpenDapGriddedPurge extends
IServiceDataSetMetaDataPurge<OpenDapGriddedDataSetMetaData> {
}

55
edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/IServiceDataSetMetaDataPurge.java
View file

@ -1,55 +0,0 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.harvester.purge;
import com.raytheon.uf.common.datadelivery.harvester.HarvesterConfig;
import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData;
/**
* Defines a purge for a {@link DataSetMetaData} type. Intentionally
* package-level as it's an internal implementation detail that should only
* reside and be accessible within this package.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Sep 4, 2012 1102 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
interface IServiceDataSetMetaDataPurge<T extends DataSetMetaData> {
/**
* Perform the purge of a {@link DataSetMetaData} instance if required.
*
* @param metaData
* the metaData
* @param harvesterConfig TODO
* @return
*/
boolean isTimeToPurge(T metaData, HarvesterConfig harvesterConfig);
}

185
edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/OpenDapGriddedPurgeImpl.java
View file

@ -1,185 +0,0 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.datadelivery.harvester.purge;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.params.ConnRoutePNames;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;
import com.google.common.annotations.VisibleForTesting;
import com.raytheon.uf.common.comm.ProxyConfiguration;
import com.raytheon.uf.common.datadelivery.harvester.HarvesterConfig;
import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData;
import com.raytheon.uf.common.datadelivery.registry.OpenDapGriddedDataSetMetaData;
import com.raytheon.uf.common.datadelivery.registry.Provider;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.edex.datadelivery.retrieval.util.ConnectionUtil;
/**
* Purges {@link OpenDapGriddedDataSetMetaData} instances that are no longer
* retrievable. Intentionally package-private as it should not be part of the
* public API.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Sep 4, 2012 1102 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
class OpenDapGriddedPurgeImpl implements IOpenDapGriddedPurge {
private static final IUFStatusHandler statusHandler = UFStatus
.getHandler(OpenDapGriddedPurgeImpl.class);
/**
* Retrieves the error response pattern from the provider for the specified
* metadata instance.
*
* @param metaData
* the metadata
* @param harvesterConfig
* the harvester configuration
* @return the pattern or null if the applicable provider cannot be found
*/
private static Pattern getProviderErrorResponsePattern(
DataSetMetaData metaData, HarvesterConfig harvesterConfig) {
Provider provider = harvesterConfig.getProvider();
final String providerName = provider.getName();
final String metadataProviderName = metaData.getProviderName();
if (statusHandler.isPriorityEnabled(Priority.DEBUG)) {
statusHandler.debug("Checking provider [" + providerName
+ "] to match metadata provider name ["
+ metadataProviderName + "].");
}
if (metadataProviderName.equals(providerName)) {
return Pattern.compile(provider.getErrorResponsePattern());
}
return null;
}
/**
* Retrieves the HttpClient implementation to use.
*
* @return the {@link HttpClient} implementation
*/
@VisibleForTesting
HttpClient getHttpClient() {
HttpClient httpClient = new DefaultHttpClient();
ProxyConfiguration proxyParameters = ConnectionUtil
.getProxyParameters();
if (proxyParameters != null) {
HttpHost proxy = new HttpHost(proxyParameters.getHost(),
proxyParameters.getPort());
httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY,
proxy);
}
return httpClient;
}
/**
* {@inheritDoc}
*/
@Override
public boolean isTimeToPurge(OpenDapGriddedDataSetMetaData metaData,
HarvesterConfig harvesterConfig) {
HttpGet request = new HttpGet();
HttpResponse response = null;
try {
final String url = metaData.getUrl();
request.setURI(new URI(url));
HttpClient httpClient = getHttpClient();
response = httpClient.execute(request);
int code = response.getStatusLine().getStatusCode();
if (statusHandler.isPriorityEnabled(Priority.DEBUG)) {
statusHandler.debug(String.format(
"Received status code [%s] from url [%s]", code, url));
}
if (code == HttpStatus.SC_OK) {
String entityContent = EntityUtils.toString(response
.getEntity());
Pattern providerErrorResponsePattern = getProviderErrorResponsePattern(
metaData, harvesterConfig);
if (providerErrorResponsePattern != null) {
Matcher matcher = providerErrorResponsePattern
.matcher(entityContent);
if (matcher.find()) {
return true;
}
} else {
statusHandler
.warn(String
.format("Unable to find a configured provider by name [%s], removing obsolete DataSetMetaData.",
metaData.getProviderName()));
}
}
return false;
} catch (URISyntaxException e) {
statusHandler
.handle(Priority.PROBLEM,
"Unable to parse URL into a URI, purging metadata since it would otherwise remain unusable!",
e);
return true;
} catch (IOException e) {
statusHandler
.handle(Priority.PROBLEM,
"Unable to contact the host, not purging metadata since it may still be valid!",
e);
request.abort();
return false;
} finally {
if (response != null) {
try {
EntityUtils.consume(response.getEntity());
} catch (IOException e) {
statusHandler.handle(Priority.PROBLEM,
e.getLocalizedMessage(), e);
}
}
}
}
}

146
edexOsgi/com.raytheon.uf.edex.datadelivery.registry/src/com/raytheon/uf/edex/datadelivery/registry/federation/RegistryFederationManager.java
View file

@ -95,6 +95,7 @@ import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.time.util.TimeUtil; import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.common.util.CollectionUtil; import com.raytheon.uf.common.util.CollectionUtil;
import com.raytheon.uf.common.util.StringUtil; import com.raytheon.uf.common.util.StringUtil;
import com.raytheon.uf.edex.core.EDEXUtil;
import com.raytheon.uf.edex.datadelivery.registry.availability.FederatedRegistryMonitor; import com.raytheon.uf.edex.datadelivery.registry.availability.FederatedRegistryMonitor;
import com.raytheon.uf.edex.datadelivery.registry.dao.ReplicationEventDao; import com.raytheon.uf.edex.datadelivery.registry.dao.ReplicationEventDao;
import com.raytheon.uf.edex.datadelivery.registry.replication.NotificationServers; import com.raytheon.uf.edex.datadelivery.registry.replication.NotificationServers;
@ -103,6 +104,7 @@ import com.raytheon.uf.edex.registry.ebxml.dao.DbInit;
import com.raytheon.uf.edex.registry.ebxml.dao.RegistryDao; import com.raytheon.uf.edex.registry.ebxml.dao.RegistryDao;
import com.raytheon.uf.edex.registry.ebxml.dao.RegistryObjectDao; import com.raytheon.uf.edex.registry.ebxml.dao.RegistryObjectDao;
import com.raytheon.uf.edex.registry.ebxml.exception.EbxmlRegistryException; import com.raytheon.uf.edex.registry.ebxml.exception.EbxmlRegistryException;
import com.raytheon.uf.edex.registry.ebxml.exception.NoReplicationServersAvailableException;
import com.raytheon.uf.edex.registry.ebxml.init.RegistryInitializedListener; import com.raytheon.uf.edex.registry.ebxml.init.RegistryInitializedListener;
import com.raytheon.uf.edex.registry.ebxml.services.query.QueryConstants; import com.raytheon.uf.edex.registry.ebxml.services.query.QueryConstants;
import com.raytheon.uf.edex.registry.ebxml.services.query.RegistryQueryUtil; import com.raytheon.uf.edex.registry.ebxml.services.query.RegistryQueryUtil;
@ -154,6 +156,7 @@ import com.raytheon.uf.edex.registry.events.CreateAuditTrailEvent;
* 2/13/2014 2769 bphillip Refactored registry sync. Created quartz tasks to monitor registry uptime as well as subscription integrity * 2/13/2014 2769 bphillip Refactored registry sync. Created quartz tasks to monitor registry uptime as well as subscription integrity
* Mar 31, 2014 2889 dhladky Added username for notification center tracking. * Mar 31, 2014 2889 dhladky Added username for notification center tracking.
* 4/11/2014 3011 bphillip Removed automatic registry sync check on startup * 4/11/2014 3011 bphillip Removed automatic registry sync check on startup
* 4/15/2014 3012 dhladky Merge fixes.
* </pre> * </pre>
* *
* @author bphillip * @author bphillip
@ -173,7 +176,7 @@ public class RegistryFederationManager implements IRegistryFederationManager,
/** Query used for synchronizing registries */ /** Query used for synchronizing registries */
private static final String SYNC_QUERY = "FROM RegistryObjectType obj where obj.id in (%s) order by obj.id asc"; private static final String SYNC_QUERY = "FROM RegistryObjectType obj where obj.id in (%s) order by obj.id asc";
/** Batch size for registry synchronization queries */ /** Batch size for registry synchronization queries */
private static final int SYNC_BATCH_SIZE = Integer.parseInt(System private static final int SYNC_BATCH_SIZE = Integer.parseInt(System
.getProperty("ebxml-notification-batch-size")); .getProperty("ebxml-notification-batch-size"));
@ -202,14 +205,7 @@ public class RegistryFederationManager implements IRegistryFederationManager,
*/ */
private static final long MAX_DOWN_TIME_DURATION = TimeUtil.MILLIS_PER_HOUR * 48; private static final long MAX_DOWN_TIME_DURATION = TimeUtil.MILLIS_PER_HOUR * 48;
private static final String SYNC_WARNING_MSG = "Registry is out of sync with federation. Registry Synchronization required. Go to: [" public static AtomicBoolean SYNC_IN_PROGRESS = new AtomicBoolean(false);
+ RegistryUtil.LOCAL_REGISTRY_ADDRESS
+ "/registry/federation/status.html] to synchronize.";
private static volatile boolean SYNC_NECESSARY = false;
public static AtomicBoolean SYNC_IN_PROGRESS = new AtomicBoolean(
false);
/** Cutoff parameter for the query to get the expired events */ /** Cutoff parameter for the query to get the expired events */
private static final String GET_EXPIRED_EVENTS_QUERY_CUTOFF_PARAMETER = "cutoff"; private static final String GET_EXPIRED_EVENTS_QUERY_CUTOFF_PARAMETER = "cutoff";
@ -327,9 +323,7 @@ public class RegistryFederationManager implements IRegistryFederationManager,
throw new EbxmlRegistryException( throw new EbxmlRegistryException(
"Error joining federation!!"); "Error joining federation!!");
} }
if (!centralRegistry) {
checkDownTime();
}
} catch (Exception e1) { } catch (Exception e1) {
throw new EbxmlRegistryException( throw new EbxmlRegistryException(
"Error initializing RegistryReplicationManager", e1); "Error initializing RegistryReplicationManager", e1);
@ -351,29 +345,6 @@ public class RegistryFederationManager implements IRegistryFederationManager,
initialized.set(true); initialized.set(true);
} }
/**
* Checks how long a registry has been down. If the registry has been down
* longer than the MAX_DOWN_TIME_DURATION, then a sync is necessary
*
* @see RegistryFederationManager.MAX_DOWN_TIME_DURATION
* @throws Exception
*/
private void checkDownTime() throws Exception {
long currentTime = TimeUtil.currentTimeMillis();
long lastKnownUp = federatedRegistryMonitor.getLastKnownUptime();
long downTime = currentTime - lastKnownUp;
statusHandler.info("Registry has been down since: "
+ new Date(currentTime - downTime));
/*
* The registry has been down for ~2 days, this requires a
* synchronization of the data from the federation
*/
if (currentTime - lastKnownUp > MAX_DOWN_TIME_DURATION) {
SYNC_NECESSARY = true;
sendSyncMessage();
}
}
public boolean joinFederation() { public boolean joinFederation() {
try { try {
final List<RegistryObjectType> objects = new ArrayList<RegistryObjectType>( final List<RegistryObjectType> objects = new ArrayList<RegistryObjectType>(
@ -510,6 +481,62 @@ public class RegistryFederationManager implements IRegistryFederationManager,
return true; return true;
} }
/**
* Checks how long a registry has been down. If the registry has been down
* for over 2 days, the registry is synchronized with one of the federation
* members
*
* @throws Exception
*/
private void synchronize() throws EbxmlRegistryException {
monitorHandler.warn("Synchronizing registry with federation...");
RegistryType registryToSyncFrom = null;
for (String remoteRegistryId : servers.getRegistryReplicationServers()) {
statusHandler.info("Checking availability of [" + remoteRegistryId
+ "]...");
RegistryType remoteRegistry = null;
try {
remoteRegistry = dataDeliveryRestClient.getRegistryObject(
ncfAddress, remoteRegistryId
+ FederationProperties.REGISTRY_SUFFIX);
} catch (Exception e) {
throw new EbxmlRegistryException(
"Error getting remote registry object!", e);
}
if (remoteRegistry == null) {
statusHandler
.warn("Registry at ["
+ remoteRegistryId
+ "] not found in federation. Unable to use as synchronization source.");
} else if (dataDeliveryRestClient
.isRegistryAvailable(remoteRegistry.getBaseURL())) {
registryToSyncFrom = remoteRegistry;
break;
} else {
statusHandler
.info("Registry at ["
+ remoteRegistryId
+ "] is not available. Unable to use as synchronization source.");
}
}
// No available registry was found!
if (registryToSyncFrom == null) {
throw new NoReplicationServersAvailableException(
"No available registries found! Registry data will not be synchronized with the federation!");
} else {
try {
synchronizeWithRegistry(registryToSyncFrom.getId());
} catch (Exception e) {
throw new EbxmlRegistryException(
"Error synchronizing with registry ["
+ registryToSyncFrom.getId() + "]", e);
}
}
}
/** /**
* Synchronizes this registry's data with the registry at the specified URL * Synchronizes this registry's data with the registry at the specified URL
* *
@ -524,6 +551,7 @@ public class RegistryFederationManager implements IRegistryFederationManager,
@Path("synchronizeWithRegistry/{registryId}") @Path("synchronizeWithRegistry/{registryId}")
public void synchronizeWithRegistry(@PathParam("registryId") public void synchronizeWithRegistry(@PathParam("registryId")
String registryId) throws Exception { String registryId) throws Exception {
if (SYNC_IN_PROGRESS.compareAndSet(false, true)) { if (SYNC_IN_PROGRESS.compareAndSet(false, true)) {
try { try {
monitorHandler.handle(Priority.WARN, monitorHandler.handle(Priority.WARN,
@ -552,7 +580,6 @@ public class RegistryFederationManager implements IRegistryFederationManager,
for (final String objectType : replicatedObjectTypes) { for (final String objectType : replicatedObjectTypes) {
syncObjectType(objectType, remoteRegistryUrl); syncObjectType(objectType, remoteRegistryUrl);
} }
SYNC_NECESSARY = false;
federatedRegistryMonitor.updateTime(); federatedRegistryMonitor.updateTime();
StringBuilder syncMsg = new StringBuilder(); StringBuilder syncMsg = new StringBuilder();
@ -566,6 +593,10 @@ public class RegistryFederationManager implements IRegistryFederationManager,
} finally { } finally {
SYNC_IN_PROGRESS.set(false); SYNC_IN_PROGRESS.set(false);
} }
} else {
statusHandler.info("Registry sync already in progress.");
} }
} }
@ -612,8 +643,9 @@ public class RegistryFederationManager implements IRegistryFederationManager,
int remainder = remoteIds.size() % SYNC_BATCH_SIZE; int remainder = remoteIds.size() % SYNC_BATCH_SIZE;
for (int currentBatch = 0; currentBatch < batches; currentBatch++) { for (int currentBatch = 0; currentBatch < batches; currentBatch++) {
statusHandler.info("Processing batch " + (currentBatch + 1)
+ "/" + batches); statusHandler.info("Processing batch "+(currentBatch+1)+"/"+batches);
persistBatch(objectType, remoteRegistryUrl, remoteIds.subList( persistBatch(objectType, remoteRegistryUrl, remoteIds.subList(
currentBatch * SYNC_BATCH_SIZE, (currentBatch + 1) currentBatch * SYNC_BATCH_SIZE, (currentBatch + 1)
* SYNC_BATCH_SIZE)); * SYNC_BATCH_SIZE));
@ -670,12 +702,6 @@ public class RegistryFederationManager implements IRegistryFederationManager,
} }
} }
private void sendSyncMessage() {
if (!SYNC_IN_PROGRESS.get()) {
statusHandler.warn(SYNC_WARNING_MSG);
monitorHandler.handle(Priority.WARN, SYNC_WARNING_MSG);
}
}
@GET @GET
@Path("isFederated") @Path("isFederated")
@ -1096,15 +1122,35 @@ public class RegistryFederationManager implements IRegistryFederationManager,
* Updates the record in the registry that keeps track of if this registry * Updates the record in the registry that keeps track of if this registry
* has been up. This method is called every minute via a quartz cron * has been up. This method is called every minute via a quartz cron
* configured in Camel * configured in Camel
*
* @throws EbxmlRegistryException
*/ */
@Transactional @Transactional
public void updateUpTime() { public void updateUpTime() throws EbxmlRegistryException {
if (initialized.get()) { if (initialized.get() && EDEXUtil.isRunning()) {
if (SYNC_NECESSARY) { long currentTime = TimeUtil.currentTimeMillis();
if (!SYNC_IN_PROGRESS.get() long lastKnownUp = federatedRegistryMonitor.getLastKnownUptime();
&& TimeUtil.newGmtCalendar().get(Calendar.MINUTE) % 15 == 0) { long downTime = currentTime - lastKnownUp;
sendSyncMessage(); if (currentTime - lastKnownUp > MAX_DOWN_TIME_DURATION
&& !centralRegistry) {
if (!SYNC_IN_PROGRESS.get()) {
statusHandler.info("Registry has been down since: "
+ new Date(currentTime - downTime));
statusHandler
.warn("Registry is out of sync with federation. Attempting to automatically synchronize");
try {
synchronize();
monitorHandler
.info("Registry successfully synchronized!");
} catch (EbxmlRegistryException e) {
monitorHandler
.error("Error synchronizing registry!", e);
throw e;
}
} }
} else { } else {
federatedRegistryMonitor.updateTime(); federatedRegistryMonitor.updateTime();
} }

5
edexOsgi/com.raytheon.uf.edex.datadelivery.registry/src/com/raytheon/uf/edex/datadelivery/registry/web/RegistryDataAccessService.java
View file

@ -80,6 +80,7 @@ import com.raytheon.uf.edex.registry.ebxml.services.query.RegistryQueryUtil;
* 10/8/2013 1682 bphillip Added query queries * 10/8/2013 1682 bphillip Added query queries
* 11/7/2013 1678 bphillip Added getCustomQueries method * 11/7/2013 1678 bphillip Added getCustomQueries method
* Mar 31, 2014 2889 dhladky Added username for notification center tracking. * Mar 31, 2014 2889 dhladky Added username for notification center tracking.
* Apr 12,2014 3012 dhladky Purge never worked, fixed to make work.
* </pre> * </pre>
* *
* @author bphillip * @author bphillip
@ -343,9 +344,9 @@ public class RegistryDataAccessService implements IRegistryDataAccessService {
try { try {
Object subObj = subscriptionJaxbManager Object subObj = subscriptionJaxbManager
.unmarshalFromXml(subscriptionXML); .unmarshalFromXml(subscriptionXML);
EDEXUtil.getMessageProducer().sendSync("scheduleSubscription",
subObj);
lcm.submitObjects(submitRequest); lcm.submitObjects(submitRequest);
EDEXUtil.getMessageProducer().sendSync("scheduleSubscription",
new Object[] {subObj, false});
subscriptionFile.delete(); subscriptionFile.delete();
response.append( response.append(
"Subscription successfully restored from file [") "Subscription successfully restored from file [")

3
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/util/RetrievalGeneratorUtilities.java
View file

@ -53,6 +53,7 @@ import com.raytheon.uf.edex.datadelivery.retrieval.handlers.SubscriptionRetrieva
* Dec 10, 2012 1259 bsteffen Switch Data Delivery from LatLon to referenced envelopes. * Dec 10, 2012 1259 bsteffen Switch Data Delivery from LatLon to referenced envelopes.
* Dec 11, 2013 2625 mpduff Remove creation of DataURI. * Dec 11, 2013 2625 mpduff Remove creation of DataURI.
* Jan 30, 2014 2686 dhladky refactor of retrieval. * Jan 30, 2014 2686 dhladky refactor of retrieval.
* Apr 09, 2014 3012 dhladky Added error message.
* Apr 21, 2014 2060 njensen Remove dependency on grid dataURI column * Apr 21, 2014 2060 njensen Remove dependency on grid dataURI column
* *
* </pre> * </pre>
@ -102,7 +103,7 @@ public class RetrievalGeneratorUtilities {
} }
} catch (Exception e) { } catch (Exception e) {
statusHandler.handle(Priority.PROBLEM, statusHandler.handle(Priority.PROBLEM,
e.getLocalizedMessage(), e); "Can't determine if URI is a duplicate", e);
} }
} }
} }

3
edexOsgi/com.raytheon.uf.edex.ogc.registry/src/com/raytheon/uf/edex/ogc/registry/RegistryCollectorAddon.java
View file

@ -63,6 +63,7 @@ import com.raytheon.uf.edex.ogc.common.db.SimpleLayer;
* Aug 08, 2013 dhladky Made operational * Aug 08, 2013 dhladky Made operational
* Jan 13, 2014 #2679 dhladky multiple layers * Jan 13, 2014 #2679 dhladky multiple layers
* Mar 31, 2014 2889 dhladky Added username for notification center tracking. * Mar 31, 2014 2889 dhladky Added username for notification center tracking.
* Apr 14, 2014 3012 dhladky Cleaned up.
* *
* </pre> * </pre>
* *
@ -120,7 +121,7 @@ public abstract class RegistryCollectorAddon<D extends SimpleDimension, L extend
final String description = metaData.getDataSetDescription(); final String description = metaData.getDataSetDescription();
statusHandler.info("Attempting store of DataSetMetaData[" + description statusHandler.info("Attempting store of DataSetMetaData[" + description
+ "]"); + "] " + "Date: "+metaData.getDate());
try { try {
handler.update(RegistryUtil.registryUser, metaData); handler.update(RegistryUtil.registryUser, metaData);

13
edexOsgi/com.raytheon.uf.edex.ogc.registry/src/com/raytheon/uf/edex/ogc/registry/WfsRegistryCollectorAddon.java
View file

@ -76,6 +76,7 @@ import com.vividsolutions.jts.geom.Envelope;
* Oct 10, 2013 1797 bgonzale Refactored registry Time objects. * Oct 10, 2013 1797 bgonzale Refactored registry Time objects.
* Nov 6, 2013 2525 dhladky Stop appending "/wfs" * Nov 6, 2013 2525 dhladky Stop appending "/wfs"
* Jan 13, 2014 2679 dhladky Multiple ingest layer windows for a single request window. * Jan 13, 2014 2679 dhladky Multiple ingest layer windows for a single request window.
* Apr 13, 2014 3012 dhladky Cleaned up.
* *
* </pre> * </pre>
* *
@ -454,13 +455,15 @@ public abstract class WfsRegistryCollectorAddon<D extends SimpleDimension, L ext
times.put(layer.getName(), new PointTime()); times.put(layer.getName(), new PointTime());
// harvests the times from the layer // harvests the times from the layer
setTime(layer); setTime(layer);
// make sure you populate the metadata // make sure you populate the metadata
setDataSetMetaData(layer); setDataSetMetaData(layer);
getDataSetMetaData(layer.getName()).setTime(getTime(layer.getName())); PointDataSetMetaData data = getDataSetMetaData(layer.getName());
ImmutableDate date = null; PointTime time = getTime(layer.getName());
date = new ImmutableDate(getTime(layer.getName()).getEnd()); data.setTime(time);
getDataSetMetaData(layer.getName()).setDate(date); ImmutableDate date = new ImmutableDate(time.getEnd());
storeMetaData(getDataSetMetaData(layer.getName())); data.setDate(date);
storeMetaData(data);
} }
} }

2
edexOsgi/com.raytheon.uf.edex.plugin.madis.registry/utility/common_static/base/datadelivery/harvester/MADIS-harvester.xml
View file

@ -19,6 +19,8 @@
<description>MADIS Test LatLon Coverage</description> <description>MADIS Test LatLon Coverage</description>
</projection> </projection>
</provider> </provider>
<!-- MADIS doesn't purge Data Set Meta Data -->
<retention>-1</retention>
<agent xsi:type="ogcAgent" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> <agent xsi:type="ogcAgent" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<dateFormat>HHddMMMyyyy</dateFormat> <dateFormat>HHddMMMyyyy</dateFormat>
<layer name="madis-conus" namespace="http://madis.edex.uf.raytheon.com"> <layer name="madis-conus" namespace="http://madis.edex.uf.raytheon.com">

9
edexOsgi/com.raytheon.uf.edex.registry.ebxml/src/com/raytheon/uf/edex/registry/ebxml/dao/SlotTypeDao.java
View file

@ -19,13 +19,16 @@
**/ **/
package com.raytheon.uf.edex.registry.ebxml.dao; package com.raytheon.uf.edex.registry.ebxml.dao;
import java.util.Collection;
import java.util.List; import java.util.List;
import oasis.names.tc.ebxml.regrep.xsd.rim.v4.SlotType; import oasis.names.tc.ebxml.regrep.xsd.rim.v4.SlotType;
import org.hibernate.SQLQuery; import org.hibernate.SQLQuery;
import org.hibernate.criterion.Property;
import com.raytheon.uf.edex.database.dao.SessionManagedDao; import com.raytheon.uf.edex.database.dao.SessionManagedDao;
import com.raytheon.uf.edex.registry.ebxml.services.query.QueryConstants;
/** /**
* *
@ -84,4 +87,10 @@ public class SlotTypeDao extends SessionManagedDao<String, SlotType> {
this.template.delete(slot); this.template.delete(slot);
} }
} }
public void deleteBySlotId(Collection<String> ids){
template.deleteAll(createCriteria().add(
Property.forName(QueryConstants.ID).in(ids))
.list());
}
} }

5
edexOsgi/com.raytheon.uf.edex.registry.ebxml/src/com/raytheon/uf/edex/registry/ebxml/services/RegistryGarbageCollector.java
View file

@ -55,6 +55,7 @@ import com.raytheon.uf.edex.registry.events.DeleteSlotEvent;
* 2/4/2014 2769 bphillip Removed flush and clear call * 2/4/2014 2769 bphillip Removed flush and clear call
* 2/13/2014 2769 bphillip Refactored to no longer use executor threads * 2/13/2014 2769 bphillip Refactored to no longer use executor threads
* 4/11/2014 3011 bphillip Added slot purging via event bus notifications * 4/11/2014 3011 bphillip Added slot purging via event bus notifications
* 4/17/2014 3011 bphillip Delete slot events now contain strings
* </pre> * </pre>
* *
* @author bphillip * @author bphillip
@ -139,15 +140,17 @@ public class RegistryGarbageCollector {
@Subscribe @Subscribe
public void deleteOrphanedSlot(DeleteSlotEvent slotEvent) { public void deleteOrphanedSlot(DeleteSlotEvent slotEvent) {
if (!CollectionUtil.isNullOrEmpty(slotEvent.getSlotsToDelete())) { if (!CollectionUtil.isNullOrEmpty(slotEvent.getSlotsToDelete())) {
long start = TimeUtil.currentTimeMillis(); long start = TimeUtil.currentTimeMillis();
statusHandler.info("Deleting " statusHandler.info("Deleting "
+ slotEvent.getSlotsToDelete().size() + " slots..."); + slotEvent.getSlotsToDelete().size() + " slots...");
slotDao.deleteAll(slotEvent.getSlotsToDelete()); slotDao.deleteBySlotId(slotEvent.getSlotsToDelete());
statusHandler.info("Deleted " + slotEvent.getSlotsToDelete().size() statusHandler.info("Deleted " + slotEvent.getSlotsToDelete().size()
+ " slots in " + (TimeUtil.currentTimeMillis() - start) + " slots in " + (TimeUtil.currentTimeMillis() - start)
+ " ms"); + " ms");
} }
} }
} }

6
edexOsgi/com.raytheon.uf.edex.registry.ebxml/src/com/raytheon/uf/edex/registry/ebxml/services/lifecycle/LifecycleManagerImpl.java
View file

@ -112,6 +112,7 @@ import com.raytheon.uf.edex.registry.events.DeleteSlotEvent;
* 2/19/2014 2769 bphillip Added current time to audit trail events * 2/19/2014 2769 bphillip Added current time to audit trail events
* Mar 31, 2014 2889 dhladky Added username for notification center tracking. * Mar 31, 2014 2889 dhladky Added username for notification center tracking.
* 4/11/2014 3011 bphillip Modified merge behavior * 4/11/2014 3011 bphillip Modified merge behavior
* 4/17/2014 3011 bphillip Delete slot events now contain strings
* *
* *
* </pre> * </pre>
@ -301,6 +302,8 @@ public class LifecycleManagerImpl implements LifecycleManager {
event.setObjectType(objectType); event.setObjectType(objectType);
EventBus.publish(event); EventBus.publish(event);
} }
DeleteSlotEvent deleteEvent = new DeleteSlotEvent(obj.getSlot());
EventBus.publish(deleteEvent);
EventBus.publish(new RegistryStatisticsEvent(obj.getObjectType(), EventBus.publish(new RegistryStatisticsEvent(obj.getObjectType(),
obj.getStatus(), obj.getOwner(), avTimePerRecord)); obj.getStatus(), obj.getOwner(), avTimePerRecord));
} }
@ -757,9 +760,8 @@ public class LifecycleManagerImpl implements LifecycleManager {
private void mergeObjects(RegistryObjectType newObject, private void mergeObjects(RegistryObjectType newObject,
RegistryObjectType existingObject) { RegistryObjectType existingObject) {
DeleteSlotEvent deleteSlotEvent = new DeleteSlotEvent(existingObject.getSlot());
registryObjectDao.merge(newObject, existingObject); registryObjectDao.merge(newObject, existingObject);
DeleteSlotEvent deleteSlotEvent = new DeleteSlotEvent(
existingObject.getSlot());
EventBus.publish(deleteSlotEvent); EventBus.publish(deleteSlotEvent);
} }

46
edexOsgi/com.raytheon.uf.edex.registry.ebxml/src/com/raytheon/uf/edex/registry/events/DeleteSlotEvent.java
View file

@ -19,11 +19,13 @@
**/ **/
package com.raytheon.uf.edex.registry.events; package com.raytheon.uf.edex.registry.events;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import oasis.names.tc.ebxml.regrep.xsd.rim.v4.SlotType; import oasis.names.tc.ebxml.regrep.xsd.rim.v4.SlotType;
import com.raytheon.uf.common.event.Event; import com.raytheon.uf.common.event.Event;
import com.raytheon.uf.common.util.CollectionUtil;
/** /**
* Event containing slots to be deleted by the registry garbage collector * Event containing slots to be deleted by the registry garbage collector
@ -35,6 +37,7 @@ import com.raytheon.uf.common.event.Event;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* 4/11/2014 3011 bphillip Initial Coding * 4/11/2014 3011 bphillip Initial Coding
* 4/17/2014 3011 bphillip Delete slot events now contain strings
* </pre> * </pre>
* *
* @author bphillip * @author bphillip
@ -42,26 +45,31 @@ import com.raytheon.uf.common.event.Event;
*/ */
public class DeleteSlotEvent extends Event { public class DeleteSlotEvent extends Event {
private static final long serialVersionUID = -2818002679753482984L; private static final long serialVersionUID = -2818002679753482984L;
private List<SlotType> slotsToDelete;
public DeleteSlotEvent(){
super();
}
public DeleteSlotEvent(List<SlotType> slotsToDelete){
this.slotsToDelete = slotsToDelete;
}
public List<SlotType> getSlotsToDelete() { private List<String> slotsToDelete;;
return slotsToDelete;
}
public void setSlotsToDelete(List<SlotType> slotsToDelete) { public DeleteSlotEvent() {
this.slotsToDelete = slotsToDelete; super();
} }
public DeleteSlotEvent(List<SlotType> slotsToDelete) {
if (CollectionUtil.isNullOrEmpty(slotsToDelete)) {
slotsToDelete = new ArrayList<SlotType>();
} else {
this.slotsToDelete = new ArrayList<String>(slotsToDelete.size());
for (SlotType slot : slotsToDelete) {
this.slotsToDelete.add(slot.getId());
}
}
}
public List<String> getSlotsToDelete() {
return slotsToDelete;
}
public void setSlotsToDelete(List<String> slotsToDelete) {
this.slotsToDelete = slotsToDelete;
}
} }