diff --git a/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/common/ui/TableDataManager.java b/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/common/ui/TableDataManager.java index 0bc6aad030..6a91ac4f00 100644 --- a/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/common/ui/TableDataManager.java +++ b/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/common/ui/TableDataManager.java @@ -31,6 +31,7 @@ package com.raytheon.uf.viz.datadelivery.common.ui; * Jun 06, 2012 lvenable Initial creation * Apr 10, 2013 1891 djohnson Declare variable as List. * Feb 07, 2014 2453 mpduff Added getSize(). + * Apr 18, 2014 3012 dhladky Null check. * * * @@ -134,7 +135,11 @@ public class TableDataManager> implements ISortTable { if (index >= 0 && index < tableData.size()) { return tableData.get(index); } else { - return tableData.get(0); + if (!tableData.isEmpty()) { + return tableData.get(0); + } else { + return null; + } } } diff --git a/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/notification/NotificationTableComp.java b/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/notification/NotificationTableComp.java index bf7f980632..ddead4d62a 100644 --- a/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/notification/NotificationTableComp.java +++ b/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/notification/NotificationTableComp.java @@ -85,6 +85,7 @@ import com.raytheon.uf.viz.datadelivery.utils.NotificationHandler; * Oct 15, 2013 2451 skorolev Get highlighted rows after message update. * Nov 01, 2013 2431 skorolev Changed labels on the table. * Feb 07, 2014 2453 mpduff Refactored. + * Apr 18, 2014 3012 dhladky Null check. * * * @author lvenable @@ -941,6 +942,9 @@ public class NotificationTableComp extends TableComp implements ITableFind { for (int index : indices) { NotificationRowData rowData = filteredTableList .getDataRow(index); + if (rowData == null) { + continue; + } selectedRowIds.add(rowData.getId()); } } diff --git a/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/SubscriptionManagerDlg.java b/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/SubscriptionManagerDlg.java index 0dd10104bf..7657c805db 100644 --- a/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/SubscriptionManagerDlg.java +++ b/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/SubscriptionManagerDlg.java @@ -159,6 +159,7 @@ import com.raytheon.viz.ui.presenter.IDisplay; * Mar 24, 2014 #2951 lvenable Added dispose checks for SWT widgets. * Mar 31, 2014 2889 dhladky Added username for notification center tracking. * Apr 2, 2014 2974 dhladky DD ID added to list for dropdowns in DD. + * Apr 18, 2014 3012 dhladky Null check. * * * @@ -1001,18 +1002,20 @@ public class SubscriptionManagerDlg extends CaveSWTDialog implements SubscriptionManagerRowData removedItem = tableComp .getSubscriptionData().getDataRow(idx); Subscription sub = removedItem.getSubscription(); - if (sub instanceof SharedSubscription) { - sub.getOfficeIDs().remove(CURRENT_SITE); - if (sub.getOfficeIDs().size() > 0) { - subsToUpdate.add(sub); + if (sub != null) { + if (sub instanceof SharedSubscription) { + sub.getOfficeIDs().remove(CURRENT_SITE); + if (sub.getOfficeIDs().size() > 0) { + subsToUpdate.add(sub); + } else { + subsToDelete.add(sub); + } } else { - subsToDelete.add(sub); + subsToDelete.add(removedItem.getSubscription()); } - } else { - subsToDelete.add(removedItem.getSubscription()); - } - deleteList.add(removedItem); + deleteList.add(removedItem); + } } String message = getMessage(subsToDelete, subsToUpdate); @@ -1189,7 +1192,9 @@ public class SubscriptionManagerDlg extends CaveSWTDialog implements int idx = selectionIndices[i]; SubscriptionManagerRowData rowData = tableComp .getSubscriptionData().getDataRow(idx); - + if (rowData == null) { + continue; + } Subscription sub = rowData.getSubscription(); if (activate) { sub.activate(); diff --git a/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/SubscriptionTableComp.java b/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/SubscriptionTableComp.java index d91bd91f6c..89927b45d9 100644 --- a/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/SubscriptionTableComp.java +++ b/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/SubscriptionTableComp.java @@ -113,7 +113,7 @@ import com.raytheon.uf.viz.datadelivery.utils.DataDeliveryUtils.TABLE_TYPE; * Feb 04, 2014 2722 mpduff Add last update time. * Feb 11, 2014 2771 bgonzale Use Data Delivery ID instead of Site. * Mar 24, 2014 #2951 lvenable Added dispose checks for SWT widgets. - * + * Apr 18, 2014 3012 dhladky Null check. * @version 1.0 */ @@ -325,6 +325,9 @@ public class SubscriptionTableComp extends TableComp implements IGroupAction { SubscriptionManagerRowData rowData = subManagerData .getDataRow(selectionIndices[i]); + if (rowData == null) { + continue; + } // get the subscription details to be displayed to the user printDetails.append(DataDeliveryUtils.formatDetails(rowData .getSubscription())); @@ -911,4 +914,4 @@ public class SubscriptionTableComp extends TableComp implements IGroupAction { public long getLastUpdateTime() { return lastUpdateTime; } -} \ No newline at end of file +} diff --git a/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/approve/SubApprovalTableComp.java b/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/approve/SubApprovalTableComp.java index 6d5520aae1..e464776963 100644 --- a/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/approve/SubApprovalTableComp.java +++ b/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/approve/SubApprovalTableComp.java @@ -74,6 +74,7 @@ import com.raytheon.uf.viz.datadelivery.utils.DataDeliveryUtils.TABLE_TYPE; * denied pending messages. * Apr 05, 2013 1841 djohnson Refresh entire table on receiving a notification of the correct type. * Apr 10, 2013 1891 djohnson Move logic to get column display text to the column definition, fix sorting. + * Apr 18, 2014 3012 dhladky Null check. * * * @author lvenable @@ -270,6 +271,9 @@ public class SubApprovalTableComp extends TableComp { SubscriptionApprovalRowData rowData = pendingSubData.getDataRow(table .getSelectionIndex()); + if (rowData == null) { + return; + } // Get the subscription object InitialPendingSubscription pendingSub = rowData.getSubscription(); diffDetails.append("Subscription Name: ") diff --git a/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/approve/SubscriptionApprovalDlg.java b/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/approve/SubscriptionApprovalDlg.java index 46e9cb0fff..c115f24bc5 100644 --- a/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/approve/SubscriptionApprovalDlg.java +++ b/cave/com.raytheon.uf.viz.datadelivery/src/com/raytheon/uf/viz/datadelivery/subscription/approve/SubscriptionApprovalDlg.java @@ -96,6 +96,7 @@ import com.raytheon.viz.ui.presenter.IDisplay; * Sep 03, 2013 2315 mpduff Add subscription name to denied approval message. * Oct 23, 2013 2292 mpduff Move subscription overlap checks to edex. * Mar 31, 2014 2889 dhladky Added username for notification center tracking. + * Apr 18, 2014 3012 dhladky Null check. * * * @@ -368,13 +369,15 @@ public class SubscriptionApprovalDlg extends CaveSWTDialog implements for (int idx : tableComp.getTable().getSelectionIndices()) { SubscriptionApprovalRowData approvedItem = pendingSubData .getDataRow(idx); - if (site) { - approveList.add(approvedItem); - } else { - if (approvedItem.isOwner(user)) { + if (approvedItem != null) { + if (site) { approveList.add(approvedItem); } else { - notApprovedSubList.add(approvedItem.getSubName()); + if (approvedItem.isOwner(user)) { + approveList.add(approvedItem); + } else { + notApprovedSubList.add(approvedItem.getSubName()); + } } } } @@ -461,12 +464,13 @@ public class SubscriptionApprovalDlg extends CaveSWTDialog implements for (int idx : tableComp.getTable().getSelectionIndices()) { SubscriptionApprovalRowData removedItem = pendingSubData .getDataRow(idx); - - if (site) { - deleteList.add(removedItem); - } else { - if (removedItem.isOwner(user)) { + if (removedItem != null) { + if (site) { deleteList.add(removedItem); + } else { + if (removedItem.isOwner(user)) { + deleteList.add(removedItem); + } } } } diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/ifpAG/ASCIIGrid.java b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/ifpAG/ASCIIGrid.java index 04276f8595..d53bfc46c3 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/ifpAG/ASCIIGrid.java +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/ifpAG/ASCIIGrid.java @@ -25,6 +25,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.PrintWriter; import java.text.DateFormat; +import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; @@ -34,6 +35,7 @@ import java.util.Scanner; import com.raytheon.edex.plugin.gfe.config.IFPServerConfig; import com.raytheon.edex.plugin.gfe.config.IFPServerConfigManager; import com.raytheon.edex.plugin.gfe.exception.GfeConfigurationException; +import com.raytheon.edex.plugin.gfe.server.IFPServer; import com.raytheon.uf.common.dataplugin.gfe.GridDataHistory; import com.raytheon.uf.common.dataplugin.gfe.GridDataHistory.OriginType; import com.raytheon.uf.common.dataplugin.gfe.RemapGrid; @@ -46,6 +48,7 @@ import com.raytheon.uf.common.dataplugin.gfe.db.objects.GridParmInfo; import com.raytheon.uf.common.dataplugin.gfe.db.objects.ParmID; import com.raytheon.uf.common.dataplugin.gfe.db.objects.TimeConstraints; import com.raytheon.uf.common.dataplugin.gfe.discrete.DiscreteKey; +import com.raytheon.uf.common.dataplugin.gfe.exception.GfeException; import com.raytheon.uf.common.dataplugin.gfe.grid.Grid2DByte; import com.raytheon.uf.common.dataplugin.gfe.grid.Grid2DFloat; import com.raytheon.uf.common.dataplugin.gfe.slice.DiscreteGridSlice; @@ -72,6 +75,7 @@ import com.vividsolutions.jts.geom.Coordinate; * Apr 13, 2011 #8393 dgilling Initial creation * 02/19/13 #1637 randerso Added exception handling for Discrete and Weather * 10/31/2013 #2508 randerso Change to use DiscreteGridSlice.getKeys() + * 04/22/2014 #3050 randerso Allow exceptions to propagate to caller from readASCIIGridData * * * @@ -353,12 +357,14 @@ public class ASCIIGrid { } public String readASCIIGridData(File aGridData) - throws FileNotFoundException { + throws FileNotFoundException, GfeException, ParseException { List gridSlices = new ArrayList(); - Scanner inputStream = new Scanner(aGridData, "US-ASCII"); - while (true) { - try { + Scanner inputStream = null; + try { + inputStream = new Scanner(aGridData, "US-ASCII"); + + while (true) { // read the ASCIIGRID keyword // if we have an ASCIIGRID to read if (!inputStream.next().equals("ASCIIGRID")) { @@ -421,8 +427,12 @@ public class ASCIIGrid { float yExtent = inputStream.nextFloat(); // make the GridLocation - IFPServerConfig config = IFPServerConfigManager - .getServerConfig(dbSiteId); + IFPServer ifpServer = IFPServer.getActiveServer(dbSiteId); + if (ifpServer == null) { + throw new GfeException("No active IFPServer for site: " + + dbSiteId); + } + IFPServerConfig config = ifpServer.getConfig(); GridLocation baseGLoc = config.dbDomain(); ProjectionData projData = config.getProjectionData(projId); GridLocation gLocation = new GridLocation(dbSiteId, projData, @@ -600,14 +610,12 @@ public class ASCIIGrid { if (!inputStream.hasNext()) { break; } - - } catch (Exception e) { - statusHandler.handle(Priority.PROBLEM, - "Caught exception in readASCIIGridData()", e); - break; + } + } finally { + if (inputStream != null) { + inputStream.close(); } } - inputStream.close(); this.gridSlices = gridSlices; return ""; diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/SaveASCIIGridsHandler.java b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/SaveASCIIGridsHandler.java index 6c1ef52bdc..02d63e30bc 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/SaveASCIIGridsHandler.java +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/SaveASCIIGridsHandler.java @@ -61,6 +61,9 @@ import com.raytheon.uf.common.status.UFStatus; * Apr 21, 2011 dgilling Initial creation * Apr 23, 2013 1949 rjpeter Removed extra lock table look up * Jun 13, 2013 #2044 randerso Refactored to use IFPServer + * Apr 21, 2014 #3050 randerso Get the IFPServer instance based on the + * site in the ParmID + * * * * @author dgilling @@ -83,9 +86,6 @@ public class SaveASCIIGridsHandler extends BaseGfeRequestHandler implements @Override public ServerResponse handleRequest(SaveASCIIGridsRequest request) throws Exception { - IFPServer ifpServer = getIfpServer(request); - GridParmManager gridParmMgr = ifpServer.getGridParmMgr(); - LockManager lockMgr = ifpServer.getLockMgr(); ServerResponse sr = new ServerResponse(); @@ -97,10 +97,23 @@ public class SaveASCIIGridsHandler extends BaseGfeRequestHandler implements sr.addMessage(msg); } + String prevSiteID = null; int ngrids = agrid.getGridSlices().size(); for (int i = 0; i < ngrids; i++) { ParmID pid = agrid.getGridSlices().get(i).getGridInfo().getParmID(); + // get the server for this site + String siteID = pid.getDbId().getSiteId(); + IFPServer ifpServer = IFPServer.getActiveServer(siteID); + if (ifpServer == null && !siteID.equals(prevSiteID)) { + sr.addMessage("No active IFPServer for site: " + siteID); + continue; + } + prevSiteID = siteID; + + GridParmManager gridParmMgr = ifpServer.getGridParmMgr(); + LockManager lockMgr = ifpServer.getLockMgr(); + // get a list of available databases, see if the grid is part of an // existing database. ServerResponse> srDbInv = gridParmMgr diff --git a/edexOsgi/com.raytheon.uf.common.datadelivery.harvester/src/com/raytheon/uf/common/datadelivery/harvester/HarvesterConfig.java b/edexOsgi/com.raytheon.uf.common.datadelivery.harvester/src/com/raytheon/uf/common/datadelivery/harvester/HarvesterConfig.java index 4765f2ff1b..e682c12869 100644 --- a/edexOsgi/com.raytheon.uf.common.datadelivery.harvester/src/com/raytheon/uf/common/datadelivery/harvester/HarvesterConfig.java +++ b/edexOsgi/com.raytheon.uf.common.datadelivery.harvester/src/com/raytheon/uf/common/datadelivery/harvester/HarvesterConfig.java @@ -19,6 +19,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; * 12 Sept, 2012 1038 dhladky Initial creation * 1 May 2013 1959 dhladky remove backup registry references * 23 Oct, 2013 2361 njensen Remove ISerializableObject + * 15 Apr, 2014 3012 dhladky Added retention time for this provider in registry. * * * @@ -37,6 +38,11 @@ public class HarvesterConfig { @XmlElement(name = "agent") @DynamicSerializeElement private Agent agent; + + /** default of 7 days **/ + @XmlElement(name = "retention") + @DynamicSerializeElement + private String retention = "7"; public HarvesterConfig() { @@ -58,4 +64,12 @@ public class HarvesterConfig { this.provider = provider; } + public String getRetention() { + return retention; + } + + public void setRetention(String retention) { + this.retention = retention; + } + } diff --git a/edexOsgi/com.raytheon.uf.common.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/NOMADS-harvester.xml b/edexOsgi/com.raytheon.uf.common.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/NOMADS-harvester.xml index f6a2be6687..0c1d4fb65e 100644 --- a/edexOsgi/com.raytheon.uf.common.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/NOMADS-harvester.xml +++ b/edexOsgi/com.raytheon.uf.common.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/NOMADS-harvester.xml @@ -28,6 +28,8 @@ http://nomads.ncep.noaa.gov:9090/dods/ + + 7 /awips2/crawl diff --git a/edexOsgi/com.raytheon.uf.common.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/OGC-harvester.xml.sample b/edexOsgi/com.raytheon.uf.common.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/OGC-harvester.xml.sample index 329561f2cd..969ddc31a8 100644 --- a/edexOsgi/com.raytheon.uf.common.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/OGC-harvester.xml.sample +++ b/edexOsgi/com.raytheon.uf.common.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/OGC-harvester.xml.sample @@ -15,6 +15,8 @@ METAR Test LatLon Coverage + + 1 HHddMMMyyyy diff --git a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/DataSetMetaData.java b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/DataSetMetaData.java index 2bd34d4968..c1ae950cb3 100644 --- a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/DataSetMetaData.java +++ b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/DataSetMetaData.java @@ -43,7 +43,8 @@ import com.raytheon.uf.common.time.util.ImmutableDate; * Nov 19, 2012 1166 djohnson Clean up JAXB representation of registry objects. * Sept, 30 2013 1797 dhladky Made generic based on Time * Dec 20, 2013 2636 mpduff Add a dataset availability offset - * jan 23, 2013 2584 dhladky Versions. + * jan 23, 2013 2584 dhladky Versions. + * Apr 14, 2013 3012 dhladky Removed unused methods. * * * @author dhladky @@ -229,13 +230,4 @@ public abstract class DataSetMetaData { return url; } - /** - * Accepts a {@link IDataSetMetaDataVisitor} which can perform arbitrary - * processing on this {@link DataSetMetaData} instance. Should be defined by - * each concrete class instance. - * - * @param visitor - * the visitor - */ - public abstract void accept(IDataSetMetaDataVisitor visitor); } diff --git a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/IDataSetMetaDataVisitor.java b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/IDataSetMetaDataVisitor.java deleted file mode 100644 index 78efaf40e5..0000000000 --- a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/IDataSetMetaDataVisitor.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * This software was developed and / or modified by Raytheon Company, - * pursuant to Contract DG133W-05-CQ-1067 with the US Government. - * - * U.S. EXPORT CONTROLLED TECHNICAL DATA - * This software product contains export-restricted data whose - * export/transfer/disclosure is restricted by U.S. law. Dissemination - * to non-U.S. persons whether in the United States or abroad requires - * an export license or other authorization. - * - * Contractor Name: Raytheon Company - * Contractor Address: 6825 Pine Street, Suite 340 - * Mail Stop B8 - * Omaha, NE 68106 - * 402.291.0100 - * - * See the AWIPS II Master Rights File ("Master Rights File.pdf") for - * further licensing information. - **/ -package com.raytheon.uf.common.datadelivery.registry; - - -/** - * Defines a type that can visit {@link DataSetMetaData} instances and perform - * some activity. - * - * 
- * 
- * SOFTWARE HISTORY
- * 
- * Date         Ticket#    Engineer    Description
- * ------------ ---------- ----------- --------------------------
- * Sep 4, 2012  1102       djohnson     Initial creation
- * 
- *
- * - * @author djohnson - * @version 1.0 - */ - -public interface IDataSetMetaDataVisitor { - - /** - * Visits an {@link OpenDapGriddedDataSetMetaData} instance. - * - * @param metaData - * the metaData - */ - void visit(OpenDapGriddedDataSetMetaData metaData); -} diff --git a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/OpenDapGriddedDataSetMetaData.java b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/OpenDapGriddedDataSetMetaData.java index faa522a6f0..f680686be9 100644 --- a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/OpenDapGriddedDataSetMetaData.java +++ b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/OpenDapGriddedDataSetMetaData.java @@ -36,7 +36,8 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Sep 4, 2012 1102 djohnson Initial creation - * jan 23, 2013 2584 dhladky Versions + * jan 23, 2014 2584 dhladky Versions + * Apr 14, 2014 3012 dhladky Unneeded method removed. * * * @@ -49,12 +50,4 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; @DynamicSerialize public class OpenDapGriddedDataSetMetaData extends GriddedDataSetMetaData { - /** - * {@inheritDoc} - */ - @Override - public void accept(IDataSetMetaDataVisitor visitor) { - visitor.visit(this); - } - } diff --git a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/PointDataSetMetaData.java b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/PointDataSetMetaData.java index 6da6f317aa..dbb53a7b53 100644 --- a/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/PointDataSetMetaData.java +++ b/edexOsgi/com.raytheon.uf.common.datadelivery.registry/src/com/raytheon/uf/common/datadelivery/registry/PointDataSetMetaData.java @@ -34,6 +34,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; * ------------ ---------- ----------- -------------------------- * Aug 20, 2012 754 dhladky Initial creation * Sept 30, 2013 1797 dhladky Generics + * Apr 14, 2014 3012 dhladky Unneeded method removed. * * * @@ -50,10 +51,5 @@ public class PointDataSetMetaData extends DataSetMetaData { public PointDataSetMetaData() { } - - @Override - public void accept(IDataSetMetaDataVisitor visitor) { - // TODO: not sure what this does? - } - + } \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/xml/DataSetInformation.java b/edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/xml/DataSetInformation.java index 2c64949b57..e6e0f2fd60 100644 --- a/edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/xml/DataSetInformation.java +++ b/edexOsgi/com.raytheon.uf.common.datadelivery.retrieval/src/com/raytheon/uf/common/datadelivery/retrieval/xml/DataSetInformation.java @@ -20,7 +20,6 @@ package com.raytheon.uf.common.datadelivery.retrieval.xml; * further licensing information. **/ - import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; @@ -28,6 +27,7 @@ import javax.xml.bind.annotation.XmlRootElement; import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; +import com.raytheon.uf.common.time.util.TimeUtil; /** * Data Set Info XML Object. @@ -39,6 +39,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Jan 14, 2014 dhladky Initial creation. + * Apr 09, 2014 #3012 dhladky Fixed incorrect default calc. * * * @@ -54,25 +55,26 @@ public class DataSetInformation { @XmlElement(name = "modelName", type = String.class) @DynamicSerializeElement protected String modelName; - + @XmlElement(name = "multiplier", type = Double.class) @DynamicSerializeElement protected Double multiplier; - + @XmlElement(name = "modelRunIncrement", type = Integer.class) @DynamicSerializeElement protected Integer modelRunIncrement; - + @XmlElement(name = "defaultOffset", type = Integer.class) @DynamicSerializeElement protected Integer defaultOffset; - + public DataSetInformation() { - + } - - public DataSetInformation(String modelName, Double multiplier, int modelRunIncrement, int defaultOffset) { - + + public DataSetInformation(String modelName, Double multiplier, + int modelRunIncrement, int defaultOffset) { + this.modelName = modelName; this.multiplier = multiplier; this.modelRunIncrement = modelRunIncrement; @@ -111,8 +113,12 @@ public class DataSetInformation { this.defaultOffset = defaultOffset; } + /** + * The range int is in minutes, so we need multiplier * model running increment * minutes per hour + * @return + */ public int getRange() { - return (int) (getMultiplier() * getModelRunIncrement()); + return (int) ((getMultiplier() * getModelRunIncrement()) * TimeUtil.MINUTES_PER_HOUR); } } diff --git a/edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/dao/SessionManagedDao.java b/edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/dao/SessionManagedDao.java index edb09248fe..9c4dce3922 100644 --- a/edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/dao/SessionManagedDao.java +++ b/edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/dao/SessionManagedDao.java @@ -21,6 +21,7 @@ package com.raytheon.uf.edex.database.dao; import java.io.Serializable; +import java.util.Calendar; import java.util.Collection; import java.util.Collections; import java.util.Iterator; @@ -41,6 +42,7 @@ import org.springframework.transaction.annotation.Transactional; import com.raytheon.uf.common.dataplugin.persist.IPersistableDataObject; import com.raytheon.uf.common.status.IUFStatusHandler; import com.raytheon.uf.common.status.UFStatus; +import com.raytheon.uf.common.status.UFStatus.Priority; import com.raytheon.uf.edex.database.DataAccessLayerException; /** @@ -65,6 +67,7 @@ import com.raytheon.uf.edex.database.DataAccessLayerException; * 12/9/2013 2613 bphillip Added flushAndClearSession method * Jan 17, 2014 2459 mpduff Added null check to prevent NPE. * 2/13/2014 2769 bphillip Added read-only flag to query methods and loadById method + * 4/18/2014 3012 dhladky Diagnostic addition. * * * @@ -214,7 +217,26 @@ public abstract class SessionManagedDao query(String queryString, Object... params) { - return executeHQLQuery(queryString, 0, params); + + List stuff = executeHQLQuery(queryString, 0, params); + + // Used for diagnostics + if (statusHandler.isPriorityEnabled(Priority.DEBUG)) { + statusHandler.debug("Query: " + queryString); + + for (Object o : params) { + if (o instanceof Calendar) { + statusHandler.debug("param: " + + ((Calendar) o).getTime().toString()); + } else { + statusHandler.debug("param: " + o.toString()); + } + } + + statusHandler.debug("return size: " + stuff.size() + "\n"); + } + + return stuff; } @Transactional(propagation = Propagation.REQUIRED, readOnly = true) diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/META-INF/MANIFEST.MF b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/META-INF/MANIFEST.MF index 49bf6012a4..b413491955 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/META-INF/MANIFEST.MF +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/META-INF/MANIFEST.MF @@ -34,7 +34,8 @@ Require-Bundle: com.raytheon.uf.common.status;bundle-version="1.12.1174", com.raytheon.uf.common.datadelivery.service;bundle-version="1.0.0", org.eclipse.jetty;bundle-version="7.6.14", com.raytheon.uf.common.serialization.comm;bundle-version="1.12.1174", - org.quartz;bundle-version="1.8.6" + org.quartz;bundle-version="1.8.6", + com.raytheon.uf.edex.datadelivery.registry;bundle-version="1.0.0" Export-Package: com.raytheon.uf.edex.datadelivery.bandwidth, com.raytheon.uf.edex.datadelivery.bandwidth.dao, com.raytheon.uf.edex.datadelivery.bandwidth.hibernate, diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/BandwidthManager.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/BandwidthManager.java index b708c44fee..4a2f79b269 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/BandwidthManager.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/BandwidthManager.java @@ -148,6 +148,7 @@ import com.raytheon.uf.edex.registry.ebxml.util.RegistryIdUtil; * Feb 11, 2014 2771 bgonzale Added handler for GET_DATADELIVERY_ID request. * Feb 10, 2014 2636 mpduff Changed how retrieval plan is updated over time. * Apr 02, 2014 2810 dhladky Priority sorting of subscriptions. + * Apr 09, 2014 3012 dhladky Range the querries for metadata checks to subscriptions. * Apr 22, 2014 2992 dhladky Ability to get list of all registry nodes containing data. * * @@ -178,8 +179,14 @@ public abstract class BandwidthManager private final IBandwidthDbInit dbInit; + /** used for min time range **/ + public static final String MIN_RANGE_TIME = "min"; private final RegistryIdUtil idUtil; + + /** used for max time range **/ + public static final String MAX_RANGE_TIME = "max"; + // Instance variable and not static, because there are multiple child // implementation classes which should each have a unique prefix private final IPerformanceStatusHandler performanceHandler = PerformanceStatus @@ -379,7 +386,9 @@ public abstract class BandwidthManager .getBaseReferenceTime(); Calendar startTime = TimeUtil.newGmtCalendar(retrievalTime .getTime()); - + + startTime.add(Calendar.MINUTE, + retrieval.getDataSetAvailablityDelay()); int maxLatency = retrieval.getSubscriptionLatency(); retrieval.setStartTime(startTime); @@ -458,8 +467,14 @@ public abstract class BandwidthManager @Override public List schedule(Subscription subscription) { - List unscheduled = null; + List unscheduled = Collections.emptyList(); + if (subscription instanceof RecurringSubscription) { + if (!((RecurringSubscription) subscription).shouldSchedule()) { + return unscheduled; + } + } + final DataType dataSetType = subscription.getDataSetType(); switch (dataSetType) { case GRID: @@ -478,7 +493,7 @@ public abstract class BandwidthManager return unscheduled; } - + /** * Update the retrieval plan for this subscription. * @@ -736,7 +751,10 @@ public abstract class BandwidthManager Subscription subscription) { List unscheduled = schedule(subscription, ((PointTime) subscription.getTime()).getInterval()); - unscheduled.addAll(getMostRecent(subscription, false)); + // add an adhoc if one exists and isn't in startup mode + if (EDEXUtil.isRunning()) { + unscheduled.addAll(getMostRecent(subscription, false)); + } return unscheduled; } @@ -760,10 +778,20 @@ public abstract class BandwidthManager if (subscribedToCycles) { unscheduled = schedule(subscription, Sets.newTreeSet(cycles)); } + // add an adhoc if one exists and isn't in startup mode + if (EDEXUtil.isRunning()) { + unscheduled.addAll(getMostRecent(subscription, true)); + } return unscheduled; } + /** + * Schedule the most recent dataset update if one exists. + * @param subscription + * @param useMostRecentDataSetUpdate + * @return + */ private List getMostRecent( Subscription subscription, boolean useMostRecentDataSetUpdate) { List unscheduled = Collections.emptyList(); @@ -1717,4 +1745,27 @@ public abstract class BandwidthManager return dataSetMetaDataTime; } + + /** + * Sets a range based on the baseReferenceTime hour. + * @param baseReferenceTime + * @return + */ + public static Map getBaseReferenceTimeDateRange(Calendar baseReferenceTime) { + + Map dates = new HashMap(2); + // Set min range to baseReferenceTime hour "00" minutes, "00" seconds + // Set max range to baseReferenceTime hour "59" minutes, "59" seconds + Calendar min = TimeUtil.newGmtCalendar(baseReferenceTime.getTime()); + min.set(Calendar.MINUTE, 0); + min.set(Calendar.SECOND, 0); + Calendar max = TimeUtil.newGmtCalendar(baseReferenceTime.getTime()); + max.set(Calendar.MINUTE, 59); + max.set(Calendar.SECOND, 59); + + dates.put(MIN_RANGE_TIME, min.getTime()); + dates.put(MAX_RANGE_TIME, max.getTime()); + + return dates; + } } diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/EdexBandwidthManager.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/EdexBandwidthManager.java index b6eb345463..b213f1deff 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/EdexBandwidthManager.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/EdexBandwidthManager.java @@ -128,6 +128,7 @@ import com.raytheon.uf.edex.registry.ebxml.util.RegistryIdUtil; * Feb 10, 2014 2636 mpduff Pass Network map to be scheduled. * Feb 21, 2014, 2636 dhladky Try catch to keep MaintTask from dying. * Mar 31, 2014 2889 dhladky Added username for notification center tracking. + * Apr 09, 2014 3012 dhladky Range the queries for metadata checks, adhoc firing prevention. * Apr 22, 2014 2992 dhladky Added IdUtil for siteList * * @@ -295,7 +296,6 @@ public abstract class EdexBandwidthManager * @param subscription * The completed subscription. */ - @SuppressWarnings("unchecked") @Subscribe public void subscriptionFulfilled( SubscriptionRetrievalFulfilled subscriptionRetrievalFulfilled) { @@ -385,6 +385,7 @@ public abstract class EdexBandwidthManager Subscription sub = getRegistryObjectById(subscriptionHandler, re.getId()); sendSubscriptionNotificationEvent(re, sub); + } } @@ -450,9 +451,9 @@ public abstract class EdexBandwidthManager + dsmd.getDataSetName() + "] to [rap_f]"); dsmd.setDataSetName("rap_f"); } - // TODO: End of hack.. BandwidthEventBus.publish(dsmd); + } else { statusHandler.error("No DataSetMetaData found for id [" + id + "]"); } @@ -492,6 +493,7 @@ public abstract class EdexBandwidthManager @Subscribe public void updateGriddedDataSetMetaData( GriddedDataSetMetaData dataSetMetaData) throws ParseException { + // Daily/Hourly/Monthly datasets if (dataSetMetaData.getCycle() == GriddedDataSetMetaData.NO_CYCLE) { updateDataSetMetaDataWithoutCycle((DataSetMetaData) dataSetMetaData); @@ -500,6 +502,7 @@ public abstract class EdexBandwidthManager else { updateDataSetMetaDataWithCycle((DataSetMetaData) dataSetMetaData); } + } /** @@ -667,19 +670,20 @@ public abstract class EdexBandwidthManager DataSetMetaData dataSetMetaData) throws ParseException { BandwidthDataSetUpdate dataset = bandwidthDao .newBandwidthDataSetUpdate(dataSetMetaData); - - // Looking for active subscriptions to the dataset. - List subscriptions = bandwidthDao - .getSubscriptionRetrievals(dataset.getProviderName(), - dataset.getDataSetName(), dataset.getDataSetBaseTime()); + + // Range the query for subscriptions within the baseReferenceTime hour. + // SOME models, RAP and RTMA, come not exactly on the hour. This causes these + // subscriptions to be missed because baseReferenceTimes are on the hour. + Map timeRange = getBaseReferenceTimeDateRange(dataset.getDataSetBaseTime()); + + final SortedSet subscriptions = bandwidthDao + .getSubscriptionRetrievals(dataset.getProviderName(), dataset.getDataSetName(), + RetrievalStatus.SCHEDULED, timeRange.get(MIN_RANGE_TIME), timeRange.get(MAX_RANGE_TIME)); if (!subscriptions.isEmpty()) { // Loop through the scheduled SubscriptionRetrievals and mark // the scheduled retrievals as ready for retrieval for (SubscriptionRetrieval retrieval : subscriptions) { - // TODO: Evaluate the state changes for receiving multiple - // dataset update messages. This seems to be happening - // quite a bit. if (RetrievalStatus.SCHEDULED.equals(retrieval.getStatus())) { // Need to update the Subscription Object in the @@ -723,14 +727,12 @@ public abstract class EdexBandwidthManager } } else { - if (statusHandler.isPriorityEnabled(Priority.DEBUG)) { - statusHandler - .debug("No Subscriptions scheduled for BandwidthDataSetUpdate [" - + dataset.getIdentifier() - + "] base time [" - + BandwidthUtil.format(dataset - .getDataSetBaseTime()) + "]"); - } + statusHandler + .debug("No Subscriptions scheduled for BandwidthDataSetUpdate [" + + dataset.getIdentifier() + + "] base time [" + + BandwidthUtil.format(dataset.getDataSetBaseTime()) + + "]"); } } @@ -867,7 +869,8 @@ public abstract class EdexBandwidthManager + plan.getPlanEnd().getTime()); statusHandler.info("MaintenanceTask: Update schedule"); } - // Find DEFERRED Allocations and load them into the plan... + // Find DEFERRED Allocations and load them into the + // plan... List deferred = bandwidthDao .getDeferred(plan.getNetwork(), plan.getPlanEnd()); if (!deferred.isEmpty()) { @@ -886,8 +889,11 @@ public abstract class EdexBandwidthManager + " Subscriptions processed."); } catch (Throwable t) { - statusHandler.error("MaintenanceTask: Subscription update scheduling has failed", t); + statusHandler + .error("MaintenanceTask: Subscription update scheduling has failed", + t); } } + } } diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/hibernate/HibernateBandwidthInitializer.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/hibernate/HibernateBandwidthInitializer.java index e8629dac0b..2e0917f7fa 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/hibernate/HibernateBandwidthInitializer.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/hibernate/HibernateBandwidthInitializer.java @@ -39,6 +39,7 @@ import com.raytheon.uf.edex.registry.ebxml.util.RegistryIdUtil; * Feb 06, 2014 2636 bgonzale Use scheduling initialization method after registry init. * Feb 11, 2014 2771 bgonzale Use Data Delivery ID instead of Site. * Feb 14, 2014 2636 mpduff Clean up logging + * Apr 09, 2014 3012 dhladky Adhoc firing prevention. * * * @author djohnson @@ -102,19 +103,22 @@ public class HibernateBandwidthInitializer implements BandwidthInitializer { @Override public void executeAfterRegistryInit() { try { + Map> subMap = findSubscriptionsStrategy .findSubscriptionsToSchedule(); List unscheduled = instance.initializeScheduling(subMap); if (!unscheduled.isEmpty()) { - StringBuilder sb = new StringBuilder("The following subscriptions could not be scheduled at startup: "); + StringBuilder sb = new StringBuilder( + "The following subscriptions could not be scheduled at startup: "); sb.append(StringUtil.NEWLINE); for (String subscription : unscheduled) { sb.append(subscription).append(" "); } statusHandler.handle(Priority.INFO, sb.toString()); } + } catch (Exception e) { statusHandler.error( "Failed to query for subscriptions to schedule", e); diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/processing/SimpleSubscriptionAggregator.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/processing/SimpleSubscriptionAggregator.java index 5d711b090f..6b63175e5f 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/processing/SimpleSubscriptionAggregator.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/processing/SimpleSubscriptionAggregator.java @@ -33,7 +33,6 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.dao.SubscriptionRetrieval; import com.raytheon.uf.edex.datadelivery.bandwidth.interfaces.ISubscriptionAggregator; import com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.RetrievalAgent; import com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.RetrievalStatus; -import com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.SubscriptionRetrievalAgent; import com.raytheon.uf.edex.datadelivery.bandwidth.util.BandwidthUtil; /** @@ -55,6 +54,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.util.BandwidthUtil; * Jul 11, 2013 2106 djohnson aggregate() signature changed. * Jan 06, 2014 2636 mpduff Changed how data set offset is set. * Jan 30, 2014 2686 dhladky refactor of retrieval. + * Apr 15, 2014 3012 dhladky help with confusing nomenclature. * * * @author jspinks @@ -87,6 +87,8 @@ public class SimpleSubscriptionAggregator implements ISubscriptionAggregator { // (i.e. has SubscriptionRetrievals associated with it) if // not, create a SubscriptionRetrieval for the subscription + // Do NOT confuse this with an actual SubscriptionRetrieval. + // This SubscriptionRetrieval object is a BandwidthAllocation object SubscriptionRetrieval subscriptionRetrieval = new SubscriptionRetrieval(); // Link this SubscriptionRetrieval with the subscription. subscriptionRetrieval.setBandwidthSubscription(subDao); diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/BandwidthDaoUtil.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/BandwidthDaoUtil.java index dbaf01baeb..9f59583046 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/BandwidthDaoUtil.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/BandwidthDaoUtil.java @@ -84,6 +84,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.retrieval.RetrievalStatus; * active period. * Jan 29, 2014 2636 mpduff Scheduling refactor. * Feb 11, 2014 2636 mpduff Change how retrieval times are calculated. + * Apr 15, 2014 3012 dhladky Fixed improper offsets. * Apr 21, 2014 2887 dhladky Missed start/end in previous call, needs shouldScheduleForTime(); * * @@ -180,6 +181,7 @@ public class BandwidthDaoUtil { // based on plan start and subscription start. Calendar subscriptionCalculatedStart = subscription .calculateStart(planStart); + // end time when when subscription is last valid for scheduling based on // plan end and subscription end. Calendar subscriptionCalculatedEnd = subscription.calculateEnd(planEnd); @@ -201,55 +203,54 @@ public class BandwidthDaoUtil { Calendar start = TimeUtil.newGmtCalendar(subscriptionCalculatedStart .getTime()); - int availabilityOffset = 0; - try { - availabilityOffset = BandwidthUtil.getDataSetAvailablityOffset( - subscription, start); - } catch (RegistryHandlerException e) { - // Error occurred querying the registry. Log and continue on - statusHandler - .handle(Priority.PROBLEM, - "Unable to retrieve data availability offset, using 0 for the offset.", - e); - } - while (!start.after(subscriptionCalculatedEnd)) { - if (!hours.isEmpty()) { - for (Integer cycle : hours) { - start.set(Calendar.HOUR_OF_DAY, cycle); - for (Integer minute : minutes) { - start.set(Calendar.MINUTE, minute); - Calendar retrievalTime = TimeUtil.newGmtCalendar(); - retrievalTime.setTimeInMillis(start.getTimeInMillis()); - retrievalTime.add(Calendar.MINUTE, availabilityOffset); + for (Integer cycle : hours) { + start.set(Calendar.HOUR_OF_DAY, cycle); - if (retrievalTime.after(planStart) - && retrievalTime.before(planEnd)) { - // Check for nonsense - /* - * Fine grain check by hour and minute, for - * subscription(start/end), activePeriod(start/end) - */ - - // TODO: IMPORTANT NOTE: WHEN 14.2.1 MERGES IN. THIS NEEDS - // TO CHECK AGAINST THE OFFSET BASE REFTIME, THE BASE REFTIME - // WILL BE WHAT IS ADDED IF THE CHECK IS TRUE. DO NOT BLINDLY - // MERGE 14.2.1's CODE OVER THIS. - if (!subscription - .shouldScheduleForTime(retrievalTime)) { - // don't schedule this retrieval time, - // outside subscription window - continue; - } - subscriptionTimes.add(retrievalTime); - } - } + // calculate the offset, every hour + int availabilityOffset = 0; + try { + availabilityOffset = BandwidthUtil + .getDataSetAvailablityOffset(subscription, start); + } catch (RegistryHandlerException e) { + // Error occurred querying the registry. Log and continue on + statusHandler + .handle(Priority.PROBLEM, + "Unable to retrieve data availability offset, using 0 for the offset.", + e); } - // Start the next day.. - start.add(Calendar.DAY_OF_YEAR, 1); - start.set(Calendar.HOUR_OF_DAY, hours.first()); + for (Integer minute : minutes) { + + start.set(Calendar.MINUTE, minute); + Calendar baseRefTime = TimeUtil.newGmtCalendar(); + baseRefTime.setTimeInMillis(start.getTimeInMillis()); + + // add the offset and check if it falls within window + Calendar offsetBaseRefTime = TimeUtil + .newGmtCalendar(baseRefTime.getTime()); + offsetBaseRefTime.add(Calendar.MINUTE, availabilityOffset); + + if (offsetBaseRefTime.after(planStart) + && offsetBaseRefTime.before(planEnd)) { + /* + * Fine grain check by hour and minute, for + * subscription(start/end), activePeriod(start/end) + */ + + if (!subscription.shouldScheduleForTime(baseRefTime)) { + // don't schedule this retrieval time, + // outside subscription window + continue; + } + subscriptionTimes.add(baseRefTime); + } + } } + + // Start the next day.. + start.add(Calendar.DAY_OF_YEAR, 1); + start.set(Calendar.HOUR_OF_DAY, hours.first()); } return subscriptionTimes; diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/BandwidthUtil.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/BandwidthUtil.java index 236a5d438d..c7d40a7e8f 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/BandwidthUtil.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth/src/com/raytheon/uf/edex/datadelivery/bandwidth/util/BandwidthUtil.java @@ -33,6 +33,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.dao.BandwidthSubscription; * Oct 30, 2013 2448 dhladky Moved methods to TimeUtil. * Dec 20, 2013 2636 mpduff Changed dataset delay to offset. * Jan 08, 2014 2615 bgonzale Moved Calendar min and max methods to TimeUtil. + * Apr 09, 2014 3012 dhladky GMT Calendar use. * * * @@ -191,7 +192,7 @@ public class BandwidthUtil { dao.setDataSetName(dataSetMetaData.getDataSetName()); dao.setProviderName(dataSetMetaData.getProviderName()); dao.setUpdateTime(BandwidthUtil.now()); - dao.setDataSetBaseTime(TimeUtil.newCalendar(dataSetMetaData.getDate())); + dao.setDataSetBaseTime(TimeUtil.newGmtCalendar(dataSetMetaData.getDate())); dao.setUrl(dataSetMetaData.getUrl()); return dao; diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/res/spring/harvester-datadelivery.xml b/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/res/spring/harvester-datadelivery.xml index 3781db8a84..b2e48a6e96 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/res/spring/harvester-datadelivery.xml +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/res/spring/harvester-datadelivery.xml @@ -10,43 +10,25 @@ - + + + - + - - - + + - - - - - - - - - - - - - java.lang.Throwable - - - - - - - + + - - + + @@ -59,4 +41,31 @@ + + + + + + + + + + + + + + + + + + + java.lang.Throwable + + + + + + \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/resources/com.raytheon.uf.edex.datadelivery.harvester.properties b/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/resources/com.raytheon.uf.edex.datadelivery.harvester.properties index c736f3e0a7..5baece9e42 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/resources/com.raytheon.uf.edex.datadelivery.harvester.properties +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/resources/com.raytheon.uf.edex.datadelivery.harvester.properties @@ -2,6 +2,6 @@ metadata-process.cron=0+*+*+*+*+? # The cron pattern for how often the DataSetMetaDataPurgeLauncher will run, e.g. # how often to check for purgeable dataset metadata instances -metadata-purge.cron=0+0+3+*+*+? +metadata-purge.cron=0+0+8+*+*+? # how many meta data process threads to use to process metadata metadata-process.threads=2 diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/DataSetMetaDataPurgeLauncher.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/DataSetMetaDataPurgeLauncher.java index 4033797805..4aeb30ada9 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/DataSetMetaDataPurgeLauncher.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/DataSetMetaDataPurgeLauncher.java @@ -24,6 +24,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import com.raytheon.uf.common.status.IUFStatusHandler; import com.raytheon.uf.common.status.UFStatus; import com.raytheon.uf.common.status.UFStatus.Priority; +import com.raytheon.uf.edex.registry.ebxml.dao.RegistryObjectDao; /** * Container class to hold the {@link IDataSetMetaDataPurgeTask} instance. It @@ -36,28 +37,32 @@ import com.raytheon.uf.common.status.UFStatus.Priority; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Sep 4, 2012 1102 djohnson Initial creation + * Apr 12,2014 3012 dhladky Purge never worked, fixed to make work. * * * * @author djohnson * @version 1.0 */ -public final class DataSetMetaDataPurgeLauncher { +public class DataSetMetaDataPurgeLauncher { + private static final IUFStatusHandler statusHandler = UFStatus .getHandler(DataSetMetaDataPurgeLauncher.class); - private static final DataSetMetaDataPurgeLauncher INSTANCE = new DataSetMetaDataPurgeLauncher(); - - private static final IDataSetMetaDataPurgeTask PURGE_TASK = new DataSetMetaDataPurgeTaskImpl(); + private IDataSetMetaDataPurgeTask PURGE_TASK = null; private static final AtomicBoolean running = new AtomicBoolean(); /** - * Disabled constructor. + * Public constructor. */ - private DataSetMetaDataPurgeLauncher() { + public DataSetMetaDataPurgeLauncher(RegistryObjectDao rdo) { + PURGE_TASK = new DataSetMetaDataPurgeTaskImpl(rdo); } + /** + * Try purging datasets. + */ public void runPurge() { try { if (running.compareAndSet(false, true)) { @@ -70,11 +75,4 @@ public final class DataSetMetaDataPurgeLauncher { running.set(false); } } - - /** - * @return the instance - */ - public static DataSetMetaDataPurgeLauncher getInstance() { - return INSTANCE; - } } diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/DataSetMetaDataPurgeTaskImpl.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/DataSetMetaDataPurgeTaskImpl.java index 0631652ebf..4cdfdca25a 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/DataSetMetaDataPurgeTaskImpl.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/DataSetMetaDataPurgeTaskImpl.java @@ -22,33 +22,29 @@ package com.raytheon.uf.edex.datadelivery.harvester.purge; import static com.google.common.base.Preconditions.checkNotNull; import java.util.ArrayList; -import java.util.Collection; +import java.util.Calendar; import java.util.Collections; -import java.util.Iterator; +import java.util.HashMap; import java.util.List; -import java.util.SortedSet; +import java.util.Map; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Multimap; -import com.google.common.collect.Ordering; -import com.google.common.collect.TreeMultimap; import com.raytheon.uf.common.datadelivery.harvester.Agent; import com.raytheon.uf.common.datadelivery.harvester.CrawlAgent; import com.raytheon.uf.common.datadelivery.harvester.HarvesterConfig; +import com.raytheon.uf.common.datadelivery.harvester.HarvesterConfigurationManager; +import com.raytheon.uf.common.datadelivery.registry.DataDeliveryRegistryObjectTypes; import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData; -import com.raytheon.uf.common.datadelivery.registry.IDataSetMetaDataVisitor; -import com.raytheon.uf.common.datadelivery.registry.OpenDapGriddedDataSetMetaData; import com.raytheon.uf.common.datadelivery.registry.handlers.DataDeliveryHandlers; import com.raytheon.uf.common.localization.LocalizationFile; -import com.raytheon.uf.common.registry.ebxml.RegistryUtil; import com.raytheon.uf.common.registry.handler.RegistryHandlerException; -import com.raytheon.uf.common.serialization.SerializationUtil; import com.raytheon.uf.common.status.IUFStatusHandler; import com.raytheon.uf.common.status.UFStatus; import com.raytheon.uf.common.status.UFStatus.Priority; import com.raytheon.uf.common.time.util.ITimer; import com.raytheon.uf.common.time.util.TimeUtil; import com.raytheon.uf.edex.datadelivery.harvester.crawler.CrawlLauncher; +import com.raytheon.uf.edex.registry.ebxml.dao.RegistryObjectDao; /** * Purges {@link DataSetMetaData} instances that are no longer accessible on @@ -66,50 +62,22 @@ import com.raytheon.uf.edex.datadelivery.harvester.crawler.CrawlLauncher; * Oct 05, 2012 1241 djohnson Replace RegistryManager calls with registry handler calls. * Dec 12, 2012 1410 dhladky multi provider configurations. * Sept 30, 2013 1797 dhladky Generics + * Apr 12,2014 3012 dhladky Purge never worked, fixed to make work. * * * * @author djohnson * @version 1.0 */ -class DataSetMetaDataPurgeTaskImpl implements IDataSetMetaDataPurgeTask, - IDataSetMetaDataVisitor { - - /** - * Maintains state for an instance of the purge task. - */ - private static class State { - /** - * This boolean flag is used to mark whether or not the DataSetMetaData - * group should be continued, it will be set to false when the purge has - * found a DataSetMetaData instance that should NOT be purged - */ - private boolean continueWithDataSet = true; - - /** - * The harvester configurations instance at the time the purge started. - */ - private List harvesterConfigs; - } +public class DataSetMetaDataPurgeTaskImpl implements IDataSetMetaDataPurgeTask { private static final IUFStatusHandler statusHandler = UFStatus .getHandler(DataSetMetaDataPurgeTaskImpl.class); - private static final IOpenDapGriddedPurge GRIDDED_OPENDAP = new OpenDapGriddedPurgeImpl(); - - /** - * This is the unique identifying key for this metadata's dataset in the - * map. - * - * @param metaData - * the metaDat - * @return the key - */ - @VisibleForTesting - static String getDatasetMetaDataMapKey(DataSetMetaData metaData) { - return metaData.getDataSetName() + metaData.getProviderName(); - } + /** Data access object for registry objects */ + private RegistryObjectDao rdo; + /** * Purges a {@link DataSetMetaData} instance. * @@ -133,83 +101,40 @@ class DataSetMetaDataPurgeTaskImpl implements IDataSetMetaDataPurgeTask, } } - private final IOpenDapGriddedPurge openDapGriddedPurge; - - // Used to maintain state on a per-thread basis, in case two purges somehow - // overrun each other - private final ThreadLocal threadState = new ThreadLocal(); - /** * Default Constructor. */ - DataSetMetaDataPurgeTaskImpl() { - this(GRIDDED_OPENDAP); + public DataSetMetaDataPurgeTaskImpl(RegistryObjectDao rdo) { + this.rdo = rdo; } - + /** - * Constructor accepting specific purge strategies. - * - * @param openDapGriddedPurge - * openDapGriddedPurge - * - */ - @VisibleForTesting - DataSetMetaDataPurgeTaskImpl(IOpenDapGriddedPurge openDapGriddedPurge) { - this.openDapGriddedPurge = openDapGriddedPurge; - } - - /** - * Clears the state for a running instance. - */ - private void clearState() { - threadState.set(null); - } - - /** - * Returns all {@link DataSetMetaData} instances that are to be checked for - * validity. - * - * @return the {@link DataSetMetaData} instances - */ - @SuppressWarnings("rawtypes") - @VisibleForTesting - List getDataSetMetaDatas() { - try { - return DataDeliveryHandlers.getDataSetMetaDataHandler().getAll(); - } catch (RegistryHandlerException e) { - statusHandler.handle(Priority.PROBLEM, - "Unable to retrieve DataSetMetaData instances!", e); - return Collections.emptyList(); - } - } - - /** - * Creates a map from the DataSetMetaData key as defined by - * {@link #getDatasetMetaDataMapKey(DataSetMetaData)} to the - * {@link SortedSet} of instances. + * Gets the entire list of DSMD ids from the registry. * * @return the map */ @VisibleForTesting - Multimap> getDataSetNameKeyedInstanceMap() { - Multimap> map = TreeMultimap.create( - Ordering. natural(), DataSetMetaData.DATE_COMPARATOR); - - for (DataSetMetaData metaData : getDataSetMetaDatas()) { - String key = getDatasetMetaDataMapKey(metaData); - map.put(key, metaData); + List getDataSetMetaDataIds() { + ArrayList ids = null; + try { + // Gets the list of all available lids for current DataSetMetaData objects + ids = (ArrayList) rdo.getRegistryObjectIdsOfType(DataDeliveryRegistryObjectTypes.DATASETMETADATA); + } catch (Exception e) { + statusHandler.handle(Priority.PROBLEM, + "Unable to retrieve DataSetMetaData ids!", e); + return Collections.emptyList(); } - return map; + return ids; } /** - * Returns the HarvesterConfig Array from localization. + * Returns the Retention times by Provider name. * * @return the {@link HarvesterConfig} */ @VisibleForTesting - List getHarvesterConfigs() { + static Map getHarvesterConfigs() { // first get the Localization directory and find all harvester // configs @@ -220,8 +145,7 @@ class DataSetMetaDataPurgeTaskImpl implements IDataSetMetaDataPurgeTask, HarvesterConfig hc = null; try { - hc = SerializationUtil.jaxbUnmarshalFromXmlFile( - HarvesterConfig.class, lf.getFile()); + hc = HarvesterConfigurationManager.getHarvesterFile(lf.getFile()); } catch (Exception se) { statusHandler.handle(Priority.PROBLEM, se.getLocalizedMessage(), se); @@ -238,102 +162,78 @@ class DataSetMetaDataPurgeTaskImpl implements IDataSetMetaDataPurgeTask, } } } + + Map configMap = null; - return configs; - } - - /** - * This method consolidates the logic of applying a purge strategy for a - * specific data type and service (e.g. OpenDAP for Gridded data) on a - * specific {@link DataSetMetaData} of that type. The generics ensure strict - * adherence to the data type mappings. - * - * @param - * the type that extends DataSetMetaData - * @param metaData - * the metadata instance - * @param purge - * the purge strategy - */ - private > void handleVisit(T metaData, - IServiceDataSetMetaDataPurge purge) { - State state = threadState.get(); - List harvesterConfigs = state.harvesterConfigs; - - for (HarvesterConfig config : harvesterConfigs) { - - if (purge.isTimeToPurge(metaData, config)) { - purgeMetaData(metaData); - } else { - // Found a non-purgeable metadata instance - state.continueWithDataSet = false; - if (statusHandler.isPriorityEnabled(Priority.DEBUG)) { - final String id = RegistryUtil - .getRegistryObjectKey(metaData); - statusHandler - .debug(String - .format("Provider: " - + config.getProvider().getName() - + " : DataSetMetaData with id [%s] does not require purging.", - id)); - } + if (!configs.isEmpty()) { + configMap = new HashMap( + configs.size()); + for (HarvesterConfig config : configs) { + configMap.put(config.getProvider().getName(), config.getRetention()); } + } else { + return Collections.emptyMap(); } + + return configMap; } - - /** - * Initializes the state for a running instance. - * - * @return the State instance - */ - @VisibleForTesting - State initializeState() { - State state = new State(); - state.harvesterConfigs = getHarvesterConfigs(); - threadState.set(state); - - return state; - } - + /** * {@inheritDoc} */ @Override public void run() { + ITimer timer = TimeUtil.getTimer(); timer.start(); - Multimap> dataSetKeyedMap = getDataSetNameKeyedInstanceMap(); + List idList = getDataSetMetaDataIds(); + Map configMap = getHarvesterConfigs(); + int deletes = 0; + + for (String id : idList) { - try { - State state = initializeState(); + try { - for (String key : dataSetKeyedMap.keySet()) { - Collection> metaDatas = dataSetKeyedMap - .get(key); - Iterator> iter = metaDatas.iterator(); + DataSetMetaData metaData = DataDeliveryHandlers + .getDataSetMetaDataHandler().getById(id); + Integer retention = Integer.valueOf(configMap.get(metaData.getProviderName())); - state.continueWithDataSet = true; - while (iter.hasNext() && state.continueWithDataSet) { - DataSetMetaData metaData = iter.next(); - metaData.accept(this); + if (retention != null) { + + if (retention == -1) { + // no purging for this DSMD type + continue; + } else { + // retention is in days + retention = retention * (-1); + // we are subtracting from current + Calendar thresholdTime = TimeUtil.newGmtCalendar(); + thresholdTime.add(Calendar.DAY_OF_YEAR, retention); + + if (thresholdTime.getTimeInMillis() >= metaData + .getDate().getTime()) { + purgeMetaData(metaData); + deletes++; + } + } + + } else { + statusHandler + .warn("No retention time set for this DataSetMetaData provider! " + + id + + "Provider: " + + metaData.getProviderName()); } + + } catch (Exception e) { + statusHandler.error("DataSetMetaData purge failed! " + id, e); } - } finally { - clearState(); } timer.stop(); statusHandler.info(String.format( "DataSetMetaData purge completed in %s ms.", - timer.getElapsedTime())); - } - - /** - * {@inheritDoc} - */ - @Override - public void visit(OpenDapGriddedDataSetMetaData metaData) { - handleVisit(metaData, openDapGriddedPurge); + timer.getElapsedTime()+" deleted: "+deletes)); } } diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/IDataSetMetaDataPurgeTask.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/IDataSetMetaDataPurgeTask.java index ef2660feaa..b155dd9667 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/IDataSetMetaDataPurgeTask.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/IDataSetMetaDataPurgeTask.java @@ -16,7 +16,9 @@ * * See the AWIPS II Master Rights File ("Master Rights File.pdf") for * further licensing information. - **/ +*/ + + package com.raytheon.uf.edex.datadelivery.harvester.purge; import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData; diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/IOpenDapGriddedPurge.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/IOpenDapGriddedPurge.java deleted file mode 100644 index fbb45e11eb..0000000000 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/IOpenDapGriddedPurge.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * This software was developed and / or modified by Raytheon Company, - * pursuant to Contract DG133W-05-CQ-1067 with the US Government. - * - * U.S. EXPORT CONTROLLED TECHNICAL DATA - * This software product contains export-restricted data whose - * export/transfer/disclosure is restricted by U.S. law. Dissemination - * to non-U.S. persons whether in the United States or abroad requires - * an export license or other authorization. - * - * Contractor Name: Raytheon Company - * Contractor Address: 6825 Pine Street, Suite 340 - * Mail Stop B8 - * Omaha, NE 68106 - * 402.291.0100 - * - * See the AWIPS II Master Rights File ("Master Rights File.pdf") for - * further licensing information. - **/ -package com.raytheon.uf.edex.datadelivery.harvester.purge; - -import com.raytheon.uf.common.datadelivery.registry.OpenDapGriddedDataSetMetaData; - -/** - * Marker interface for the gridded OpenDAP purge strategy. Intentionally - * package-level as it's an internal implementation detail that should only - * reside and be accessible within this package. - * - * 
- * 
- * SOFTWARE HISTORY
- * 
- * Date         Ticket#    Engineer    Description
- * ------------ ---------- ----------- --------------------------
- * Sep 4, 2012  1102       djohnson     Initial creation
- * 
- *
- * - * @author djohnson - * @version 1.0 - */ -interface IOpenDapGriddedPurge extends - IServiceDataSetMetaDataPurge { -} diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/IServiceDataSetMetaDataPurge.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/IServiceDataSetMetaDataPurge.java deleted file mode 100644 index 7e8aef5e52..0000000000 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/IServiceDataSetMetaDataPurge.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * This software was developed and / or modified by Raytheon Company, - * pursuant to Contract DG133W-05-CQ-1067 with the US Government. - * - * U.S. EXPORT CONTROLLED TECHNICAL DATA - * This software product contains export-restricted data whose - * export/transfer/disclosure is restricted by U.S. law. Dissemination - * to non-U.S. persons whether in the United States or abroad requires - * an export license or other authorization. - * - * Contractor Name: Raytheon Company - * Contractor Address: 6825 Pine Street, Suite 340 - * Mail Stop B8 - * Omaha, NE 68106 - * 402.291.0100 - * - * See the AWIPS II Master Rights File ("Master Rights File.pdf") for - * further licensing information. - **/ -package com.raytheon.uf.edex.datadelivery.harvester.purge; - -import com.raytheon.uf.common.datadelivery.harvester.HarvesterConfig; -import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData; - -/** - * Defines a purge for a {@link DataSetMetaData} type. Intentionally - * package-level as it's an internal implementation detail that should only - * reside and be accessible within this package. - * - * 
- * 
- * SOFTWARE HISTORY
- * 
- * Date         Ticket#    Engineer    Description
- * ------------ ---------- ----------- --------------------------
- * Sep 4, 2012  1102      djohnson     Initial creation
- * 
- *
- * - * @author djohnson - * @version 1.0 - */ - -interface IServiceDataSetMetaDataPurge { - - /** - * Perform the purge of a {@link DataSetMetaData} instance if required. - * - * @param metaData - * the metaData - * @param harvesterConfig TODO - * @return - */ - boolean isTimeToPurge(T metaData, HarvesterConfig harvesterConfig); -} diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/OpenDapGriddedPurgeImpl.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/OpenDapGriddedPurgeImpl.java deleted file mode 100644 index ac5f9cd53a..0000000000 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/OpenDapGriddedPurgeImpl.java +++ /dev/null @@ -1,185 +0,0 @@ -/** - * This software was developed and / or modified by Raytheon Company, - * pursuant to Contract DG133W-05-CQ-1067 with the US Government. - * - * U.S. EXPORT CONTROLLED TECHNICAL DATA - * This software product contains export-restricted data whose - * export/transfer/disclosure is restricted by U.S. law. Dissemination - * to non-U.S. persons whether in the United States or abroad requires - * an export license or other authorization. - * - * Contractor Name: Raytheon Company - * Contractor Address: 6825 Pine Street, Suite 340 - * Mail Stop B8 - * Omaha, NE 68106 - * 402.291.0100 - * - * See the AWIPS II Master Rights File ("Master Rights File.pdf") for - * further licensing information. - **/ -package com.raytheon.uf.edex.datadelivery.harvester.purge; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.http.HttpHost; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.conn.params.ConnRoutePNames; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.util.EntityUtils; - -import com.google.common.annotations.VisibleForTesting; -import com.raytheon.uf.common.comm.ProxyConfiguration; -import com.raytheon.uf.common.datadelivery.harvester.HarvesterConfig; -import com.raytheon.uf.common.datadelivery.registry.DataSetMetaData; -import com.raytheon.uf.common.datadelivery.registry.OpenDapGriddedDataSetMetaData; -import com.raytheon.uf.common.datadelivery.registry.Provider; -import com.raytheon.uf.common.status.IUFStatusHandler; -import com.raytheon.uf.common.status.UFStatus; -import com.raytheon.uf.common.status.UFStatus.Priority; -import com.raytheon.uf.edex.datadelivery.retrieval.util.ConnectionUtil; - -/** - * Purges {@link OpenDapGriddedDataSetMetaData} instances that are no longer - * retrievable. Intentionally package-private as it should not be part of the - * public API. - * - * 
- * 
- * SOFTWARE HISTORY
- * 
- * Date         Ticket#    Engineer    Description
- * ------------ ---------- ----------- --------------------------
- * Sep 4, 2012  1102      djohnson     Initial creation
- * 
- *
- * - * @author djohnson - * @version 1.0 - */ -class OpenDapGriddedPurgeImpl implements IOpenDapGriddedPurge { - - private static final IUFStatusHandler statusHandler = UFStatus - .getHandler(OpenDapGriddedPurgeImpl.class); - - /** - * Retrieves the error response pattern from the provider for the specified - * metadata instance. - * - * @param metaData - * the metadata - * @param harvesterConfig - * the harvester configuration - * @return the pattern or null if the applicable provider cannot be found - */ - private static Pattern getProviderErrorResponsePattern( - DataSetMetaData metaData, HarvesterConfig harvesterConfig) { - - Provider provider = harvesterConfig.getProvider(); - final String providerName = provider.getName(); - final String metadataProviderName = metaData.getProviderName(); - - if (statusHandler.isPriorityEnabled(Priority.DEBUG)) { - statusHandler.debug("Checking provider [" + providerName - + "] to match metadata provider name [" - + metadataProviderName + "]."); - } - - if (metadataProviderName.equals(providerName)) { - return Pattern.compile(provider.getErrorResponsePattern()); - } - - return null; - } - - /** - * Retrieves the HttpClient implementation to use. - * - * @return the {@link HttpClient} implementation - */ - @VisibleForTesting - HttpClient getHttpClient() { - HttpClient httpClient = new DefaultHttpClient(); - ProxyConfiguration proxyParameters = ConnectionUtil - .getProxyParameters(); - if (proxyParameters != null) { - HttpHost proxy = new HttpHost(proxyParameters.getHost(), - proxyParameters.getPort()); - httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, - proxy); - } - return httpClient; - } - - /** - * {@inheritDoc} - */ - @Override - public boolean isTimeToPurge(OpenDapGriddedDataSetMetaData metaData, - HarvesterConfig harvesterConfig) { - HttpGet request = new HttpGet(); - HttpResponse response = null; - try { - final String url = metaData.getUrl(); - request.setURI(new URI(url)); - - HttpClient httpClient = getHttpClient(); - - response = httpClient.execute(request); - int code = response.getStatusLine().getStatusCode(); - - if (statusHandler.isPriorityEnabled(Priority.DEBUG)) { - statusHandler.debug(String.format( - "Received status code [%s] from url [%s]", code, url)); - } - - if (code == HttpStatus.SC_OK) { - String entityContent = EntityUtils.toString(response - .getEntity()); - Pattern providerErrorResponsePattern = getProviderErrorResponsePattern( - metaData, harvesterConfig); - if (providerErrorResponsePattern != null) { - Matcher matcher = providerErrorResponsePattern - .matcher(entityContent); - if (matcher.find()) { - return true; - } - } else { - statusHandler - .warn(String - .format("Unable to find a configured provider by name [%s], removing obsolete DataSetMetaData.", - metaData.getProviderName())); - } - } - return false; - } catch (URISyntaxException e) { - statusHandler - .handle(Priority.PROBLEM, - "Unable to parse URL into a URI, purging metadata since it would otherwise remain unusable!", - e); - return true; - } catch (IOException e) { - statusHandler - .handle(Priority.PROBLEM, - "Unable to contact the host, not purging metadata since it may still be valid!", - e); - request.abort(); - return false; - } finally { - if (response != null) { - try { - EntityUtils.consume(response.getEntity()); - } catch (IOException e) { - statusHandler.handle(Priority.PROBLEM, - e.getLocalizedMessage(), e); - } - } - } - } -} diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.registry/src/com/raytheon/uf/edex/datadelivery/registry/federation/RegistryFederationManager.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.registry/src/com/raytheon/uf/edex/datadelivery/registry/federation/RegistryFederationManager.java index d1811a2ac3..7a4021af70 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.registry/src/com/raytheon/uf/edex/datadelivery/registry/federation/RegistryFederationManager.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.registry/src/com/raytheon/uf/edex/datadelivery/registry/federation/RegistryFederationManager.java @@ -95,6 +95,7 @@ import com.raytheon.uf.common.status.UFStatus.Priority; import com.raytheon.uf.common.time.util.TimeUtil; import com.raytheon.uf.common.util.CollectionUtil; import com.raytheon.uf.common.util.StringUtil; +import com.raytheon.uf.edex.core.EDEXUtil; import com.raytheon.uf.edex.datadelivery.registry.availability.FederatedRegistryMonitor; import com.raytheon.uf.edex.datadelivery.registry.dao.ReplicationEventDao; import com.raytheon.uf.edex.datadelivery.registry.replication.NotificationServers; @@ -103,6 +104,7 @@ import com.raytheon.uf.edex.registry.ebxml.dao.DbInit; import com.raytheon.uf.edex.registry.ebxml.dao.RegistryDao; import com.raytheon.uf.edex.registry.ebxml.dao.RegistryObjectDao; import com.raytheon.uf.edex.registry.ebxml.exception.EbxmlRegistryException; +import com.raytheon.uf.edex.registry.ebxml.exception.NoReplicationServersAvailableException; import com.raytheon.uf.edex.registry.ebxml.init.RegistryInitializedListener; import com.raytheon.uf.edex.registry.ebxml.services.query.QueryConstants; import com.raytheon.uf.edex.registry.ebxml.services.query.RegistryQueryUtil; @@ -154,6 +156,7 @@ import com.raytheon.uf.edex.registry.events.CreateAuditTrailEvent; * 2/13/2014 2769 bphillip Refactored registry sync. Created quartz tasks to monitor registry uptime as well as subscription integrity * Mar 31, 2014 2889 dhladky Added username for notification center tracking. * 4/11/2014 3011 bphillip Removed automatic registry sync check on startup + * 4/15/2014 3012 dhladky Merge fixes. * * * @author bphillip @@ -173,7 +176,7 @@ public class RegistryFederationManager implements IRegistryFederationManager, /** Query used for synchronizing registries */ private static final String SYNC_QUERY = "FROM RegistryObjectType obj where obj.id in (%s) order by obj.id asc"; - + /** Batch size for registry synchronization queries */ private static final int SYNC_BATCH_SIZE = Integer.parseInt(System .getProperty("ebxml-notification-batch-size")); @@ -202,14 +205,7 @@ public class RegistryFederationManager implements IRegistryFederationManager, */ private static final long MAX_DOWN_TIME_DURATION = TimeUtil.MILLIS_PER_HOUR * 48; - private static final String SYNC_WARNING_MSG = "Registry is out of sync with federation. Registry Synchronization required. Go to: [" - + RegistryUtil.LOCAL_REGISTRY_ADDRESS - + "/registry/federation/status.html] to synchronize."; - - private static volatile boolean SYNC_NECESSARY = false; - - public static AtomicBoolean SYNC_IN_PROGRESS = new AtomicBoolean( - false); + public static AtomicBoolean SYNC_IN_PROGRESS = new AtomicBoolean(false); /** Cutoff parameter for the query to get the expired events */ private static final String GET_EXPIRED_EVENTS_QUERY_CUTOFF_PARAMETER = "cutoff"; @@ -327,9 +323,7 @@ public class RegistryFederationManager implements IRegistryFederationManager, throw new EbxmlRegistryException( "Error joining federation!!"); } - if (!centralRegistry) { - checkDownTime(); - } + } catch (Exception e1) { throw new EbxmlRegistryException( "Error initializing RegistryReplicationManager", e1); @@ -351,29 +345,6 @@ public class RegistryFederationManager implements IRegistryFederationManager, initialized.set(true); } - /** - * Checks how long a registry has been down. If the registry has been down - * longer than the MAX_DOWN_TIME_DURATION, then a sync is necessary - * - * @see RegistryFederationManager.MAX_DOWN_TIME_DURATION - * @throws Exception - */ - private void checkDownTime() throws Exception { - long currentTime = TimeUtil.currentTimeMillis(); - long lastKnownUp = federatedRegistryMonitor.getLastKnownUptime(); - long downTime = currentTime - lastKnownUp; - statusHandler.info("Registry has been down since: " - + new Date(currentTime - downTime)); - /* - * The registry has been down for ~2 days, this requires a - * synchronization of the data from the federation - */ - if (currentTime - lastKnownUp > MAX_DOWN_TIME_DURATION) { - SYNC_NECESSARY = true; - sendSyncMessage(); - } - } - public boolean joinFederation() { try { final List objects = new ArrayList( @@ -510,6 +481,62 @@ public class RegistryFederationManager implements IRegistryFederationManager, return true; } + /** + * Checks how long a registry has been down. If the registry has been down + * for over 2 days, the registry is synchronized with one of the federation + * members + * + * @throws Exception + */ + private void synchronize() throws EbxmlRegistryException { + + monitorHandler.warn("Synchronizing registry with federation..."); + RegistryType registryToSyncFrom = null; + for (String remoteRegistryId : servers.getRegistryReplicationServers()) { + statusHandler.info("Checking availability of [" + remoteRegistryId + + "]..."); + RegistryType remoteRegistry = null; + try { + remoteRegistry = dataDeliveryRestClient.getRegistryObject( + ncfAddress, remoteRegistryId + + FederationProperties.REGISTRY_SUFFIX); + } catch (Exception e) { + throw new EbxmlRegistryException( + "Error getting remote registry object!", e); + } + if (remoteRegistry == null) { + statusHandler + .warn("Registry at [" + + remoteRegistryId + + "] not found in federation. Unable to use as synchronization source."); + } else if (dataDeliveryRestClient + .isRegistryAvailable(remoteRegistry.getBaseURL())) { + registryToSyncFrom = remoteRegistry; + break; + } else { + statusHandler + .info("Registry at [" + + remoteRegistryId + + "] is not available. Unable to use as synchronization source."); + } + } + + // No available registry was found! + if (registryToSyncFrom == null) { + throw new NoReplicationServersAvailableException( + "No available registries found! Registry data will not be synchronized with the federation!"); + } else { + try { + synchronizeWithRegistry(registryToSyncFrom.getId()); + } catch (Exception e) { + throw new EbxmlRegistryException( + "Error synchronizing with registry [" + + registryToSyncFrom.getId() + "]", e); + } + } + + } + /** * Synchronizes this registry's data with the registry at the specified URL * @@ -524,6 +551,7 @@ public class RegistryFederationManager implements IRegistryFederationManager, @Path("synchronizeWithRegistry/{registryId}") public void synchronizeWithRegistry(@PathParam("registryId") String registryId) throws Exception { + if (SYNC_IN_PROGRESS.compareAndSet(false, true)) { try { monitorHandler.handle(Priority.WARN, @@ -552,7 +580,6 @@ public class RegistryFederationManager implements IRegistryFederationManager, for (final String objectType : replicatedObjectTypes) { syncObjectType(objectType, remoteRegistryUrl); } - SYNC_NECESSARY = false; federatedRegistryMonitor.updateTime(); StringBuilder syncMsg = new StringBuilder(); @@ -566,6 +593,10 @@ public class RegistryFederationManager implements IRegistryFederationManager, } finally { SYNC_IN_PROGRESS.set(false); } + + } else { + statusHandler.info("Registry sync already in progress."); + } } @@ -612,8 +643,9 @@ public class RegistryFederationManager implements IRegistryFederationManager, int remainder = remoteIds.size() % SYNC_BATCH_SIZE; for (int currentBatch = 0; currentBatch < batches; currentBatch++) { - statusHandler.info("Processing batch " + (currentBatch + 1) - + "/" + batches); + + statusHandler.info("Processing batch "+(currentBatch+1)+"/"+batches); + persistBatch(objectType, remoteRegistryUrl, remoteIds.subList( currentBatch * SYNC_BATCH_SIZE, (currentBatch + 1) * SYNC_BATCH_SIZE)); @@ -670,12 +702,6 @@ public class RegistryFederationManager implements IRegistryFederationManager, } } - private void sendSyncMessage() { - if (!SYNC_IN_PROGRESS.get()) { - statusHandler.warn(SYNC_WARNING_MSG); - monitorHandler.handle(Priority.WARN, SYNC_WARNING_MSG); - } - } @GET @Path("isFederated") @@ -1096,15 +1122,35 @@ public class RegistryFederationManager implements IRegistryFederationManager, * Updates the record in the registry that keeps track of if this registry * has been up. This method is called every minute via a quartz cron * configured in Camel + * + * @throws EbxmlRegistryException */ @Transactional - public void updateUpTime() { - if (initialized.get()) { - if (SYNC_NECESSARY) { - if (!SYNC_IN_PROGRESS.get() - && TimeUtil.newGmtCalendar().get(Calendar.MINUTE) % 15 == 0) { - sendSyncMessage(); + public void updateUpTime() throws EbxmlRegistryException { + if (initialized.get() && EDEXUtil.isRunning()) { + long currentTime = TimeUtil.currentTimeMillis(); + long lastKnownUp = federatedRegistryMonitor.getLastKnownUptime(); + long downTime = currentTime - lastKnownUp; + if (currentTime - lastKnownUp > MAX_DOWN_TIME_DURATION + && !centralRegistry) { + if (!SYNC_IN_PROGRESS.get()) { + statusHandler.info("Registry has been down since: " + + new Date(currentTime - downTime)); + statusHandler + .warn("Registry is out of sync with federation. Attempting to automatically synchronize"); + try { + synchronize(); + monitorHandler + .info("Registry successfully synchronized!"); + } catch (EbxmlRegistryException e) { + monitorHandler + .error("Error synchronizing registry!", e); + throw e; + } + } + + } else { federatedRegistryMonitor.updateTime(); } diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.registry/src/com/raytheon/uf/edex/datadelivery/registry/web/RegistryDataAccessService.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.registry/src/com/raytheon/uf/edex/datadelivery/registry/web/RegistryDataAccessService.java index 5b0af16e0d..398caabab1 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.registry/src/com/raytheon/uf/edex/datadelivery/registry/web/RegistryDataAccessService.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.registry/src/com/raytheon/uf/edex/datadelivery/registry/web/RegistryDataAccessService.java @@ -80,6 +80,7 @@ import com.raytheon.uf.edex.registry.ebxml.services.query.RegistryQueryUtil; * 10/8/2013 1682 bphillip Added query queries * 11/7/2013 1678 bphillip Added getCustomQueries method * Mar 31, 2014 2889 dhladky Added username for notification center tracking. + * Apr 12,2014 3012 dhladky Purge never worked, fixed to make work. * * * @author bphillip @@ -343,9 +344,9 @@ public class RegistryDataAccessService implements IRegistryDataAccessService { try { Object subObj = subscriptionJaxbManager .unmarshalFromXml(subscriptionXML); - EDEXUtil.getMessageProducer().sendSync("scheduleSubscription", - subObj); lcm.submitObjects(submitRequest); + EDEXUtil.getMessageProducer().sendSync("scheduleSubscription", + new Object[] {subObj, false}); subscriptionFile.delete(); response.append( "Subscription successfully restored from file [") diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/util/RetrievalGeneratorUtilities.java b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/util/RetrievalGeneratorUtilities.java index cac0b9dc73..1ec242036b 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/util/RetrievalGeneratorUtilities.java +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/src/com/raytheon/uf/edex/datadelivery/retrieval/util/RetrievalGeneratorUtilities.java @@ -53,6 +53,7 @@ import com.raytheon.uf.edex.datadelivery.retrieval.handlers.SubscriptionRetrieva * Dec 10, 2012 1259 bsteffen Switch Data Delivery from LatLon to referenced envelopes. * Dec 11, 2013 2625 mpduff Remove creation of DataURI. * Jan 30, 2014 2686 dhladky refactor of retrieval. + * Apr 09, 2014 3012 dhladky Added error message. * Apr 21, 2014 2060 njensen Remove dependency on grid dataURI column * * @@ -102,7 +103,7 @@ public class RetrievalGeneratorUtilities { } } catch (Exception e) { statusHandler.handle(Priority.PROBLEM, - e.getLocalizedMessage(), e); + "Can't determine if URI is a duplicate", e); } } } diff --git a/edexOsgi/com.raytheon.uf.edex.ogc.registry/src/com/raytheon/uf/edex/ogc/registry/RegistryCollectorAddon.java b/edexOsgi/com.raytheon.uf.edex.ogc.registry/src/com/raytheon/uf/edex/ogc/registry/RegistryCollectorAddon.java index d441a5a077..841283d8e3 100644 --- a/edexOsgi/com.raytheon.uf.edex.ogc.registry/src/com/raytheon/uf/edex/ogc/registry/RegistryCollectorAddon.java +++ b/edexOsgi/com.raytheon.uf.edex.ogc.registry/src/com/raytheon/uf/edex/ogc/registry/RegistryCollectorAddon.java @@ -63,6 +63,7 @@ import com.raytheon.uf.edex.ogc.common.db.SimpleLayer; * Aug 08, 2013 dhladky Made operational * Jan 13, 2014 #2679 dhladky multiple layers * Mar 31, 2014 2889 dhladky Added username for notification center tracking. + * Apr 14, 2014 3012 dhladky Cleaned up. * * * @@ -120,7 +121,7 @@ public abstract class RegistryCollectorAddon * @@ -454,13 +455,15 @@ public abstract class WfsRegistryCollectorAddonMADIS Test LatLon Coverage + + -1 HHddMMMyyyy diff --git a/edexOsgi/com.raytheon.uf.edex.registry.ebxml/src/com/raytheon/uf/edex/registry/ebxml/dao/SlotTypeDao.java b/edexOsgi/com.raytheon.uf.edex.registry.ebxml/src/com/raytheon/uf/edex/registry/ebxml/dao/SlotTypeDao.java index 6995bd5db0..ebe6ba4096 100644 --- a/edexOsgi/com.raytheon.uf.edex.registry.ebxml/src/com/raytheon/uf/edex/registry/ebxml/dao/SlotTypeDao.java +++ b/edexOsgi/com.raytheon.uf.edex.registry.ebxml/src/com/raytheon/uf/edex/registry/ebxml/dao/SlotTypeDao.java @@ -19,13 +19,16 @@ **/ package com.raytheon.uf.edex.registry.ebxml.dao; +import java.util.Collection; import java.util.List; import oasis.names.tc.ebxml.regrep.xsd.rim.v4.SlotType; import org.hibernate.SQLQuery; +import org.hibernate.criterion.Property; import com.raytheon.uf.edex.database.dao.SessionManagedDao; +import com.raytheon.uf.edex.registry.ebxml.services.query.QueryConstants; /** * @@ -84,4 +87,10 @@ public class SlotTypeDao extends SessionManagedDao { this.template.delete(slot); } } + + public void deleteBySlotId(Collection ids){ + template.deleteAll(createCriteria().add( + Property.forName(QueryConstants.ID).in(ids)) + .list()); + } } diff --git a/edexOsgi/com.raytheon.uf.edex.registry.ebxml/src/com/raytheon/uf/edex/registry/ebxml/services/RegistryGarbageCollector.java b/edexOsgi/com.raytheon.uf.edex.registry.ebxml/src/com/raytheon/uf/edex/registry/ebxml/services/RegistryGarbageCollector.java index 17cd6c76ab..39fe07f2f0 100644 --- a/edexOsgi/com.raytheon.uf.edex.registry.ebxml/src/com/raytheon/uf/edex/registry/ebxml/services/RegistryGarbageCollector.java +++ b/edexOsgi/com.raytheon.uf.edex.registry.ebxml/src/com/raytheon/uf/edex/registry/ebxml/services/RegistryGarbageCollector.java @@ -55,6 +55,7 @@ import com.raytheon.uf.edex.registry.events.DeleteSlotEvent; * 2/4/2014 2769 bphillip Removed flush and clear call * 2/13/2014 2769 bphillip Refactored to no longer use executor threads * 4/11/2014 3011 bphillip Added slot purging via event bus notifications + * 4/17/2014 3011 bphillip Delete slot events now contain strings * * * @author bphillip @@ -139,15 +140,17 @@ public class RegistryGarbageCollector { @Subscribe public void deleteOrphanedSlot(DeleteSlotEvent slotEvent) { + if (!CollectionUtil.isNullOrEmpty(slotEvent.getSlotsToDelete())) { long start = TimeUtil.currentTimeMillis(); statusHandler.info("Deleting " + slotEvent.getSlotsToDelete().size() + " slots..."); - slotDao.deleteAll(slotEvent.getSlotsToDelete()); + slotDao.deleteBySlotId(slotEvent.getSlotsToDelete()); statusHandler.info("Deleted " + slotEvent.getSlotsToDelete().size() + " slots in " + (TimeUtil.currentTimeMillis() - start) + " ms"); } + } } diff --git a/edexOsgi/com.raytheon.uf.edex.registry.ebxml/src/com/raytheon/uf/edex/registry/ebxml/services/lifecycle/LifecycleManagerImpl.java b/edexOsgi/com.raytheon.uf.edex.registry.ebxml/src/com/raytheon/uf/edex/registry/ebxml/services/lifecycle/LifecycleManagerImpl.java index 73002427f1..aec58708a4 100644 --- a/edexOsgi/com.raytheon.uf.edex.registry.ebxml/src/com/raytheon/uf/edex/registry/ebxml/services/lifecycle/LifecycleManagerImpl.java +++ b/edexOsgi/com.raytheon.uf.edex.registry.ebxml/src/com/raytheon/uf/edex/registry/ebxml/services/lifecycle/LifecycleManagerImpl.java @@ -112,6 +112,7 @@ import com.raytheon.uf.edex.registry.events.DeleteSlotEvent; * 2/19/2014 2769 bphillip Added current time to audit trail events * Mar 31, 2014 2889 dhladky Added username for notification center tracking. * 4/11/2014 3011 bphillip Modified merge behavior + * 4/17/2014 3011 bphillip Delete slot events now contain strings * * * @@ -301,6 +302,8 @@ public class LifecycleManagerImpl implements LifecycleManager { event.setObjectType(objectType); EventBus.publish(event); } + DeleteSlotEvent deleteEvent = new DeleteSlotEvent(obj.getSlot()); + EventBus.publish(deleteEvent); EventBus.publish(new RegistryStatisticsEvent(obj.getObjectType(), obj.getStatus(), obj.getOwner(), avTimePerRecord)); } @@ -757,9 +760,8 @@ public class LifecycleManagerImpl implements LifecycleManager { private void mergeObjects(RegistryObjectType newObject, RegistryObjectType existingObject) { + DeleteSlotEvent deleteSlotEvent = new DeleteSlotEvent(existingObject.getSlot()); registryObjectDao.merge(newObject, existingObject); - DeleteSlotEvent deleteSlotEvent = new DeleteSlotEvent( - existingObject.getSlot()); EventBus.publish(deleteSlotEvent); } diff --git a/edexOsgi/com.raytheon.uf.edex.registry.ebxml/src/com/raytheon/uf/edex/registry/events/DeleteSlotEvent.java b/edexOsgi/com.raytheon.uf.edex.registry.ebxml/src/com/raytheon/uf/edex/registry/events/DeleteSlotEvent.java index 11868fc1fa..fa9e94a9ff 100644 --- a/edexOsgi/com.raytheon.uf.edex.registry.ebxml/src/com/raytheon/uf/edex/registry/events/DeleteSlotEvent.java +++ b/edexOsgi/com.raytheon.uf.edex.registry.ebxml/src/com/raytheon/uf/edex/registry/events/DeleteSlotEvent.java @@ -19,11 +19,13 @@ **/ package com.raytheon.uf.edex.registry.events; +import java.util.ArrayList; import java.util.List; import oasis.names.tc.ebxml.regrep.xsd.rim.v4.SlotType; import com.raytheon.uf.common.event.Event; +import com.raytheon.uf.common.util.CollectionUtil; /** * Event containing slots to be deleted by the registry garbage collector @@ -35,6 +37,7 @@ import com.raytheon.uf.common.event.Event; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * 4/11/2014 3011 bphillip Initial Coding + * 4/17/2014 3011 bphillip Delete slot events now contain strings * * * @author bphillip @@ -42,26 +45,31 @@ import com.raytheon.uf.common.event.Event; */ public class DeleteSlotEvent extends Event { - private static final long serialVersionUID = -2818002679753482984L; - - private List slotsToDelete; - - public DeleteSlotEvent(){ - super(); - } - - public DeleteSlotEvent(List slotsToDelete){ - this.slotsToDelete = slotsToDelete; - } + private static final long serialVersionUID = -2818002679753482984L; - public List getSlotsToDelete() { - return slotsToDelete; - } + private List slotsToDelete;; - public void setSlotsToDelete(List slotsToDelete) { - this.slotsToDelete = slotsToDelete; - } - - + public DeleteSlotEvent() { + super(); + } + + public DeleteSlotEvent(List slotsToDelete) { + if (CollectionUtil.isNullOrEmpty(slotsToDelete)) { + slotsToDelete = new ArrayList(); + } else { + this.slotsToDelete = new ArrayList(slotsToDelete.size()); + for (SlotType slot : slotsToDelete) { + this.slotsToDelete.add(slot.getId()); + } + } + } + + public List getSlotsToDelete() { + return slotsToDelete; + } + + public void setSlotsToDelete(List slotsToDelete) { + this.slotsToDelete = slotsToDelete; + } }