diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/res/spring/gfe-common.xml b/edexOsgi/com.raytheon.edex.plugin.gfe/res/spring/gfe-common.xml index 3fc55ea83a..e9c6931aee 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/res/spring/gfe-common.xml +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/res/spring/gfe-common.xml @@ -31,27 +31,10 @@ depends-on="commonTimeRegistered, gfeRegistered"> - - - - - - - - - - - - java.lang.Throwable - - - - - diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/GridParmManager.java b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/GridParmManager.java index c0f39b727a..ef2b251a95 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/GridParmManager.java +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/GridParmManager.java @@ -129,6 +129,10 @@ import com.raytheon.uf.edex.database.purge.PurgeLogger; * 10/07/2014 #3684 randerso Restructured IFPServer start up. * Reordered handling of DbInvChangeNotification * Don't process GFENotifications sent by this JVM + * Prevent createDB from creating databases that are immediately purged + * Send DBInvChangeNotifications at site activation so new D2D data + * ingested while deactivated gets recognized + * * * * @author bphillip @@ -890,6 +894,10 @@ public class GridParmManager { if (notify) { createDbNotification(Arrays.asList(dbId), purged); } + // always notify on purges + else if (!purged.isEmpty()) { + createDbNotification(null, purged); + } } } @@ -913,7 +921,10 @@ public class GridParmManager { GridDatabase db = this.getDatabase(dbId); if (db != null) { + sr.setPayload(db); return sr; // database already exists + } else { + sr.addMessage("Unable to create database: " + dbId); } // is it a singleton database? @@ -1051,42 +1062,64 @@ public class GridParmManager { return sr; } - private ServerResponse createDB(DatabaseID id) { + private ServerResponse createDB(DatabaseID dbId) { // TODO: consider merging this into getDatabase() ServerResponse status = new ServerResponse(); - if (!id.isValid() || !id.getFormat().equals(DataType.GRID)) { + if (!dbId.isValid() || !dbId.getFormat().equals(DataType.GRID)) { status.addMessage("Database id " - + id + + dbId + " is not valid, or is not a grid-type. Cannot create database."); return status; } - // create the grid database + // get the dbConfig IFPGridDatabase db = null; - GridDbConfig dbConfig = this.config.gridDbConfig(id); + GridDbConfig dbConfig = this.config.gridDbConfig(dbId); if (dbConfig == null) { status.addMessage("Unable to obtain GridDbConfig information for creation" - + " in createDB() for " + id); + + " in createDB() for " + dbId); } else { + // don't create if this version would be immediately purgeable + List existing = new ArrayList(); + for (DatabaseID id : this.dbMap.keySet()) { + if (id.sameModel(dbId)) { + existing.add(id); + } + } + + if (!existing.contains(dbId)) { + existing.add(dbId); + } + + // sort by model time (most recent first) + Collections.sort(existing); + + int desiredVersions = this.config.desiredDbVersions(dbId); + if (existing.indexOf(dbId) >= desiredVersions) { + status.addMessage("Unable to create " + dbId + + " as it would exceed the desired number of versions"); + return status; + } + // attempt to create the GridDatabase - db = new IFPGridDatabase(id, dbConfig); + db = new IFPGridDatabase(dbId, dbConfig); if (db.databaseIsValid()) { // get databaseID object from database - DatabaseID dbId = db.getDbId(); + DatabaseID id = db.getDbId(); - if (dbId.getRemovedDate() != null) { + if (id.getRemovedDate() != null) { // mark database as not removed try { GFEDao gfeDao = new GFEDao(); - gfeDao.setDatabaseRemovedDate(dbId, null); - statusHandler.info("Database " + dbId + " restored"); + gfeDao.setDatabaseRemovedDate(id, null); + statusHandler.info("Database " + id + " restored"); } catch (Exception e) { statusHandler.handle(Priority.PROBLEM, - "Unable to mark database restored: " + dbId, e); + "Unable to mark database restored: " + id, e); } } } else { - status.addMessage("Database " + id + " is not valid."); + status.addMessage("Database " + dbId + " is not valid."); db = null; } } @@ -1095,12 +1128,13 @@ public class GridParmManager { // mark database as removed try { GFEDao gfeDao = new GFEDao(); - gfeDao.setDatabaseRemovedDate(id, new Date()); - statusHandler.warn("Database " + id + " marked for removal in " + gfeDao.setDatabaseRemovedDate(dbId, new Date()); + statusHandler.warn("Database " + dbId + + " marked for removal in " + GFEDao.REMOVED_DB_PURGE_TIME + " days."); } catch (Exception e) { statusHandler.handle(Priority.PROBLEM, - "Unable to mark database removed: " + id, e); + "Unable to mark database removed: " + dbId, e); } } @@ -1142,8 +1176,12 @@ public class GridParmManager { } // create the databases (the list should now only contain GRID dbs) + List added = new ArrayList(inventory.size()); for (DatabaseID dbId : inventory) { - getDatabase(dbId, false); + GridDatabase db = getDatabase(dbId, false); + if (db != null) { + added.add(db.getDbId()); + } } NetCDFDatabaseManager.initializeNetCDFDatabases(config); @@ -1156,7 +1194,8 @@ public class GridParmManager { D2DSatDatabase satDb = new D2DSatDatabase(config); addDB(satDb); - initD2DDbs(); + added.addAll(initD2DDbs()); + createDbNotification(added, null); // only fire smartInits if queue is instantiated SmartInitQueue queue = SmartInitQueue.getQueue(); @@ -1197,7 +1236,8 @@ public class GridParmManager { } } - private void initD2DDbs() { + private List initD2DDbs() { + List added = new ArrayList(); for (String d2dModelName : config.getD2dModels()) { try { // get dbId to get desiredDbVersions (date doesn't matter) @@ -1209,13 +1249,17 @@ public class GridParmManager { d2dModelName, desiredVersions)) { dbId = D2DGridDatabase.getDbId(d2dModelName, refTime, config); - getDatabase(dbId, true); + GridDatabase db = getDatabase(dbId, false); + if (db != null) { + added.add(db.getDbId()); + } } } catch (Exception e) { statusHandler.error("Error initializing D2D model: " + d2dModelName, e); } } + return (added); } /** @@ -1236,6 +1280,9 @@ public class GridParmManager { } D2DGridDatabase db = (D2DGridDatabase) getDatabase(dbId, true); + if (db == null) { + continue; + } GridUpdateNotification gun = db.update(record); if (gun != null) { @@ -1416,7 +1463,8 @@ public class GridParmManager { private void createDbNotification(List additions, List deletions) { - if (!additions.isEmpty() || !deletions.isEmpty()) { + if ((additions != null && !additions.isEmpty()) + || (deletions != null && !deletions.isEmpty())) { DBInvChangeNotification notify = new DBInvChangeNotification( additions, deletions, siteID); SendNotifications.send(notify); @@ -1550,14 +1598,9 @@ public class GridParmManager { // process the inventory looking for "old" unwanted databases List purged = new ArrayList(); - String model = modelToPurge.getModelName(); - String site = modelToPurge.getSiteId(); - String type = modelToPurge.getDbType(); int count = 0; for (DatabaseID dbId : currentInv) { - // new series? - if (dbId.getSiteId().equals(site) && dbId.getDbType().equals(type) - && dbId.getModelName().equals(model)) { + if (dbId.sameModel(modelToPurge)) { // process the id and determine whether it should be purged count++; diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/notify/GfeNotificationFilter.java b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/notify/GfeNotificationFilter.java deleted file mode 100644 index 1896056e5e..0000000000 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/notify/GfeNotificationFilter.java +++ /dev/null @@ -1,53 +0,0 @@ -/** - * This software was developed and / or modified by Raytheon Company, - * pursuant to Contract DG133W-05-CQ-1067 with the US Government. - * - * U.S. EXPORT CONTROLLED TECHNICAL DATA - * This software product contains export-restricted data whose - * export/transfer/disclosure is restricted by U.S. law. Dissemination - * to non-U.S. persons whether in the United States or abroad requires - * an export license or other authorization. - * - * Contractor Name: Raytheon Company - * Contractor Address: 6825 Pine Street, Suite 340 - * Mail Stop B8 - * Omaha, NE 68106 - * 402.291.0100 - * - * See the AWIPS II Master Rights File ("Master Rights File.pdf") for - * further licensing information. - **/ -package com.raytheon.edex.plugin.gfe.server.notify; - -import java.util.List; - -import com.raytheon.uf.common.dataplugin.gfe.server.notify.GfeNotification; - -/** - * TODO Add Description - * - * 
- * SOFTWARE HISTORY
- * Date         Ticket#    Engineer    Description
- * ------------ ---------- ----------- --------------------------
- * Dec 3, 2008            njensen     Initial creation
- *
- * - * @author njensen - * @version 1.0 - */ - -public class GfeNotificationFilter { - - public boolean isGfeNotification(Object body) { - Object obj = body; - if (body instanceof List) { - List list = (List) body; - if (list.size() > 0) { - obj = list.get(0); - } - } - return obj instanceof GfeNotification; - } - -} diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/smartinit/InitClient.java b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/smartinit/InitClient.java index cff2bd5eb1..c8db1a71ad 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/smartinit/InitClient.java +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/smartinit/InitClient.java @@ -25,6 +25,7 @@ import com.raytheon.edex.plugin.gfe.config.IFPServerConfig; import com.raytheon.edex.plugin.gfe.config.IFPServerConfigManager; import com.raytheon.edex.plugin.gfe.reference.ReferenceMgr; import com.raytheon.edex.plugin.gfe.server.IFPServer; +import com.raytheon.edex.plugin.gfe.server.database.GridDatabase; import com.raytheon.edex.plugin.gfe.util.SendNotifications; import com.raytheon.uf.common.dataplugin.gfe.db.objects.DatabaseID; import com.raytheon.uf.common.dataplugin.gfe.db.objects.GridLocation; @@ -49,6 +50,7 @@ import com.raytheon.uf.common.status.UFStatus.Priority; * Jul 25, 2012 #957 dgilling Implement getEditAreaNames(). * Jun 13, 2013 #2044 randerso Refactored to use IFPServer * Nov 20, 2013 #2331 randerso Changed return type of getTopoData + * Oct 08, 2014 #3684 randerso Changed createDB to return status * * * @@ -98,13 +100,11 @@ public class InitClient { * * @param key */ - public void createDB(String key) { + public ServerResponse createDB(String key) { DatabaseID id = new DatabaseID(key); - ServerResponse sr = ifpServer.getGridParmMgr().createNewDb(id); - if (!sr.isOkay()) { - statusHandler.error("Error creating database " + id + ": " - + sr.message()); - } + ServerResponse sr = ifpServer.getGridParmMgr() + .createNewDb(id); + return sr; } /** diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/util/SendNotifications.java b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/util/SendNotifications.java index 76cce585ff..7c7686a421 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/util/SendNotifications.java +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/util/SendNotifications.java @@ -28,9 +28,8 @@ import org.apache.commons.logging.LogFactory; import com.raytheon.uf.common.dataplugin.gfe.server.message.ServerResponse; import com.raytheon.uf.common.dataplugin.gfe.server.notify.GfeNotification; -import com.raytheon.uf.common.dataplugin.gfe.util.GfeUtil; +import com.raytheon.uf.common.serialization.SerializationUtil; import com.raytheon.uf.edex.core.EDEXUtil; -import com.raytheon.uf.edex.core.EdexException; /** * Sends GFE notifications to the GFE notify JMS topic. @@ -43,6 +42,8 @@ import com.raytheon.uf.edex.core.EdexException; * 09/22/09 3058 rjpeter changed to utility. * 06/12/13 2099 dgilling Remove error when passed empty list of * notifications. + * 10/08/14 #3684 randerso Changed to send directly to JMS topic + * * * * @author bphillip @@ -66,11 +67,12 @@ public class SendNotifications { } try { - EDEXUtil.getMessageProducer().sendAsync(GfeUtil.NOTIFY, - notifications); + EDEXUtil.getMessageProducer().sendAsyncUri( + "jms-generic:topic:edex.alerts.gfe?timeToLive=60000", + SerializationUtil.transformToThrift(notifications)); // logger.info("Sending " + notifications.size() + " " // + notifications.get(0).getClass().getSimpleName()); - } catch (EdexException e) { + } catch (Exception e) { logger.error("Error sending gfe notification", e); sr.addMessage("Error sending gfe notification"); } diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/smartinit/Init.py b/edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/smartinit/Init.py index bb8c4fd59e..c8d5a40b37 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/smartinit/Init.py +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/smartinit/Init.py @@ -325,7 +325,9 @@ class Forecaster(GridUtilities): msg = "No databases for " + self._srcName LogStream.logProblem(msg) return - #sys.exit(1) + + if self.newdb() is None: + return self.__topo = self.getTopo() * .3048 srcdbkeys = self.srcdb().getKeys() @@ -547,6 +549,9 @@ class Forecaster(GridUtilities): start = time.time() self.__init() + if self.newdb() is None: + return + msgDest = "Destination database:" + self.newdb().getModelIdentifier() if validTime is not None: @@ -643,10 +648,15 @@ class Forecaster(GridUtilities): break if singletonNeeded: newdb = newdb[:-13] + '00000000_0000' + newdb = self.getDb(newdb) else: - client.createDB(newdb) - - newdb = self.getDb(newdb) + sr = client.createDB(newdb) + if sr.isOkay(): + newdb = self.getDb(newdb) + else: + msg = "Unable to create database for " + str(newdb) + LogStream.logProblem(msg) + newdb = None return srcdb, newdb diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/db/objects/DatabaseID.java b/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/db/objects/DatabaseID.java index 6f49908cf8..ec1cf3bf09 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/db/objects/DatabaseID.java +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/db/objects/DatabaseID.java @@ -67,6 +67,8 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeTypeAdap * 06/20/13 2127 rjpeter Removed unused bidirectional relationship. * 06/13/13 2044 randerso Code cleanup * 07/31/13 2057 randerso Added removedDate + * 10/08/14 #3684 randerso Added sameModel() + * * * * @author bphillip @@ -598,6 +600,23 @@ public class DatabaseID implements Comparable { return cal.getTime(); } + public boolean sameModel(DatabaseID other) { + + if (!this.siteId.equals(other.getSiteId())) { + return false; + } + + if (!this.format.equals(other.getFormat())) { + return false; + } + + if (!this.dbType.equals(other.getDbType())) { + return false; + } + + return this.modelName.equals(other.getModelName()); + } + /* * (non-Javadoc) * diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/util/GfeUtil.java b/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/util/GfeUtil.java index 2a42ebcc99..391da4b188 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/util/GfeUtil.java +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/util/GfeUtil.java @@ -67,6 +67,7 @@ import com.vividsolutions.jts.operation.polygonize.Polygonizer; * favor of new GridLocation constructor * 06/24/13 #2044 randerso Changed format of hdf5 group to include * minutes for satellite data + * 10/08/14 #3684 randerso Removed NOTIFY * * * @@ -105,8 +106,6 @@ public class GfeUtil { public static final String HAZARDS_KEY = "Hazards"; - public static final String NOTIFY = "gfeNotify"; - private static Pattern DISCRETE_PATTERN = Pattern .compile("'[\\w<>+/^\\[\\]\\.,:-]*'");