xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #3684 Fix some issues found after #3684 with certain GFE configurations

Browse source 
Change-Id: I92b45588d9d4fe9169708c07119767017426d468

Former-commit-id: 6dc55b51414414aa416c9ad0c688d1d34ba08c17
This commit is contained in:
Ron Anderson 2014-10-08 18:05:27 -05:00
parent bf5225906f
commit a64a36d182
8 changed files with 117 additions and 114 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
edexOsgi/com.raytheon.edex.plugin.gfe/res/spring/gfe-common.xml
View file

@ -31,27 +31,10 @@
depends-on="commonTimeRegistered, gfeRegistered">
</bean>
<bean id="gfeNotifyFilter" class="com.raytheon.edex.plugin.gfe.server.notify.GfeNotificationFilter"/>
<bean id="ifpServer" class="com.raytheon.edex.plugin.gfe.server.IFPServer.Wrapper"/>
<camelContext id="gfe-common-camel" xmlns="http://camel.apache.org/schema/spring" errorHandlerRef="errorHandler">
<route id="gfeNotify">
<from uri="vm:edex.gfeNotification?size=5000"/>
<doTry>
<filter>
<method bean="gfeNotifyFilter" method="isGfeNotification"/>
<bean ref="serializationUtil" method="transformToThrift"/>
<to uri="jms-generic:topic:edex.alerts.gfe?timeToLive=60000"/>
</filter>
<doCatch>
<exception>java.lang.Throwable</exception>
<to uri="log:gfeNotify?level=ERROR"/>
</doCatch>
</doTry>
</route>
<route id="notifyIfpServer">
<from uri="jms-generic:topic:edex.alerts.gfe?threadName=notifyIfpServer-edex.alerts.gfe" />
<doTry>

97
edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/GridParmManager.java
View file

@ -129,6 +129,10 @@ import com.raytheon.uf.edex.database.purge.PurgeLogger;
* 10/07/2014 #3684 randerso Restructured IFPServer start up.
* Reordered handling of DbInvChangeNotification
* Don't process GFENotifications sent by this JVM
* Prevent createDB from creating databases that are immediately purged
* Send DBInvChangeNotifications at site activation so new D2D data
* ingested while deactivated gets recognized
*
* </pre>
*
* @author bphillip
@ -890,6 +894,10 @@ public class GridParmManager {
if (notify) {
createDbNotification(Arrays.asList(dbId), purged);
}
// always notify on purges
else if (!purged.isEmpty()) {
createDbNotification(null, purged);
}
}
}
@ -913,7 +921,10 @@ public class GridParmManager {
GridDatabase db = this.getDatabase(dbId);
if (db != null) {
sr.setPayload(db);
return sr; // database already exists
} else {
sr.addMessage("Unable to create database: " + dbId);
}
// is it a singleton database?
@ -1051,42 +1062,64 @@ public class GridParmManager {
return sr;
}
private ServerResponse<GridDatabase> createDB(DatabaseID id) {
private ServerResponse<GridDatabase> createDB(DatabaseID dbId) {
// TODO: consider merging this into getDatabase()
ServerResponse<GridDatabase> status = new ServerResponse<GridDatabase>();
if (!id.isValid() || !id.getFormat().equals(DataType.GRID)) {
if (!dbId.isValid() || !dbId.getFormat().equals(DataType.GRID)) {
status.addMessage("Database id "
+ id
+ dbId
+ " is not valid, or is not a grid-type. Cannot create database.");
return status;
}
// create the grid database
// get the dbConfig
IFPGridDatabase db = null;
GridDbConfig dbConfig = this.config.gridDbConfig(id);
GridDbConfig dbConfig = this.config.gridDbConfig(dbId);
if (dbConfig == null) {
status.addMessage("Unable to obtain GridDbConfig information for creation"
+ " in createDB() for " + id);
+ " in createDB() for " + dbId);
} else {
// don't create if this version would be immediately purgeable
List<DatabaseID> existing = new ArrayList<DatabaseID>();
for (DatabaseID id : this.dbMap.keySet()) {
if (id.sameModel(dbId)) {
existing.add(id);
}
}
if (!existing.contains(dbId)) {
existing.add(dbId);
}
// sort by model time (most recent first)
Collections.sort(existing);
int desiredVersions = this.config.desiredDbVersions(dbId);
if (existing.indexOf(dbId) >= desiredVersions) {
status.addMessage("Unable to create " + dbId
+ " as it would exceed the desired number of versions");
return status;
}
// attempt to create the GridDatabase
db = new IFPGridDatabase(id, dbConfig);
db = new IFPGridDatabase(dbId, dbConfig);
if (db.databaseIsValid()) {
// get databaseID object from database
DatabaseID dbId = db.getDbId();
DatabaseID id = db.getDbId();
if (dbId.getRemovedDate() != null) {
if (id.getRemovedDate() != null) {
// mark database as not removed
try {
GFEDao gfeDao = new GFEDao();
gfeDao.setDatabaseRemovedDate(dbId, null);
statusHandler.info("Database " + dbId + " restored");
gfeDao.setDatabaseRemovedDate(id, null);
statusHandler.info("Database " + id + " restored");
} catch (Exception e) {
statusHandler.handle(Priority.PROBLEM,
"Unable to mark database restored: " + dbId, e);
"Unable to mark database restored: " + id, e);
}
}
} else {
status.addMessage("Database " + id + " is not valid.");
status.addMessage("Database " + dbId + " is not valid.");
db = null;
}
}
@ -1095,12 +1128,13 @@ public class GridParmManager {
// mark database as removed
try {
GFEDao gfeDao = new GFEDao();
gfeDao.setDatabaseRemovedDate(id, new Date());
statusHandler.warn("Database " + id + " marked for removal in "
gfeDao.setDatabaseRemovedDate(dbId, new Date());
statusHandler.warn("Database " + dbId
+ " marked for removal in "
+ GFEDao.REMOVED_DB_PURGE_TIME + " days.");
} catch (Exception e) {
statusHandler.handle(Priority.PROBLEM,
"Unable to mark database removed: " + id, e);
"Unable to mark database removed: " + dbId, e);
}
}
@ -1142,8 +1176,12 @@ public class GridParmManager {
}
// create the databases (the list should now only contain GRID dbs)
List<DatabaseID> added = new ArrayList<DatabaseID>(inventory.size());
for (DatabaseID dbId : inventory) {
getDatabase(dbId, false);
GridDatabase db = getDatabase(dbId, false);
if (db != null) {
added.add(db.getDbId());
}
}
NetCDFDatabaseManager.initializeNetCDFDatabases(config);
@ -1156,7 +1194,8 @@ public class GridParmManager {
D2DSatDatabase satDb = new D2DSatDatabase(config);
addDB(satDb);
initD2DDbs();
added.addAll(initD2DDbs());
createDbNotification(added, null);
// only fire smartInits if queue is instantiated
SmartInitQueue queue = SmartInitQueue.getQueue();
@ -1197,7 +1236,8 @@ public class GridParmManager {
}
}
private void initD2DDbs() {
private List<DatabaseID> initD2DDbs() {
List<DatabaseID> added = new ArrayList<DatabaseID>();
for (String d2dModelName : config.getD2dModels()) {
try {
// get dbId to get desiredDbVersions (date doesn't matter)
@ -1209,13 +1249,17 @@ public class GridParmManager {
d2dModelName, desiredVersions)) {
dbId = D2DGridDatabase.getDbId(d2dModelName, refTime,
config);
getDatabase(dbId, true);
GridDatabase db = getDatabase(dbId, false);
if (db != null) {
added.add(db.getDbId());
}
}
} catch (Exception e) {
statusHandler.error("Error initializing D2D model: "
+ d2dModelName, e);
}
}
return (added);
}
/**
@ -1236,6 +1280,9 @@ public class GridParmManager {
}
D2DGridDatabase db = (D2DGridDatabase) getDatabase(dbId, true);
if (db == null) {
continue;
}
GridUpdateNotification gun = db.update(record);
if (gun != null) {
@ -1416,7 +1463,8 @@ public class GridParmManager {
private void createDbNotification(List<DatabaseID> additions,
List<DatabaseID> deletions) {
if (!additions.isEmpty() || !deletions.isEmpty()) {
if ((additions != null && !additions.isEmpty())
|| (deletions != null && !deletions.isEmpty())) {
DBInvChangeNotification notify = new DBInvChangeNotification(
additions, deletions, siteID);
SendNotifications.send(notify);
@ -1550,14 +1598,9 @@ public class GridParmManager {
// process the inventory looking for "old" unwanted databases
List<DatabaseID> purged = new ArrayList<DatabaseID>();
String model = modelToPurge.getModelName();
String site = modelToPurge.getSiteId();
String type = modelToPurge.getDbType();
int count = 0;
for (DatabaseID dbId : currentInv) {
// new series?
if (dbId.getSiteId().equals(site) && dbId.getDbType().equals(type)
&& dbId.getModelName().equals(model)) {
if (dbId.sameModel(modelToPurge)) {
// process the id and determine whether it should be purged
count++;

53
edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/notify/GfeNotificationFilter.java
View file

@ -1,53 +0,0 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.edex.plugin.gfe.server.notify;
import java.util.List;
import com.raytheon.uf.common.dataplugin.gfe.server.notify.GfeNotification;
/**
* TODO Add Description
*
* <pre>
* SOFTWARE HISTORY
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Dec 3, 2008 njensen Initial creation
* </pre>
*
* @author njensen
* @version 1.0
*/
public class GfeNotificationFilter {
public boolean isGfeNotification(Object body) {
Object obj = body;
if (body instanceof List) {
List<?> list = (List<?>) body;
if (list.size() > 0) {
obj = list.get(0);
}
}
return obj instanceof GfeNotification;
}
}

12
edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/smartinit/InitClient.java
View file

@ -25,6 +25,7 @@ import com.raytheon.edex.plugin.gfe.config.IFPServerConfig;
import com.raytheon.edex.plugin.gfe.config.IFPServerConfigManager;
import com.raytheon.edex.plugin.gfe.reference.ReferenceMgr;
import com.raytheon.edex.plugin.gfe.server.IFPServer;
import com.raytheon.edex.plugin.gfe.server.database.GridDatabase;
import com.raytheon.edex.plugin.gfe.util.SendNotifications;
import com.raytheon.uf.common.dataplugin.gfe.db.objects.DatabaseID;
import com.raytheon.uf.common.dataplugin.gfe.db.objects.GridLocation;
@ -49,6 +50,7 @@ import com.raytheon.uf.common.status.UFStatus.Priority;
* Jul 25, 2012 #957 dgilling Implement getEditAreaNames().
* Jun 13, 2013 #2044 randerso Refactored to use IFPServer
* Nov 20, 2013 #2331 randerso Changed return type of getTopoData
* Oct 08, 2014 #3684 randerso Changed createDB to return status
*
* </pre>
*
@ -98,13 +100,11 @@ public class InitClient {
*
* @param key
*/
public void createDB(String key) {
public ServerResponse<GridDatabase> createDB(String key) {
DatabaseID id = new DatabaseID(key);
ServerResponse<?> sr = ifpServer.getGridParmMgr().createNewDb(id);
if (!sr.isOkay()) {
statusHandler.error("Error creating database " + id + ": "
+ sr.message());
}
ServerResponse<GridDatabase> sr = ifpServer.getGridParmMgr()
.createNewDb(id);
return sr;
}
/**

12
edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/util/SendNotifications.java
View file

@ -28,9 +28,8 @@ import org.apache.commons.logging.LogFactory;
import com.raytheon.uf.common.dataplugin.gfe.server.message.ServerResponse;
import com.raytheon.uf.common.dataplugin.gfe.server.notify.GfeNotification;
import com.raytheon.uf.common.dataplugin.gfe.util.GfeUtil;
import com.raytheon.uf.common.serialization.SerializationUtil;
import com.raytheon.uf.edex.core.EDEXUtil;
import com.raytheon.uf.edex.core.EdexException;
/**
* Sends GFE notifications to the GFE notify JMS topic.
@ -43,6 +42,8 @@ import com.raytheon.uf.edex.core.EdexException;
* 09/22/09 3058 rjpeter changed to utility.
* 06/12/13 2099 dgilling Remove error when passed empty list of
* notifications.
* 10/08/14 #3684 randerso Changed to send directly to JMS topic
*
* </pre>
*
* @author bphillip
@ -66,11 +67,12 @@ public class SendNotifications {
}
try {
EDEXUtil.getMessageProducer().sendAsync(GfeUtil.NOTIFY,
notifications);
EDEXUtil.getMessageProducer().sendAsyncUri(
"jms-generic:topic:edex.alerts.gfe?timeToLive=60000",
SerializationUtil.transformToThrift(notifications));
// logger.info("Sending " + notifications.size() + " "
// + notifications.get(0).getClass().getSimpleName());
} catch (EdexException e) {
} catch (Exception e) {
logger.error("Error sending gfe notification", e);
sr.addMessage("Error sending gfe notification");
}

18
edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/smartinit/Init.py
View file

@ -325,7 +325,9 @@ class Forecaster(GridUtilities):
msg = "No databases for " + self._srcName
LogStream.logProblem(msg)
return
#sys.exit(1)
if self.newdb() is None:
return
self.__topo = self.getTopo() * .3048
srcdbkeys = self.srcdb().getKeys()
@ -547,6 +549,9 @@ class Forecaster(GridUtilities):
start = time.time()
self.__init()
if self.newdb() is None:
return
msgDest = "Destination database:" + self.newdb().getModelIdentifier()
if validTime is not None:
@ -643,10 +648,15 @@ class Forecaster(GridUtilities):
break
if singletonNeeded:
newdb = newdb[:-13] + '00000000_0000'
newdb = self.getDb(newdb)
else:
client.createDB(newdb)
newdb = self.getDb(newdb)
sr = client.createDB(newdb)
if sr.isOkay():
newdb = self.getDb(newdb)
else:
msg = "Unable to create database for " + str(newdb)
LogStream.logProblem(msg)
newdb = None
return srcdb, newdb

19
edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/db/objects/DatabaseID.java
View file

@ -67,6 +67,8 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeTypeAdap
* 06/20/13 2127 rjpeter Removed unused bidirectional relationship.
* 06/13/13 2044 randerso Code cleanup
* 07/31/13 2057 randerso Added removedDate
* 10/08/14 #3684 randerso Added sameModel()
*
* </pre>
*
* @author bphillip
@ -598,6 +600,23 @@ public class DatabaseID implements Comparable<DatabaseID> {
return cal.getTime();
}
public boolean sameModel(DatabaseID other) {
if (!this.siteId.equals(other.getSiteId())) {
return false;
}
if (!this.format.equals(other.getFormat())) {
return false;
}
if (!this.dbType.equals(other.getDbType())) {
return false;
}
return this.modelName.equals(other.getModelName());
}
/*
* (non-Javadoc)
*

3
edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/util/GfeUtil.java
View file

@ -67,6 +67,7 @@ import com.vividsolutions.jts.operation.polygonize.Polygonizer;
* favor of new GridLocation constructor
* 06/24/13 #2044 randerso Changed format of hdf5 group to include
* minutes for satellite data
* 10/08/14 #3684 randerso Removed NOTIFY
*
* </pre>
*
@ -105,8 +106,6 @@ public class GfeUtil {
public static final String HAZARDS_KEY = "Hazards";
public static final String NOTIFY = "gfeNotify";
private static Pattern DISCRETE_PATTERN = Pattern
.compile("'[\\w<>+/^\\[\\]\\.,:-]*'");