xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge "Issue #2057 Added support for marking obsolete databases for later removal." into development

Browse source 
Former-commit-id: c43cdaeb76 [formerly af826f283b] [formerly c43cdaeb76 [formerly af826f283b] [formerly f964cc3e65 [formerly 5e67c228fea2f719bfb0561132701097bf785a80]]]
Former-commit-id: f964cc3e65
Former-commit-id: 0d69b079ee [formerly 9a6015efcb]
Former-commit-id: c3fd34895c
This commit is contained in:
Richard Peter 2013-08-05 09:51:14 -05:00 • committed by Gerrit Code Review
commit 4bd15f7789
6 changed files with 182 additions and 214 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
deltaScripts/13.6.1/UpdateGfeDbid.sh Normal file
View file

@ -0,0 +1,11 @@
#!/bin/bash
# run the update
/awips2/psql/bin/psql -U awips -d metadata -c "ALTER TABLE gfe_dbid ADD COLUMN removeddate timestamp without time zone;"
if [ $? -ne 0 ]; then
echo "FATAL: the update has failed!"
exit 1
fi
echo "INFO: the update has completed successfully!"
exit 0

101
edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/db/dao/GFEDao.java
View file

@ -44,6 +44,7 @@ import com.raytheon.edex.db.dao.DefaultPluginDao;
import com.raytheon.edex.plugin.gfe.server.GridParmManager;
import com.raytheon.edex.plugin.gfe.server.IFPServer;
import com.raytheon.edex.plugin.gfe.server.database.GridDatabase;
import com.raytheon.edex.plugin.gfe.server.database.IFPGridDatabase;
import com.raytheon.edex.plugin.gfe.util.SendNotifications;
import com.raytheon.uf.common.dataplugin.PluginException;
import com.raytheon.uf.common.dataplugin.gfe.GridDataHistory;
@ -59,6 +60,7 @@ import com.raytheon.uf.common.datastorage.DataStoreFactory;
import com.raytheon.uf.common.datastorage.IDataStore;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.time.TimeRange;
import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.common.util.CollectionUtil;
import com.raytheon.uf.common.util.Pair;
import com.raytheon.uf.edex.database.DataAccessLayerException;
@ -93,12 +95,16 @@ import com.raytheon.uf.edex.database.query.DatabaseQuery;
* GetLatestDbTimeRequest and GetLatestModelDbIdRequest.
* 05/20/13 #2127 rjpeter Set session's to read only and switched to stateless where possible.
* 06/13/13 #2044 randerso Refactored to use IFPServer, code cleanup
* 07/30/13 #2057 randerso Added support marking and eventually purging obsolete databases
* </pre>
*
* @author bphillip
* @version 1.0
*/
public class GFEDao extends DefaultPluginDao {
/** Removed DB purge time in days */
public static final int REMOVED_DB_PURGE_TIME = 7;
// Number of retries on insert of a new DatabaseID
private static final int QUERY_RETRY = 2;
@ -374,6 +380,7 @@ public class GFEDao extends DefaultPluginDao {
.purgeExpiredPending();
PurgeLogger.logInfo("Purged " + requestsPurged
+ " expired pending isc send requests.", "gfe");
purgeRemovedDbs();
} catch (DataAccessLayerException e) {
throw new PluginException(
"Error purging expired send ISC records!", e);
@ -384,6 +391,39 @@ public class GFEDao extends DefaultPluginDao {
}
}
private void purgeRemovedDbs() throws DataAccessLayerException {
List<DatabaseID> removed = null;
try {
removed = txTemplate
.execute(new TransactionCallback<List<DatabaseID>>() {
@Override
public List<DatabaseID> doInTransaction(
TransactionStatus status) {
Date purgeDate = new Date(
System.currentTimeMillis()
- (REMOVED_DB_PURGE_TIME * TimeUtil.MILLIS_PER_DAY));
@SuppressWarnings("unchecked")
List<DatabaseID> removed = getHibernateTemplate()
.find("FROM DatabaseID where removedDate < ?",
purgeDate);
return removed;
}
});
} catch (Exception e) {
throw new DataAccessLayerException(
"Error purging removed databases", e);
}
if (removed != null) {
for (DatabaseID dbId : removed) {
IFPGridDatabase.deleteDatabase(dbId);
PurgeLogger.logInfo("Purging removed database: " + dbId, "gfe");
}
}
}
/**
* Purge all DatabaseIDs for a site
*
@ -559,8 +599,9 @@ public class GFEDao extends DefaultPluginDao {
TransactionStatus status) {
@SuppressWarnings("unchecked")
List<DatabaseID> result = getHibernateTemplate()
.find("FROM DatabaseID WHERE siteId = ?",
.find("FROM DatabaseID WHERE siteId = ? AND removeddate is null",
siteId);
return result;
}
});
@ -1210,4 +1251,62 @@ public class GFEDao extends DefaultPluginDao {
return null;
}
}
/**
* Set the database removed date
*
* @param dbId
* databaseID to be updated
* @param removedDate
* date database was removed or null if not removed (restored)
* @throws DataAccessLayerException
*/
public void setDatabaseRemovedDate(DatabaseID dbId, Date removedDate)
throws DataAccessLayerException {
dbId.setRemovedDate(removedDate);
Session sess = null;
try {
sess = getHibernateTemplate().getSessionFactory().openSession();
int tries = 0;
Transaction tx = null;
try {
tx = sess.beginTransaction();
sess.update(dbId);
tx.commit();
} catch (ConstraintViolationException e) {
if (tx != null) {
try {
tx.rollback();
} catch (Exception e1) {
logger.error("Error occurred rolling back transaction",
e1);
}
}
// database may have been inserted on another process, redo
// the look up
if (tries < 2) {
logger.info("Constraint violation on save, attempting to look up database id again");
} else {
throw new DataAccessLayerException(
"Unable to look up DatabaseID: " + dbId.toString(),
e);
}
}
} catch (Exception e) {
throw new DataAccessLayerException("Unable to look up DatabaseID: "
+ dbId.toString(), e);
} finally {
if (sess != null) {
try {
sess.close();
} catch (Exception e) {
statusHandler.error(
"Error occurred closing database session", e);
}
}
}
}
}

126
edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/GridParmManager.java
View file

@ -105,6 +105,8 @@ import com.raytheon.uf.edex.database.purge.PurgeLogger;
* 05/03/13 #1974 randerso Fixed error logging to include stack trace
* 05/14/13 #2004 randerso Added methods to synch GridParmManager across JVMs
* 05/30/13 #2044 randerso Refactored to better match A1 design. Removed D2DParmIDCache.
* 07/30/13 #2057 randerso Added support for marking obsoleted databases for removal and
* eventually purging them
* </pre>
*
* @author bphillip
@ -1028,97 +1030,45 @@ public class GridParmManager {
if (dbConfig == null) {
status.addMessage("Unable to obtain GridDbConfig information for creation"
+ " in createDB() for " + id);
// TODO: implement A2 equivalent
// make list of files that match this "bad" database
// TextString dir = PathMgr::dirname(filename) + '/';
// TextString baseName = PathMgr::basename(filename);
// TextString dbName = PathMgr::stripExtension(baseName);
// SeqOf<TextString> filenames = PathMgr::listDir(dir);
// unsigned pos = 0;
// TextString badTime =
// AbsTime::current().string("/BADDB-%Y%m%d_%H%M-");
// for (int i = 0; i < filenames.length(); i++)
// if (PathMgr::isFile(dir + filenames[i]) &&
// filenames[i].found(dbName, pos))
// {
// logVerbose << "Deleting Bogus Database:" << filenames[i]
// << std::endl;
// TextString bdir = _config.baseDir() + "/BAD";
// PathMgr::mkdir(bdir);
// TextString sourceF = dir + filenames[i];
// TextString destF = bdir + badTime + filenames[i];
// TextString cmd = "mv " + sourceF + ' ' + destF;
// system(cmd.stringPtr());
// logProblem << "Bad database moved to: " << destF << std::endl;
// }
return status;
}
// we attempt to create the GridDatabase twice, if fails the 1st time,
// we delete it and try again. Singleton databases get two tries,
// non-singletons are simply deleted if invalid.
int trys = 0;
int maxtrys = 2;
if (id.getModelTime().equals(DatabaseID.NO_MODEL_TIME)) {
maxtrys = 1;
}
while (trys < maxtrys) {
if (trys != 0) {
statusHandler.error("Attempting to recreate: ");
}
} else {
// attempt to create the GridDatabase
db = new IFPGridDatabase(id, dbConfig);
// ServerResponse sr;
// sr = db->databaseIsValid();
// if (sr.okay())
if (db.databaseIsValid()) {
break; // database is okay - continue processing
}
//
// // error on creating database
// logProblem << "Database invalid with id: " << id << std::endl;
// logProblem << sr << std::endl;
// logProblem << "Deleting bogus database: " << id << std::endl;
//
// // make list of files that match this "bad" database
// TextString dir = PathMgr::dirname(filename) + '/';
// TextString baseName = PathMgr::basename(filename);
// TextString dbName = PathMgr::stripExtension(baseName);
// SeqOf<TextString> filenames = PathMgr::listDir(dir);
// TextString badTime =
// AbsTime::current().string("/BADDB-%Y%m%d_%H%M-");
// unsigned int pos = 0;
// for (int i = 0; i < filenames.length(); i++)
// if (PathMgr::isFile(dir + filenames[i])
// && filenames[i].found(dbName, pos))
// {
// TextString bdir = _config.baseDir() + "/BAD";
// PathMgr::mkdir(bdir);
// TextString sourceF = dir + filenames[i];
// TextString destF = bdir + badTime + filenames[i];
// TextString cmd = "mv " + sourceF + ' ' + destF;
// system(cmd.stringPtr());
// logProblem << "Bad database moved to: " << destF << std::endl;
// }
// //db->deleteDb(); // delete the database (not needed since we
// move
// // files to a "BAD" directory
// delete db;
// db = NULL;
if ((trys + 1) == maxtrys) { // final loop
// get databaseID object from database
DatabaseID dbId = db.getDbId();
if (dbId.getRemovedDate() != null) {
// mark database as not removed
try {
GFEDao gfeDao = new GFEDao();
gfeDao.setDatabaseRemovedDate(dbId, null);
statusHandler.info("Database " + dbId + " restored");
} catch (Exception e) {
statusHandler.handle(Priority.PROBLEM,
"Unable to mark database restored: " + dbId, e);
}
}
// add to list of databases
addDB(db);
} else {
status.addMessage("Database " + id + " is not valid.");
// status += sr;
return null;
db = null;
}
trys++;
}
if (db != null) {
addDB(db);
} // add to list of databases if not-NULL
if (db == null) {
// mark database as removed
try {
GFEDao gfeDao = new GFEDao();
gfeDao.setDatabaseRemovedDate(id, new Date());
statusHandler.warn("Database " + id + " marked for removal in "
+ GFEDao.REMOVED_DB_PURGE_TIME + " days.");
} catch (Exception e) {
statusHandler.handle(Priority.PROBLEM,
"Unable to mark database removed: " + id, e);
}
}
status.setPayload(db);
return status;
@ -1151,9 +1101,9 @@ public class GridParmManager {
ServerResponse<GridDatabase> sr = new ServerResponse<GridDatabase>();
for (DatabaseID dbId : inventory) {
sr = createDB(dbId);
}
if (!sr.isOkay()) {
statusHandler.error(sr.message());
if (!sr.isOkay()) {
statusHandler.error(sr.message());
}
}
NetCDFDatabaseManager.initializeNetCDFDatabases(config);

15
edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/database/IFPGridDatabase.java
View file

@ -120,6 +120,7 @@ import com.vividsolutions.jts.geom.Coordinate;
* 04/08/13 #1949 rjpeter Updated to work with normalized database.
* 05/02/13 #1969 randerso Removed updateDbs from parent class
* 06/13/13 #2044 randerso Pass in GridDbConfig as construction parameter
* 07/30/13 #2057 randerso Added a static deleteDatabase method
* </pre>
*
* @author bphillip
@ -2462,6 +2463,16 @@ public class IFPGridDatabase extends GridDatabase {
@Override
public void deleteDb() {
DatabaseID id = getDbId();
deleteDatabase(id);
}
/**
* Delete a database
*
* @param id
* the DatabaseID of the datbase to be deleted
*/
public static void deleteDatabase(DatabaseID id) {
try {
GFEDao gfeDao = new GFEDao();
gfeDao.purgeGFEGrids(id);
@ -2470,7 +2481,7 @@ public class IFPGridDatabase extends GridDatabase {
"Unable to delete model database: " + id, e);
}
this.deleteModelHDF5();
deleteModelHDF5(id);
}
/**
@ -2537,7 +2548,7 @@ public class IFPGridDatabase extends GridDatabase {
}
}
private void deleteModelHDF5() {
private static void deleteModelHDF5(DatabaseID dbId) {
File hdf5File = GfeUtil.getHdf5Dir(GridDatabase.gfeBaseDataDir, dbId);
IDataStore ds = DataStoreFactory.getDataStore(hdf5File);
try {

22
edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/db/objects/DatabaseID.java
View file

@ -42,7 +42,6 @@ import javax.persistence.Table;
import javax.persistence.Transient;
import javax.persistence.UniqueConstraint;
import org.hibernate.annotations.Immutable;
import org.hibernate.annotations.OnDelete;
import org.hibernate.annotations.OnDeleteAction;
@ -67,6 +66,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeTypeAdap
* 03/28/13 1949 rjpeter Normalized database structure.
* 06/20/13 2127 rjpeter Removed unused bidirectional relationship.
* 06/13/13 2044 randerso Code cleanup
* 07/31/13 2057 randerso Added removedDate
* </pre>
*
* @author bphillip
@ -75,7 +75,6 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeTypeAdap
@Entity
@Table(name = "gfe_dbid", uniqueConstraints = { @UniqueConstraint(columnNames = {
"siteId", "modelName", "modelTime", "dbType" }) })
@Immutable
@DynamicSerialize
@DynamicSerializeTypeAdapter(factory = DatabaseIDAdapter.class)
public class DatabaseID implements Comparable<DatabaseID> {
@ -147,6 +146,10 @@ public class DatabaseID implements Comparable<DatabaseID> {
@DataURI(position = 2)
private String modelTime = NO_MODEL_TIME;
/** Date database was removed from localConfig.py. */
@Column(nullable = true)
private Date removedDate;
/** The model identifier */
@Transient
private String modelId;
@ -478,6 +481,21 @@ public class DatabaseID implements Comparable<DatabaseID> {
return modelTime;
}
/**
* @return the removedDate
*/
public Date getRemovedDate() {
return removedDate;
}
/**
* @param removedDate
* the removedDate to set
*/
public void setRemovedDate(Date removedDate) {
this.removedDate = removedDate;
}
/**
* @return the modelId
*/

121
edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/db/type/DatabaseIdType.java
View file

@ -1,121 +0,0 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.dataplugin.gfe.db.type;
import java.io.Serializable;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Types;
import org.hibernate.HibernateException;
import org.hibernate.usertype.UserType;
import com.raytheon.uf.common.dataplugin.gfe.db.objects.DatabaseID;
/**
* Custom Hibernate type for the DatabaseID class. This class defines how a
* DatabaseID object is saved and retrieved from the database.
*
*
* <pre>
* SOFTWARE HISTORY
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 4/10/08 875 bphillip Initial Creation
*
* </pre>
*
* @author bphillip
* @version 1.0
*/
public class DatabaseIdType implements UserType {
private static final int[] SQL_TYPES = { Types.VARCHAR };
@Override
public Object assemble(Serializable cached, Object owner)
throws HibernateException {
return cached;
}
@Override
public Object deepCopy(Object value) throws HibernateException {
return value;
}
@Override
public Serializable disassemble(Object value) throws HibernateException {
return (Serializable) value;
}
@Override
public boolean equals(Object x, Object y) throws HibernateException {
if (x instanceof DatabaseID && y instanceof DatabaseID) {
return x.equals(y);
} else {
return false;
}
}
@Override
public int hashCode(Object value) throws HibernateException {
return 0;
}
@Override
public boolean isMutable() {
return false;
}
@Override
public Object nullSafeGet(ResultSet resultSet, String[] names, Object owner)
throws HibernateException, SQLException {
return new DatabaseID(resultSet.getString(names[0]));
}
@Override
public void nullSafeSet(PreparedStatement statement, Object value, int index)
throws HibernateException, SQLException {
if (value == null) {
statement.setString(index, null);
} else {
statement.setString(index, value.toString());
}
}
@Override
public Object replace(Object original, Object target, Object owner)
throws HibernateException {
return original;
}
@Override
public Class<?> returnedClass() {
return DatabaseID.class;
}
@Override
public int[] sqlTypes() {
return DatabaseIdType.SQL_TYPES;
}
}