diff --git a/deltaScripts/13.6.1/UpdateGfeDbid.sh b/deltaScripts/13.6.1/UpdateGfeDbid.sh new file mode 100644 index 0000000000..1538860cfb --- /dev/null +++ b/deltaScripts/13.6.1/UpdateGfeDbid.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# run the update +/awips2/psql/bin/psql -U awips -d metadata -c "ALTER TABLE gfe_dbid ADD COLUMN removeddate timestamp without time zone;" +if [ $? -ne 0 ]; then + echo "FATAL: the update has failed!" + exit 1 +fi + +echo "INFO: the update has completed successfully!" + +exit 0 diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/db/dao/GFEDao.java b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/db/dao/GFEDao.java index 987e28e24b..382b9bebc0 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/db/dao/GFEDao.java +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/db/dao/GFEDao.java @@ -44,6 +44,7 @@ import com.raytheon.edex.db.dao.DefaultPluginDao; import com.raytheon.edex.plugin.gfe.server.GridParmManager; import com.raytheon.edex.plugin.gfe.server.IFPServer; import com.raytheon.edex.plugin.gfe.server.database.GridDatabase; +import com.raytheon.edex.plugin.gfe.server.database.IFPGridDatabase; import com.raytheon.edex.plugin.gfe.util.SendNotifications; import com.raytheon.uf.common.dataplugin.PluginException; import com.raytheon.uf.common.dataplugin.gfe.GridDataHistory; @@ -59,6 +60,7 @@ import com.raytheon.uf.common.datastorage.DataStoreFactory; import com.raytheon.uf.common.datastorage.IDataStore; import com.raytheon.uf.common.status.UFStatus.Priority; import com.raytheon.uf.common.time.TimeRange; +import com.raytheon.uf.common.time.util.TimeUtil; import com.raytheon.uf.common.util.CollectionUtil; import com.raytheon.uf.common.util.Pair; import com.raytheon.uf.edex.database.DataAccessLayerException; @@ -93,12 +95,16 @@ import com.raytheon.uf.edex.database.query.DatabaseQuery; * GetLatestDbTimeRequest and GetLatestModelDbIdRequest. * 05/20/13 #2127 rjpeter Set session's to read only and switched to stateless where possible. * 06/13/13 #2044 randerso Refactored to use IFPServer, code cleanup + * 07/30/13 #2057 randerso Added support marking and eventually purging obsolete databases * * * @author bphillip * @version 1.0 */ public class GFEDao extends DefaultPluginDao { + /** Removed DB purge time in days */ + public static final int REMOVED_DB_PURGE_TIME = 7; + // Number of retries on insert of a new DatabaseID private static final int QUERY_RETRY = 2; @@ -374,6 +380,7 @@ public class GFEDao extends DefaultPluginDao { .purgeExpiredPending(); PurgeLogger.logInfo("Purged " + requestsPurged + " expired pending isc send requests.", "gfe"); + purgeRemovedDbs(); } catch (DataAccessLayerException e) { throw new PluginException( "Error purging expired send ISC records!", e); @@ -384,6 +391,39 @@ public class GFEDao extends DefaultPluginDao { } } + private void purgeRemovedDbs() throws DataAccessLayerException { + List removed = null; + try { + removed = txTemplate + .execute(new TransactionCallback>() { + @Override + public List doInTransaction( + TransactionStatus status) { + Date purgeDate = new Date( + System.currentTimeMillis() + - (REMOVED_DB_PURGE_TIME * TimeUtil.MILLIS_PER_DAY)); + @SuppressWarnings("unchecked") + List removed = getHibernateTemplate() + .find("FROM DatabaseID where removedDate < ?", + purgeDate); + + return removed; + } + }); + } catch (Exception e) { + throw new DataAccessLayerException( + "Error purging removed databases", e); + } + + if (removed != null) { + for (DatabaseID dbId : removed) { + IFPGridDatabase.deleteDatabase(dbId); + PurgeLogger.logInfo("Purging removed database: " + dbId, "gfe"); + } + } + + } + /** * Purge all DatabaseIDs for a site * @@ -559,8 +599,9 @@ public class GFEDao extends DefaultPluginDao { TransactionStatus status) { @SuppressWarnings("unchecked") List result = getHibernateTemplate() - .find("FROM DatabaseID WHERE siteId = ?", + .find("FROM DatabaseID WHERE siteId = ? AND removeddate is null", siteId); + return result; } }); @@ -1210,4 +1251,62 @@ public class GFEDao extends DefaultPluginDao { return null; } } + + /** + * Set the database removed date + * + * @param dbId + * databaseID to be updated + * @param removedDate + * date database was removed or null if not removed (restored) + * @throws DataAccessLayerException + */ + public void setDatabaseRemovedDate(DatabaseID dbId, Date removedDate) + throws DataAccessLayerException { + + dbId.setRemovedDate(removedDate); + Session sess = null; + try { + sess = getHibernateTemplate().getSessionFactory().openSession(); + int tries = 0; + Transaction tx = null; + try { + tx = sess.beginTransaction(); + sess.update(dbId); + tx.commit(); + + } catch (ConstraintViolationException e) { + if (tx != null) { + try { + tx.rollback(); + } catch (Exception e1) { + logger.error("Error occurred rolling back transaction", + e1); + } + } + + // database may have been inserted on another process, redo + // the look up + if (tries < 2) { + logger.info("Constraint violation on save, attempting to look up database id again"); + } else { + throw new DataAccessLayerException( + "Unable to look up DatabaseID: " + dbId.toString(), + e); + } + } + } catch (Exception e) { + throw new DataAccessLayerException("Unable to look up DatabaseID: " + + dbId.toString(), e); + } finally { + if (sess != null) { + try { + sess.close(); + } catch (Exception e) { + statusHandler.error( + "Error occurred closing database session", e); + } + } + } + } } diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/GridParmManager.java b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/GridParmManager.java index 6365862127..a269386645 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/GridParmManager.java +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/GridParmManager.java @@ -105,6 +105,8 @@ import com.raytheon.uf.edex.database.purge.PurgeLogger; * 05/03/13 #1974 randerso Fixed error logging to include stack trace * 05/14/13 #2004 randerso Added methods to synch GridParmManager across JVMs * 05/30/13 #2044 randerso Refactored to better match A1 design. Removed D2DParmIDCache. + * 07/30/13 #2057 randerso Added support for marking obsoleted databases for removal and + * eventually purging them * * * @author bphillip @@ -1028,97 +1030,45 @@ public class GridParmManager { if (dbConfig == null) { status.addMessage("Unable to obtain GridDbConfig information for creation" + " in createDB() for " + id); - - // TODO: implement A2 equivalent - // make list of files that match this "bad" database - // TextString dir = PathMgr::dirname(filename) + '/'; - // TextString baseName = PathMgr::basename(filename); - // TextString dbName = PathMgr::stripExtension(baseName); - // SeqOf filenames = PathMgr::listDir(dir); - // unsigned pos = 0; - // TextString badTime = - // AbsTime::current().string("/BADDB-%Y%m%d_%H%M-"); - // for (int i = 0; i < filenames.length(); i++) - // if (PathMgr::isFile(dir + filenames[i]) && - // filenames[i].found(dbName, pos)) - // { - // logVerbose << "Deleting Bogus Database:" << filenames[i] - // << std::endl; - // TextString bdir = _config.baseDir() + "/BAD"; - // PathMgr::mkdir(bdir); - // TextString sourceF = dir + filenames[i]; - // TextString destF = bdir + badTime + filenames[i]; - // TextString cmd = "mv " + sourceF + ' ' + destF; - // system(cmd.stringPtr()); - // logProblem << "Bad database moved to: " << destF << std::endl; - // } - - return status; - } - - // we attempt to create the GridDatabase twice, if fails the 1st time, - // we delete it and try again. Singleton databases get two tries, - // non-singletons are simply deleted if invalid. - int trys = 0; - int maxtrys = 2; - if (id.getModelTime().equals(DatabaseID.NO_MODEL_TIME)) { - maxtrys = 1; - } - - while (trys < maxtrys) { - if (trys != 0) { - statusHandler.error("Attempting to recreate: "); - } - + } else { + // attempt to create the GridDatabase db = new IFPGridDatabase(id, dbConfig); - // ServerResponse sr; - // sr = db->databaseIsValid(); - // if (sr.okay()) if (db.databaseIsValid()) { - break; // database is okay - continue processing - } - // - // // error on creating database - // logProblem << "Database invalid with id: " << id << std::endl; - // logProblem << sr << std::endl; - // logProblem << "Deleting bogus database: " << id << std::endl; - // - // // make list of files that match this "bad" database - // TextString dir = PathMgr::dirname(filename) + '/'; - // TextString baseName = PathMgr::basename(filename); - // TextString dbName = PathMgr::stripExtension(baseName); - // SeqOf filenames = PathMgr::listDir(dir); - // TextString badTime = - // AbsTime::current().string("/BADDB-%Y%m%d_%H%M-"); - // unsigned int pos = 0; - // for (int i = 0; i < filenames.length(); i++) - // if (PathMgr::isFile(dir + filenames[i]) - // && filenames[i].found(dbName, pos)) - // { - // TextString bdir = _config.baseDir() + "/BAD"; - // PathMgr::mkdir(bdir); - // TextString sourceF = dir + filenames[i]; - // TextString destF = bdir + badTime + filenames[i]; - // TextString cmd = "mv " + sourceF + ' ' + destF; - // system(cmd.stringPtr()); - // logProblem << "Bad database moved to: " << destF << std::endl; - // } - // //db->deleteDb(); // delete the database (not needed since we - // move - // // files to a "BAD" directory - // delete db; - // db = NULL; - if ((trys + 1) == maxtrys) { // final loop + // get databaseID object from database + DatabaseID dbId = db.getDbId(); + + if (dbId.getRemovedDate() != null) { + // mark database as not removed + try { + GFEDao gfeDao = new GFEDao(); + gfeDao.setDatabaseRemovedDate(dbId, null); + statusHandler.info("Database " + dbId + " restored"); + } catch (Exception e) { + statusHandler.handle(Priority.PROBLEM, + "Unable to mark database restored: " + dbId, e); + } + } + + // add to list of databases + addDB(db); + } else { status.addMessage("Database " + id + " is not valid."); - // status += sr; - return null; + db = null; } - trys++; } - if (db != null) { - addDB(db); - } // add to list of databases if not-NULL + if (db == null) { + // mark database as removed + try { + GFEDao gfeDao = new GFEDao(); + gfeDao.setDatabaseRemovedDate(id, new Date()); + statusHandler.warn("Database " + id + " marked for removal in " + + GFEDao.REMOVED_DB_PURGE_TIME + " days."); + } catch (Exception e) { + statusHandler.handle(Priority.PROBLEM, + "Unable to mark database removed: " + id, e); + } + } status.setPayload(db); return status; @@ -1151,9 +1101,9 @@ public class GridParmManager { ServerResponse sr = new ServerResponse(); for (DatabaseID dbId : inventory) { sr = createDB(dbId); - } - if (!sr.isOkay()) { - statusHandler.error(sr.message()); + if (!sr.isOkay()) { + statusHandler.error(sr.message()); + } } NetCDFDatabaseManager.initializeNetCDFDatabases(config); diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/database/IFPGridDatabase.java b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/database/IFPGridDatabase.java index 53c798b1d3..2b63370bfd 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/database/IFPGridDatabase.java +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/database/IFPGridDatabase.java @@ -120,6 +120,7 @@ import com.vividsolutions.jts.geom.Coordinate; * 04/08/13 #1949 rjpeter Updated to work with normalized database. * 05/02/13 #1969 randerso Removed updateDbs from parent class * 06/13/13 #2044 randerso Pass in GridDbConfig as construction parameter + * 07/30/13 #2057 randerso Added a static deleteDatabase method * * * @author bphillip @@ -2462,6 +2463,16 @@ public class IFPGridDatabase extends GridDatabase { @Override public void deleteDb() { DatabaseID id = getDbId(); + deleteDatabase(id); + } + + /** + * Delete a database + * + * @param id + * the DatabaseID of the datbase to be deleted + */ + public static void deleteDatabase(DatabaseID id) { try { GFEDao gfeDao = new GFEDao(); gfeDao.purgeGFEGrids(id); @@ -2470,7 +2481,7 @@ public class IFPGridDatabase extends GridDatabase { "Unable to delete model database: " + id, e); } - this.deleteModelHDF5(); + deleteModelHDF5(id); } /** @@ -2537,7 +2548,7 @@ public class IFPGridDatabase extends GridDatabase { } } - private void deleteModelHDF5() { + private static void deleteModelHDF5(DatabaseID dbId) { File hdf5File = GfeUtil.getHdf5Dir(GridDatabase.gfeBaseDataDir, dbId); IDataStore ds = DataStoreFactory.getDataStore(hdf5File); try { diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/db/objects/DatabaseID.java b/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/db/objects/DatabaseID.java index 1cd5a95464..6f49908cf8 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/db/objects/DatabaseID.java +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/db/objects/DatabaseID.java @@ -42,7 +42,6 @@ import javax.persistence.Table; import javax.persistence.Transient; import javax.persistence.UniqueConstraint; -import org.hibernate.annotations.Immutable; import org.hibernate.annotations.OnDelete; import org.hibernate.annotations.OnDeleteAction; @@ -67,6 +66,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeTypeAdap * 03/28/13 1949 rjpeter Normalized database structure. * 06/20/13 2127 rjpeter Removed unused bidirectional relationship. * 06/13/13 2044 randerso Code cleanup + * 07/31/13 2057 randerso Added removedDate * * * @author bphillip @@ -75,7 +75,6 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeTypeAdap @Entity @Table(name = "gfe_dbid", uniqueConstraints = { @UniqueConstraint(columnNames = { "siteId", "modelName", "modelTime", "dbType" }) }) -@Immutable @DynamicSerialize @DynamicSerializeTypeAdapter(factory = DatabaseIDAdapter.class) public class DatabaseID implements Comparable { @@ -147,6 +146,10 @@ public class DatabaseID implements Comparable { @DataURI(position = 2) private String modelTime = NO_MODEL_TIME; + /** Date database was removed from localConfig.py. */ + @Column(nullable = true) + private Date removedDate; + /** The model identifier */ @Transient private String modelId; @@ -478,6 +481,21 @@ public class DatabaseID implements Comparable { return modelTime; } + /** + * @return the removedDate + */ + public Date getRemovedDate() { + return removedDate; + } + + /** + * @param removedDate + * the removedDate to set + */ + public void setRemovedDate(Date removedDate) { + this.removedDate = removedDate; + } + /** * @return the modelId */ diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/db/type/DatabaseIdType.java b/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/db/type/DatabaseIdType.java deleted file mode 100644 index 2f574ba018..0000000000 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/db/type/DatabaseIdType.java +++ /dev/null @@ -1,121 +0,0 @@ -/** - * This software was developed and / or modified by Raytheon Company, - * pursuant to Contract DG133W-05-CQ-1067 with the US Government. - * - * U.S. EXPORT CONTROLLED TECHNICAL DATA - * This software product contains export-restricted data whose - * export/transfer/disclosure is restricted by U.S. law. Dissemination - * to non-U.S. persons whether in the United States or abroad requires - * an export license or other authorization. - * - * Contractor Name: Raytheon Company - * Contractor Address: 6825 Pine Street, Suite 340 - * Mail Stop B8 - * Omaha, NE 68106 - * 402.291.0100 - * - * See the AWIPS II Master Rights File ("Master Rights File.pdf") for - * further licensing information. - **/ - -package com.raytheon.uf.common.dataplugin.gfe.db.type; - -import java.io.Serializable; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Types; - -import org.hibernate.HibernateException; -import org.hibernate.usertype.UserType; - -import com.raytheon.uf.common.dataplugin.gfe.db.objects.DatabaseID; - -/** - * Custom Hibernate type for the DatabaseID class. This class defines how a - * DatabaseID object is saved and retrieved from the database. - * - * - * 
- * SOFTWARE HISTORY
- * Date         Ticket#    Engineer    Description
- * ------------ ---------- ----------- --------------------------
- * 4/10/08      875        bphillip    Initial Creation
- * 
- *
- * - * @author bphillip - * @version 1.0 - */ -public class DatabaseIdType implements UserType { - - private static final int[] SQL_TYPES = { Types.VARCHAR }; - - @Override - public Object assemble(Serializable cached, Object owner) - throws HibernateException { - return cached; - } - - @Override - public Object deepCopy(Object value) throws HibernateException { - return value; - } - - @Override - public Serializable disassemble(Object value) throws HibernateException { - return (Serializable) value; - } - - @Override - public boolean equals(Object x, Object y) throws HibernateException { - if (x instanceof DatabaseID && y instanceof DatabaseID) { - return x.equals(y); - } else { - return false; - } - } - - @Override - public int hashCode(Object value) throws HibernateException { - return 0; - } - - @Override - public boolean isMutable() { - return false; - } - - @Override - public Object nullSafeGet(ResultSet resultSet, String[] names, Object owner) - throws HibernateException, SQLException { - return new DatabaseID(resultSet.getString(names[0])); - } - - @Override - public void nullSafeSet(PreparedStatement statement, Object value, int index) - throws HibernateException, SQLException { - if (value == null) { - statement.setString(index, null); - } else { - statement.setString(index, value.toString()); - } - - } - - @Override - public Object replace(Object original, Object target, Object owner) - throws HibernateException { - return original; - } - - @Override - public Class returnedClass() { - return DatabaseID.class; - } - - @Override - public int[] sqlTypes() { - return DatabaseIdType.SQL_TYPES; - } -}