Merge "Issue #2057 Added support for marking obsolete databases for later removal." into development

Former-commit-id: af826f283b [formerly af826f283b [formerly 5e67c228fea2f719bfb0561132701097bf785a80]]
Former-commit-id: f964cc3e65
Former-commit-id: 9a6015efcb

deltaScripts/13.6.1/UpdateGfeDbid.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+# run the update
+/awips2/psql/bin/psql -U awips -d metadata -c "ALTER TABLE gfe_dbid ADD COLUMN removeddate timestamp without time zone;"
+if [ $? -ne 0 ]; then
+   echo "FATAL: the update has failed!"
+   exit 1
+fi
+
+echo "INFO: the update has completed successfully!"
+
+exit 0

edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/db/dao/GFEDao.java

@@ -44,6 +44,7 @@ import com.raytheon.edex.db.dao.DefaultPluginDao;
 import com.raytheon.edex.plugin.gfe.server.GridParmManager;
 import com.raytheon.edex.plugin.gfe.server.IFPServer;
 import com.raytheon.edex.plugin.gfe.server.database.GridDatabase;
+import com.raytheon.edex.plugin.gfe.server.database.IFPGridDatabase;
 import com.raytheon.edex.plugin.gfe.util.SendNotifications;
 import com.raytheon.uf.common.dataplugin.PluginException;
 import com.raytheon.uf.common.dataplugin.gfe.GridDataHistory;
@@ -59,6 +60,7 @@ import com.raytheon.uf.common.datastorage.DataStoreFactory;
 import com.raytheon.uf.common.datastorage.IDataStore;
 import com.raytheon.uf.common.status.UFStatus.Priority;
 import com.raytheon.uf.common.time.TimeRange;
+import com.raytheon.uf.common.time.util.TimeUtil;
 import com.raytheon.uf.common.util.CollectionUtil;
 import com.raytheon.uf.common.util.Pair;
 import com.raytheon.uf.edex.database.DataAccessLayerException;
@@ -93,12 +95,16 @@ import com.raytheon.uf.edex.database.query.DatabaseQuery;
  *                                     GetLatestDbTimeRequest and GetLatestModelDbIdRequest.
  * 05/20/13     #2127      rjpeter     Set session's to read only and switched to stateless where possible.
  * 06/13/13     #2044      randerso    Refactored to use IFPServer, code cleanup
+ * 07/30/13     #2057      randerso    Added support marking and eventually purging obsolete databases
  * </pre>
  * 
  * @author bphillip
  * @version 1.0
  */
 public class GFEDao extends DefaultPluginDao {
+    /** Removed DB purge time in days */
+    public static final int REMOVED_DB_PURGE_TIME = 7;
+
     // Number of retries on insert of a new DatabaseID
     private static final int QUERY_RETRY = 2;
 
@@ -374,6 +380,7 @@ public class GFEDao extends DefaultPluginDao {
                         .purgeExpiredPending();
                 PurgeLogger.logInfo("Purged " + requestsPurged
                         + " expired pending isc send requests.", "gfe");
+                purgeRemovedDbs();
             } catch (DataAccessLayerException e) {
                 throw new PluginException(
                         "Error purging expired send ISC records!", e);
@@ -384,6 +391,39 @@ public class GFEDao extends DefaultPluginDao {
         }
     }
 
+    private void purgeRemovedDbs() throws DataAccessLayerException {
+        List<DatabaseID> removed = null;
+        try {
+            removed = txTemplate
+                    .execute(new TransactionCallback<List<DatabaseID>>() {
+                        @Override
+                        public List<DatabaseID> doInTransaction(
+                                TransactionStatus status) {
+                            Date purgeDate = new Date(
+                                    System.currentTimeMillis()
+                                            - (REMOVED_DB_PURGE_TIME * TimeUtil.MILLIS_PER_DAY));
+                            @SuppressWarnings("unchecked")
+                            List<DatabaseID> removed = getHibernateTemplate()
+                                    .find("FROM DatabaseID where removedDate < ?",
+                                            purgeDate);
+
+                            return removed;
+                        }
+                    });
+        } catch (Exception e) {
+            throw new DataAccessLayerException(
+                    "Error purging removed databases", e);
+        }
+
+        if (removed != null) {
+            for (DatabaseID dbId : removed) {
+                IFPGridDatabase.deleteDatabase(dbId);
+                PurgeLogger.logInfo("Purging removed database: " + dbId, "gfe");
+            }
+        }
+
+    }
+
     /**
      * Purge all DatabaseIDs for a site
      * 
@@ -559,8 +599,9 @@ public class GFEDao extends DefaultPluginDao {
                                 TransactionStatus status) {
                             @SuppressWarnings("unchecked")
                             List<DatabaseID> result = getHibernateTemplate()
-                                    .find("FROM DatabaseID WHERE siteId = ?",
+                                    .find("FROM DatabaseID WHERE siteId = ? AND removeddate is null",
                                             siteId);
+
                             return result;
                         }
                     });
@@ -1210,4 +1251,62 @@ public class GFEDao extends DefaultPluginDao {
             return null;
         }
     }
+
+    /**
+     * Set the database removed date
+     * 
+     * @param dbId
+     *            databaseID to be updated
+     * @param removedDate
+     *            date database was removed or null if not removed (restored)
+     * @throws DataAccessLayerException
+     */
+    public void setDatabaseRemovedDate(DatabaseID dbId, Date removedDate)
+            throws DataAccessLayerException {
+
+        dbId.setRemovedDate(removedDate);
+        Session sess = null;
+        try {
+            sess = getHibernateTemplate().getSessionFactory().openSession();
+            int tries = 0;
+            Transaction tx = null;
+            try {
+                tx = sess.beginTransaction();
+                sess.update(dbId);
+                tx.commit();
+
+            } catch (ConstraintViolationException e) {
+                if (tx != null) {
+                    try {
+                        tx.rollback();
+                    } catch (Exception e1) {
+                        logger.error("Error occurred rolling back transaction",
+                                e1);
+                    }
+                }
+
+                // database may have been inserted on another process, redo
+                // the look up
+                if (tries < 2) {
+                    logger.info("Constraint violation on save, attempting to look up database id again");
+                } else {
+                    throw new DataAccessLayerException(
+                            "Unable to look up DatabaseID: " + dbId.toString(),
+                            e);
+                }
+            }
+        } catch (Exception e) {
+            throw new DataAccessLayerException("Unable to look up DatabaseID: "
+                    + dbId.toString(), e);
+        } finally {
+            if (sess != null) {
+                try {
+                    sess.close();
+                } catch (Exception e) {
+                    statusHandler.error(
+                            "Error occurred closing database session", e);
+                }
+            }
+        }
+    }
 }

edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/GridParmManager.java

@@ -105,6 +105,8 @@ import com.raytheon.uf.edex.database.purge.PurgeLogger;
  * 05/03/13     #1974      randerso    Fixed error logging to include stack trace
  * 05/14/13     #2004      randerso    Added methods to synch GridParmManager across JVMs
  * 05/30/13     #2044      randerso    Refactored to better match A1 design. Removed D2DParmIDCache.
+ * 07/30/13     #2057      randerso    Added support for marking obsoleted databases for removal and
+ *                                     eventually purging them
  * </pre>
  * 
  * @author bphillip
@@ -1028,97 +1030,45 @@ public class GridParmManager {
         if (dbConfig == null) {
             status.addMessage("Unable to obtain GridDbConfig information for creation"
                     + " in createDB() for " + id);
-
+        } else {
-            // TODO: implement A2 equivalent
+            // attempt to create the GridDatabase
-            // make list of files that match this "bad" database
-            // TextString dir = PathMgr::dirname(filename) + '/';
-            // TextString baseName = PathMgr::basename(filename);
-            // TextString dbName = PathMgr::stripExtension(baseName);
-            // SeqOf<TextString> filenames = PathMgr::listDir(dir);
-            // unsigned pos = 0;
-            // TextString badTime =
-            // AbsTime::current().string("/BADDB-%Y%m%d_%H%M-");
-            // for (int i = 0; i < filenames.length(); i++)
-            // if (PathMgr::isFile(dir + filenames[i]) &&
-            // filenames[i].found(dbName, pos))
-            // {
-            // logVerbose << "Deleting Bogus Database:" << filenames[i]
-            // << std::endl;
-            // TextString bdir = _config.baseDir() + "/BAD";
-            // PathMgr::mkdir(bdir);
-            // TextString sourceF = dir + filenames[i];
-            // TextString destF = bdir + badTime + filenames[i];
-            // TextString cmd = "mv " + sourceF + ' ' + destF;
-            // system(cmd.stringPtr());
-            // logProblem << "Bad database moved to: " << destF << std::endl;
-            // }
-
-            return status;
-        }
-
-        // we attempt to create the GridDatabase twice, if fails the 1st time,
-        // we delete it and try again. Singleton databases get two tries,
-        // non-singletons are simply deleted if invalid.
-        int trys = 0;
-        int maxtrys = 2;
-        if (id.getModelTime().equals(DatabaseID.NO_MODEL_TIME)) {
-            maxtrys = 1;
-        }
-
-        while (trys < maxtrys) {
-            if (trys != 0) {
-                statusHandler.error("Attempting to recreate: ");
-            }
-
             db = new IFPGridDatabase(id, dbConfig);
-            // ServerResponse sr;
-            // sr = db->databaseIsValid();
-            // if (sr.okay())
             if (db.databaseIsValid()) {
-                break; // database is okay - continue processing
+                // get databaseID object from database
-            }
+                DatabaseID dbId = db.getDbId();
-            //
+
-            // // error on creating database
+                if (dbId.getRemovedDate() != null) {
-            // logProblem << "Database invalid with id: " << id << std::endl;
+                    // mark database as not removed
-            // logProblem << sr << std::endl;
+                    try {
-            // logProblem << "Deleting bogus database: " << id << std::endl;
+                        GFEDao gfeDao = new GFEDao();
-            //
+                        gfeDao.setDatabaseRemovedDate(dbId, null);
-            // // make list of files that match this "bad" database
+                        statusHandler.info("Database " + dbId + " restored");
-            // TextString dir = PathMgr::dirname(filename) + '/';
+                    } catch (Exception e) {
-            // TextString baseName = PathMgr::basename(filename);
+                        statusHandler.handle(Priority.PROBLEM,
-            // TextString dbName = PathMgr::stripExtension(baseName);
+                                "Unable to mark database restored: " + dbId, e);
-            // SeqOf<TextString> filenames = PathMgr::listDir(dir);
+                    }
-            // TextString badTime =
+                }
-            // AbsTime::current().string("/BADDB-%Y%m%d_%H%M-");
+
-            // unsigned int pos = 0;
+                // add to list of databases
-            // for (int i = 0; i < filenames.length(); i++)
+                addDB(db);
-            // if (PathMgr::isFile(dir + filenames[i])
+            } else {
-            // && filenames[i].found(dbName, pos))
-            // {
-            // TextString bdir = _config.baseDir() + "/BAD";
-            // PathMgr::mkdir(bdir);
-            // TextString sourceF = dir + filenames[i];
-            // TextString destF = bdir + badTime + filenames[i];
-            // TextString cmd = "mv " + sourceF + ' ' + destF;
-            // system(cmd.stringPtr());
-            // logProblem << "Bad database moved to: " << destF << std::endl;
-            // }
-            // //db->deleteDb(); // delete the database (not needed since we
-            // move
-            // // files to a "BAD" directory
-            // delete db;
-            // db = NULL;
-            if ((trys + 1) == maxtrys) { // final loop
                 status.addMessage("Database " + id + " is not valid.");
-                // status += sr;
+                db = null;
-                return null;
             }
-            trys++;
         }
 
-        if (db != null) {
+        if (db == null) {
-            addDB(db);
+            // mark database as removed
-        } // add to list of databases if not-NULL
+            try {
+                GFEDao gfeDao = new GFEDao();
+                gfeDao.setDatabaseRemovedDate(id, new Date());
+                statusHandler.warn("Database " + id + " marked for removal in "
+                        + GFEDao.REMOVED_DB_PURGE_TIME + " days.");
+            } catch (Exception e) {
+                statusHandler.handle(Priority.PROBLEM,
+                        "Unable to mark database removed: " + id, e);
+            }
+        }
 
         status.setPayload(db);
         return status;
@@ -1151,9 +1101,9 @@ public class GridParmManager {
         ServerResponse<GridDatabase> sr = new ServerResponse<GridDatabase>();
         for (DatabaseID dbId : inventory) {
             sr = createDB(dbId);
-        }
+            if (!sr.isOkay()) {
-        if (!sr.isOkay()) {
+                statusHandler.error(sr.message());
-            statusHandler.error(sr.message());
+            }
         }
 
         NetCDFDatabaseManager.initializeNetCDFDatabases(config);

edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/database/IFPGridDatabase.java

@@ -120,6 +120,7 @@ import com.vividsolutions.jts.geom.Coordinate;
  * 04/08/13     #1949      rjpeter     Updated to work with normalized database.
  * 05/02/13     #1969      randerso    Removed updateDbs from parent class
  * 06/13/13     #2044      randerso    Pass in GridDbConfig as construction parameter
+ * 07/30/13     #2057      randerso    Added a static deleteDatabase method
  * </pre>
  * 
  * @author bphillip
@@ -2462,6 +2463,16 @@ public class IFPGridDatabase extends GridDatabase {
     @Override
     public void deleteDb() {
         DatabaseID id = getDbId();
+        deleteDatabase(id);
+    }
+
+    /**
+     * Delete a database
+     * 
+     * @param id
+     *            the DatabaseID of the datbase to be deleted
+     */
+    public static void deleteDatabase(DatabaseID id) {
         try {
             GFEDao gfeDao = new GFEDao();
             gfeDao.purgeGFEGrids(id);
@@ -2470,7 +2481,7 @@ public class IFPGridDatabase extends GridDatabase {
                     "Unable to delete model database: " + id, e);
         }
 
-        this.deleteModelHDF5();
+        deleteModelHDF5(id);
     }
 
     /**
@@ -2537,7 +2548,7 @@ public class IFPGridDatabase extends GridDatabase {
         }
     }
 
-    private void deleteModelHDF5() {
+    private static void deleteModelHDF5(DatabaseID dbId) {
         File hdf5File = GfeUtil.getHdf5Dir(GridDatabase.gfeBaseDataDir, dbId);
         IDataStore ds = DataStoreFactory.getDataStore(hdf5File);
         try {

edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/db/objects/DatabaseID.java

@@ -42,7 +42,6 @@ import javax.persistence.Table;
 import javax.persistence.Transient;
 import javax.persistence.UniqueConstraint;
 
-import org.hibernate.annotations.Immutable;
 import org.hibernate.annotations.OnDelete;
 import org.hibernate.annotations.OnDeleteAction;
 
@@ -67,6 +66,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeTypeAdap
  * 03/28/13     1949       rjpeter     Normalized database structure.
  * 06/20/13     2127       rjpeter     Removed unused bidirectional relationship.
  * 06/13/13     2044       randerso    Code cleanup
+ * 07/31/13     2057       randerso    Added removedDate
  * </pre>
  * 
  * @author bphillip
@@ -75,7 +75,6 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeTypeAdap
 @Entity
 @Table(name = "gfe_dbid", uniqueConstraints = { @UniqueConstraint(columnNames = {
         "siteId", "modelName", "modelTime", "dbType" }) })
-@Immutable
 @DynamicSerialize
 @DynamicSerializeTypeAdapter(factory = DatabaseIDAdapter.class)
 public class DatabaseID implements Comparable<DatabaseID> {
@@ -147,6 +146,10 @@ public class DatabaseID implements Comparable<DatabaseID> {
     @DataURI(position = 2)
     private String modelTime = NO_MODEL_TIME;
 
+    /** Date database was removed from localConfig.py. */
+    @Column(nullable = true)
+    private Date removedDate;
+
     /** The model identifier */
     @Transient
     private String modelId;
@@ -478,6 +481,21 @@ public class DatabaseID implements Comparable<DatabaseID> {
         return modelTime;
     }
 
+    /**
+     * @return the removedDate
+     */
+    public Date getRemovedDate() {
+        return removedDate;
+    }
+
+    /**
+     * @param removedDate
+     *            the removedDate to set
+     */
+    public void setRemovedDate(Date removedDate) {
+        this.removedDate = removedDate;
+    }
+
     /**
      * @return the modelId
      */

edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/db/type/DatabaseIdType.java

@@ -1,121 +0,0 @@
-/**
- * This software was developed and / or modified by Raytheon Company,
- * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
- * 
- * U.S. EXPORT CONTROLLED TECHNICAL DATA
- * This software product contains export-restricted data whose
- * export/transfer/disclosure is restricted by U.S. law. Dissemination
- * to non-U.S. persons whether in the United States or abroad requires
- * an export license or other authorization.
- * 
- * Contractor Name:        Raytheon Company
- * Contractor Address:     6825 Pine Street, Suite 340
- *                         Mail Stop B8
- *                         Omaha, NE 68106
- *                         402.291.0100
- * 
- * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
- * further licensing information.
- **/
-
-package com.raytheon.uf.common.dataplugin.gfe.db.type;
-
-import java.io.Serializable;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Types;
-
-import org.hibernate.HibernateException;
-import org.hibernate.usertype.UserType;
-
-import com.raytheon.uf.common.dataplugin.gfe.db.objects.DatabaseID;
-
-/**
- * Custom Hibernate type for the DatabaseID class. This class defines how a
- * DatabaseID object is saved and retrieved from the database.
- * 
- * 
- * <pre>
- * SOFTWARE HISTORY
- * Date         Ticket#    Engineer    Description
- * ------------ ---------- ----------- --------------------------
- * 4/10/08      875        bphillip    Initial Creation
- * 
- * </pre>
- * 
- * @author bphillip
- * @version 1.0
- */
-public class DatabaseIdType implements UserType {
-
-    private static final int[] SQL_TYPES = { Types.VARCHAR };
-
-    @Override
-    public Object assemble(Serializable cached, Object owner)
-            throws HibernateException {
-        return cached;
-    }
-
-    @Override
-    public Object deepCopy(Object value) throws HibernateException {
-        return value;
-    }
-
-    @Override
-    public Serializable disassemble(Object value) throws HibernateException {
-        return (Serializable) value;
-    }
-
-    @Override
-    public boolean equals(Object x, Object y) throws HibernateException {
-        if (x instanceof DatabaseID && y instanceof DatabaseID) {
-            return x.equals(y);
-        } else {
-            return false;
-        }
-    }
-
-    @Override
-    public int hashCode(Object value) throws HibernateException {
-        return 0;
-    }
-
-    @Override
-    public boolean isMutable() {
-        return false;
-    }
-
-    @Override
-    public Object nullSafeGet(ResultSet resultSet, String[] names, Object owner)
-            throws HibernateException, SQLException {
-        return new DatabaseID(resultSet.getString(names[0]));
-    }
-
-    @Override
-    public void nullSafeSet(PreparedStatement statement, Object value, int index)
-            throws HibernateException, SQLException {
-        if (value == null) {
-            statement.setString(index, null);
-        } else {
-            statement.setString(index, value.toString());
-        }
-
-    }
-
-    @Override
-    public Object replace(Object original, Object target, Object owner)
-            throws HibernateException {
-        return original;
-    }
-
-    @Override
-    public Class<?> returnedClass() {
-        return DatabaseID.class;
-    }
-
-    @Override
-    public int[] sqlTypes() {
-        return DatabaseIdType.SQL_TYPES;
-    }
-}