From 7a734502b3c9533ac3977444193f1cc44aa0190f Mon Sep 17 00:00:00 2001 From: Nate Jensen Date: Fri, 15 Mar 2013 16:10:11 -0500 Subject: [PATCH] Issue #1795 speed up publish by optimizing publish time updates in grid histories Change-Id: I43c3f63d60832c49c6d02dab97c47d2b1f12f1c8 Former-commit-id: 3b331be9f66f481ac0ef8a10b5d8e43550b22c4d [formerly 30ddd909bf976caf0e7359bc894080c89bd30c93] [formerly e65267d5f304cd98f326b75ffd1f6393c8476e33] [formerly 3b331be9f66f481ac0ef8a10b5d8e43550b22c4d [formerly 30ddd909bf976caf0e7359bc894080c89bd30c93] [formerly e65267d5f304cd98f326b75ffd1f6393c8476e33] [formerly 235952a115b60d93382c020728e108f3ecd7288c [formerly e65267d5f304cd98f326b75ffd1f6393c8476e33 [formerly 788242460621edfffaf2a2c62c0c5b01da6c0e09]]]] Former-commit-id: 235952a115b60d93382c020728e108f3ecd7288c Former-commit-id: 568f988ef982407cb18539f7fb82a92a91aa2d93 [formerly 4ea1508baa29fd70542e7dce178eadd2ece449f7] [formerly 322940c9800f51152a2cd23a4f14a10dee589147 [formerly dad08657aa8c822126634d048b81c8087e824c12]] Former-commit-id: 3e00bfb092b747d3a0792b45f6f63ff35470bf34 [formerly d622b8c12ec587f346dd79862e1beacb25c7edff] Former-commit-id: 499209754a17c2975768ed309a740182a4bef17d --- .../edex/plugin/gfe/db/dao/GFEDao.java | 58 +++++++++++++++++++ .../edex/plugin/gfe/server/GridParm.java | 30 +++++++++- .../plugin/gfe/server/GridParmManager.java | 14 ++++- .../gfe/server/database/GridDatabase.java | 18 ++++++ .../gfe/server/database/IFPGridDatabase.java | 23 +++++++- 5 files changed, 136 insertions(+), 7 deletions(-) diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/db/dao/GFEDao.java b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/db/dao/GFEDao.java index 869f3d725e..b5ad4e1fa8 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/db/dao/GFEDao.java +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/db/dao/GFEDao.java @@ -28,6 +28,7 @@ import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -111,6 +112,7 @@ import com.raytheon.uf.edex.database.query.DatabaseQuery; * 01/21/12 #1504 randerso Back ported change to use ParameterMapper into 13.1.2 * 02/10/13 #1603 randerso Eliminated unnecessary conversion from lists to arrays * 02/12/13 #1608 randerso Changed to use explicit deletes for groups and datasets + * 03/15/13 #1795 njensen Added updatePublishTime() * * * @@ -478,6 +480,7 @@ public class GFEDao extends DefaultPluginDao { } @SuppressWarnings("unchecked") + @Deprecated public void updateGridHistories(ParmID parmId, Map> history) throws DataAccessLayerException { @@ -1138,4 +1141,59 @@ public class GFEDao extends DefaultPluginDao { && (id.getParmName().startsWith("mxt") || id.getParmName() .startsWith("mnt")); } + + /** + * Updates the publish times in the database of all provided + * GridDataHistories. Does not alter the publish times in memory. + * + * @param history + * the histories to alter in the database + * @param publishTime + * the publish time to update to + * @throws DataAccessLayerException + */ + public void updatePublishTime(List history, + Date publishTime) throws DataAccessLayerException { + StringBuilder query = new StringBuilder(); + query.append("update gfe_gridhistory set publishtime=:publishtime where key in ("); + Iterator itr = history.iterator(); + while (itr.hasNext()) { + query.append(itr.next().getKey()); + if (itr.hasNext()) { + query.append(","); + } + } + query.append(");"); + + Session sess = null; + Transaction tx = null; + + try { + sess = getHibernateTemplate().getSessionFactory().openSession(); + tx = sess.beginTransaction(); + Query q = sess.createSQLQuery(query.toString()); + q.setTimestamp("publishtime", publishTime); + q.executeUpdate(); + tx.commit(); + } catch (Exception e) { + if (tx != null) { + try { + tx.rollback(); + tx = null; + } catch (Exception e1) { + logger.error("Error occurred rolling back transaction", e1); + } + } + throw new DataAccessLayerException("Error updating history", e); + } finally { + if (sess != null) { + try { + sess.close(); + } catch (Exception e) { + statusHandler.error( + "Error occurred closing database session", e); + } + } + } + } } diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/GridParm.java b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/GridParm.java index b76ae4d73f..4cff93fdb1 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/GridParm.java +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/GridParm.java @@ -21,6 +21,7 @@ package com.raytheon.edex.plugin.gfe.server; import java.util.ArrayList; +import java.util.Collection; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -64,6 +65,7 @@ import com.raytheon.uf.edex.database.plugin.PluginFactory; * 04/08/08 #875 bphillip Initial Creation * 06/17/08 #940 bphillip Implemented GFE Locking * 02/10/13 #1603 randerso Returned number of records purged from timePurge + * 03/15/13 #1795 njensen Added updatePublishTime() * * * @@ -131,10 +133,34 @@ public class GridParm { return db.getGridHistory(id, trs); } + @Deprecated public ServerResponse updateGridHistory( Map> history) { - ServerResponse sr = new ServerResponse(); - db.updateGridHistory(id, history); + return db.updateGridHistory(id, history); + } + + /** + * Updates the publish times in the database of all provided + * GridDataHistories. Does not alter the publish times in memory. + * + * @param history + * the histories to alter in the database + * @param publishTime + * the publish time to update to + * @return + */ + public ServerResponse updatePublishTime( + Collection> history, Date publishTime) { + ServerResponse sr = null; + List oneList = new ArrayList(); + for (List innerList : history) { + oneList.addAll(innerList); + } + if (!oneList.isEmpty()) { + sr = db.updatePublishTime(oneList, publishTime); + } else { + sr = new ServerResponse(); + } return sr; } diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/GridParmManager.java b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/GridParmManager.java index 81f2b65070..5233f53657 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/GridParmManager.java +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/GridParmManager.java @@ -90,6 +90,7 @@ import com.raytheon.uf.edex.database.purge.PurgeLogger; * fixed error which caused D2D purging to remove * smartInit hdf5 data * 03/07/13 #1773 njensen Logged commitGrid() times + * 03/15/13 #1795 njensen Sped up commitGrid() * * * @@ -711,10 +712,11 @@ public class GridParmManager { histories.put(tr, histList); } - // update the histories into the source database, update the + // update the publish times in the source database, update the // notifications historyUpdateTimer.start(); - sr.addMessages(sourceGP.updateGridHistory(histories)); + sr.addMessages(sourceGP.updatePublishTime(histories.values(), + (Date) nowTime.clone())); // System.out.println("Updated " + histories.size() + " histories"); historyUpdateTimer.stop(); @@ -737,8 +739,14 @@ public class GridParmManager { destHistList.get(i).replaceValues(srcHistList.get(i)); } } + + // only need to update the publish time on the destination histories + // of grids that are not being saved (due to no changes), because + // the saveGridSlices() call below will update the publish time + // of the ones with changes historyUpdateTimer.start(); - destGP.updateGridHistory(destHistory); + destGP.updatePublishTime(destHistory.values(), + (Date) nowTime.clone()); historyUpdateTimer.stop(); // save data directly to the official database (bypassing diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/database/GridDatabase.java b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/database/GridDatabase.java index cb585a4a47..b9076247f2 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/database/GridDatabase.java +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/database/GridDatabase.java @@ -22,6 +22,7 @@ package com.raytheon.edex.plugin.gfe.server.database; import java.io.File; import java.util.Arrays; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -71,6 +72,7 @@ import com.raytheon.uf.common.time.TimeRange; * in the gfeBaseDataDir. * 02/10/13 #1603 randerso Moved removeFromDb, removeFromHDF5 and deleteModelHDF5 * methods down to IFPGridDatabase + * 03/15/13 #1795 njensen Added updatePublishTime() * * * @@ -442,6 +444,22 @@ public abstract class GridDatabase { + this.getClass().getName()); } + /** + * Updates the publish times in the database of all provided + * GridDataHistories. Does not alter the publish times in memory. + * + * @param history + * the histories to alter in the database + * @param publishTime + * the publish time to update to + * @return + */ + public ServerResponse updatePublishTime(List history, + Date publishTime) { + throw new UnsupportedOperationException("Not implemented for class " + + this.getClass().getName()); + } + public ServerResponse saveGridSlices(ParmID parmId, TimeRange tr, List sliceData, WsId requestor, List skipDelete) { diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/database/IFPGridDatabase.java b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/database/IFPGridDatabase.java index 1df012da14..a25c00e1ec 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/database/IFPGridDatabase.java +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/database/IFPGridDatabase.java @@ -25,6 +25,7 @@ import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -113,6 +114,7 @@ import com.vividsolutions.jts.io.WKTReader; * Added performance logging * 02/12/13 #1608 randerso Changed to explicitly call deleteGroups * 03/07/13 #1737 njensen Logged getGridData times + * 03/15/13 #1795 njensen Added updatePublishTime() * * * @@ -1174,6 +1176,7 @@ public class IFPGridDatabase extends GridDatabase { } @Override + @Deprecated public ServerResponse updateGridHistory(ParmID parmId, Map> history) { ServerResponse sr = new ServerResponse(); @@ -1188,7 +1191,6 @@ public class IFPGridDatabase extends GridDatabase { "Unable to update grid history!!", e); } return sr; - } /** @@ -2481,8 +2483,25 @@ public class IFPGridDatabase extends GridDatabase { Priority.PROBLEM, "Error deleting GFE model data from hdf5 for " + dbId.toString(), e); - } } + @Override + public ServerResponse updatePublishTime(List history, + Date publishTime) { + ServerResponse sr = new ServerResponse(); + GFEDao dao = null; + try { + dao = (GFEDao) PluginFactory.getInstance().getPluginDao("gfe"); + dao.updatePublishTime(history, publishTime); + } catch (PluginException e1) { + statusHandler.handle(Priority.PROBLEM, "Unable to get gfe dao", e1); + } catch (DataAccessLayerException e) { + statusHandler.handle(Priority.PROBLEM, + "Unable to update grid history!", e); + sr.addMessage("Error updating history"); + } + return sr; + } + }