diff --git a/edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/plugin/PluginDao.java b/edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/plugin/PluginDao.java index 67e49c078a..82b2418d9b 100644 --- a/edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/plugin/PluginDao.java +++ b/edexOsgi/com.raytheon.uf.edex.database/src/com/raytheon/uf/edex/database/plugin/PluginDao.java @@ -112,6 +112,7 @@ import com.raytheon.uf.edex.database.query.DatabaseQuery; * May 07, 2013 1869 bsteffen Remove dataURI column from * PluginDataObject. * May 16, 2013 1869 bsteffen Rewrite dataURI property mappings. + * Jun 11, 2013 2090 djohnson Separate the hdf5 purge by ref time for reuse. * * * @@ -1353,47 +1354,13 @@ public abstract class PluginDao extends CoreDao { dataQuery.setMaxResults(500); - // fields for hdf5 purge - String previousFile = null; - StringBuilder pathBuilder = new StringBuilder(); - do { pdos = (List) this.queryByCriteria(dataQuery); if ((pdos != null) && !pdos.isEmpty()) { this.delete(pdos); if (trackHdf5 && (hdf5FileToUriPurged != null)) { - for (PluginDataObject pdo : pdos) { - pathBuilder.setLength(0); - IPersistable persist = (IPersistable) pdo; - pathBuilder - .append(PLUGIN_HDF5_DIR) - .append(pathProvider.getHDFPath( - this.pluginName, persist)) - .append(File.separatorChar) - .append(pathProvider.getHDFFileName( - this.pluginName, persist)); - String file = pathBuilder.toString(); - - if (trackToUri) { - List uriList = hdf5FileToUriPurged - .get(file); - if (uriList == null) { - // sizing to 50 as most data types have numerous - // entries in a file - uriList = new ArrayList(50); - hdf5FileToUriPurged.put(file, uriList); - } - uriList.add(file); - } else { - // only need to track file, tracking last file - // instead of constantly indexing hashMap - if (!file.equals(previousFile)) { - hdf5FileToUriPurged.put(file, null); - previousFile = file; - } - } - } + purgeHdf5ForPdos(trackToUri, hdf5FileToUriPurged, pdos); } results += pdos.size(); @@ -1404,6 +1371,59 @@ public abstract class PluginDao extends CoreDao { return results; } + /** + * Purge HDF5 data for a list of PDOs. Extracted as is from + * {@link #purgeDataByRefTime} so it can be reused. + * + * @param trackToUri + * If true will track each URI that needs to be deleted from + * HDF5, if false will only track the hdf5 files that need to be + * deleted. + * @param hdf5FileToUriPurged + * Map to be populated by purgeDataByRefTime of all the hdf5 + * files that need to be updated. If trackToUri is true, each + * file will have the exact data URI's to be removed from each + * file. If trackToUri is false, the map will have a null entry + * for the list and only track the files. + * @param pdos + * the pdos + */ + protected void purgeHdf5ForPdos(boolean trackToUri, + Map> hdf5FileToUriPurged, + List pdos) { + // fields for hdf5 purge + String previousFile = null; + for (PluginDataObject pdo : pdos) { + StringBuilder pathBuilder = new StringBuilder(); + IPersistable persist = (IPersistable) pdo; + pathBuilder + .append(PLUGIN_HDF5_DIR) + .append(pathProvider.getHDFPath(this.pluginName, persist)) + .append(File.separatorChar) + .append(pathProvider.getHDFFileName(this.pluginName, + persist)); + String file = pathBuilder.toString(); + + if (trackToUri) { + List uriList = hdf5FileToUriPurged.get(file); + if (uriList == null) { + // sizing to 50 as most data types have numerous + // entries in a file + uriList = new ArrayList(50); + hdf5FileToUriPurged.put(file, uriList); + } + uriList.add(file); + } else { + // only need to track file, tracking last file + // instead of constantly indexing hashMap + if (!file.equals(previousFile)) { + hdf5FileToUriPurged.put(file, null); + previousFile = file; + } + } + } + } + /** * Purges the HDF5 data according to the provided time and key. * diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.madis/src/com/raytheon/uf/edex/plugin/madis/MadisDao.java b/edexOsgi/com.raytheon.uf.edex.plugin.madis/src/com/raytheon/uf/edex/plugin/madis/MadisDao.java index f92377f55f..92cef73d5a 100644 --- a/edexOsgi/com.raytheon.uf.edex.plugin.madis/src/com/raytheon/uf/edex/plugin/madis/MadisDao.java +++ b/edexOsgi/com.raytheon.uf.edex.plugin.madis/src/com/raytheon/uf/edex/plugin/madis/MadisDao.java @@ -23,11 +23,13 @@ package com.raytheon.uf.edex.plugin.madis; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; +import java.util.Date; import java.util.List; import java.util.Map; import javax.xml.bind.JAXBException; +import com.raytheon.uf.common.dataplugin.PluginDataObject; import com.raytheon.uf.common.dataplugin.PluginException; import com.raytheon.uf.common.dataplugin.madis.MadisRecord; import com.raytheon.uf.common.dataquery.db.QueryParam; @@ -36,14 +38,16 @@ import com.raytheon.uf.common.pointdata.spatial.ObStation; import com.raytheon.uf.common.status.IUFStatusHandler; import com.raytheon.uf.common.status.UFStatus; import com.raytheon.uf.common.status.UFStatus.Priority; +import com.raytheon.uf.common.util.CollectionUtil; import com.raytheon.uf.edex.database.DataAccessLayerException; +import com.raytheon.uf.edex.database.plugin.PluginDao; import com.raytheon.uf.edex.database.query.DatabaseQuery; import com.raytheon.uf.edex.pointdata.PointDataDbDescription; import com.raytheon.uf.edex.pointdata.PointDataPluginDao; import com.raytheon.uf.edex.pointdata.spatial.ObStationDao; /** - * MadisDao MADIS data DAO + * MadisDao MADIS data DAO * * 
  * 
@@ -51,6 +55,7 @@ import com.raytheon.uf.edex.pointdata.spatial.ObStationDao;
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * MAR 27, 2013 1746       dhladky     MADIS data record creation
+ * Jun 11, 2013 2090       djohnson    Override purgeDataByRefTime to improve purge performance.
  * 
  *
* @@ -59,22 +64,22 @@ import com.raytheon.uf.edex.pointdata.spatial.ObStationDao; */ public class MadisDao extends PointDataPluginDao { - + /** The station dao */ private ObStationDao obDao = new ObStationDao(); - + private static final IUFStatusHandler statusHandler = UFStatus - .getHandler(MadisDao.class); - + .getHandler(MadisDao.class); + public List queryBySpatialBox(double upperLeftLat, double upperLeftLon, double lowerRightLat, double lowerRightLon) throws DataAccessLayerException { List stationList = obDao.queryBySpatialBox(upperLeftLat, upperLeftLon, lowerRightLat, lowerRightLon); - + List stationNames = new ArrayList(); - for (ObStation ob: stationList) { + for (ObStation ob : stationList) { stationNames.add(ob.getIcao()); } @@ -104,8 +109,8 @@ public class MadisDao extends PointDataPluginDao { */ public Object[] queryDataUriColumn(final String dataUri) { - String sql = "select datauri from awips.madis where datauri='" + dataUri - + "';"; + String sql = "select datauri from awips.madis where datauri='" + + dataUri + "';"; Object[] results = executeSQLQuery(sql); @@ -128,7 +133,8 @@ public class MadisDao extends PointDataPluginDao { } catch (JAXBException e) { statusHandler.error("Unable to load madis Point Data Description", e); - throw new PluginException("Unable to load madis Point Data Description!", e); + throw new PluginException( + "Unable to load madis Point Data Description!", e); } } @@ -138,7 +144,6 @@ public class MadisDao extends PointDataPluginDao { return hdf5DataDescription; } - public ObStationDao getObDao() { return obDao; } @@ -161,7 +166,7 @@ public class MadisDao extends PointDataPluginDao { public MadisRecord newObject() { return new MadisRecord(); } - + /* * (non-Javadoc) * @@ -196,5 +201,51 @@ public class MadisDao extends PointDataPluginDao { return dbDataDescription; } + /** + * Overridden because {@link PluginDao} retrieves all PDO instances prior to + * purging them, in order to calculute HDF5 paths for each one. In the case + * of {@link Madis} objects, the granularity is only down to the hour level + * therefore we can just pull one, calculate the HDF5 path, and purge all + * entities without retrieving them. + */ + @Override + public int purgeDataByRefTime(Date refTime, + Map productKeys, boolean trackHdf5, + boolean trackToUri, Map> hdf5FileToUriPurged) + throws DataAccessLayerException { + + DatabaseQuery dataQuery = new DatabaseQuery(this.daoClass); + + if (refTime != null) { + dataQuery.addQueryParam(PURGE_VERSION_FIELD, refTime); + } + + if ((productKeys != null) && (productKeys.size() > 0)) { + for (Map.Entry pair : productKeys.entrySet()) { + dataQuery.addQueryParam(pair.getKey(), pair.getValue()); + } + } + + // Keep track of the old max results so we can use the same query to + // find an example pdo to get the HDF5 path + final Integer oldMaxResults = dataQuery.getMaxResults(); + dataQuery.setMaxResults(1); + @SuppressWarnings("unchecked") + final List pdos = (List) this + .queryByCriteria(dataQuery); + if (CollectionUtil.isNullOrEmpty(pdos)) { + return 0; + } + + // Restore the old max results so it targets all entities + dataQuery.setMaxResults(oldMaxResults); + int numberDeleted = this.deleteByCriteria(dataQuery); + + if (trackHdf5 && (hdf5FileToUriPurged != null)) { + purgeHdf5ForPdos(trackToUri, hdf5FileToUriPurged, pdos); + } + + return numberDeleted; + } } diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.madis/utility/common_static/base/purge/madisPurgeRules.xml b/edexOsgi/com.raytheon.uf.edex.plugin.madis/utility/common_static/base/purge/madisPurgeRules.xml index 04291d909c..cbf056065f 100644 --- a/edexOsgi/com.raytheon.uf.edex.plugin.madis/utility/common_static/base/purge/madisPurgeRules.xml +++ b/edexOsgi/com.raytheon.uf.edex.plugin.madis/utility/common_static/base/purge/madisPurgeRules.xml @@ -1,6 +1,8 @@ + 00-03:00:00