From 84c906ccb284ebfb5ebee6d012a6eb593a44a06a Mon Sep 17 00:00:00 2001 From: Richard Peter Date: Fri, 18 Apr 2014 16:39:45 -0500 Subject: [PATCH] Issue #2681: Update stats aggregate to work in discrete time chunks. Change-Id: I999c6676d5b2464eb8f5467b5155f842c365798e Former-commit-id: 23a71c7f13b8adc409ac06c7c6f452b9f816cb18 [formerly 23a71c7f13b8adc409ac06c7c6f452b9f816cb18 [formerly 1d18f8f5b3fb1986f197c27b765dabec84343dac]] Former-commit-id: 78ab2738d078d882a7031eeedcfc863655bd3ccb Former-commit-id: 2b86613c4fbd29940bba2bd200cd22c4abe51cf9 --- .../uf/edex/stats/AggregateManager.java | 75 +++++++++++++------ .../raytheon/uf/edex/stats/dao/StatsDao.java | 35 +++++++-- 2 files changed, 82 insertions(+), 28 deletions(-) diff --git a/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/AggregateManager.java b/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/AggregateManager.java index 5536df811d..cc72935dfe 100644 --- a/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/AggregateManager.java +++ b/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/AggregateManager.java @@ -78,6 +78,7 @@ import com.raytheon.uf.edex.stats.util.ConfigLoader; * Mar 27, 2013 1802 bphillip Made jaxb manager static and changed visibility of a method * May 22, 2013 1917 rjpeter Added ability to save raw and aggregate stats, to reclaimSpace every scan call, * and to not pretty print xml grouping information. + * Apr 18, 2014 2681 rjpeter Updated scan to process in distinct chunks of time. * * * @author jsanchez @@ -243,35 +244,65 @@ public class AggregateManager { String type = entry.getKey(); StatisticsEventConfig event = entry.getValue(); List records = null; + Calendar minTime = statsRecordDao.retrieveMinTime(type); - do { - // retrieve stats in blocks of 1000 - records = statsRecordDao.retrieveRecords(timeToProcess, type, - 2000); + if ((minTime != null) && minTime.before(timeToProcess)) { + /* + * process in minute chunks to avoid overwhelming the database + * and having consistent results if stats is down for a period + * of time. + */ + Calendar maxTime = (Calendar) minTime.clone(); + maxTime.add(Calendar.MINUTE, 1); - if (!CollectionUtil.isNullOrEmpty(records)) { - // sort events into time buckets - Map> timeMap = sort( - event, records); + // not checking before since we want before or equal to + while (!maxTime.after(timeToProcess) && maxTime.after(minTime)) { + records = statsRecordDao.retrieveRecords(type, minTime, + maxTime); - for (Map.Entry> timeMapEntry : timeMap - .entrySet()) { - aggregate(event, timeMapEntry.getKey(), - timeMapEntry.getValue()); - } + if (!CollectionUtil.isNullOrEmpty(records)) { + // sort events into time buckets + Map> timeMap = sort( + event, records); - try { - statsRecordDao.deleteAll(records); - } catch (Exception e) { - statusHandler.error("Error deleting stat records", e); - } + for (Map.Entry> timeMapEntry : timeMap + .entrySet()) { + aggregate(event, timeMapEntry.getKey(), + timeMapEntry.getValue()); + } - count += records.size(); - if (event.getRawOfflineRetentionDays() >= 0) { - offline.writeStatsToDisk(event, timeMap); + try { + statsRecordDao.deleteAll(records); + } catch (Exception e) { + statusHandler.error("Error deleting stat records", + e); + } + + count += records.size(); + if (event.getRawOfflineRetentionDays() >= 0) { + offline.writeStatsToDisk(event, timeMap); + } + + // increment to next interval + minTime.add(Calendar.MINUTE, 1); + maxTime.add(Calendar.MINUTE, 1); + } else { + maxTime.add(Calendar.MINUTE, 1); + + // check if at end of interval + if (maxTime.before(timeToProcess)) { + // no records found in interval, find next interval + minTime = statsRecordDao.retrieveMinTime(type); + if (minTime == null) { + break; + } + + maxTime.setTimeInMillis(minTime.getTimeInMillis()); + maxTime.add(Calendar.MINUTE, 1); + } } } - } while (!CollectionUtil.isNullOrEmpty(records)); + } } statsRecordDao.reclaimSpace(); diff --git a/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/dao/StatsDao.java b/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/dao/StatsDao.java index 0093675017..6965d305eb 100644 --- a/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/dao/StatsDao.java +++ b/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/dao/StatsDao.java @@ -42,6 +42,7 @@ import com.raytheon.uf.edex.database.dao.SessionManagedDao; * Aug 21, 2012 jsanchez Initial creation * Mar 18, 2013 1082 bphillip Modified to extend sessionmanagedDao and use spring injection * May 22, 2013 1917 rjpeter Added reclaimSpace. + * Apr 18, 2014 2681 rjpeter Added retrieveMinTime. * * * @author jsanchez @@ -56,7 +57,29 @@ public class StatsDao extends SessionManagedDao { } /** - * Retrieves stat records that has a date before the limit. + * Retrieves the earliest time in the stats table for a given data type. + * + * @param eventType + * @return + * @throws DataAccessLayerException + */ + public Calendar retrieveMinTime(String eventType) + throws DataAccessLayerException { + String hql = "select min(rec.date) from StatsRecord rec where rec.eventType = :eventType"; + List results = this + .executeHQLQuery(hql, "eventType", eventType); + if ((results != null) && !results.isEmpty()) { + Object time = results.get(0); + if (time != null) { + return (Calendar) time; + } + } + + return null; + } + + /** + * Retrieves stat records that has a date in the time range. * * @param limit * @param eventType @@ -66,11 +89,11 @@ public class StatsDao extends SessionManagedDao { * size 0 will be returned. * @throws DataAccessLayerException */ - public List retrieveRecords(Calendar limit, String eventType, - int maxResults) throws DataAccessLayerException { - String hql = "from StatsRecord rec where rec.eventType = :eventType and rec.date < :date order by rec.date asc"; - return this.query(hql, maxResults, "eventType", eventType, "date", - limit); + public List retrieveRecords(String eventType, + Calendar minTime, Calendar maxTime) throws DataAccessLayerException { + String hql = "from StatsRecord rec where rec.eventType = :eventType and rec.date >= :minDate and rec.date < :maxDate order by rec.date asc"; + return this.query(hql, "eventType", eventType, "minDate", minTime, + "maxDate", maxTime); } @Override