xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge "Issue #2681: Update stats aggregate to work in discrete time chunks." into development

Browse source 
Former-commit-id: be9fffd922 [formerly 68cb07ffdb] [formerly 76442c8d0c] [formerly 0bb34b225f [formerly 76442c8d0c [formerly 4ec6b3cfc640600335fbeb3f3e5377ff62b2b5a9]]]
Former-commit-id: 0bb34b225f
Former-commit-id: 8eed7c669eaef414bb0341039c14313355a2da4a [formerly 5ae5323d74]
Former-commit-id: eb7d29b84c
This commit is contained in:
Richard Peter 2014-04-21 13:57:28 -05:00 • committed by Gerrit Code Review
commit c74c860742
2 changed files with 82 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

75
edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/AggregateManager.java
View file

@ -78,6 +78,7 @@ import com.raytheon.uf.edex.stats.util.ConfigLoader;
* Mar 27, 2013 1802 bphillip Made jaxb manager static and changed visibility of a method * Mar 27, 2013 1802 bphillip Made jaxb manager static and changed visibility of a method
* May 22, 2013 1917 rjpeter Added ability to save raw and aggregate stats, to reclaimSpace every scan call, * May 22, 2013 1917 rjpeter Added ability to save raw and aggregate stats, to reclaimSpace every scan call,
* and to not pretty print xml grouping information. * and to not pretty print xml grouping information.
* Apr 18, 2014 2681 rjpeter Updated scan to process in distinct chunks of time.
* </pre> * </pre>
* *
* @author jsanchez * @author jsanchez
@ -243,35 +244,65 @@ public class AggregateManager {
String type = entry.getKey(); String type = entry.getKey();
StatisticsEventConfig event = entry.getValue(); StatisticsEventConfig event = entry.getValue();
List<StatsRecord> records = null; List<StatsRecord> records = null;
Calendar minTime = statsRecordDao.retrieveMinTime(type);
do { if ((minTime != null) && minTime.before(timeToProcess)) {
// retrieve stats in blocks of 1000 /*
records = statsRecordDao.retrieveRecords(timeToProcess, type, * process in minute chunks to avoid overwhelming the database
2000); * and having consistent results if stats is down for a period
* of time.
*/
Calendar maxTime = (Calendar) minTime.clone();
maxTime.add(Calendar.MINUTE, 1);
if (!CollectionUtil.isNullOrEmpty(records)) { // not checking before since we want before or equal to
// sort events into time buckets while (!maxTime.after(timeToProcess) && maxTime.after(minTime)) {
Map<TimeRange, Multimap<StatsGroupingColumn, StatisticsEvent>> timeMap = sort( records = statsRecordDao.retrieveRecords(type, minTime,
event, records); maxTime);
for (Map.Entry<TimeRange, Multimap<StatsGroupingColumn, StatisticsEvent>> timeMapEntry : timeMap if (!CollectionUtil.isNullOrEmpty(records)) {
.entrySet()) { // sort events into time buckets
aggregate(event, timeMapEntry.getKey(), Map<TimeRange, Multimap<StatsGroupingColumn, StatisticsEvent>> timeMap = sort(
timeMapEntry.getValue()); event, records);
}
try { for (Map.Entry<TimeRange, Multimap<StatsGroupingColumn, StatisticsEvent>> timeMapEntry : timeMap
statsRecordDao.deleteAll(records); .entrySet()) {
} catch (Exception e) { aggregate(event, timeMapEntry.getKey(),
statusHandler.error("Error deleting stat records", e); timeMapEntry.getValue());
} }
count += records.size(); try {
if (event.getRawOfflineRetentionDays() >= 0) { statsRecordDao.deleteAll(records);
offline.writeStatsToDisk(event, timeMap); } catch (Exception e) {
statusHandler.error("Error deleting stat records",
e);
}
count += records.size();
if (event.getRawOfflineRetentionDays() >= 0) {
offline.writeStatsToDisk(event, timeMap);
}
// increment to next interval
minTime.add(Calendar.MINUTE, 1);
maxTime.add(Calendar.MINUTE, 1);
} else {
maxTime.add(Calendar.MINUTE, 1);
// check if at end of interval
if (maxTime.before(timeToProcess)) {
// no records found in interval, find next interval
minTime = statsRecordDao.retrieveMinTime(type);
if (minTime == null) {
break;
}
maxTime.setTimeInMillis(minTime.getTimeInMillis());
maxTime.add(Calendar.MINUTE, 1);
}
} }
} }
} while (!CollectionUtil.isNullOrEmpty(records)); }
} }
statsRecordDao.reclaimSpace(); statsRecordDao.reclaimSpace();

35
edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/dao/StatsDao.java
View file

@ -42,6 +42,7 @@ import com.raytheon.uf.edex.database.dao.SessionManagedDao;
* Aug 21, 2012 jsanchez Initial creation * Aug 21, 2012 jsanchez Initial creation
* Mar 18, 2013 1082 bphillip Modified to extend sessionmanagedDao and use spring injection * Mar 18, 2013 1082 bphillip Modified to extend sessionmanagedDao and use spring injection
* May 22, 2013 1917 rjpeter Added reclaimSpace. * May 22, 2013 1917 rjpeter Added reclaimSpace.
* Apr 18, 2014 2681 rjpeter Added retrieveMinTime.
* </pre> * </pre>
* *
* @author jsanchez * @author jsanchez
@ -56,7 +57,29 @@ public class StatsDao extends SessionManagedDao<Integer, StatsRecord> {
} }
/** /**
* Retrieves stat records that has a date before the limit. * Retrieves the earliest time in the stats table for a given data type.
*
* @param eventType
* @return
* @throws DataAccessLayerException
*/
public Calendar retrieveMinTime(String eventType)
throws DataAccessLayerException {
String hql = "select min(rec.date) from StatsRecord rec where rec.eventType = :eventType";
List<Object> results = this
.executeHQLQuery(hql, "eventType", eventType);
if ((results != null) && !results.isEmpty()) {
Object time = results.get(0);
if (time != null) {
return (Calendar) time;
}
}
return null;
}
/**
* Retrieves stat records that has a date in the time range.
* *
* @param limit * @param limit
* @param eventType * @param eventType
@ -66,11 +89,11 @@ public class StatsDao extends SessionManagedDao<Integer, StatsRecord> {
* size 0 will be returned. * size 0 will be returned.
* @throws DataAccessLayerException * @throws DataAccessLayerException
*/ */
public List<StatsRecord> retrieveRecords(Calendar limit, String eventType, public List<StatsRecord> retrieveRecords(String eventType,
int maxResults) throws DataAccessLayerException { Calendar minTime, Calendar maxTime) throws DataAccessLayerException {
String hql = "from StatsRecord rec where rec.eventType = :eventType and rec.date < :date order by rec.date asc"; String hql = "from StatsRecord rec where rec.eventType = :eventType and rec.date >= :minDate and rec.date < :maxDate order by rec.date asc";
return this.query(hql, maxResults, "eventType", eventType, "date", return this.query(hql, "eventType", eventType, "minDate", minTime,
limit); "maxDate", maxTime);
} }
@Override @Override