diff --git a/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFFGForceUtil.java b/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFFGForceUtil.java index 3eeb5ec014..dac07a4582 100644 --- a/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFFGForceUtil.java +++ b/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFFGForceUtil.java @@ -24,7 +24,6 @@ import java.util.List; import com.raytheon.uf.common.dataplugin.ffmp.FFMPBasin; import com.raytheon.uf.common.dataplugin.ffmp.FFMPGuidanceInterpolation; -import com.raytheon.uf.common.dataplugin.ffmp.FFMPRecord; import com.raytheon.uf.common.dataplugin.ffmp.FFMPTemplates; import com.raytheon.uf.common.monitor.config.FFFGDataMgr; import com.raytheon.uf.common.monitor.config.FFMPSourceConfigurationManager; @@ -44,6 +43,7 @@ import com.raytheon.uf.common.monitor.xml.SourceXML; * 01/14/13 1569 dhladky changed arraylist to list * 04/15/13 1890 dhladky Changed COUNTY to use constant * 05/10/13 1919 mpduff If there are forced pfafs then the aggregate is forced. + * 05/22/13 1902 mpduff Added methods to get forced values. * * * @@ -135,10 +135,8 @@ public class FFFGForceUtil { pfafList = ft.getAggregatePfafs(cBasin.getPfaf(), resource.getSiteKey(), resource.getHuc()); } else if (!domain.equals("NA")) { - if (!resource.getHuc().equals(FFMPRecord.ALL)) { - pfafList = ft.getAggregatePfafsByDomain(cBasin.getPfaf(), - resource.getSiteKey(), domain, resource.getHuc()); - } + pfafList = ft.getAggregatePfafsByDomain(cBasin.getPfaf(), + resource.getSiteKey(), domain, resource.getHuc()); } // else use the existing pfaf list // Add current pfaf to the list @@ -253,7 +251,7 @@ public class FFFGForceUtil { float tvalue = 0.0f; float value; int i = 0; - if (interpolation.isInterpolate() == false) { + if (!interpolation.isInterpolate()) { FFFGDataMgr dman = FFFGDataMgr.getInstance(); for (long pfaf : forcedPfafs) { long countyFips = templates.getCountyFipsByPfaf(pfaf); @@ -266,6 +264,49 @@ public class FFFGForceUtil { } return tvalue / i; + } else { + // TODO interpolated code under new ticket + } + + return Float.NaN; + } + + /** + * Get the max forced value (max is smallest number for FFG) + * + * @param pfafList + * list of pfaf ids + * @param forcedPfafs + * list of forced pfaf ids + * @param interpolation + * FFMPGuidanceInterpolation object + * @param expiration + * force expiration + * @param templates + * ffmp templates + * @return max forced value + */ + public float getMaxForcedValue(List pfafList, List forcedPfafs, + FFMPGuidanceInterpolation interpolation, long expiration, + FFMPTemplates templates) { + float tvalue = 0.0f; + float value; + if (!interpolation.isInterpolate()) { + FFFGDataMgr dman = FFFGDataMgr.getInstance(); + for (long pfaf : forcedPfafs) { + long countyFips = templates.getCountyFipsByPfaf(pfaf); + templates.getCountyFipsByPfaf(pfaf); + value = dman.adjustValue(Float.NaN, + interpolation.getStandardSource(), pfaf, countyFips); + + if (value < tvalue) { + tvalue = value; + } + } + + return tvalue; + } else { + // TODO interpolated code } return Float.NaN; @@ -315,4 +356,40 @@ public class FFFGForceUtil { public void setSliderTime(double sliderTime) { this.sliderTime = sliderTime; } + + /** + * Get the forced values for the pfaf list. + * + * @param pfafList + * list of pfaf ids + * @param forcedPfafs + * list of forced pfafs + * @param ffmpGuidanceInterpolation + * FFMPGuidanceInterpolation object + * @param guidSourceExpiration + * expiration time + * @param ft + * ffmp templates + * @return list of forced guidance values + */ + public List getForcedGuidValues(List pfafList, + List forcedPfafs, + FFMPGuidanceInterpolation ffmpGuidanceInterpolation, + long guidSourceExpiration, FFMPTemplates ft) { + List guidList = new ArrayList(); + if (pfafList != null) { + for (Long pfaf : pfafList) { + if (pfaf == null) { + continue; + } + + List pl = new ArrayList(); + pl.add(pfaf); + float val = getAvgForcedValue(pl, forcedPfafs, + ffmpGuidanceInterpolation, guidSourceExpiration, ft); + guidList.add(val); + } + } + return guidList; + } } diff --git a/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPDataGenerator.java b/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPDataGenerator.java index eee705cf71..3ce931953a 100644 --- a/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPDataGenerator.java +++ b/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPDataGenerator.java @@ -79,6 +79,7 @@ import com.raytheon.uf.viz.monitor.ffmp.ui.dialogs.FfmpTableConfigData; * Apr 26, 2013 1954 bsteffen Minor code cleanup throughout FFMP. * May 07, 2013 1986 njensen Removed unnecessary sort * May 10, 2013 1919 mpduff Fixed problem with VGBs + * May 22, 2013 1902 mpduff Code cleanup. * * * @@ -207,10 +208,8 @@ public class FFMPDataGenerator { setFFMPRow(fbd.get(key), tData, false, cwa); } catch (Exception e) { - statusHandler - .handle(Priority.PROBLEM, - "Couldn't create table row" - + e); + statusHandler.handle(Priority.PROBLEM, + "Couldn't create table row", e); } if (virtualBasin != null) { for (Long id : ft @@ -257,10 +256,8 @@ public class FFMPDataGenerator { setFFMPRow(fbd.get(key), tData, isVGB, null); } catch (Exception e) { - statusHandler - .handle(Priority.PROBLEM, - "Couldn't create table row" - + e); + statusHandler.handle(Priority.PROBLEM, + "Couldn't create table row", e); } } } @@ -293,10 +290,10 @@ public class FFMPDataGenerator { virtualBasin.get(id), tData, true, null); } catch (Exception e) { - statusHandler.handle( - Priority.PROBLEM, - "Couldn't create table row" - + e); + statusHandler + .handle(Priority.PROBLEM, + "Couldn't create table row", + e); } } } @@ -414,6 +411,11 @@ public class FFMPDataGenerator { if (guidCellData == null) { // check for forcing even if no data are available guidance = getForcedAvg(domain, cBasin, guidType); + boolean forced = !guidance.isNaN(); + guidCellData = new FFMPTableCellData( + FIELDS.GUIDANCE, guidance, forced); + } else { + guidance = guidCellData.getValueAsFloat(); } trd.setTableCellData(i + 4, guidCellData); @@ -440,7 +442,6 @@ public class FFMPDataGenerator { } } else { displayName = getDisplayName(cBasin); - if (displayName != null) { long cBasinPfaf = cBasin.getPfaf(); String cBasinPfafStr = Long.toString(cBasinPfaf); @@ -498,6 +499,9 @@ public class FFMPDataGenerator { if (guidCellData == null) { // check for forcing even if no data are available guidance = getForcedAvg(domain, cBasin, guidType); + boolean forced = !guidance.isNaN(); + guidCellData = new FFMPTableCellData( + FIELDS.GUIDANCE, guidance, forced); } else { guidance = guidCellData.getValueAsFloat(); } @@ -587,11 +591,13 @@ public class FFMPDataGenerator { guidance, forcedPfafs, resource.getGuidSourceExpiration(guidType)); } else { - guidance = resource.getGuidanceValue(ffmpGuidBasin, paintRefTime, - guidType); + if (ffmpGuidBasin != null) { + guidance = resource.getGuidanceValue(ffmpGuidBasin, + paintRefTime, guidType); - if (guidance < 0.0f) { - guidance = Float.NaN; + if (guidance < 0.0f) { + guidance = Float.NaN; + } } } @@ -783,31 +789,30 @@ public class FFMPDataGenerator { FFMPBasinData guidBasin = guidBasins.get(guidType); List pfafList = new ArrayList(); + if (cBasin.getAggregated()) { + pfafList = ft.getAggregatePfafs(cBasin.getPfaf(), + siteKey, huc); + pfafList.add(ft.getAggregatedPfaf(cBasin.getPfaf(), + siteKey, huc)); + } + + boolean forced = false; + List forcedPfafs = new ArrayList(); + FFFGDataMgr fdm = FFFGDataMgr.getInstance(); + + if (fdm.isForcingConfigured()) { + forceUtil.calculateForcings(pfafList, ft, cBasin); + forcedPfafs = forceUtil.getForcedPfafList(); + forced = forceUtil.isForced(); + } + + if (!forced) { + if ((forcedPfafs != null) && (!forcedPfafs.isEmpty())) { + forced = true; + } + } if ((guidBasin != null) && (!guidBasin.getBasins().isEmpty())) { - if (cBasin.getAggregated()) { - pfafList = ft.getAggregatePfafs(cBasin.getPfaf(), - siteKey, huc); - pfafList.add(ft.getAggregatedPfaf(cBasin.getPfaf(), - siteKey, huc)); - } - - boolean forced = false; - List forcedPfafs = new ArrayList(); - FFFGDataMgr fdm = FFFGDataMgr.getInstance(); - - if (fdm.isForcingConfigured()) { - forceUtil.calculateForcings(pfafList, ft, cBasin); - forcedPfafs = forceUtil.getForcedPfafList(); - forced = forceUtil.isForced(); - } - - if (!forced) { - if ((forcedPfafs != null) - && (!forcedPfafs.isEmpty())) { - forced = true; - } - } if (isWorstCase) { guidance = guidRecords @@ -830,8 +835,19 @@ public class FFMPDataGenerator { trd.setTableCellData(i + 4, new FFMPTableCellData( FIELDS.GUIDANCE, guidance, forced)); } else { + if (forced) { + // Recalculate guidance using the forced value(s) + guidance = forceUtil.getMaxForcedValue( + pfafList, + forcedPfafs, + resource.getGuidanceInterpolators().get( + guidType), resource + .getGuidSourceExpiration(guidType), + ft); + } + trd.setTableCellData(i + 4, new FFMPTableCellData( - FIELDS.GUIDANCE, Float.NaN)); + FIELDS.GUIDANCE, guidance, forced)); } // If guidance is NaN then it cannot be > 0 @@ -846,6 +862,14 @@ public class FFMPDataGenerator { guids = guidBasin.getGuidanceValues(pfafs, resource .getGuidanceInterpolators().get(guidType), resource.getGuidSourceExpiration(guidType)); + } else if (forced) { + guids = forceUtil.getForcedGuidValues( + pfafList, + forcedPfafs, + resource.getGuidanceInterpolators().get( + guidType), resource + .getGuidSourceExpiration(guidType), + ft); } if ((!qpes.isEmpty()) diff --git a/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPDataLoader.java b/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPDataLoader.java index 79c025cc12..6128e1fb79 100644 --- a/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPDataLoader.java +++ b/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPDataLoader.java @@ -72,6 +72,7 @@ import com.raytheon.uf.viz.monitor.ffmp.ui.listeners.FFMPLoaderEvent; * Apr 9, 2013 1890 dhladky removed loading of phantom Virtual template and cache file processing. * Apr 18, 2013 1912 bsteffen Increase bulk requests to pypies. * Apr 26, 2013 1954 bsteffen Minor code cleanup throughout FFMP. + * May 22, 2013 1902 mpduff Check for null times. * * * @@ -105,9 +106,9 @@ public class FFMPDataLoader extends Thread { private FFMPConfig config = null; - private ArrayList loadListeners = new ArrayList(); + private final ArrayList loadListeners = new ArrayList(); - private CountDownLatch latch; + private final CountDownLatch latch; public FFMPDataLoader(FFMPResourceData resourceData, Date timeBack, Date mostRecentTime, LOADER_TYPE loadType, List hucsToLoad) { @@ -195,9 +196,8 @@ public class FFMPDataLoader extends Thread { } if ((loadType == LOADER_TYPE.INITIAL || loadType == LOADER_TYPE.GENERAL) && !product.getRate().equals(product.getQpe())) { - Map> rateURIs = monitor - .getAvailableUris(siteKey, dataKey, product.getRate(), - mostRecentTime); + Map> rateURIs = monitor.getAvailableUris( + siteKey, dataKey, product.getRate(), mostRecentTime); if (rateURIs.containsKey(mostRecentTime)) { rateURI = rateURIs.get(mostRecentTime).get(0); } @@ -243,11 +243,13 @@ public class FFMPDataLoader extends Thread { NavigableMap> iguidURIs = null; Date guidTime = timeBack; - if (loadType == LOADER_TYPE.GENERAL) { guidTime = monitor.getPreviousQueryTime(siteKey, guidSource.getSourceName()); } + if (guidTime == null) { + continue; + } iguidURIs = monitor.getAvailableUris(siteKey, dataKey, guidSource.getSourceName(), guidTime); @@ -292,10 +294,11 @@ public class FFMPDataLoader extends Thread { SourceXML source = sourceConfig.getSource(product.getQpe()); - qpeCache = readAggregateRecord(source, dataKey, wfo); + qpeCache = readAggregateRecord(source, dataKey, wfo); if (qpeCache != null) { - monitor.insertFFMPData(qpeCache, qpeURIs, siteKey, product.getQpe()); + monitor.insertFFMPData(qpeCache, qpeURIs, siteKey, + product.getQpe()); } } diff --git a/cave/com.raytheon.uf.viz.stats/src/com/raytheon/uf/viz/stats/utils/StatsUiUtils.java b/cave/com.raytheon.uf.viz.stats/src/com/raytheon/uf/viz/stats/utils/StatsUiUtils.java index db4fb8618b..f6f453efad 100644 --- a/cave/com.raytheon.uf.viz.stats/src/com/raytheon/uf/viz/stats/utils/StatsUiUtils.java +++ b/cave/com.raytheon.uf.viz.stats/src/com/raytheon/uf/viz/stats/utils/StatsUiUtils.java @@ -28,7 +28,7 @@ import com.google.common.annotations.VisibleForTesting; import com.raytheon.uf.common.stats.data.StatsEventData; import com.raytheon.uf.common.stats.xml.StatisticsAggregate; import com.raytheon.uf.common.stats.xml.StatisticsConfig; -import com.raytheon.uf.common.stats.xml.StatisticsEvent; +import com.raytheon.uf.common.stats.xml.StatisticsEventConfig; import com.raytheon.uf.common.stats.xml.StatisticsGroup; /** @@ -40,7 +40,8 @@ import com.raytheon.uf.common.stats.xml.StatisticsGroup; * * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- - * Nov 8, 2012 728 mpduff Initial creation + * Nov 8, 2012 728 mpduff Initial creation + * May 22, 2013 1917 rjpeter Renamed StatisticsEvent to StatisticsEventConfig * * * @@ -83,7 +84,7 @@ public class StatsUiUtils { */ @VisibleForTesting void processConfig(StatisticsConfig config) { - for (StatisticsEvent event: config.getEvents()) { + for (StatisticsEventConfig event: config.getEvents()) { processEvent(event); } } @@ -94,7 +95,7 @@ public class StatsUiUtils { * @param event event config object */ @VisibleForTesting - void processEvent(StatisticsEvent event) { + void processEvent(StatisticsEventConfig event) { if (!eventMap.containsKey(event.getCategory())) { eventMap.put(event.getCategory(), new HashMap()); } @@ -143,7 +144,7 @@ public class StatsUiUtils { public Map getEventAttributes(String category, String type) { Map attMap = new TreeMap(); for (StatisticsConfig config: configList) { - for (StatisticsEvent event: config.getEvents()) { + for (StatisticsEventConfig event: config.getEvents()) { if (event.getCategory().equals(category) && event.getDisplayName().equals(type)) { for (StatisticsAggregate agg: event.getAggregateList()) { attMap.put(agg.getDisplayName(), agg.getField()); @@ -186,7 +187,7 @@ public class StatsUiUtils { public StatisticsAggregate getAggregateConfig(String category, String typeID, String attributeDisplayName) { for (StatisticsConfig config : configList) { - for (StatisticsEvent event: config.getEvents()) { + for (StatisticsEventConfig event: config.getEvents()) { if (event.getCategory().equals(category) && event.getType().equals(typeID)) { for (StatisticsAggregate agg: event.getAggregateList()) { if (agg.getDisplayName().equals(attributeDisplayName)) { diff --git a/cave/com.raytheon.viz.mpe.ui/src/com/raytheon/viz/mpe/ui/dialogs/gagetable/GageTableDataManager.java b/cave/com.raytheon.viz.mpe.ui/src/com/raytheon/viz/mpe/ui/dialogs/gagetable/GageTableDataManager.java index f870f3f8e3..7bd34caf8e 100644 --- a/cave/com.raytheon.viz.mpe.ui/src/com/raytheon/viz/mpe/ui/dialogs/gagetable/GageTableDataManager.java +++ b/cave/com.raytheon.viz.mpe.ui/src/com/raytheon/viz/mpe/ui/dialogs/gagetable/GageTableDataManager.java @@ -59,6 +59,8 @@ import com.vividsolutions.jts.geom.Coordinate; * May 29, 2009 2476 mpduff Initial creation. * Jan 28, 2010 4415 mpduff Fixed problem with column * header creation. + * May 20, 2013 15962 lbousaidi Added a new routine getRadarIdsTrue() + * for Radar Sites dialog. * * * @@ -252,6 +254,30 @@ public class GageTableDataManager { return radarIds; } + + /** + * Get the list of Radar Ids from radarloc. + * only the one with use_radar= T + * @return the radarIds + * @throws VizException + */ + public String[] getRadarIdsTrue() throws VizException { + + if (radarIds == null) { + String query = "select radid from radarloc where use_radar='T' " + + "order by radid asc"; + List rs = DirectDbQuery.executeQuery(query, + HydroConstants.IHFS, QueryLanguage.SQL); + + radarIds = new String[rs.size()]; + for (int i = 0; i < rs.size(); i++) { + Object[] oa = rs.get(i); + radarIds[i] = (String) oa[0]; + } + } + + return radarIds; + } /** * Lookup the Radar Id for the gage. diff --git a/cave/com.raytheon.viz.mpe.ui/src/com/raytheon/viz/mpe/ui/dialogs/hourlyradar/RadarSiteSelectionDlg.java b/cave/com.raytheon.viz.mpe.ui/src/com/raytheon/viz/mpe/ui/dialogs/hourlyradar/RadarSiteSelectionDlg.java index b000266087..22b29b456c 100644 --- a/cave/com.raytheon.viz.mpe.ui/src/com/raytheon/viz/mpe/ui/dialogs/hourlyradar/RadarSiteSelectionDlg.java +++ b/cave/com.raytheon.viz.mpe.ui/src/com/raytheon/viz/mpe/ui/dialogs/hourlyradar/RadarSiteSelectionDlg.java @@ -48,7 +48,8 @@ import com.raytheon.viz.ui.dialogs.CaveSWTDialog; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Jul 21, 2009 mpduff Initial creation - * + * May 20, 2013 15962 lbousaidi changed getActiveRadarIds() call to + * getRadarIdsTrue(). * * * @author mpduff @@ -180,7 +181,7 @@ public class RadarSiteSelectionDlg extends CaveSWTDialog { private void populateBox() { String[] radarIds = null; try { - radarIds = GageTableDataManager.getInstance().getActiveRadarIds(); + radarIds = GageTableDataManager.getInstance().getRadarIdsTrue(); for (String s : radarIds) { radarListBox.add(s); } diff --git a/deltaScripts/13.4.1/removeOldStatAggregates.sh b/deltaScripts/13.4.1/removeOldStatAggregates.sh new file mode 100644 index 0000000000..734b2dbbf6 --- /dev/null +++ b/deltaScripts/13.4.1/removeOldStatAggregates.sh @@ -0,0 +1,11 @@ +#!/bin/bash + + +# 1917 Removes old aggregate format/layout +echo "Removing old stat aggregates" +rm -rf /awips2/edex/data/utility/common_static/site/*/stats/aggregates + +# run full vacuum on stats table, code keeps table more stable +PSQL="/awips2/psql/bin/psql" +echo "Running full vacuum on stats" +${PSQL} -U awips -d metadata -c "VACUUM FULL ANALYZE events.stats;" diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/res/spring/gfe-request.xml b/edexOsgi/com.raytheon.edex.plugin.gfe/res/spring/gfe-request.xml index 1779967557..4d0a5a14d1 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/res/spring/gfe-request.xml +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/res/spring/gfe-request.xml @@ -309,6 +309,20 @@ value="com.raytheon.uf.common.dataplugin.gfe.request.CreateNewDbRequest" /> + + + + + + + + + + diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/db/dao/GFEDao.java b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/db/dao/GFEDao.java index 75c9c60c19..5d03b2df1e 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/db/dao/GFEDao.java +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/db/dao/GFEDao.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; import java.util.Collection; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -55,6 +56,7 @@ import com.raytheon.uf.common.dataplugin.gfe.server.notify.LockNotification; import com.raytheon.uf.common.dataplugin.gfe.type.Pair; import com.raytheon.uf.common.dataplugin.gfe.util.GfeUtil; import com.raytheon.uf.common.dataplugin.persist.IPersistable; +import com.raytheon.uf.common.dataquery.db.QueryParam.QueryOperand; import com.raytheon.uf.common.datastorage.DataStoreFactory; import com.raytheon.uf.common.datastorage.IDataStore; import com.raytheon.uf.common.status.UFStatus.Priority; @@ -62,6 +64,7 @@ import com.raytheon.uf.common.time.TimeRange; import com.raytheon.uf.common.util.CollectionUtil; import com.raytheon.uf.edex.database.DataAccessLayerException; import com.raytheon.uf.edex.database.purge.PurgeLogger; +import com.raytheon.uf.edex.database.query.DatabaseQuery; /** * Data access object for manipulating GFE Records @@ -87,6 +90,8 @@ import com.raytheon.uf.edex.database.purge.PurgeLogger; * 03/15/13 #1795 njensen Added updatePublishTime() * 03/21/13 #1774 randerso Moved D2D routines into {@link com.raytheon.edex.plugin.gfe.db.dao.GFED2DDao} * 04/08/13 #1949 rjpeter Normalized GFE Database. + * 05/22/13 #2025 dgilling Re-implement functions needed by + * GetLatestDbTimeRequest and GetLatestModelDbIdRequest. * * * @author bphillip @@ -1100,4 +1105,52 @@ public class GFEDao extends DefaultPluginDao { } } } + + @SuppressWarnings("unchecked") + public Date getMaxInsertTimeByDbId(final DatabaseID dbId) + throws DataAccessLayerException { + DatabaseQuery query = new DatabaseQuery(this.daoClass); + query.addQueryParam("parmId.dbId", getDatabaseId(dbId), + QueryOperand.EQUALS); + query.addReturnedField("insertTime"); + query.addOrder("insertTime", false); + query.setMaxResults(1); + + List result = (List) this.queryByCriteria(query); + if (!result.isEmpty()) { + return result.get(0).getTime(); + } else { + return null; + } + } + + @SuppressWarnings("unchecked") + public DatabaseID getLatestDbIdByModelName(final String siteId, + final String modelName) throws DataAccessLayerException { + // TODO: Should this be done from GridParmManager? + List results = Collections.emptyList(); + try { + final String[] queryParams = { siteId, modelName }; + results = (List) txTemplate + .execute(new TransactionCallback() { + @Override + public List doInTransaction( + TransactionStatus status) { + return getHibernateTemplate() + .find("FROM DatabaseID WHERE siteId = ? AND modelName = ? ORDER BY modelTime DESC LIMIT 1", + queryParams); + } + }); + } catch (Exception e) { + throw new DataAccessLayerException( + "Unable to look up database inventory for site " + siteId, + e); + } + + if (!results.isEmpty()) { + return results.get(0); + } else { + return null; + } + } } diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/isc/IscScript.java b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/isc/IscScript.java index fa710dbd69..d16e4827b1 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/isc/IscScript.java +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/isc/IscScript.java @@ -46,6 +46,8 @@ import com.raytheon.uf.common.util.FileUtil; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Mar 11, 2013 dgilling Initial creation + * May 22, 2013 #1759 dgilling Ensure addSitePath() also adds base + * path. * * * @@ -127,6 +129,7 @@ public class IscScript extends PythonScript { .getValue("sys.path.index('" + basePath + "')"); } else { index = (Integer) jep.getValue("len(sys.path)"); + jep.eval("sys.path.insert(" + index + ", '" + basePath + "')"); } jep.eval("sys.path.insert(" + index + ", '" + sitePath + "')"); } diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/GetLatestDbTimeHandler.java b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/GetLatestDbTimeHandler.java new file mode 100644 index 0000000000..7b728ecdb1 --- /dev/null +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/GetLatestDbTimeHandler.java @@ -0,0 +1,61 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.edex.plugin.gfe.server.handler; + +import java.util.Date; + +import com.raytheon.edex.plugin.gfe.db.dao.GFEDao; +import com.raytheon.uf.common.dataplugin.gfe.request.GetLatestDbTimeRequest; +import com.raytheon.uf.common.serialization.comm.IRequestHandler; + +/** + * Handler for getting the latest insert time for a given database ID. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Aug 16, 2010  6349      bphillip     Initial creation
+ * May 22, 2013  2025      dgilling     Re-implement for new GFE db schema.
+ * 
+ *
+ * + * @author bphillip + * @version 1.0 + */ + +public class GetLatestDbTimeHandler implements + IRequestHandler { + + /* + * (non-Javadoc) + * + * @see + * com.raytheon.uf.common.serialization.comm.IRequestHandler#handleRequest + * (com.raytheon.uf.common.serialization.comm.IServerRequest) + */ + @Override + public Date handleRequest(GetLatestDbTimeRequest request) throws Exception { + GFEDao dao = new GFEDao(); + return dao.getMaxInsertTimeByDbId(request.getDbId()); + } +} diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/GetLatestModelDbIdHandler.java b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/GetLatestModelDbIdHandler.java new file mode 100644 index 0000000000..cf38ceca2d --- /dev/null +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/GetLatestModelDbIdHandler.java @@ -0,0 +1,63 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.edex.plugin.gfe.server.handler; + +import com.raytheon.edex.plugin.gfe.db.dao.GFEDao; +import com.raytheon.uf.common.dataplugin.gfe.db.objects.DatabaseID; +import com.raytheon.uf.common.dataplugin.gfe.request.GetLatestModelDbIdRequest; +import com.raytheon.uf.common.serialization.comm.IRequestHandler; + +/** + * Handler for getting the latest DatabaseID for a given model name and site ID. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Aug 17, 2010            dgilling     Initial creation
+ * May 22, 2013  2025      dgilling     Re-implement for new GFE db schema.
+ * 
+ *
+ * + * @author dgilling + * @version 1.0 + */ + +public class GetLatestModelDbIdHandler implements + IRequestHandler { + + /* + * (non-Javadoc) + * + * @see + * com.raytheon.uf.common.serialization.comm.IRequestHandler#handleRequest + * (com.raytheon.uf.common.serialization.comm.IServerRequest) + */ + @Override + public DatabaseID handleRequest(GetLatestModelDbIdRequest request) + throws Exception { + GFEDao dao = new GFEDao(); + return dao.getLatestDbIdByModelName(request.getSiteID(), + request.getModelName()); + } + +} diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/gfe/isc/IrtServer.py b/edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/gfe/isc/IrtServer.py index 437cf96733..72512699e9 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/gfe/isc/IrtServer.py +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/gfe/isc/IrtServer.py @@ -42,6 +42,8 @@ from com.raytheon.edex.plugin.gfe.isc import IRTManager # 03/13/13 1759 dgilling Move siteConfig imports into # functions where module is used # to interact better with IscScript. +# 05/22/13 1759 dgilling Add missing import to +# makeISCrequest(). # # # @@ -244,6 +246,7 @@ def irtGetServers(ancfURL, bncfURL, iscWfosWanted): # xmlRequest is the original request from the GFE's ISCRequestDialog. def makeISCrequest(xmlRequest, gridDims, gridProj, gridBoundBox, mhs, host, port, protocol, site, xmtScript): import IrtAccess + import siteConfig import xml from xml.etree import ElementTree from xml.etree.ElementTree import Element, SubElement diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/gfe/isc/ifpnetCDF.py b/edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/gfe/isc/ifpnetCDF.py index bad2e53d98..448fc1e692 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/gfe/isc/ifpnetCDF.py +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/gfe/isc/ifpnetCDF.py @@ -64,7 +64,7 @@ from com.raytheon.uf.common.localization import LocalizationContext_Localization # 03/11/13 1759 dgilling Removed unneeded methods. # 04/23/13 1937 dgilling Reimplement WECache to match # A1, big perf improvement. -# +# 05/23/13 1759 dgilling Remove unnecessary imports. # # @@ -77,7 +77,6 @@ ifpNetcdfLogger=None ## Logging methods ## def initLogger(logFile=None): global ifpNetcdfLogger - import logging, siteConfig ifpNetcdfLogger = iscUtil.getLogger("ifpnetCDF",logFile) def logEvent(*msg): diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/gfe/isc/iscMosaic.py b/edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/gfe/isc/iscMosaic.py index d60d1c825e..5ac2bcf270 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/gfe/isc/iscMosaic.py +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/gfe/isc/iscMosaic.py @@ -79,6 +79,7 @@ from com.raytheon.uf.edex.database.cluster import ClusterTask # 04/24/13 1941 dgilling Re-port WECache to match A1. # 05/08/13 1988 dgilling Fix history handling bug in # __getDbGrid(). +# 05/23/13 1759 dgilling Remove unnecessary imports. # # @@ -248,7 +249,6 @@ class IscMosaic: ## Logging methods ## def __initLogger(self): - import logging, siteConfig self.__logger=iscUtil.getLogger("iscMosaic",self.__logFile) def __init__(self, args): diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/utility/common_static/base/stats/ffmpStats.xml b/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/utility/common_static/base/stats/ffmpStats.xml index 52d8172725..15035ea734 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/utility/common_static/base/stats/ffmpStats.xml +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/utility/common_static/base/stats/ffmpStats.xml @@ -1,7 +1,8 @@ + displayName="Load Time" category="FFMP Load Times" + rawOfflineRetentionDays="90" aggregateOfflineRetentionDays="90"> diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/request/GetLatestDbTimeRequest.java b/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/request/GetLatestDbTimeRequest.java new file mode 100644 index 0000000000..26d7234c9d --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/request/GetLatestDbTimeRequest.java @@ -0,0 +1,84 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.common.dataplugin.gfe.request; + +import com.raytheon.uf.common.dataplugin.gfe.db.objects.DatabaseID; +import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; +import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; + +/** + * Request object for getting the latest insert time for a given database ID. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Aug 16, 2010  6349      bphillip     Initial creation
+ * May 22, 2013  2025      dgilling     Add DynamicSerialize support.
+ * 
+ *
+ * + * @author bphillip + * @version 1.0 + */ + +@DynamicSerialize +public class GetLatestDbTimeRequest extends AbstractGfeRequest { + + @DynamicSerializeElement + /** The database ID to get the latest insert time for */ + private DatabaseID dbId; + + public GetLatestDbTimeRequest() { + // no-op + } + + /** + * Creates a new GetLatestDbTimeRequest + * + * @param dbId + * The database ID to get the latest insert time for + */ + public GetLatestDbTimeRequest(DatabaseID dbId) { + super(); + this.dbId = dbId; + } + + /** + * Creates a new GetLatestDbTimeRequest + * + * @param dbId + * The database ID to get the latest insert time for + */ + public GetLatestDbTimeRequest(String dbId) { + super(); + this.dbId = new DatabaseID(dbId); + } + + public DatabaseID getDbId() { + return dbId; + } + + public void setDbId(DatabaseID dbId) { + this.dbId = dbId; + } +} diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/request/GetLatestModelDbIdRequest.java b/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/request/GetLatestModelDbIdRequest.java new file mode 100644 index 0000000000..b5137a6b5d --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/request/GetLatestModelDbIdRequest.java @@ -0,0 +1,87 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.common.dataplugin.gfe.request; + +import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; +import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; + +/** + * Request object for getting the latest database ID for a given model name and + * site ID. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Aug 17, 2010            dgilling     Initial creation
+ * May 22, 2013  2025      dgilling     Add DynamicSerialize support.
+ * 
+ *
+ * + * @author dgilling + * @version 1.0 + */ + +@DynamicSerialize +public class GetLatestModelDbIdRequest extends AbstractGfeRequest { + + /** + * The model name to perform the request for. + */ + @DynamicSerializeElement + private String modelName; + + public GetLatestModelDbIdRequest() { + // no-op + } + + /** + * Creates a new GetLatestModelDbIdRequest object given a model name and + * site identifier. + * + * @param siteId + * The site identifier to search for. + * @param modelName + * The name of the model to search for. + */ + public GetLatestModelDbIdRequest(String siteId, String modelName) { + super(); + this.modelName = modelName; + this.siteID = siteId; + } + + public String getSiteId() { + return getSiteID(); + } + + public void setSiteId(String siteId) { + setSiteID(siteId); + } + + public String getModelName() { + return modelName; + } + + public void setModelName(String modelName) { + this.modelName = modelName; + } +} diff --git a/edexOsgi/com.raytheon.uf.common.serialization/src/com/raytheon/uf/common/serialization/JAXBManager.java b/edexOsgi/com.raytheon.uf.common.serialization/src/com/raytheon/uf/common/serialization/JAXBManager.java index 2e3c859a8e..10e4f79754 100644 --- a/edexOsgi/com.raytheon.uf.common.serialization/src/com/raytheon/uf/common/serialization/JAXBManager.java +++ b/edexOsgi/com.raytheon.uf.common.serialization/src/com/raytheon/uf/common/serialization/JAXBManager.java @@ -49,7 +49,7 @@ import com.raytheon.uf.common.status.UFStatus.Priority; * ------------ ---------- ----------- -------------------------- * Sep 24, 2008 chammack Initial creation * Nov 13, 2008 njensen Added thrift methods - * + * May 22, 2013 1917 rjpeter Added non-pretty print option to jaxb serialize methods. * * * @author chammack @@ -81,7 +81,7 @@ public class JAXBManager { private static class MaintainEventsValidationHandler implements ValidationEventHandler { - private ArrayList events = new ArrayList( + private final ArrayList events = new ArrayList( 0); @Override @@ -105,9 +105,9 @@ public class JAXBManager { private final JAXBContext jaxbContext; - private Queue unmarshallers = new ConcurrentLinkedQueue(); + private final Queue unmarshallers = new ConcurrentLinkedQueue(); - private Queue marshallers = new ConcurrentLinkedQueue(); + private final Queue marshallers = new ConcurrentLinkedQueue(); public JAXBManager(Class... clazz) throws JAXBException { jaxbContext = JAXBContext.newInstance(clazz); @@ -164,7 +164,7 @@ public class JAXBManager { return obj; } finally { handleEvents(msh, null); - if (msh != null && unmarshallers.size() < QUEUE_SIZE) { + if ((msh != null) && (unmarshallers.size() < QUEUE_SIZE)) { unmarshallers.add(msh); } } @@ -221,8 +221,8 @@ public class JAXBManager { } /** - * Convert an instance of a class to an XML representation in a string. Uses - * JAXB. + * Convert an instance of a class to an XML pretty print representation in a + * string. Uses JAXB. * * @param obj * Object being marshalled @@ -230,19 +230,51 @@ public class JAXBManager { * @throws JAXBException */ public String marshalToXml(Object obj) throws JAXBException { + return marshalToXml(obj, true); + } + + /** + * Convert an instance of a class to an XML representation in a string. Uses + * JAXB. + * + * @param obj + * Object being marshalled + * @param formattedOutput + * True if the output should be xml pretty print. + * @return XML string representation of the object + * @throws JAXBException + */ + public String marshalToXml(Object obj, boolean formatedOutput) + throws JAXBException { Marshaller msh = getMarshaller(); try { StringWriter writer = new StringWriter(); - msh.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, new Boolean(true)); + msh.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, new Boolean( + formatedOutput)); msh.marshal(obj, writer); return writer.toString(); } finally { - if (msh != null && marshallers.size() < QUEUE_SIZE) { + if ((msh != null) && (marshallers.size() < QUEUE_SIZE)) { marshallers.add(msh); } } } + /** + * Convert an instance of a class to an XML representation and writes pretty + * print formatted XML to file. Uses JAXB. + * + * @param obj + * Object to be marshaled + * @param filePath + * Path to the output file + * @throws SerializationException + */ + public void jaxbMarshalToXmlFile(Object obj, String filePath) + throws SerializationException { + jaxbMarshalToXmlFile(obj, filePath, true); + } + /** * Convert an instance of a class to an XML representation and write XML to * file. Uses JAXB. @@ -251,21 +283,24 @@ public class JAXBManager { * Object to be marshaled * @param filePath * Path to the output file + * @param formattedOutput + * True if the output should be xml pretty print. * @throws SerializationException */ - public void jaxbMarshalToXmlFile(Object obj, String filePath) - throws SerializationException { + public void jaxbMarshalToXmlFile(Object obj, String filePath, + boolean formattedOutput) throws SerializationException { FileWriter writer = null; Marshaller msh = null; try { msh = getMarshaller(); - msh.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, new Boolean(true)); + msh.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, new Boolean( + formattedOutput)); writer = new FileWriter(new File(filePath)); msh.marshal(obj, writer); } catch (Exception e) { throw new SerializationException(e); } finally { - if (msh != null && marshallers.size() < QUEUE_SIZE) { + if ((msh != null) && (marshallers.size() < QUEUE_SIZE)) { marshallers.add(msh); } if (writer != null) { @@ -315,7 +350,7 @@ public class JAXBManager { if (msh != null) { handleEvents(msh, file.getName()); } - if (msh != null && unmarshallers.size() < QUEUE_SIZE) { + if ((msh != null) && (unmarshallers.size() < QUEUE_SIZE)) { unmarshallers.add(msh); } if (reader != null) { @@ -350,7 +385,7 @@ public class JAXBManager { if (msh != null) { handleEvents(msh, null); } - if (msh != null && unmarshallers.size() < QUEUE_SIZE) { + if ((msh != null) && (unmarshallers.size() < QUEUE_SIZE)) { unmarshallers.add(msh); } if (is != null) { diff --git a/edexOsgi/com.raytheon.uf.common.stats/META-INF/MANIFEST.MF b/edexOsgi/com.raytheon.uf.common.stats/META-INF/MANIFEST.MF index b2726a09ef..b93347bb29 100644 --- a/edexOsgi/com.raytheon.uf.common.stats/META-INF/MANIFEST.MF +++ b/edexOsgi/com.raytheon.uf.common.stats/META-INF/MANIFEST.MF @@ -20,4 +20,5 @@ Require-Bundle: com.raytheon.uf.common.time;bundle-version="1.12.1174", com.raytheon.uf.common.status;bundle-version="1.12.1174", javax.measure;bundle-version="1.0.0", com.raytheon.uf.common.units;bundle-version="1.0.0", - org.apache.commons.lang;bundle-version="2.3.0" + org.apache.commons.lang;bundle-version="2.3.0", + org.hibernate diff --git a/edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/StatsGrouping.java b/edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/StatsGrouping.java index 3995883cbc..c1d063d831 100644 --- a/edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/StatsGrouping.java +++ b/edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/StatsGrouping.java @@ -33,8 +33,8 @@ import javax.xml.bind.annotation.XmlRootElement; * * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- - * Jan 15, 2013 1487 djohnson Initial creation - * + * Jan 15, 2013 1487 djohnson Initial creation + * May 22, 2013 1917 rjpeter Added hashCode and equals. * * * @author djohnson @@ -98,4 +98,41 @@ public class StatsGrouping { this.value = value; } + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((name == null) ? 0 : name.hashCode()); + result = prime * result + ((value == null) ? 0 : value.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + StatsGrouping other = (StatsGrouping) obj; + if (name == null) { + if (other.name != null) { + return false; + } + } else if (!name.equals(other.name)) { + return false; + } + if (value == null) { + if (other.value != null) { + return false; + } + } else if (!value.equals(other.value)) { + return false; + } + return true; + } } diff --git a/edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/StatsGroupingColumn.java b/edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/StatsGroupingColumn.java index b63902407a..b75eeb47af 100644 --- a/edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/StatsGroupingColumn.java +++ b/edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/StatsGroupingColumn.java @@ -37,8 +37,8 @@ import com.google.common.collect.Lists; * * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- - * Jan 15, 2013 1487 djohnson Initial creation - * + * Jan 15, 2013 1487 djohnson Initial creation + * May 22, 2013 1917 rjpeter Added hashCode and equals. * * * @author djohnson @@ -84,4 +84,34 @@ public class StatsGroupingColumn { return column; } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((group == null) ? 0 : group.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + StatsGroupingColumn other = (StatsGroupingColumn) obj; + if (group == null) { + if (other.group != null) { + return false; + } + } else if (!group.equals(other.group)) { + return false; + } + return true; + } } diff --git a/edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/StatsRecord.java b/edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/StatsRecord.java index 7c23b9f571..51c3f5aa65 100644 --- a/edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/StatsRecord.java +++ b/edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/StatsRecord.java @@ -31,6 +31,8 @@ import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; +import org.hibernate.annotations.BatchSize; + import com.raytheon.uf.common.dataplugin.persist.PersistableDataObject; import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; @@ -43,14 +45,14 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; * SOFTWARE HISTORY * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- - * Aug 21, 2012 jsanchez Initial creation - * + * Aug 21, 2012 jsanchez Initial creation + * May 22, 2013 1917 rjpeter Added BatchSize annotation. * * * @author jsanchez - * */ @Entity +@BatchSize(size = 500) @Table(name = "stats", schema = "events") @XmlRootElement @XmlAccessorType(XmlAccessType.NONE) diff --git a/edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/xml/StatisticsConfig.java b/edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/xml/StatisticsConfig.java index 580683cda6..4c65f26db5 100644 --- a/edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/xml/StatisticsConfig.java +++ b/edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/xml/StatisticsConfig.java @@ -43,8 +43,8 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; * * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- - * Nov 6, 2012 728 mpduff Initial creation. - * + * Nov 6, 2012 728 mpduff Initial creation. + * May 22, 2013 1917 rjpeter Renamed StatisticsEvent to StatisticsEventConfig. * * * @author mpduff @@ -54,14 +54,14 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; @XmlRootElement(name = "statisticsConfig") @XmlAccessorType(XmlAccessType.NONE) public class StatisticsConfig implements ISerializableObject { - @XmlElements({ @XmlElement(name = "statisticsEvent", type = StatisticsEvent.class) }) + @XmlElements({ @XmlElement(name = "statisticsEvent", type = StatisticsEventConfig.class) }) @DynamicSerializeElement - private List events; + private List events; /** * @return the events */ - public List getEvents() { + public List getEvents() { return events; } @@ -69,7 +69,7 @@ public class StatisticsConfig implements ISerializableObject { * @param events * the events to set */ - public void setEvents(List events) { + public void setEvents(List events) { this.events = events; } @@ -81,7 +81,7 @@ public class StatisticsConfig implements ISerializableObject { public List getCategories() { Set categories = new HashSet(); if (events != null && events.size() > 0) { - for (StatisticsEvent event : events) { + for (StatisticsEventConfig event : events) { categories.add(event.getCategory()); } } diff --git a/edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/xml/StatisticsEvent.java b/edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/xml/StatisticsEventConfig.java similarity index 77% rename from edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/xml/StatisticsEvent.java rename to edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/xml/StatisticsEventConfig.java index 5f58b5a3d5..1c9c729aae 100644 --- a/edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/xml/StatisticsEvent.java +++ b/edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/xml/StatisticsEventConfig.java @@ -42,8 +42,9 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; * * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- - * Nov 6, 2012 728 mpduff Initial creation. - * + * Nov 6, 2012 728 mpduff Initial creation. + * May 22, 2013 1917 rjpeter Renamed to StatisticsEventConfig and + * added offline retention settings. * * * @author mpduff @@ -52,7 +53,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; @DynamicSerialize @XmlRootElement(name = "event") @XmlAccessorType(XmlAccessType.NONE) -public class StatisticsEvent { +public class StatisticsEventConfig { @XmlAttribute @DynamicSerializeElement @@ -66,6 +67,22 @@ public class StatisticsEvent { @DynamicSerializeElement private String category; + /** + * Retention period for the raw offline statistic to be saved. Value < 0 do + * not retain, value = 0 retain all, value > 0 retain for value days. + */ + @XmlAttribute + @DynamicSerializeElement + private int rawOfflineRetentionDays = -1; + + /** + * Retention period for the aggregate offline statistic to be saved. Value < + * 0 do not retain, value = 0 retain all, value > 0 retain for value days. + */ + @XmlAttribute + @DynamicSerializeElement + private int aggregateOfflineRetentionDays; + @XmlElements({ @XmlElement(name = "statisticsGroup", type = StatisticsGroup.class) }) @DynamicSerializeElement private List groupList; @@ -179,4 +196,20 @@ public class StatisticsEvent { this.aggregateMethods = aggregateMethods; } + public int getRawOfflineRetentionDays() { + return rawOfflineRetentionDays; + } + + public void setRawOfflineRetentionDays(int rawOfflineRetentionDays) { + this.rawOfflineRetentionDays = rawOfflineRetentionDays; + } + + public int getAggregateOfflineRetentionDays() { + return aggregateOfflineRetentionDays; + } + + public void setAggregateOfflineRetentionDays( + int aggregateOfflineRetentionDays) { + this.aggregateOfflineRetentionDays = aggregateOfflineRetentionDays; + } } diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.event/resources/stats.properties b/edexOsgi/com.raytheon.uf.edex.datadelivery.event/resources/stats.properties deleted file mode 100644 index e3dddc196a..0000000000 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.event/resources/stats.properties +++ /dev/null @@ -1,4 +0,0 @@ -# scan interval of stats table in minutes -stats.scanInterval=15 -# bucket interval or period of when to aggregate in minutes -stats.period=5 \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/utility/common_static/base/stats/retrievalProcessStats.xml b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/utility/common_static/base/stats/retrievalProcessStats.xml index 2759dde1d3..8cf4608dc5 100644 --- a/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/utility/common_static/base/stats/retrievalProcessStats.xml +++ b/edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/utility/common_static/base/stats/retrievalProcessStats.xml @@ -1,7 +1,8 @@ + displayName="Subscription Retrieval" category="Data Delivery" + rawOfflineRetentionDays="-1" aggregateOfflineRetentionDays="90"> diff --git a/edexOsgi/com.raytheon.uf.edex.registry.ebxml/utility/common_static/base/stats/registryProcessStats.xml b/edexOsgi/com.raytheon.uf.edex.registry.ebxml/utility/common_static/base/stats/registryProcessStats.xml index 7c3ed01ee8..f545649602 100644 --- a/edexOsgi/com.raytheon.uf.edex.registry.ebxml/utility/common_static/base/stats/registryProcessStats.xml +++ b/edexOsgi/com.raytheon.uf.edex.registry.ebxml/utility/common_static/base/stats/registryProcessStats.xml @@ -1,7 +1,8 @@ + displayName="Registry Statistics" category="Registry" + rawOfflineRetentionDays="-1" aggregateOfflineRetentionDays="90"> diff --git a/edexOsgi/com.raytheon.uf.edex.stats/META-INF/MANIFEST.MF b/edexOsgi/com.raytheon.uf.edex.stats/META-INF/MANIFEST.MF index 2c0c6cafe7..a2b6bf81d5 100644 --- a/edexOsgi/com.raytheon.uf.edex.stats/META-INF/MANIFEST.MF +++ b/edexOsgi/com.raytheon.uf.edex.stats/META-INF/MANIFEST.MF @@ -10,6 +10,7 @@ Require-Bundle: com.raytheon.uf.common.serialization;bundle-version="1.12.1174", com.raytheon.uf.common.event;bundle-version="1.0.0", com.google.guava;bundle-version="1.0.0", com.raytheon.uf.edex.database;bundle-version="1.0.0", + com.raytheon.edex.common, com.raytheon.uf.common.localization;bundle-version="1.12.1174", com.raytheon.uf.common.dataquery;bundle-version="1.0.0", com.raytheon.uf.common.time;bundle-version="1.12.1174", diff --git a/edexOsgi/com.raytheon.uf.edex.stats/res/spring/edex-process-stats.xml b/edexOsgi/com.raytheon.uf.edex.stats/res/spring/edex-process-stats.xml deleted file mode 100644 index cf2b57d112..0000000000 --- a/edexOsgi/com.raytheon.uf.edex.stats/res/spring/edex-process-stats.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - java.lang.Throwable - - - - - - - diff --git a/edexOsgi/com.raytheon.uf.edex.stats/res/spring/stats-graph-request.xml b/edexOsgi/com.raytheon.uf.edex.stats/res/spring/stats-graph-request.xml deleted file mode 100644 index 9748fb9b8f..0000000000 --- a/edexOsgi/com.raytheon.uf.edex.stats/res/spring/stats-graph-request.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - diff --git a/edexOsgi/com.raytheon.uf.edex.stats/res/spring/stats-ingest.xml b/edexOsgi/com.raytheon.uf.edex.stats/res/spring/stats-ingest.xml new file mode 100644 index 0000000000..4bf8289144 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.stats/res/spring/stats-ingest.xml @@ -0,0 +1,66 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + java.lang.Throwable + + + + + + + + + + + java.lang.Throwable + + + + + + + + + + + java.lang.Throwable + + + + + + diff --git a/edexOsgi/com.raytheon.uf.edex.stats/res/spring/stats-request.xml b/edexOsgi/com.raytheon.uf.edex.stats/res/spring/stats-request.xml index 41e6dd11eb..0a25914fd6 100644 --- a/edexOsgi/com.raytheon.uf.edex.stats/res/spring/stats-request.xml +++ b/edexOsgi/com.raytheon.uf.edex.stats/res/spring/stats-request.xml @@ -1,19 +1,17 @@ - - - - + - - + + + + + + + + diff --git a/edexOsgi/com.raytheon.uf.edex.stats/resources/edexProcessStats.properties b/edexOsgi/com.raytheon.uf.edex.stats/resources/edexProcessStats.properties deleted file mode 100644 index fe31fc60f3..0000000000 --- a/edexOsgi/com.raytheon.uf.edex.stats/resources/edexProcessStats.properties +++ /dev/null @@ -1,4 +0,0 @@ -# scan interval of stats table in minutes -stats.scanInterval=2 -# bucket interval or period of when to aggregate in minutes -stats.period=5 \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.edex.stats/resources/stats.properties b/edexOsgi/com.raytheon.uf.edex.stats/resources/stats.properties new file mode 100644 index 0000000000..b4ce41010f --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.stats/resources/stats.properties @@ -0,0 +1,8 @@ +# scan interval of stats table in minutes +stats.scanInterval=2 + +# When to save off aggregate data to csv format +stats.aggregateToCsv.cron=0+10+*+*+*+? + +# When to run purge of aggregate tables and csv files +stats.purge.cron=0+15+*+*+*+? diff --git a/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/AggregateManager.java b/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/AggregateManager.java index 40af314884..30687edd42 100644 --- a/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/AggregateManager.java +++ b/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/AggregateManager.java @@ -24,6 +24,7 @@ import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Calendar; import java.util.Collection; +import java.util.Date; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -35,15 +36,15 @@ import javax.xml.bind.JAXBException; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; -import com.raytheon.uf.common.event.Event; import com.raytheon.uf.common.serialization.JAXBManager; import com.raytheon.uf.common.serialization.SerializationUtil; import com.raytheon.uf.common.stats.AggregateRecord; +import com.raytheon.uf.common.stats.StatisticsEvent; import com.raytheon.uf.common.stats.StatsGrouping; import com.raytheon.uf.common.stats.StatsGroupingColumn; import com.raytheon.uf.common.stats.StatsRecord; import com.raytheon.uf.common.stats.xml.StatisticsAggregate; -import com.raytheon.uf.common.stats.xml.StatisticsEvent; +import com.raytheon.uf.common.stats.xml.StatisticsEventConfig; import com.raytheon.uf.common.stats.xml.StatisticsGroup; import com.raytheon.uf.common.status.IUFStatusHandler; import com.raytheon.uf.common.status.UFStatus; @@ -66,10 +67,12 @@ import com.raytheon.uf.edex.stats.util.ConfigLoader; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Aug 21, 2012 jsanchez Stored the aggregate buckets in the db. - * Nov 07, 2012 1317 mpduff Updated Configuration Files. - * Nov 28, 2012 1350 rjpeter Simplied aggregation and added aggregation with current db aggregate records. + * Nov 07, 2012 1317 mpduff Updated Configuration Files. + * Nov 28, 2012 1350 rjpeter Simplied aggregation and added aggregation with current db aggregate records. * Jan 07, 2013 1451 djohnson Use newGmtCalendar(). * Jan 15, 2013 1487 djohnson Use xml for the grouping information on an {@link AggregateRecord}. + * May 22, 2013 1917 rjpeter Added ability to save raw and aggregate stats, to reclaimSpace every scan call, + * and to not pretty print xml grouping information. * * * @author jsanchez @@ -96,9 +99,6 @@ public class AggregateManager { /** default value */ private static final int defaultBucketInterval = 5; - /** default value */ - private static final int defaultScanInterval = 15; - public AggregateManager(String bucketInterval) { validateIntervals(bucketInterval); } @@ -112,8 +112,10 @@ public class AggregateManager { * @param timeRange * @param groupedEvents */ - private void aggregate(AggregateRecordDao dao, StatisticsEvent statsEvent, - TimeRange timeRange, Multimap groupedEvents) { + private void aggregate(AggregateRecordDao dao, + StatisticsEventConfig statsEvent, TimeRange timeRange, + Multimap groupedEvents) + throws JAXBException { Calendar start = TimeUtil.newGmtCalendar(); start.setTime(timeRange.getStart()); @@ -121,8 +123,10 @@ public class AggregateManager { end.setTime(timeRange.getEnd()); // perform aggregate functions on the grouped data - for (String groupKey : groupedEvents.keySet()) { - Collection groupData = groupedEvents.get(groupKey); + for (StatsGroupingColumn group : groupedEvents.keySet()) { + Collection groupData = groupedEvents.get(group); + String groupKey = JAXB_MANAGER.marshalToXml(group, false); + Iterator aggrMethodIter = statsEvent.getAggregateMethods() .iterator(); Iterator statAggrIter = statsEvent @@ -138,7 +142,7 @@ public class AggregateManager { double min = Double.MAX_VALUE; double sum = 0; - for (Event event : groupData) { + for (StatisticsEvent event : groupData) { Number number = (Number) m.invoke(event, new Object[0]); double value = number.doubleValue(); sum += value; @@ -217,9 +221,10 @@ public class AggregateManager { long t0 = System.currentTimeMillis(); ConfigLoader configLoader = ConfigLoader.getInstance(); StatsDao statsRecordDao = new StatsDao(); + OfflineStatsManager offline = new OfflineStatsManager(); AggregateRecordDao aggrRecordDao = new AggregateRecordDao(); - - Map statsMap = configLoader.getTypeView(); + Map statsMap = configLoader + .getTypeView(); // latest time to pull Calendar timeToProcess = Calendar.getInstance(TimeZone @@ -227,9 +232,10 @@ public class AggregateManager { int count = 0; // process the events by type - for (Map.Entry entry : statsMap.entrySet()) { + for (Map.Entry entry : statsMap + .entrySet()) { String type = entry.getKey(); - StatisticsEvent event = entry.getValue(); + StatisticsEventConfig event = entry.getValue(); List records = null; do { @@ -239,10 +245,10 @@ public class AggregateManager { if (!CollectionUtil.isNullOrEmpty(records)) { // sort events into time buckets - Map> timeMap = sort( + Map> timeMap = sort( event, records); - for (Map.Entry> timeMapEntry : timeMap + for (Map.Entry> timeMapEntry : timeMap .entrySet()) { aggregate(aggrRecordDao, event, timeMapEntry.getKey(), timeMapEntry.getValue()); @@ -255,10 +261,14 @@ public class AggregateManager { } count += records.size(); + if (event.getRawOfflineRetentionDays() >= 0) { + offline.writeStatsToDisk(event, timeMap); + } } } while (!CollectionUtil.isNullOrEmpty(records)); } + statsRecordDao.reclaimSpace(); long t1 = System.currentTimeMillis(); statusHandler.info("Aggregated " + count + " stat events in " + (t1 - t0) + " ms"); @@ -270,11 +280,11 @@ public class AggregateManager { * @param records * @return */ - private Map> sort( - StatisticsEvent statEvent, List records) { - Map> rval = new HashMap>(); + private Map> sort( + StatisticsEventConfig statEvent, List records) { + Map> rval = new HashMap>(); TimeRange timeRange = null; - Multimap eventsByGroup = null; + Multimap eventsByGroup = null; for (StatsRecord record : records) { if ((timeRange == null) @@ -290,13 +300,13 @@ public class AggregateManager { try { // get underlying event - Event event = SerializationUtil.transformFromThrift( - Event.class, record.getEvent()); + StatisticsEvent event = SerializationUtil.transformFromThrift( + StatisticsEvent.class, record.getEvent()); - String groupAsString = determineGroupRepresentationForEvent( + StatsGroupingColumn group = determineGroupRepresentationForEvent( statEvent, event); - if (groupAsString != null) { - eventsByGroup.put(groupAsString, event); + if (group != null) { + eventsByGroup.put(group, event); } } catch (Exception e) { statusHandler @@ -309,10 +319,9 @@ public class AggregateManager { } @VisibleForTesting - static String determineGroupRepresentationForEvent( - StatisticsEvent statEvent, Event event) - throws IllegalAccessException, InvocationTargetException, - JAXBException { + static StatsGroupingColumn determineGroupRepresentationForEvent( + StatisticsEventConfig statEvent, StatisticsEvent event) + throws IllegalAccessException, InvocationTargetException { Iterator gMethodIter = statEvent.getGroupByMethods().iterator(); Iterator gFieldNameIter = statEvent.getGroupList() .iterator(); @@ -322,14 +331,13 @@ public class AggregateManager { Method m = gMethodIter.next(); String field = gFieldNameIter.next().getName(); String gVal = String.valueOf(m.invoke(event, EMPTY_OBJ_ARR)); - groupings.add(new StatsGrouping(field, gVal)); } StatsGroupingColumn column = new StatsGroupingColumn(); column.setGroup(groupings); - return JAXB_MANAGER.marshalToXml(column); + return column; } /** @@ -361,4 +369,68 @@ public class AggregateManager { + bucketInterval + "'"); } } + + /** + * Scans the aggregate table for aggregate statistics to offline. It doesn't + * process any aggregate from within the 12 hours. + */ + public void offlineAggregates() { + ConfigLoader configLoader = ConfigLoader.getInstance(); + OfflineStatsManager offline = new OfflineStatsManager(); + AggregateRecordDao aggrRecordDao = new AggregateRecordDao(); + Map statsMap = configLoader + .getTypeView(); + + // offline aggregate data older than 6 hours + long maxTime = (System.currentTimeMillis() / TimeUtil.MILLIS_PER_HOUR - 6) + * TimeUtil.MILLIS_PER_HOUR; + + for (StatisticsEventConfig conf : statsMap.values()) { + if (conf.getAggregateOfflineRetentionDays() >= 0) { + String eventType = conf.getType(); + + try { + Date oldestAggregateDate = aggrRecordDao + .getOldestAggregateDate(eventType); + if (oldestAggregateDate != null) { + Date mostRecentOfflineDate = offline + .getMostRecentOfflinedAggregate(conf); + + long startHour = oldestAggregateDate.getTime() + / TimeUtil.MILLIS_PER_HOUR; + + if (mostRecentOfflineDate != null) { + // move ahead one hour from most recent time on disk + long offlineHour = mostRecentOfflineDate.getTime() + / TimeUtil.MILLIS_PER_HOUR + 1; + if (offlineHour > startHour) { + startHour = offlineHour; + } + } + + Date startDate = new Date(startHour + * TimeUtil.MILLIS_PER_HOUR); + // process an hour at a time + Date endDate = new Date(startDate.getTime() + + TimeUtil.MILLIS_PER_HOUR); + while (endDate.getTime() <= maxTime) { + List records = aggrRecordDao + .getAggregates(eventType, startDate, + endDate); + offline.writeAggregatesToDisk(conf, records); + startDate = endDate; + endDate = new Date(startDate.getTime() + + TimeUtil.MILLIS_PER_HOUR); + } + } + } catch (Exception e) { + statusHandler.error( + "Error occured generating offline aggregates for event " + + conf.getType(), e); + } + } + } + + // zip up old data? + } } diff --git a/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/OfflineStatsManager.java b/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/OfflineStatsManager.java new file mode 100644 index 0000000000..d5f4fd46e9 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/OfflineStatsManager.java @@ -0,0 +1,599 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.edex.stats; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.lang.reflect.Method; +import java.text.DecimalFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Collection; +import java.util.Date; +import java.util.Iterator; +import java.util.Map; +import java.util.TimeZone; + +import javax.xml.bind.JAXBException; + +import com.google.common.collect.Multimap; +import com.raytheon.edex.util.Util; +import com.raytheon.uf.common.localization.IPathManager; +import com.raytheon.uf.common.localization.LocalizationContext; +import com.raytheon.uf.common.localization.LocalizationContext.LocalizationLevel; +import com.raytheon.uf.common.localization.LocalizationContext.LocalizationType; +import com.raytheon.uf.common.localization.LocalizationFile; +import com.raytheon.uf.common.localization.PathManagerFactory; +import com.raytheon.uf.common.localization.exception.LocalizationException; +import com.raytheon.uf.common.stats.AggregateRecord; +import com.raytheon.uf.common.stats.StatisticsEvent; +import com.raytheon.uf.common.stats.StatsGrouping; +import com.raytheon.uf.common.stats.StatsGroupingColumn; +import com.raytheon.uf.common.stats.xml.StatisticsAggregate; +import com.raytheon.uf.common.stats.xml.StatisticsEventConfig; +import com.raytheon.uf.common.stats.xml.StatisticsGroup; +import com.raytheon.uf.common.status.IUFStatusHandler; +import com.raytheon.uf.common.status.UFStatus; +import com.raytheon.uf.common.status.UFStatus.Priority; +import com.raytheon.uf.common.time.TimeRange; +import com.raytheon.uf.common.time.util.TimeUtil; +import com.raytheon.uf.common.util.FileUtil; +import com.raytheon.uf.edex.stats.data.StatsDataAccumulator; + +/** + * Offlines data to csv format for long term comparison. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Aug 21, 2012            jsanchez    Initial creation.
+ * Nov 09, 2012            dhladky     Changed to CSV output
+ * Jan 24, 2013 1357       mpduff      Fix comma output and paths.
+ * May 22, 2013 1917       rjpeter     Renamed from Archiver, added generation of raw statistics,
+ *                                     added method to purge statistics, moved saving of statistics
+ *                                     to configured instead of site level.
+ *
+ * + * @author jsanchez + * + */ +public class OfflineStatsManager { + + private class StatisticsKey { + private final long epochHours; + + public StatisticsKey(Date time) { + this.epochHours = time.getTime() / TimeUtil.MILLIS_PER_HOUR; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + (int) (epochHours ^ (epochHours >>> 32)); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + StatisticsKey other = (StatisticsKey) obj; + if (!getOuterType().equals(other.getOuterType())) { + return false; + } + if (epochHours != other.epochHours) { + return false; + } + return true; + } + + private OfflineStatsManager getOuterType() { + return OfflineStatsManager.this; + } + } + + private static final String COMMA = ","; + + private static final IUFStatusHandler statusHandler = UFStatus + .getHandler(OfflineStatsManager.class); + + private final IPathManager pm = PathManagerFactory.getPathManager(); + + private final LocalizationContext configuredContext = pm.getContext( + LocalizationType.COMMON_STATIC, LocalizationLevel.CONFIGURED); + + private final SimpleDateFormat fieldSdf; + + private final SimpleDateFormat directorySdf; + + private final SimpleDateFormat fileSdf; + + private final DecimalFormat avgFormatter = new DecimalFormat("0.######"); + + public OfflineStatsManager() { + TimeZone gmt = TimeZone.getTimeZone("GMT"); + fieldSdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); + fieldSdf.setTimeZone(gmt); + directorySdf = new SimpleDateFormat("yyyyMMdd"); + directorySdf.setTimeZone(gmt); + fileSdf = new SimpleDateFormat("yyyyMMddHH"); + fileSdf.setTimeZone(gmt); + } + + /** + * Gets a directory name in the format stats/[rawStats|aggregates]/StatType + * + * @param conf + * @param isAggregate + * @return + */ + private String getBaseDirectory(StatisticsEventConfig conf, + boolean isAggregate) { + StringBuffer sb = new StringBuffer(40); + sb.append("stats").append(File.separatorChar); + + if (isAggregate) { + sb.append("aggregates"); + } else { + sb.append("rawStats"); + } + + sb.append(File.separatorChar).append(conf.getTypeClass().getName()); + return sb.toString(); + } + + /** + * Creates a filename in the format + * stats/[rawStats|aggregates]/StatType/yyyyMMdd/StatType_yyyyMMddHH.csv + * + * @param conf + * @param isAggregate + * @param epochHours + * @return + */ + private String getStatFilename(StatisticsEventConfig conf, + boolean isAggregate, long epochHours) { + String baseName = getBaseDirectory(conf, isAggregate); + StringBuilder sb = new StringBuilder(baseName.length() + 40); + Date time = new Date(epochHours * TimeUtil.MILLIS_PER_HOUR); + sb.append(baseName).append(File.separatorChar) + .append(directorySdf.format(time)).append(File.separatorChar) + .append(conf.getTypeClass().getSimpleName()).append("_") + .append(fileSdf.format(time)).append(".csv"); + return sb.toString(); + } + + /** + * Writes a raw statistic in CSV format to the passed BufferedWriter. + * + * @param bw + * @param conf + * @param grouping + * @param event + * @throws IOException + */ + private void writeCSVOutput(BufferedWriter bw, StatisticsEventConfig conf, + StatsGroupingColumn grouping, StatisticsEvent event) + throws IOException { + + Calendar time = event.getDate(); + + if (time != null) { + bw.write(fieldSdf.format(time.getTime())); + } + + for (StatsGrouping group : grouping.getGroup()) { + bw.write(COMMA); + bw.write(group.getValue()); + } + + for (Method m : conf.getAggregateMethods()) { + try { + bw.write(COMMA); + Number number = (Number) m.invoke(event, new Object[0]); + bw.write(number.toString()); + } catch (Exception e) { + statusHandler.error( + "Unable to aggregate '" + m.getName() + "'", e); + } + } + + bw.newLine(); + } + + /** + * Writes the aggregate statistic to the passed BufferedWriter. + * + * @param bw + * @param conf + * @param agg + * @throws IOException + */ + private void writeCSVOutput(BufferedWriter bw, StatisticsEventConfig conf, + AggregateRecord agg) throws IOException { + + Calendar startDate = agg.getStartDate(); + Calendar endDate = agg.getEndDate(); + double sum = agg.getSum(); + double count = agg.getCount(); + + if (startDate != null) { + bw.write(fieldSdf.format(startDate.getTime())); + } + bw.write(COMMA); + + if (endDate != null) { + bw.write(fieldSdf.format(endDate.getTime())); + } + + StatsGroupingColumn grouping = StatsDataAccumulator + .unmarshalGroupingColumnFromRecord(agg); + for (StatsGrouping group : grouping.getGroup()) { + bw.write(COMMA); + bw.write(group.getValue()); + } + + bw.write(COMMA); + bw.write(agg.getField()); + bw.write(COMMA); + + if (count > 0) { + bw.write(avgFormatter.format(sum / count)); + } else { + bw.write("0"); + } + + bw.write(COMMA); + bw.write(String.valueOf(agg.getMin())); + bw.write(COMMA); + bw.write(String.valueOf(agg.getMax())); + bw.write(COMMA); + bw.write(String.valueOf(sum)); + bw.write(COMMA); + bw.write(String.valueOf(count)); + bw.newLine(); + } + + /** + * Opens a buffered writer for the given StatisticsKey and + * StatisticsEventConfig. If its a new CSV file a header is also added to + * the file. + * + * @param key + * @param conf + * @return + * @throws IOException + */ + private BufferedWriter getStatEventBufferedWriter(StatisticsKey key, + StatisticsEventConfig conf) throws IOException { + BufferedWriter bw = null; + LocalizationFile siteLocalization = pm + .getLocalizationFile(configuredContext, + getStatFilename(conf, false, key.epochHours)); + File outFile = siteLocalization.getFile(); + boolean addHeader = outFile.length() == 0; + + if (addHeader) { + // pre-create directories if necessary + outFile.getParentFile().mkdirs(); + } + + bw = new BufferedWriter(new FileWriter(outFile, true)); + + if (addHeader) { + bw.write("Time"); + for (StatisticsGroup group : conf.getGroupList()) { + bw.write(COMMA); + bw.write(group.getDisplayName()); + } + for (StatisticsAggregate aggr : conf.getAggregateList()) { + bw.write(COMMA); + bw.write(aggr.getDisplayName()); + } + bw.newLine(); + } + + return bw; + } + + /** + * Opens a buffered writer for the given StatisticsKey and + * StatisticsEventConfig. If its a new CSV file a header is also added to + * the file. + * + * @param key + * @param conf + * @return + * @throws IOException + */ + private BufferedWriter getAggregateBufferedWriter(StatisticsKey key, + StatisticsEventConfig conf) throws IOException { + BufferedWriter bw = null; + LocalizationFile siteLocalization = pm.getLocalizationFile( + configuredContext, getStatFilename(conf, true, key.epochHours)); + File outFile = siteLocalization.getFile(); + boolean addHeader = outFile.length() == 0; + + if (addHeader) { + // pre-create directories if necessary + outFile.getParentFile().mkdirs(); + } + + bw = new BufferedWriter(new FileWriter(outFile, true)); + + if (addHeader) { + bw.write("Start,End,"); + for (StatisticsGroup group : conf.getGroupList()) { + bw.write(group.getDisplayName()); + bw.write(COMMA); + } + bw.write("Field,Avg,Min,Max,Sum,Count"); + bw.newLine(); + } + + return bw; + } + + /** + * Writes the raw statistics to disk in CSV format. + * + * @param conf + * @param timeMap + */ + public void writeStatsToDisk( + StatisticsEventConfig conf, + Map> timeMap) { + if (!timeMap.isEmpty()) { + String outfilePath = null; + BufferedWriter bw = null; + + try { + for (Multimap groupedEvents : timeMap + .values()) { + for (StatsGroupingColumn group : groupedEvents.keySet()) { + Iterator iter = groupedEvents.get( + group).iterator(); + StatisticsKey prevKey = null; + + while (iter.hasNext()) { + StatisticsEvent event = iter.next(); + StatisticsKey curKey = new StatisticsKey(event + .getDate().getTime()); + + if (!curKey.equals(prevKey)) { + Util.close(bw); + bw = getStatEventBufferedWriter(curKey, conf); + } + + writeCSVOutput(bw, conf, group, event); + } + } + } + } catch (IOException e) { + statusHandler.handle(Priority.ERROR, "Failed to write File: " + + outfilePath, e); + } finally { + Util.close(bw); + } + } + } + + /** + * Writes the aggregate records to disk in CSV format. + * + * @param conf + * The StatisticsEventConfig the aggregates belong to + * @param aggregateRecords + * The aggregate records + * @throws JAXBException + */ + public void writeAggregatesToDisk(StatisticsEventConfig conf, + Collection aggregateRecords) { + if (!aggregateRecords.isEmpty()) { + + String outfilePath = null; + BufferedWriter bw = null; + + try { + + Iterator iter = aggregateRecords.iterator(); + StatisticsKey prevKey = null; + + while (iter.hasNext()) { + AggregateRecord agg = iter.next(); + StatisticsKey curKey = new StatisticsKey(agg.getStartDate() + .getTime()); + + if (!curKey.equals(prevKey)) { + Util.close(bw); + bw = getAggregateBufferedWriter(curKey, conf); + } + + writeCSVOutput(bw, conf, agg); + } + } catch (IOException e) { + statusHandler.handle(Priority.ERROR, "Failed to write File: " + + outfilePath, e); + } finally { + Util.close(bw); + } + } + } + + /** + * Returns the most recent offlined date for the given + * StatisticsEventConfig. + * + * @param conf + * @return + * @throws LocalizationException + * @throws IOException + */ + public Date getMostRecentOfflinedAggregate(StatisticsEventConfig conf) + throws LocalizationException, IOException { + Date rval = null; + + LocalizationFile siteLocalization = pm.getLocalizationFile( + configuredContext, getBaseDirectory(conf, true)); + File eventDir = siteLocalization.getFile(true); + + if (eventDir.exists() && eventDir.isDirectory()) { + File latestDir = null; + for (File handle : eventDir.listFiles()) { + if (handle.isDirectory()) { + try { + Date handleDate = directorySdf.parse(handle.getName()); + + if ((rval == null) || rval.before(handleDate)) { + rval = handleDate; + latestDir = handle; + } + } catch (ParseException e) { + statusHandler.handle(Priority.WARN, "Directory [" + + handle.getAbsolutePath() + + "] is not in expected date format [" + + directorySdf.toPattern() + "]"); + } + } + } + + // found latest directory date + if (latestDir != null) { + for (File csv : latestDir.listFiles()) { + String name = csv.getName(); + if (csv.isFile() && name.endsWith(".csv")) { + // StatType_yyyyMMddHH.csv + int index = name.indexOf('_'); + if (index >= 0) { + try { + Date handleDate = fileSdf.parse(name.substring( + index + 1, index + 11)); + if ((rval == null) || rval.before(handleDate)) { + rval = handleDate; + } + } catch (ParseException e) { + statusHandler.handle(Priority.WARN, "File [" + + csv.getAbsolutePath() + + "] is not in expected date format [" + + fileSdf.toPattern() + "]"); + } + } + } + } + } + } + + return rval; + } + + /** + * Handle retention day rules, -1 keep nothing, 0 keep everything, any + * positive number keep that many full days. + * + * @param retentionDays + * @return + */ + private long getMinTime(int retentionDays) { + long currentDay = System.currentTimeMillis() / TimeUtil.MILLIS_PER_DAY; + + if (retentionDays == 0) { + return 0; + } else if (retentionDays < 0) { + return currentDay * TimeUtil.MILLIS_PER_DAY; + } else { + // add 1 day to not include current day + return (currentDay - (retentionDays + 1)) * TimeUtil.MILLIS_PER_DAY; + } + } + + /** + * Purges offline statistics directories for the given + * StatisticsEventConfig. + * + * @param conf + * @return + */ + public void purgeOffline(StatisticsEventConfig conf) { + // purge aggregates + long minTime = getMinTime(conf.getAggregateOfflineRetentionDays()); + + if (minTime > 0) { + purgeDir(getBaseDirectory(conf, true), minTime); + } + + // purge raw + minTime = getMinTime(conf.getRawOfflineRetentionDays()); + + if (minTime > 0) { + purgeDir(getBaseDirectory(conf, false), minTime); + } + } + + /** + * Purges a given stat event dir keeping any directories newer than minTime. + * + * @param dir + * @param minTime + */ + private void purgeDir(String dir, long minTime) { + LocalizationFile siteLocalization = pm.getLocalizationFile( + configuredContext, dir); + File eventDir = siteLocalization.getFile(); + + if (eventDir.exists() && eventDir.isDirectory()) { + try { + for (File handle : eventDir.listFiles()) { + if (handle.isDirectory()) { + try { + Date handleDate = directorySdf.parse(handle + .getName()); + + if (handleDate.getTime() <= minTime) { + FileUtil.deleteDir(handle); + } + } catch (ParseException e) { + statusHandler.warn("Directory [" + + handle.getAbsolutePath() + + "] is not in expected date format [" + + directorySdf.toPattern() + "]"); + } + } + } + } catch (Exception e) { + statusHandler.error( + "Error occurred purging " + eventDir.getAbsolutePath(), + e); + } + } + } +} diff --git a/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/StatsPurge.java b/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/StatsPurge.java index 14a9fa4364..b686136290 100644 --- a/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/StatsPurge.java +++ b/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/StatsPurge.java @@ -25,14 +25,13 @@ import java.util.Calendar; import java.util.List; import java.util.TimeZone; -import javax.xml.bind.JAXBException; - import com.raytheon.uf.common.dataquery.db.QueryParam.QueryOperand; import com.raytheon.uf.common.localization.PathManagerFactory; import com.raytheon.uf.common.serialization.SerializationException; import com.raytheon.uf.common.serialization.SerializationUtil; import com.raytheon.uf.common.stats.AggregateRecord; import com.raytheon.uf.common.stats.StatsRecord; +import com.raytheon.uf.common.stats.xml.StatisticsEventConfig; import com.raytheon.uf.common.status.IUFStatusHandler; import com.raytheon.uf.common.status.UFStatus; import com.raytheon.uf.edex.database.DataAccessLayerException; @@ -41,21 +40,18 @@ import com.raytheon.uf.edex.database.dao.DaoConfig; import com.raytheon.uf.edex.database.purge.PurgeRule; import com.raytheon.uf.edex.database.purge.PurgeRuleSet; import com.raytheon.uf.edex.database.query.DatabaseQuery; -import com.raytheon.uf.edex.stats.util.Archiver; +import com.raytheon.uf.edex.stats.util.ConfigLoader; /** - * Purges the stats table of expired/unused stat records. Purges the aggregate - * table and write it to disk. - * - * * + * Purges the stats table of expired/unused stat records. * * 
  * 
  * SOFTWARE HISTORY
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
- * Aug 21, 2012            jsanchez     Initial creation.
- * 
+ * Aug 21, 2012            jsanchez    Initial creation.
+ * May 22, 2013 1917       rjpeter     Added purging off offline statistics.
  *
* * @author jsanchez @@ -66,8 +62,6 @@ public class StatsPurge { private static final transient IUFStatusHandler statusHandler = UFStatus .getHandler(StatsPurge.class); - private Archiver archiver; - private final CoreDao aggregateRecordDao = new CoreDao(DaoConfig.forClass( "metadata", AggregateRecord.class)); @@ -81,57 +75,53 @@ public class StatsPurge { public StatsPurge() { aggregatePurgeRules = readPurgeRules("aggregatePurgeRules.xml"); statsPurgeRules = readPurgeRules("statsPurgeRules.xml"); - try { - archiver = new Archiver(); - purgeStats(); - } catch (DataAccessLayerException e) { - statusHandler - .error("Error purging stats on start up. Stats will not be purged. ", - e); + } + + public void purge() { + purgeAggregates(); + purgeStats(); + + // purge offline stats + OfflineStatsManager offlineStats = new OfflineStatsManager(); + ConfigLoader loader = ConfigLoader.getInstance(); + for (StatisticsEventConfig conf : loader.getTypeView().values()) { + offlineStats.purgeOffline(conf); } } /** * Purges records from the aggregate table and writes them to disk. */ - public void purgeAggregates() throws JAXBException, - DataAccessLayerException { + public void purgeAggregates() { if (aggregatePurgeRules != null) { - Calendar expiration = Calendar.getInstance(TimeZone - .getTimeZone("GMT")); - DatabaseQuery query = new DatabaseQuery(AggregateRecord.class); - List allRules = new ArrayList(); + try { + Calendar expiration = Calendar.getInstance(TimeZone + .getTimeZone("GMT")); + DatabaseQuery deleteStmt = new DatabaseQuery( + AggregateRecord.class); + List allRules = new ArrayList(); - // check for specific rules, if none, apply defaults - if (!aggregatePurgeRules.getRules().isEmpty()) { - allRules.addAll(aggregatePurgeRules.getRules()); - } else if (!aggregatePurgeRules.getDefaultRules().isEmpty()) { - allRules.addAll(aggregatePurgeRules.getDefaultRules()); - } + // check for specific rules, if none, apply defaults + if (!aggregatePurgeRules.getRules().isEmpty()) { + allRules.addAll(aggregatePurgeRules.getRules()); + } else if (!aggregatePurgeRules.getDefaultRules().isEmpty()) { + allRules.addAll(aggregatePurgeRules.getDefaultRules()); + } - for (PurgeRule rule : allRules) { - if (rule.isPeriodSpecified()) { - long ms = rule.getPeriodInMillis(); - int minutes = new Long(ms / (1000 * 60)).intValue(); - expiration.add(Calendar.MINUTE, -minutes); + for (PurgeRule rule : allRules) { + if (rule.isPeriodSpecified()) { + long ms = rule.getPeriodInMillis(); + int minutes = new Long(ms / (1000 * 60)).intValue(); + expiration.add(Calendar.MINUTE, -minutes); - query.addQueryParam("endDate", expiration, - QueryOperand.LESSTHAN); + deleteStmt.addQueryParam("endDate", expiration, + QueryOperand.LESSTHAN); - List objects = aggregateRecordDao.queryByCriteria(query); - - if (!objects.isEmpty()) { - AggregateRecord[] aggregateRecords = new AggregateRecord[objects - .size()]; - - for (int i = 0; i < aggregateRecords.length; i++) { - aggregateRecords[i] = (AggregateRecord) objects - .get(i); - } - archiver.writeToDisk(aggregateRecords); - aggregateRecordDao.deleteAll(objects); + aggregateRecordDao.deleteByCriteria(deleteStmt); } } + } catch (DataAccessLayerException e) { + statusHandler.error("Error purging stats aggregates", e); } } } @@ -140,21 +130,25 @@ public class StatsPurge { * Purges records from the stats table if they are older than the expiration * time. */ - private void purgeStats() throws DataAccessLayerException { + private void purgeStats() { if (statsPurgeRules != null) { - Calendar expiration = Calendar.getInstance(TimeZone - .getTimeZone("GMT")); - DatabaseQuery deleteStmt = new DatabaseQuery(StatsRecord.class); + try { + Calendar expiration = Calendar.getInstance(TimeZone + .getTimeZone("GMT")); + DatabaseQuery deleteStmt = new DatabaseQuery(StatsRecord.class); - for (PurgeRule rule : statsPurgeRules.getRules()) { - if (rule.isPeriodSpecified()) { - long ms = rule.getPeriodInMillis(); - int minutes = new Long(ms / (1000 * 60)).intValue(); - expiration.add(Calendar.MINUTE, -minutes); - deleteStmt.addQueryParam("date", expiration, - QueryOperand.LESSTHAN); - statsRecordDao.deleteByCriteria(deleteStmt); + for (PurgeRule rule : statsPurgeRules.getRules()) { + if (rule.isPeriodSpecified()) { + long ms = rule.getPeriodInMillis(); + int minutes = new Long(ms / (1000 * 60)).intValue(); + expiration.add(Calendar.MINUTE, -minutes); + deleteStmt.addQueryParam("date", expiration, + QueryOperand.LESSTHAN); + statsRecordDao.deleteByCriteria(deleteStmt); + } } + } catch (DataAccessLayerException e) { + statusHandler.error("Error purging stats aggregates", e); } } } diff --git a/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/dao/AggregateRecordDao.java b/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/dao/AggregateRecordDao.java index 622ed63451..0f24974397 100644 --- a/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/dao/AggregateRecordDao.java +++ b/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/dao/AggregateRecordDao.java @@ -1,16 +1,60 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ package com.raytheon.uf.edex.stats.dao; +import java.util.Calendar; +import java.util.Date; import java.util.List; +import org.hibernate.Query; +import org.hibernate.Session; +import org.hibernate.Transaction; + import com.raytheon.uf.common.dataquery.db.QueryParam.QueryOperand; import com.raytheon.uf.common.stats.AggregateRecord; +import com.raytheon.uf.common.status.IUFStatusHandler; +import com.raytheon.uf.common.status.UFStatus; import com.raytheon.uf.common.util.CollectionUtil; import com.raytheon.uf.edex.database.DataAccessLayerException; import com.raytheon.uf.edex.database.dao.CoreDao; import com.raytheon.uf.edex.database.dao.DaoConfig; import com.raytheon.uf.edex.database.query.DatabaseQuery; +/** + * Record class for stats waiting to be stored in the appropriate bucket. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Aug 21, 2012            jsanchez    Initial creation
+ * May 22, 2013 1917       rjpeter     Added query methods for retrieving data about aggregates.
+ *
+ * + * @author jsanchez + */ public class AggregateRecordDao extends CoreDao { + private static final IUFStatusHandler statusHandler = UFStatus + .getHandler(AggregateRecordDao.class); + /** * Creates a new data access object */ @@ -61,4 +105,109 @@ public class AggregateRecordDao extends CoreDao { persist(newRecord); } } + + /** + * Returns the oldest start date for a given aggregate eventType. + * + * @param eventType + * @return + * @throws DataAccessLayerException + */ + public Date getOldestAggregateDate(final String eventType) + throws DataAccessLayerException { + Session sess = null; + Transaction tx = null; + + try { + sess = getHibernateTemplate().getSessionFactory().openSession(); + tx = sess.beginTransaction(); + + Query query = sess + .createQuery("SELECT MIN(startDate) FROM AggregateRecord WHERE eventType = ?"); + query.setString(0, eventType); + Calendar rval = (Calendar) query.uniqueResult(); + tx.commit(); + if (rval != null) { + return rval.getTime(); + } + + return null; + } catch (Exception e) { + if (tx != null) { + try { + tx.rollback(); + } catch (Exception e1) { + statusHandler.error( + "Error occurred rolling back transaction", e1); + } + } + + throw new DataAccessLayerException( + "Unable to look up min start date for event [" + eventType + + "]", e); + } finally { + if (sess != null) { + try { + sess.close(); + } catch (Exception e) { + statusHandler.error( + "Error occurred closing database session", e); + } + } + } + } + + /** + * Returns all aggregates of a given type and such that startDate >= + * event.startDate < endDate. + * + * @param eventType + * @param startDate + * @param endDate + * @return + * @throws DataAccessLayerException + */ + public List getAggregates(final String eventType, + final Date startDate, final Date endDate) + throws DataAccessLayerException { + Session sess = null; + Transaction tx = null; + + try { + sess = getHibernateTemplate().getSessionFactory().openSession(); + tx = sess.beginTransaction(); + + Query query = sess + .createQuery("FROM AggregateRecord WHERE eventType = ? AND startDate >= ? AND startDate < ? ORDER BY startDate"); + query.setString(0, eventType); + query.setTimestamp(1, startDate); + query.setTimestamp(2, endDate); + @SuppressWarnings("unchecked") + List rval = query.list(); + tx.commit(); + return rval; + } catch (Exception e) { + if (tx != null) { + try { + tx.rollback(); + } catch (Exception e1) { + statusHandler.error( + "Error occurred rolling back transaction", e1); + } + } + + throw new DataAccessLayerException( + "Unable to look up aggregates for event [" + eventType + + "]", e); + } finally { + if (sess != null) { + try { + sess.close(); + } catch (Exception e) { + statusHandler.error( + "Error occurred closing database session", e); + } + } + } + } } diff --git a/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/dao/StatsDao.java b/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/dao/StatsDao.java index b1e1df0439..be185423b5 100644 --- a/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/dao/StatsDao.java +++ b/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/dao/StatsDao.java @@ -1,8 +1,30 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ package com.raytheon.uf.edex.stats.dao; import java.util.Calendar; import java.util.List; +import org.hibernate.Query; +import org.hibernate.StatelessSession; + import com.raytheon.uf.common.dataquery.db.QueryParam.QueryOperand; import com.raytheon.uf.common.stats.StatsRecord; import com.raytheon.uf.edex.database.DataAccessLayerException; @@ -10,6 +32,20 @@ import com.raytheon.uf.edex.database.dao.CoreDao; import com.raytheon.uf.edex.database.dao.DaoConfig; import com.raytheon.uf.edex.database.query.DatabaseQuery; +/** + * Data access object for raw statistics. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Aug 21, 2012            jsanchez    Initial creation
+ * May 22, 2013 1917       rjpeter     Added reclaimSpace.
+ *
+ * + * @author jsanchez + */ public class StatsDao extends CoreDao { /** * Creates a new data access object @@ -43,4 +79,35 @@ public class StatsDao extends CoreDao { return (List) queryByCriteria(query); } + + /** + * Manually runs vacuum due to large numbers of inserts and deletes to keep + * table size to a minimum. + */ + public void reclaimSpace() { + StatelessSession sess = null; + + try { + sess = getHibernateTemplate().getSessionFactory() + .openStatelessSession(); + // vacuum can't run within a transaction, hack to allow vacuum to + // run from within hibernate + Query query = sess + .createSQLQuery("rollback; VACUUM ANALYZE events.stats"); + query.executeUpdate(); + statusHandler.info("stats vacuumed"); + } catch (Exception e) { + statusHandler.error( + "Error occurred running VACUUM on events.stats", e); + } finally { + if (sess != null) { + try { + sess.close(); + } catch (Exception e) { + statusHandler.error( + "Error occurred closing database session", e); + } + } + } + } } diff --git a/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/data/StatsDataAccumulator.java b/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/data/StatsDataAccumulator.java index d207d2d13d..59659615dd 100644 --- a/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/data/StatsDataAccumulator.java +++ b/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/data/StatsDataAccumulator.java @@ -56,10 +56,10 @@ import com.raytheon.uf.common.util.CollectionUtil; * * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- - * Nov 15, 2012 728 mpduff Initial creation + * Nov 15, 2012 728 mpduff Initial creation * Jan 15, 2013 1487 djohnson Use xml for the grouping information on an {@link AggregateRecord}. - * Jan 17, 2013 1357 mpduff Remove unit conversions, add time step, other cleanup. - * + * Jan 17, 2013 1357 mpduff Remove unit conversions, add time step, other cleanup. + * May 22, 2013 1917 rjpeter Made unmarshalGroupingColumnFromRecord public. * * * @author mpduff @@ -268,7 +268,7 @@ public class StatsDataAccumulator { * @return the unmarshalled column, or an empty column if unable to * unmarshal */ - private static StatsGroupingColumn unmarshalGroupingColumnFromRecord( + public static StatsGroupingColumn unmarshalGroupingColumnFromRecord( AggregateRecord record) { String groupingXmlAsString = record.getGrouping(); try { diff --git a/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/handler/GraphDataHandler.java b/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/handler/GraphDataHandler.java index 6ad60bff07..319e25a8a4 100644 --- a/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/handler/GraphDataHandler.java +++ b/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/handler/GraphDataHandler.java @@ -32,7 +32,7 @@ import com.raytheon.uf.common.stats.GraphDataResponse; import com.raytheon.uf.common.stats.data.GraphData; import com.raytheon.uf.common.stats.xml.StatisticsAggregate; import com.raytheon.uf.common.stats.xml.StatisticsConfig; -import com.raytheon.uf.common.stats.xml.StatisticsEvent; +import com.raytheon.uf.common.stats.xml.StatisticsEventConfig; import com.raytheon.uf.common.time.util.TimeUtil; import com.raytheon.uf.edex.database.dao.CoreDao; import com.raytheon.uf.edex.database.dao.DaoConfig; @@ -49,9 +49,9 @@ import com.raytheon.uf.edex.stats.util.ConfigLoader; * * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- - * Sep 11, 2012 728 mpduff Initial creation + * Sep 11, 2012 728 mpduff Initial creation * Jan 07, 2013 1451 djohnson Use newGmtCalendar(). - * + * May 22, 2013 1917 rjpeter Renamed StatisticsEvent to StatisticsEventConfig. * * * @author mpduff @@ -199,7 +199,7 @@ public class GraphDataHandler implements IRequestHandler { for (StatisticsConfig config : configList) { for (String cat : config.getCategories()) { if (cat.equals(category)) { - for (StatisticsEvent event : config.getEvents()) { + for (StatisticsEventConfig event : config.getEvents()) { if (event.getType().equals(type)) { for (StatisticsAggregate agg : event .getAggregateList()) { diff --git a/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/handler/StatsHandler.java b/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/handler/StatsHandler.java index 1b3b552f6e..14918a9266 100644 --- a/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/handler/StatsHandler.java +++ b/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/handler/StatsHandler.java @@ -32,7 +32,7 @@ import com.raytheon.uf.common.serialization.SerializationException; import com.raytheon.uf.common.serialization.SerializationUtil; import com.raytheon.uf.common.stats.StatsRecord; import com.raytheon.uf.common.stats.xml.StatisticsConfig; -import com.raytheon.uf.common.stats.xml.StatisticsEvent; +import com.raytheon.uf.common.stats.xml.StatisticsEventConfig; import com.raytheon.uf.common.status.IUFStatusHandler; import com.raytheon.uf.common.status.UFStatus; import com.raytheon.uf.edex.database.dao.CoreDao; @@ -75,7 +75,7 @@ public class StatsHandler { public static void setValidEventTypes(List configurations) { validEventTypes = new HashSet(); for (StatisticsConfig config : configurations) { - for (StatisticsEvent event : config.getEvents()) { + for (StatisticsEventConfig event : config.getEvents()) { validEventTypes.add(event.getType()); } } @@ -103,7 +103,7 @@ public class StatsHandler { HashSet myValidEventTypes = new HashSet(); for (StatisticsConfig config : configLoader.getConfigurations()) { - for (StatisticsEvent event : config.getEvents()) { + for (StatisticsEventConfig event : config.getEvents()) { myValidEventTypes.add(event.getType()); } } diff --git a/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/util/Archiver.java b/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/util/Archiver.java deleted file mode 100644 index 59dd45cd2e..0000000000 --- a/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/util/Archiver.java +++ /dev/null @@ -1,276 +0,0 @@ -/** - * This software was developed and / or modified by Raytheon Company, - * pursuant to Contract DG133W-05-CQ-1067 with the US Government. - * - * U.S. EXPORT CONTROLLED TECHNICAL DATA - * This software product contains export-restricted data whose - * export/transfer/disclosure is restricted by U.S. law. Dissemination - * to non-U.S. persons whether in the United States or abroad requires - * an export license or other authorization. - * - * Contractor Name: Raytheon Company - * Contractor Address: 6825 Pine Street, Suite 340 - * Mail Stop B8 - * Omaha, NE 68106 - * 402.291.0100 - * - * See the AWIPS II Master Rights File ("Master Rights File.pdf") for - * further licensing information. - **/ -package com.raytheon.uf.edex.stats.util; - -import java.io.BufferedWriter; -import java.io.FileWriter; -import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.regex.Pattern; - -import javax.xml.bind.JAXBException; - -import com.raytheon.uf.common.localization.IPathManager; -import com.raytheon.uf.common.localization.LocalizationContext; -import com.raytheon.uf.common.localization.LocalizationContext.LocalizationLevel; -import com.raytheon.uf.common.localization.LocalizationContext.LocalizationType; -import com.raytheon.uf.common.localization.LocalizationFile; -import com.raytheon.uf.common.localization.PathManagerFactory; -import com.raytheon.uf.common.stats.AggregateRecord; -import com.raytheon.uf.common.status.IUFStatusHandler; -import com.raytheon.uf.common.status.UFStatus; -import com.raytheon.uf.common.status.UFStatus.Priority; -import com.raytheon.uf.common.time.TimeRange; - -/** - * Archives the data in the aggregate_bucket table to an xml file. - * - * 
- * 
- * SOFTWARE HISTORY
- * Date         Ticket#    Engineer    Description
- * ------------ ---------- ----------- --------------------------
- * Aug 21, 2012            jsanchez     Initial creation.
- * Nov 09, 2012            dhladky      Changed to CSV output
- * Jan 24, 2013   1357     mpduff       Fix comma output and paths.
- * 
- *
- * - * @author jsanchez - * - */ -public class Archiver { - - private class StatisticsKey { - public String eventType; - - public String grouping; - - public TimeRange timeRange; - - @Override - public boolean equals(Object o) { - if (o != null && o instanceof StatisticsKey) { - StatisticsKey other = (StatisticsKey) o; - - return eventType.equals(other.eventType) - && timeRange.getStart().equals( - other.timeRange.getStart()) - && timeRange.getEnd().equals(other.timeRange.getEnd()); - } - - return false; - } - - @Override - public int hashCode() { - return 1; - } - } - - private static final String COMMA = ","; - - private static final Pattern NLPattern = Pattern.compile("[\\n\\r]+"); - - private static final IUFStatusHandler statusHandler = UFStatus - .getHandler(Archiver.class); - - private final IPathManager pm = PathManagerFactory.getPathManager(); - - private final LocalizationContext context = pm.getContext( - LocalizationType.COMMON_STATIC, LocalizationLevel.SITE); - - private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; - - private static final String FILE_DATE_FORMAT = "yyyyMMdd_HHmm"; - - private static final Pattern PERIOD_PATTERN = Pattern.compile("\\."); - - public Archiver() { - - } - - /** - * Creates a filename in the format /stats/aggregates/group... - * /eventType.start-end.dat - * - * @param items - * @return - */ - private String createFilename(TimeRange tr, String eventType) { - - SimpleDateFormat fileDateFormatter = new SimpleDateFormat( - FILE_DATE_FORMAT); - StringBuilder sb = new StringBuilder("stats/aggregates"); - String[] chunks = PERIOD_PATTERN.split(eventType); - sb.append("/"); - sb.append(chunks[chunks.length - 1]); - sb.append("."); - sb.append(fileDateFormatter.format(tr.getStart())); - sb.append("-"); - sb.append(fileDateFormatter.format(tr.getEnd())); - sb.append(".csv"); - - return sb.toString(); - } - - /** - * Used for outputting the stats as CSV - * - * @return - */ - private String getCSVOutput(AggregateRecord agrec, - SimpleDateFormat dateFormat) { - - StringBuilder sb = new StringBuilder(); - - String eventType = agrec.getEventType(); - Calendar startDate = agrec.getStartDate(); - Calendar endDate = agrec.getEndDate(); - String grouping = agrec.getGrouping(); - String field = agrec.getField(); - double max = agrec.getMax(); - double min = agrec.getMin(); - double sum = agrec.getSum(); - double count = agrec.getCount(); - - if (eventType != null) { - sb.append(eventType); - } - sb.append(COMMA); - - if (startDate != null) { - sb.append(dateFormat.format(startDate.getTime())); - } - sb.append(COMMA); - - if (endDate != null) { - sb.append(dateFormat.format(endDate.getTime())); - } - sb.append(COMMA); - - if (grouping != null) { - sb.append(NLPattern.matcher(grouping).replaceAll("")); - } - sb.append(COMMA); - - if (field != null) { - sb.append(field); - } - sb.append(COMMA); - - sb.append(max).append(COMMA); - sb.append(min).append(COMMA); - sb.append(sum).append(COMMA); - sb.append(count); - - return sb.toString(); - } - - /** - * Writes the aggregate records to disk. - * - * @param aggregateRecords - * @throws JAXBException - */ - public void writeToDisk(AggregateRecord[] aggregateRecords) { - - Map> statisticsMap = new HashMap>(); - - for (AggregateRecord record : aggregateRecords) { - StatisticsKey key = new StatisticsKey(); - key.eventType = record.getEventType(); - key.grouping = record.getGrouping(); - key.timeRange = new TimeRange(record.getStartDate(), - record.getEndDate()); - - List aggregateRecordList = statisticsMap.get(key); - if (aggregateRecordList == null) { - aggregateRecordList = new ArrayList(); - statisticsMap.put(key, aggregateRecordList); - } - - aggregateRecordList.add(record); - } - - for (StatisticsKey key : statisticsMap.keySet()) { - - String eventType = key.eventType; - List records = statisticsMap.get(key); - - String filename = createFilename(key.timeRange, eventType); - try { - writeToFile(filename, records); - } catch (JAXBException e) { - statusHandler.error("Unable to write statistics file " - + filename, e); - } - } - } - - /** - * Writes the statistics xml to disk. - * - * @param statistics - * @throws JAXBException - */ - public void writeToFile(String filename, List records) - throws JAXBException { - - BufferedWriter bw = null; - SimpleDateFormat dateFormatter = new SimpleDateFormat(DATE_FORMAT); - LocalizationFile siteLocalization = pm.getLocalizationFile(context, - filename); - String outputFilePath = siteLocalization.getFile().getAbsolutePath(); - // pre-create directories if necessary - siteLocalization.getFile().getParentFile().mkdirs(); - // Write this to output CSV - try { - bw = new BufferedWriter(new FileWriter(outputFilePath)); - if (bw != null) { - for (AggregateRecord agrec : records) { - bw.write(getCSVOutput(agrec, dateFormatter)); - bw.newLine(); - } - } - - } catch (IOException e) { - - statusHandler.handle(Priority.ERROR, "Failed to write File: " - + outputFilePath, e); - } finally { - if (bw != null) { - try { - bw.close(); - } catch (IOException e) { - statusHandler.handle(Priority.PROBLEM, - "failed to close CSV output file stream. " - + filename, e); - } - } - } - - } -} diff --git a/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/util/ConfigLoader.java b/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/util/ConfigLoader.java index 551610480c..465273ce10 100644 --- a/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/util/ConfigLoader.java +++ b/edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/util/ConfigLoader.java @@ -41,7 +41,7 @@ import com.raytheon.uf.common.localization.exception.LocalizationException; import com.raytheon.uf.common.serialization.JAXBManager; import com.raytheon.uf.common.stats.xml.StatisticsAggregate; import com.raytheon.uf.common.stats.xml.StatisticsConfig; -import com.raytheon.uf.common.stats.xml.StatisticsEvent; +import com.raytheon.uf.common.stats.xml.StatisticsEventConfig; import com.raytheon.uf.common.stats.xml.StatisticsGroup; import com.raytheon.uf.common.status.IUFStatusHandler; import com.raytheon.uf.common.status.UFStatus; @@ -58,11 +58,12 @@ import com.raytheon.uf.common.util.ReflectionUtil; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Aug 21, 2012 jsanchez Updated error handling and validated config files. - * Nov 07, 2012 1317 mpduff Update config files. - * Nov 29, 2012 1350 rjpeter Updated to static, fixed localization, increased validation. - * Jan 15, 2013 1487 djohnson Make validate() static and public, so it can be run independently. - * Mar 27, 2013 1834 mpduff Filter for xml files on localization file read, wrap unmarshall and + * Nov 07, 2012 1317 mpduff Update config files. + * Nov 29, 2012 1350 rjpeter Updated to static, fixed localization, increased validation. + * Jan 15, 2013 1487 djohnson Make validate() static and public, so it can be run independently. + * Mar 27, 2013 1834 mpduff Filter for xml files on localization file read, wrap unmarshall and * log error if one occurs + * May 22, 2013 1917 rjpeter Updated validate to save typeClass back to StatisticsEventConfig. * * * @author jsanchez @@ -81,7 +82,7 @@ public class ConfigLoader { private List configurations = Collections.emptyList(); - private Map classToEventMap = Collections + private Map classToEventMap = Collections .emptyMap(); private static final String STATS_DIR = "stats"; @@ -113,7 +114,7 @@ public class ConfigLoader { * * @return */ - public Map getTypeView() { + public Map getTypeView() { return classToEventMap; } @@ -144,7 +145,7 @@ public class ConfigLoader { if (!statConfs.isEmpty()) { List myConfigurations = new ArrayList( statConfs.size()); - Map myEvents = new HashMap(); + Map myEvents = new HashMap(); for (LocalizationFile lf : statConfs.values()) { try { @@ -174,17 +175,17 @@ public class ConfigLoader { * @param config */ @VisibleForTesting - public static void validate(Map eventMap, + public static void validate(Map eventMap, StatisticsConfig config) { - for (Iterator iter = config.getEvents().iterator(); iter - .hasNext();) { - StatisticsEvent event = iter.next(); + for (Iterator iter = config.getEvents() + .iterator(); iter.hasNext();) { + StatisticsEventConfig event = iter.next(); String eventType = event.getType(); if (!eventMap.containsKey(eventType)) { try { Class clazz = Class.forName(eventType); // verify the type is an Event - clazz.asSubclass(Event.class); + event.setTypeClass(clazz.asSubclass(Event.class)); // validate groupBy fields can be found List groups = event.getGroupList(); diff --git a/edexOsgi/com.raytheon.uf.edex.stats/utility/common_static/base/stats/edexProcessStats.xml b/edexOsgi/com.raytheon.uf.edex.stats/utility/common_static/base/stats/edexProcessStats.xml index 13264c5463..09c150760e 100644 --- a/edexOsgi/com.raytheon.uf.edex.stats/utility/common_static/base/stats/edexProcessStats.xml +++ b/edexOsgi/com.raytheon.uf.edex.stats/utility/common_static/base/stats/edexProcessStats.xml @@ -1,7 +1,9 @@ + + displayName="Processing Events" category="Data Ingest Events" + rawOfflineRetentionDays="-1" aggregateOfflineRetentionDays="90"> diff --git a/pythonPackages/dynamicserialize/dstypes/__init__.py b/pythonPackages/dynamicserialize/dstypes/__init__.py index cca3b60ab0..8a203a734a 100644 --- a/pythonPackages/dynamicserialize/dstypes/__init__.py +++ b/pythonPackages/dynamicserialize/dstypes/__init__.py @@ -22,6 +22,7 @@ __all__ = [ 'com', + 'gov', 'java' ] diff --git a/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/GetLatestDbTimeRequest.py b/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/GetLatestDbTimeRequest.py new file mode 100644 index 0000000000..85e495ebb0 --- /dev/null +++ b/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/GetLatestDbTimeRequest.py @@ -0,0 +1,70 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# File auto-generated against equivalent DynamicSerialize Java class +# and then modified post-generation to use AbstractGfeRequest and +# implement str(), repr() +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 05/22/13 2025 dgilling Initial Creation. +# +# + +from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import AbstractGfeRequest +from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import DatabaseID + + +class GetLatestDbTimeRequest(AbstractGfeRequest): + + def __init__(self, dbId=None): + super(GetLatestDbTimeRequest, self).__init__() + if dbId is not None and isinstance(dbId, DatabaseID): + self.dbId = dbId + self.siteID = dbId.getSiteId() + elif dbId is not None and not isinstance(dbId, DatabaseID): + raise TypeError( + "Attempt to construct GetLatestDbTimeRequest without providing a valid DatabaseID.") + + def __str__(self): + retVal = "GetLatestDbTimeRequest[" + retVal += "wokstationID: " + str(self.workstationID) + ", " + retVal += "siteID: " + str(self.siteID) + ", " + retVal += "dbId: " + str(self.dbId) + "]" + return retVal + + def __repr__(self): + retVal = "ExecuteIfpNetCDFGridRequest(" + retVal += "wokstationID=" + repr(self.workstationID) + ", " + retVal += "siteID=" + repr(self.siteID) + ", " + retVal += "dbId=" + repr(self.dbId) + ")" + return retVal + + def getDbId(self): + return self.dbId + + def setDbId(self, dbId): + if isinstance(dbId, DatabaseID): + self.dbId = dbId + else: + raise TypeError( + "Attempt to call GetLatestDbTimeRequest.setDbId() without providing a valid DatabaseID.") diff --git a/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/GetLatestModelDbIdRequest.py b/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/GetLatestModelDbIdRequest.py new file mode 100644 index 0000000000..e9f0fa2c8a --- /dev/null +++ b/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/GetLatestModelDbIdRequest.py @@ -0,0 +1,63 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# File auto-generated against equivalent DynamicSerialize Java class +# and then modified post-generation to use AbstractGfeRequest and +# implement str(), repr() +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 05/22/13 2025 dgilling Initial Creation. +# +# + +from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import AbstractGfeRequest + + +class GetLatestModelDbIdRequest(AbstractGfeRequest): + + def __init__(self, siteId=None, modelName=None): + super(GetLatestModelDbIdRequest, self).__init__() + if siteId is not None: + self.siteID = str(siteId) + if modelName is not None: + self.modelName = str(modelName) + + def __str__(self): + retVal = "GetLatestModelDbIdRequest[" + retVal += "wokstationID: " + str(self.workstationID) + ", " + retVal += "siteID: " + str(self.siteID) + ", " + retVal += "modelName: " + str(self.modelName) + "]" + return retVal + + def __repr__(self): + retVal = "ExecuteIfpNetCDFGridRequest(" + retVal += "wokstationID=" + repr(self.workstationID) + ", " + retVal += "siteID=" + repr(self.siteID) + ", " + retVal += "modelName=" + repr(self.modelName) + ")" + return retVal + + def getModelName(self): + return self.modelName + + def setModelName(self, modelName): + self.modelName = str(modelName) diff --git a/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/__init__.py b/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/__init__.py index 6efb901e5b..10b57e8c0b 100644 --- a/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/__init__.py +++ b/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/__init__.py @@ -30,6 +30,8 @@ __all__ = [ 'GetASCIIGridsRequest', 'GetGridDataRequest', 'GetGridInventoryRequest', + 'GetLatestDbTimeRequest', + 'GetLatestModelDbIdRequest', 'GetLockTablesRequest', 'GetOfficialDbNameRequest', 'GetParmListRequest', @@ -58,6 +60,8 @@ from ExecuteIscMosaicRequest import ExecuteIscMosaicRequest from GetASCIIGridsRequest import GetASCIIGridsRequest from GetGridDataRequest import GetGridDataRequest from GetGridInventoryRequest import GetGridInventoryRequest +from GetLatestDbTimeRequest import GetLatestDbTimeRequest +from GetLatestModelDbIdRequest import GetLatestModelDbIdRequest from GetLockTablesRequest import GetLockTablesRequest from GetOfficialDbNameRequest import GetOfficialDbNameRequest from GetParmListRequest import GetParmListRequest diff --git a/rpms/awips2.core/Installer.edex-environment/edex/component.spec b/rpms/awips2.core/Installer.edex-environment/edex/component.spec index 5ed344b2b8..4efdeee75a 100644 --- a/rpms/awips2.core/Installer.edex-environment/edex/component.spec +++ b/rpms/awips2.core/Installer.edex-environment/edex/component.spec @@ -19,9 +19,9 @@ AutoReq: no provides: awips2-edex-environment requires: awips2-edex-base requires: awips2-postgresql -requires: qpid-java-broker -requires: qpid-java-client -requires: qpid-java-common +requires: awips2-qpid-java-broker +requires: awips2-qpid-java-client +requires: awips2-qpid-java-common requires: awips2-python requires: awips2-java requires: awips2-psql diff --git a/rpms/awips2.core/Installer.ldm/component.spec b/rpms/awips2.core/Installer.ldm/component.spec index 8d7a82ac97..11ee5080f2 100644 --- a/rpms/awips2.core/Installer.ldm/component.spec +++ b/rpms/awips2.core/Installer.ldm/component.spec @@ -21,6 +21,7 @@ Packager: Bryan Kowal AutoReq: no Requires: awips2-notification +Requires: qpid-cpp-client-devel Requires: zlib-devel provides: awips2-ldm provides: awips2-base-component @@ -178,6 +179,52 @@ rm -f %{_ldm_src_tar} if [ $? -ne 0 ]; then exit 1 fi + +# create .bash_profile +if [ ! -f /usr/local/ldm/.bash_profile ]; then + echo 'export PATH=$HOME/decoders:$HOME/util:$HOME/bin:$PATH' > \ + /usr/local/ldm/.bash_profile + echo 'export MANPATH=$HOME/share/man:/usr/share/man' >> \ + /usr/local/ldm/.bash_profile + /bin/chown ldm:fxalpha /usr/local/ldm/.bash_profile +fi + +pushd . > /dev/null 2>&1 +# build ldm +rm -f ~ldm/runtime +cd ${_ldm_root_dir}/src +if [ $? -ne 0 ]; then + exit 1 +fi +export _current_dir=`pwd` +su ldm -lc "cd ${_current_dir}; ./configure --disable-max-size --with-noaaport --disable-root-actions --prefix=${_ldm_root_dir} CFLAGS='-g -O0'" \ + > configure.log 2>&1 +if [ $? -ne 0 ]; then + echo "FATAL: ldm configure has failed!" + exit 1 +fi +export _current_dir=`pwd` +su ldm -lc "cd ${_current_dir}; make install" > install.log 2>&1 +if [ $? -ne 0 ]; then + echo "FATAL: make install has failed!" + exit 1 +fi + +su ldm -lc "cd ${_current_dir}; /bin/bash my-install" > my-install.log 2>&1 +if [ $? -ne 0 ]; then + echo "FATAL: my-install has failed!" + exit 1 +fi +popd > /dev/null 2>&1 +pushd . > /dev/null 2>&1 +cd ${_ldm_root_dir}/src +make root-actions > root-actions.log 2>&1 +if [ $? -ne 0 ]; then + echo "FATAL: root-actions has failed!" + exit 1 +fi +popd > /dev/null 2>&1 + # unpack bin, decoders, and etc. _PATCH_DIRS=( 'bin' 'decoders' 'etc' ) for patchDir in ${_PATCH_DIRS[*]}; @@ -191,21 +238,10 @@ do exit 1 fi done -/bin/chown -R ldm:fxalpha ${_ldm_dir} -if [ $? -ne 0 ]; then - exit 1 -fi +/bin/chmod a+x ${_ldm_dir}/bin/* +/bin/chown -R ldm:fxalpha ${_ldm_dir}/etc ${_ldm_dir}/decoders popd > /dev/null 2>&1 -# create .bash_profile -if [ ! -f /usr/local/ldm/.bash_profile ]; then - echo 'export PATH=$HOME/decoders:$HOME/util:$HOME/bin:$PATH' > \ - /usr/local/ldm/.bash_profile - echo 'export MANPATH=$HOME/share/man:/usr/share/man' >> \ - /usr/local/ldm/.bash_profile - /bin/chown ldm:fxalpha /usr/local/ldm/.bash_profile -fi - # construct pqact pushd . > /dev/null 2>&1 cd ${_ldm_dir}/etc @@ -235,47 +271,6 @@ if [ ${_myHost} != "cpsbn1" -a ${_myHost} != "cpsbn2" -a ${_myHost} != "dx1" -a fi popd > /dev/null 2>&1 -pushd . > /dev/null 2>&1 -# build ldm -cd ${_ldm_root_dir}/src -if [ $? -ne 0 ]; then - exit 1 -fi -export _current_dir=`pwd` -su ldm -lc "cd ${_current_dir}; ./configure --disable-max-size --with-noaaport --disable-root-actions --prefix=${_ldm_dir}" \ - > configure.log 2>&1 -if [ $? -ne 0 ]; then - echo "FATAL: ldm configure has failed!" - exit 1 -fi -export _current_dir=`pwd` -su ldm -lc "cd ${_current_dir}; make install" > install.log 2>&1 -if [ $? -ne 0 ]; then - echo "FATAL: make install has failed!" - exit 1 -fi -popd > /dev/null 2>&1 -pushd . > /dev/null 2>&1 -cd ${_ldm_root_dir}/src/noaaport -if [ $? -ne 0 ]; then - exit 1 -fi -export _current_dir=`pwd` -su ldm -lc "cd ${_current_dir}; /bin/bash my-make" > my-make.log 2>&1 -if [ $? -ne 0 ]; then - echo "FATAL: my-make has failed!" - exit 1 -fi -popd > /dev/null 2>&1 -pushd . > /dev/null 2>&1 -cd ${_ldm_root_dir}/src -make root-actions > root-actions.log 2>&1 -if [ $? -ne 0 ]; then - echo "FATAL: root-actions has failed!" - exit 1 -fi -popd > /dev/null 2>&1 - # build decrypt_file & edexBridge pushd . > /dev/null 2>&1 cd ${_ldm_dir}/SOURCES @@ -349,7 +344,7 @@ fi for _file in $( ls /tmp/ldm/etc/pqact.conf.* | grep -wE "pqact.conf.[a-z]{3,4}" | grep -v pqact.conf.dev | xargs ) ; do if [[ ! -f /usr/local/ldm/etc/${_file} ]]; then - scp -qp /tmp/ldm/etc/${_file} /usr/local/ldm/etc/ + scp -qp ${_file} /usr/local/ldm/etc/ fi done #if a remote CP site, copy over the filtered data configuration @@ -432,5 +427,5 @@ rm -rf ${RPM_BUILD_ROOT} %attr(755,root,root) /etc/profile.d/awipsLDM.csh %attr(755,root,root) /etc/ld.so.conf.d/awips2-ldm-i386.conf -%attr(755,root,root) /etc/ld.so.conf.d/ldm.log +%attr(755,root,root) /etc/logrotate.d/ldm.log %attr(755,root,root) /etc/init.d/ldmcp diff --git a/rpms/awips2.core/Installer.ldm/src/ldm-6.11.5.tar.gz b/rpms/awips2.core/Installer.ldm/src/ldm-6.11.5.tar.gz index a28676c4ff..373d4b4089 100644 Binary files a/rpms/awips2.core/Installer.ldm/src/ldm-6.11.5.tar.gz and b/rpms/awips2.core/Installer.ldm/src/ldm-6.11.5.tar.gz differ diff --git a/tests/unit/com/raytheon/uf/edex/stats/AggregateManagerTest.java b/tests/unit/com/raytheon/uf/edex/stats/AggregateManagerTest.java index 5590fa147e..937d37f7ba 100644 --- a/tests/unit/com/raytheon/uf/edex/stats/AggregateManagerTest.java +++ b/tests/unit/com/raytheon/uf/edex/stats/AggregateManagerTest.java @@ -44,7 +44,7 @@ import com.raytheon.uf.common.serialization.JAXBManager; import com.raytheon.uf.common.stats.StatsGrouping; import com.raytheon.uf.common.stats.StatsGroupingColumn; import com.raytheon.uf.common.stats.xml.StatisticsConfig; -import com.raytheon.uf.common.stats.xml.StatisticsEvent; +import com.raytheon.uf.common.stats.xml.StatisticsEventConfig; import com.raytheon.uf.common.util.FileUtil; import com.raytheon.uf.edex.stats.util.ConfigLoader; @@ -70,8 +70,7 @@ public class AggregateManagerTest { @BeforeClass public static void classSetUp() throws JAXBException { - jaxbManager = new JAXBManager(StatisticsConfig.class, - StatsGroupingColumn.class); + jaxbManager = new JAXBManager(StatisticsConfig.class); } @Before @@ -90,7 +89,8 @@ public class AggregateManagerTest { final StatisticsConfig statisticsConfig = lf.jaxbUnmarshal( StatisticsConfig.class, jaxbManager); - ConfigLoader.validate(Maps. newHashMap(), + ConfigLoader.validate( + Maps. newHashMap(), statisticsConfig); MockEvent mockEvent = new MockEvent(); @@ -102,15 +102,13 @@ public class AggregateManagerTest { List groupList = new ArrayList(); groupList.add(new StatsGrouping("pluginName", "somePlugin")); groupList.add(new StatsGrouping("fileName", "someFileName")); - StatsGroupingColumn column = new StatsGroupingColumn(); - column.setGroup(groupList); + StatsGroupingColumn expectedGroupingColumn = new StatsGroupingColumn(); + expectedGroupingColumn.setGroup(groupList); - final String expectedGroupRepresentation = jaxbManager - .marshalToXml(column); - final String actualGroupRepresentation = AggregateManager.determineGroupRepresentationForEvent( - statisticsConfig.getEvents().iterator().next(), mockEvent); - assertThat(actualGroupRepresentation, - is(equalTo(expectedGroupRepresentation))); + final StatsGroupingColumn actualGroupingColumn = AggregateManager + .determineGroupRepresentationForEvent(statisticsConfig + .getEvents().iterator().next(), mockEvent); + assertThat(actualGroupingColumn, is(equalTo(expectedGroupingColumn))); } }