xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

13.4.1-13 baseline

Browse source 
Former-commit-id: de476791fb [formerly e3b8086bd0 [formerly 87b9712009] [formerly de476791fb [formerly f79327d3865cb0fa7812d5fba896c1a46013badf]]]
Former-commit-id: e3b8086bd0 [formerly 87b9712009]
Former-commit-id: e3b8086bd0
Former-commit-id: 9a7ef48967
This commit is contained in:
Steve Harris 2013-05-29 16:30:09 -04:00
parent 6bf996603a
commit 03eed8f3de
54 changed files with 2045 additions and 644 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

89
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFFGForceUtil.java
View file

@ -24,7 +24,6 @@ import java.util.List;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPBasin;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPGuidanceInterpolation;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPRecord;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPTemplates;
import com.raytheon.uf.common.monitor.config.FFFGDataMgr;
import com.raytheon.uf.common.monitor.config.FFMPSourceConfigurationManager;
@ -44,6 +43,7 @@ import com.raytheon.uf.common.monitor.xml.SourceXML;
* 01/14/13 1569 dhladky changed arraylist to list
* 04/15/13 1890 dhladky Changed COUNTY to use constant
* 05/10/13 1919 mpduff If there are forced pfafs then the aggregate is forced.
* 05/22/13 1902 mpduff Added methods to get forced values.
*
* </pre>
*
@ -135,10 +135,8 @@ public class FFFGForceUtil {
pfafList = ft.getAggregatePfafs(cBasin.getPfaf(),
resource.getSiteKey(), resource.getHuc());
} else if (!domain.equals("NA")) {
if (!resource.getHuc().equals(FFMPRecord.ALL)) {
pfafList = ft.getAggregatePfafsByDomain(cBasin.getPfaf(),
resource.getSiteKey(), domain, resource.getHuc());
}
pfafList = ft.getAggregatePfafsByDomain(cBasin.getPfaf(),
resource.getSiteKey(), domain, resource.getHuc());
} // else use the existing pfaf list
// Add current pfaf to the list
@ -253,7 +251,7 @@ public class FFFGForceUtil {
float tvalue = 0.0f;
float value;
int i = 0;
if (interpolation.isInterpolate() == false) {
if (!interpolation.isInterpolate()) {
FFFGDataMgr dman = FFFGDataMgr.getInstance();
for (long pfaf : forcedPfafs) {
long countyFips = templates.getCountyFipsByPfaf(pfaf);
@ -266,6 +264,49 @@ public class FFFGForceUtil {
}
return tvalue / i;
} else {
// TODO interpolated code under new ticket
}
return Float.NaN;
}
/**
* Get the max forced value (max is smallest number for FFG)
*
* @param pfafList
* list of pfaf ids
* @param forcedPfafs
* list of forced pfaf ids
* @param interpolation
* FFMPGuidanceInterpolation object
* @param expiration
* force expiration
* @param templates
* ffmp templates
* @return max forced value
*/
public float getMaxForcedValue(List<Long> pfafList, List<Long> forcedPfafs,
FFMPGuidanceInterpolation interpolation, long expiration,
FFMPTemplates templates) {
float tvalue = 0.0f;
float value;
if (!interpolation.isInterpolate()) {
FFFGDataMgr dman = FFFGDataMgr.getInstance();
for (long pfaf : forcedPfafs) {
long countyFips = templates.getCountyFipsByPfaf(pfaf);
templates.getCountyFipsByPfaf(pfaf);
value = dman.adjustValue(Float.NaN,
interpolation.getStandardSource(), pfaf, countyFips);
if (value < tvalue) {
tvalue = value;
}
}
return tvalue;
} else {
// TODO interpolated code
}
return Float.NaN;
@ -315,4 +356,40 @@ public class FFFGForceUtil {
public void setSliderTime(double sliderTime) {
this.sliderTime = sliderTime;
}
/**
* Get the forced values for the pfaf list.
*
* @param pfafList
* list of pfaf ids
* @param forcedPfafs
* list of forced pfafs
* @param ffmpGuidanceInterpolation
* FFMPGuidanceInterpolation object
* @param guidSourceExpiration
* expiration time
* @param ft
* ffmp templates
* @return list of forced guidance values
*/
public List<Float> getForcedGuidValues(List<Long> pfafList,
List<Long> forcedPfafs,
FFMPGuidanceInterpolation ffmpGuidanceInterpolation,
long guidSourceExpiration, FFMPTemplates ft) {
List<Float> guidList = new ArrayList<Float>();
if (pfafList != null) {
for (Long pfaf : pfafList) {
if (pfaf == null) {
continue;
}
List<Long> pl = new ArrayList<Long>();
pl.add(pfaf);
float val = getAvgForcedValue(pl, forcedPfafs,
ffmpGuidanceInterpolation, guidSourceExpiration, ft);
guidList.add(val);
}
}
return guidList;
}
}

106
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPDataGenerator.java
View file

@ -79,6 +79,7 @@ import com.raytheon.uf.viz.monitor.ffmp.ui.dialogs.FfmpTableConfigData;
* Apr 26, 2013 1954 bsteffen Minor code cleanup throughout FFMP.
* May 07, 2013 1986 njensen Removed unnecessary sort
* May 10, 2013 1919 mpduff Fixed problem with VGBs
* May 22, 2013 1902 mpduff Code cleanup.
*
* </pre>
*
@ -207,10 +208,8 @@ public class FFMPDataGenerator {
setFFMPRow(fbd.get(key), tData, false,
cwa);
} catch (Exception e) {
statusHandler
.handle(Priority.PROBLEM,
"Couldn't create table row"
+ e);
statusHandler.handle(Priority.PROBLEM,
"Couldn't create table row", e);
}
if (virtualBasin != null) {
for (Long id : ft
@ -257,10 +256,8 @@ public class FFMPDataGenerator {
setFFMPRow(fbd.get(key), tData, isVGB,
null);
} catch (Exception e) {
statusHandler
.handle(Priority.PROBLEM,
"Couldn't create table row"
+ e);
statusHandler.handle(Priority.PROBLEM,
"Couldn't create table row", e);
}
}
}
@ -293,10 +290,10 @@ public class FFMPDataGenerator {
virtualBasin.get(id),
tData, true, null);
} catch (Exception e) {
statusHandler.handle(
Priority.PROBLEM,
"Couldn't create table row"
+ e);
statusHandler
.handle(Priority.PROBLEM,
"Couldn't create table row",
e);
}
}
}
@ -414,6 +411,11 @@ public class FFMPDataGenerator {
if (guidCellData == null) {
// check for forcing even if no data are available
guidance = getForcedAvg(domain, cBasin, guidType);
boolean forced = !guidance.isNaN();
guidCellData = new FFMPTableCellData(
FIELDS.GUIDANCE, guidance, forced);
} else {
guidance = guidCellData.getValueAsFloat();
}
trd.setTableCellData(i + 4, guidCellData);
@ -440,7 +442,6 @@ public class FFMPDataGenerator {
}
} else {
displayName = getDisplayName(cBasin);
if (displayName != null) {
long cBasinPfaf = cBasin.getPfaf();
String cBasinPfafStr = Long.toString(cBasinPfaf);
@ -498,6 +499,9 @@ public class FFMPDataGenerator {
if (guidCellData == null) {
// check for forcing even if no data are available
guidance = getForcedAvg(domain, cBasin, guidType);
boolean forced = !guidance.isNaN();
guidCellData = new FFMPTableCellData(
FIELDS.GUIDANCE, guidance, forced);
} else {
guidance = guidCellData.getValueAsFloat();
}
@ -587,11 +591,13 @@ public class FFMPDataGenerator {
guidance, forcedPfafs,
resource.getGuidSourceExpiration(guidType));
} else {
guidance = resource.getGuidanceValue(ffmpGuidBasin, paintRefTime,
guidType);
if (ffmpGuidBasin != null) {
guidance = resource.getGuidanceValue(ffmpGuidBasin,
paintRefTime, guidType);
if (guidance < 0.0f) {
guidance = Float.NaN;
if (guidance < 0.0f) {
guidance = Float.NaN;
}
}
}
@ -783,31 +789,30 @@ public class FFMPDataGenerator {
FFMPBasinData guidBasin = guidBasins.get(guidType);
List<Long> pfafList = new ArrayList<Long>();
if (cBasin.getAggregated()) {
pfafList = ft.getAggregatePfafs(cBasin.getPfaf(),
siteKey, huc);
pfafList.add(ft.getAggregatedPfaf(cBasin.getPfaf(),
siteKey, huc));
}
boolean forced = false;
List<Long> forcedPfafs = new ArrayList<Long>();
FFFGDataMgr fdm = FFFGDataMgr.getInstance();
if (fdm.isForcingConfigured()) {
forceUtil.calculateForcings(pfafList, ft, cBasin);
forcedPfafs = forceUtil.getForcedPfafList();
forced = forceUtil.isForced();
}
if (!forced) {
if ((forcedPfafs != null) && (!forcedPfafs.isEmpty())) {
forced = true;
}
}
if ((guidBasin != null)
&& (!guidBasin.getBasins().isEmpty())) {
if (cBasin.getAggregated()) {
pfafList = ft.getAggregatePfafs(cBasin.getPfaf(),
siteKey, huc);
pfafList.add(ft.getAggregatedPfaf(cBasin.getPfaf(),
siteKey, huc));
}
boolean forced = false;
List<Long> forcedPfafs = new ArrayList<Long>();
FFFGDataMgr fdm = FFFGDataMgr.getInstance();
if (fdm.isForcingConfigured()) {
forceUtil.calculateForcings(pfafList, ft, cBasin);
forcedPfafs = forceUtil.getForcedPfafList();
forced = forceUtil.isForced();
}
if (!forced) {
if ((forcedPfafs != null)
&& (!forcedPfafs.isEmpty())) {
forced = true;
}
}
if (isWorstCase) {
guidance = guidRecords
@ -830,8 +835,19 @@ public class FFMPDataGenerator {
trd.setTableCellData(i + 4, new FFMPTableCellData(
FIELDS.GUIDANCE, guidance, forced));
} else {
if (forced) {
// Recalculate guidance using the forced value(s)
guidance = forceUtil.getMaxForcedValue(
pfafList,
forcedPfafs,
resource.getGuidanceInterpolators().get(
guidType), resource
.getGuidSourceExpiration(guidType),
ft);
}
trd.setTableCellData(i + 4, new FFMPTableCellData(
FIELDS.GUIDANCE, Float.NaN));
FIELDS.GUIDANCE, guidance, forced));
}
// If guidance is NaN then it cannot be > 0
@ -846,6 +862,14 @@ public class FFMPDataGenerator {
guids = guidBasin.getGuidanceValues(pfafs, resource
.getGuidanceInterpolators().get(guidType),
resource.getGuidSourceExpiration(guidType));
} else if (forced) {
guids = forceUtil.getForcedGuidValues(
pfafList,
forcedPfafs,
resource.getGuidanceInterpolators().get(
guidType), resource
.getGuidSourceExpiration(guidType),
ft);
}
if ((!qpes.isEmpty())

19
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPDataLoader.java
View file

@ -72,6 +72,7 @@ import com.raytheon.uf.viz.monitor.ffmp.ui.listeners.FFMPLoaderEvent;
* Apr 9, 2013 1890 dhladky removed loading of phantom Virtual template and cache file processing.
* Apr 18, 2013 1912 bsteffen Increase bulk requests to pypies.
* Apr 26, 2013 1954 bsteffen Minor code cleanup throughout FFMP.
* May 22, 2013 1902 mpduff Check for null times.
*
* </pre>
*
@ -105,9 +106,9 @@ public class FFMPDataLoader extends Thread {
private FFMPConfig config = null;
private ArrayList<FFMPLoadListener> loadListeners = new ArrayList<FFMPLoadListener>();
private final ArrayList<FFMPLoadListener> loadListeners = new ArrayList<FFMPLoadListener>();
private CountDownLatch latch;
private final CountDownLatch latch;
public FFMPDataLoader(FFMPResourceData resourceData, Date timeBack,
Date mostRecentTime, LOADER_TYPE loadType, List<String> hucsToLoad) {
@ -195,9 +196,8 @@ public class FFMPDataLoader extends Thread {
}
if ((loadType == LOADER_TYPE.INITIAL || loadType == LOADER_TYPE.GENERAL)
&& !product.getRate().equals(product.getQpe())) {
Map<Date, List<String>> rateURIs = monitor
.getAvailableUris(siteKey, dataKey, product.getRate(),
mostRecentTime);
Map<Date, List<String>> rateURIs = monitor.getAvailableUris(
siteKey, dataKey, product.getRate(), mostRecentTime);
if (rateURIs.containsKey(mostRecentTime)) {
rateURI = rateURIs.get(mostRecentTime).get(0);
}
@ -243,11 +243,13 @@ public class FFMPDataLoader extends Thread {
NavigableMap<Date, List<String>> iguidURIs = null;
Date guidTime = timeBack;
if (loadType == LOADER_TYPE.GENERAL) {
guidTime = monitor.getPreviousQueryTime(siteKey,
guidSource.getSourceName());
}
if (guidTime == null) {
continue;
}
iguidURIs = monitor.getAvailableUris(siteKey, dataKey,
guidSource.getSourceName(), guidTime);
@ -292,10 +294,11 @@ public class FFMPDataLoader extends Thread {
SourceXML source = sourceConfig.getSource(product.getQpe());
qpeCache = readAggregateRecord(source, dataKey, wfo);
qpeCache = readAggregateRecord(source, dataKey, wfo);
if (qpeCache != null) {
monitor.insertFFMPData(qpeCache, qpeURIs, siteKey, product.getQpe());
monitor.insertFFMPData(qpeCache, qpeURIs, siteKey,
product.getQpe());
}
}

13
cave/com.raytheon.uf.viz.stats/src/com/raytheon/uf/viz/stats/utils/StatsUiUtils.java
View file

@ -28,7 +28,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.raytheon.uf.common.stats.data.StatsEventData;
import com.raytheon.uf.common.stats.xml.StatisticsAggregate;
import com.raytheon.uf.common.stats.xml.StatisticsConfig;
import com.raytheon.uf.common.stats.xml.StatisticsEvent;
import com.raytheon.uf.common.stats.xml.StatisticsEventConfig;
import com.raytheon.uf.common.stats.xml.StatisticsGroup;
/**
@ -40,7 +40,8 @@ import com.raytheon.uf.common.stats.xml.StatisticsGroup;
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Nov 8, 2012 728 mpduff Initial creation
* Nov 8, 2012 728 mpduff Initial creation
* May 22, 2013 1917 rjpeter Renamed StatisticsEvent to StatisticsEventConfig
*
* </pre>
*
@ -83,7 +84,7 @@ public class StatsUiUtils {
*/
@VisibleForTesting
void processConfig(StatisticsConfig config) {
for (StatisticsEvent event: config.getEvents()) {
for (StatisticsEventConfig event: config.getEvents()) {
processEvent(event);
}
}
@ -94,7 +95,7 @@ public class StatsUiUtils {
* @param event event config object
*/
@VisibleForTesting
void processEvent(StatisticsEvent event) {
void processEvent(StatisticsEventConfig event) {
if (!eventMap.containsKey(event.getCategory())) {
eventMap.put(event.getCategory(), new HashMap<String, StatsEventData>());
}
@ -143,7 +144,7 @@ public class StatsUiUtils {
public Map<String, String> getEventAttributes(String category, String type) {
Map<String, String> attMap = new TreeMap<String, String>();
for (StatisticsConfig config: configList) {
for (StatisticsEvent event: config.getEvents()) {
for (StatisticsEventConfig event: config.getEvents()) {
if (event.getCategory().equals(category) && event.getDisplayName().equals(type)) {
for (StatisticsAggregate agg: event.getAggregateList()) {
attMap.put(agg.getDisplayName(), agg.getField());
@ -186,7 +187,7 @@ public class StatsUiUtils {
public StatisticsAggregate getAggregateConfig(String category,
String typeID, String attributeDisplayName) {
for (StatisticsConfig config : configList) {
for (StatisticsEvent event: config.getEvents()) {
for (StatisticsEventConfig event: config.getEvents()) {
if (event.getCategory().equals(category) && event.getType().equals(typeID)) {
for (StatisticsAggregate agg: event.getAggregateList()) {
if (agg.getDisplayName().equals(attributeDisplayName)) {

26
cave/com.raytheon.viz.mpe.ui/src/com/raytheon/viz/mpe/ui/dialogs/gagetable/GageTableDataManager.java
View file

@ -59,6 +59,8 @@ import com.vividsolutions.jts.geom.Coordinate;
* May 29, 2009 2476 mpduff Initial creation.
* Jan 28, 2010 4415 mpduff Fixed problem with column
* header creation.
* May 20, 2013 15962 lbousaidi Added a new routine getRadarIdsTrue()
* for Radar Sites dialog.
*
* </pre>
*
@ -252,6 +254,30 @@ public class GageTableDataManager {
return radarIds;
}
/**
* Get the list of Radar Ids from radarloc.
* only the one with use_radar= T
* @return the radarIds
* @throws VizException
*/
public String[] getRadarIdsTrue() throws VizException {
if (radarIds == null) {
String query = "select radid from radarloc where use_radar='T' " +
"order by radid asc";
List<Object[]> rs = DirectDbQuery.executeQuery(query,
HydroConstants.IHFS, QueryLanguage.SQL);
radarIds = new String[rs.size()];
for (int i = 0; i < rs.size(); i++) {
Object[] oa = rs.get(i);
radarIds[i] = (String) oa[0];
}
}
return radarIds;
}
/**
* Lookup the Radar Id for the gage.

5
cave/com.raytheon.viz.mpe.ui/src/com/raytheon/viz/mpe/ui/dialogs/hourlyradar/RadarSiteSelectionDlg.java
View file

@ -48,7 +48,8 @@ import com.raytheon.viz.ui.dialogs.CaveSWTDialog;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jul 21, 2009 mpduff Initial creation
*
* May 20, 2013 15962 lbousaidi changed getActiveRadarIds() call to
* getRadarIdsTrue().
* </pre>
*
* @author mpduff
@ -180,7 +181,7 @@ public class RadarSiteSelectionDlg extends CaveSWTDialog {
private void populateBox() {
String[] radarIds = null;
try {
radarIds = GageTableDataManager.getInstance().getActiveRadarIds();
radarIds = GageTableDataManager.getInstance().getRadarIdsTrue();
for (String s : radarIds) {
radarListBox.add(s);
}

11
deltaScripts/13.4.1/removeOldStatAggregates.sh Normal file
View file

@ -0,0 +1,11 @@
#!/bin/bash
# 1917 Removes old aggregate format/layout
echo "Removing old stat aggregates"
rm -rf /awips2/edex/data/utility/common_static/site/*/stats/aggregates
# run full vacuum on stats table, code keeps table more stable
PSQL="/awips2/psql/bin/psql"
echo "Running full vacuum on stats"
${PSQL} -U awips -d metadata -c "VACUUM FULL ANALYZE events.stats;"

14
edexOsgi/com.raytheon.edex.plugin.gfe/res/spring/gfe-request.xml
View file

@ -309,6 +309,20 @@
value="com.raytheon.uf.common.dataplugin.gfe.request.CreateNewDbRequest" />
<constructor-arg ref="createNewDbHandler" />
</bean>
<bean id="getLatestDbInsertTimeHandler"
class="com.raytheon.edex.plugin.gfe.server.handler.GetLatestDbTimeHandler" />
<bean factory-bean="handlerRegistry" factory-method="register">
<constructor-arg
value="com.raytheon.uf.common.dataplugin.gfe.request.GetLatestDbTimeRequest" />
<constructor-arg ref="getLatestDbInsertTimeHandler" />
</bean>
<bean id="getLatestDbIdHandler"
class="com.raytheon.edex.plugin.gfe.server.handler.GetLatestModelDbIdHandler" />
<bean factory-bean="handlerRegistry" factory-method="register">
<constructor-arg
value="com.raytheon.uf.common.dataplugin.gfe.request.GetLatestModelDbIdRequest" />
<constructor-arg ref="getLatestDbIdHandler" />
</bean>
<!-- Service Backup Handlers -->

53
edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/db/dao/GFEDao.java
View file

@ -25,6 +25,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@ -55,6 +56,7 @@ import com.raytheon.uf.common.dataplugin.gfe.server.notify.LockNotification;
import com.raytheon.uf.common.dataplugin.gfe.type.Pair;
import com.raytheon.uf.common.dataplugin.gfe.util.GfeUtil;
import com.raytheon.uf.common.dataplugin.persist.IPersistable;
import com.raytheon.uf.common.dataquery.db.QueryParam.QueryOperand;
import com.raytheon.uf.common.datastorage.DataStoreFactory;
import com.raytheon.uf.common.datastorage.IDataStore;
import com.raytheon.uf.common.status.UFStatus.Priority;
@ -62,6 +64,7 @@ import com.raytheon.uf.common.time.TimeRange;
import com.raytheon.uf.common.util.CollectionUtil;
import com.raytheon.uf.edex.database.DataAccessLayerException;
import com.raytheon.uf.edex.database.purge.PurgeLogger;
import com.raytheon.uf.edex.database.query.DatabaseQuery;
/**
* Data access object for manipulating GFE Records
@ -87,6 +90,8 @@ import com.raytheon.uf.edex.database.purge.PurgeLogger;
* 03/15/13 #1795 njensen Added updatePublishTime()
* 03/21/13 #1774 randerso Moved D2D routines into {@link com.raytheon.edex.plugin.gfe.db.dao.GFED2DDao}
* 04/08/13 #1949 rjpeter Normalized GFE Database.
* 05/22/13 #2025 dgilling Re-implement functions needed by
* GetLatestDbTimeRequest and GetLatestModelDbIdRequest.
* </pre>
*
* @author bphillip
@ -1100,4 +1105,52 @@ public class GFEDao extends DefaultPluginDao {
}
}
}
@SuppressWarnings("unchecked")
public Date getMaxInsertTimeByDbId(final DatabaseID dbId)
throws DataAccessLayerException {
DatabaseQuery query = new DatabaseQuery(this.daoClass);
query.addQueryParam("parmId.dbId", getDatabaseId(dbId),
QueryOperand.EQUALS);
query.addReturnedField("insertTime");
query.addOrder("insertTime", false);
query.setMaxResults(1);
List<Calendar> result = (List<Calendar>) this.queryByCriteria(query);
if (!result.isEmpty()) {
return result.get(0).getTime();
} else {
return null;
}
}
@SuppressWarnings("unchecked")
public DatabaseID getLatestDbIdByModelName(final String siteId,
final String modelName) throws DataAccessLayerException {
// TODO: Should this be done from GridParmManager?
List<DatabaseID> results = Collections.emptyList();
try {
final String[] queryParams = { siteId, modelName };
results = (List<DatabaseID>) txTemplate
.execute(new TransactionCallback() {
@Override
public List<DatabaseID> doInTransaction(
TransactionStatus status) {
return getHibernateTemplate()
.find("FROM DatabaseID WHERE siteId = ? AND modelName = ? ORDER BY modelTime DESC LIMIT 1",
queryParams);
}
});
} catch (Exception e) {
throw new DataAccessLayerException(
"Unable to look up database inventory for site " + siteId,
e);
}
if (!results.isEmpty()) {
return results.get(0);
} else {
return null;
}
}
}

3
edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/isc/IscScript.java
View file

@ -46,6 +46,8 @@ import com.raytheon.uf.common.util.FileUtil;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Mar 11, 2013 dgilling Initial creation
* May 22, 2013 #1759 dgilling Ensure addSitePath() also adds base
* path.
*
* </pre>
*
@ -127,6 +129,7 @@ public class IscScript extends PythonScript {
.getValue("sys.path.index('" + basePath + "')");
} else {
index = (Integer) jep.getValue("len(sys.path)");
jep.eval("sys.path.insert(" + index + ", '" + basePath + "')");
}
jep.eval("sys.path.insert(" + index + ", '" + sitePath + "')");
}

61
edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/GetLatestDbTimeHandler.java Normal file
View file

@ -0,0 +1,61 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.edex.plugin.gfe.server.handler;
import java.util.Date;
import com.raytheon.edex.plugin.gfe.db.dao.GFEDao;
import com.raytheon.uf.common.dataplugin.gfe.request.GetLatestDbTimeRequest;
import com.raytheon.uf.common.serialization.comm.IRequestHandler;
/**
* Handler for getting the latest insert time for a given database ID.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Aug 16, 2010 6349 bphillip Initial creation
* May 22, 2013 2025 dgilling Re-implement for new GFE db schema.
*
* </pre>
*
* @author bphillip
* @version 1.0
*/
public class GetLatestDbTimeHandler implements
IRequestHandler<GetLatestDbTimeRequest> {
/*
* (non-Javadoc)
*
* @see
* com.raytheon.uf.common.serialization.comm.IRequestHandler#handleRequest
* (com.raytheon.uf.common.serialization.comm.IServerRequest)
*/
@Override
public Date handleRequest(GetLatestDbTimeRequest request) throws Exception {
GFEDao dao = new GFEDao();
return dao.getMaxInsertTimeByDbId(request.getDbId());
}
}

63
edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/GetLatestModelDbIdHandler.java Normal file
View file

@ -0,0 +1,63 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.edex.plugin.gfe.server.handler;
import com.raytheon.edex.plugin.gfe.db.dao.GFEDao;
import com.raytheon.uf.common.dataplugin.gfe.db.objects.DatabaseID;
import com.raytheon.uf.common.dataplugin.gfe.request.GetLatestModelDbIdRequest;
import com.raytheon.uf.common.serialization.comm.IRequestHandler;
/**
* Handler for getting the latest DatabaseID for a given model name and site ID.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Aug 17, 2010 dgilling Initial creation
* May 22, 2013 2025 dgilling Re-implement for new GFE db schema.
*
* </pre>
*
* @author dgilling
* @version 1.0
*/
public class GetLatestModelDbIdHandler implements
IRequestHandler<GetLatestModelDbIdRequest> {
/*
* (non-Javadoc)
*
* @see
* com.raytheon.uf.common.serialization.comm.IRequestHandler#handleRequest
* (com.raytheon.uf.common.serialization.comm.IServerRequest)
*/
@Override
public DatabaseID handleRequest(GetLatestModelDbIdRequest request)
throws Exception {
GFEDao dao = new GFEDao();
return dao.getLatestDbIdByModelName(request.getSiteID(),
request.getModelName());
}
}

3
edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/gfe/isc/IrtServer.py
View file

@ -42,6 +42,8 @@ from com.raytheon.edex.plugin.gfe.isc import IRTManager
# 03/13/13 1759 dgilling Move siteConfig imports into
# functions where module is used
# to interact better with IscScript.
# 05/22/13 1759 dgilling Add missing import to
# makeISCrequest().
#
#
#
@ -244,6 +246,7 @@ def irtGetServers(ancfURL, bncfURL, iscWfosWanted):
# xmlRequest is the original request from the GFE's ISCRequestDialog.
def makeISCrequest(xmlRequest, gridDims, gridProj, gridBoundBox, mhs, host, port, protocol, site, xmtScript):
import IrtAccess
import siteConfig
import xml
from xml.etree import ElementTree
from xml.etree.ElementTree import Element, SubElement

3
edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/gfe/isc/ifpnetCDF.py
View file

@ -64,7 +64,7 @@ from com.raytheon.uf.common.localization import LocalizationContext_Localization
# 03/11/13 1759 dgilling Removed unneeded methods.
# 04/23/13 1937 dgilling Reimplement WECache to match
# A1, big perf improvement.
#
# 05/23/13 1759 dgilling Remove unnecessary imports.
#
#
@ -77,7 +77,6 @@ ifpNetcdfLogger=None
## Logging methods ##
def initLogger(logFile=None):
global ifpNetcdfLogger
import logging, siteConfig
ifpNetcdfLogger = iscUtil.getLogger("ifpnetCDF",logFile)
def logEvent(*msg):

2
edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/gfe/isc/iscMosaic.py
View file

@ -79,6 +79,7 @@ from com.raytheon.uf.edex.database.cluster import ClusterTask
# 04/24/13 1941 dgilling Re-port WECache to match A1.
# 05/08/13 1988 dgilling Fix history handling bug in
# __getDbGrid().
# 05/23/13 1759 dgilling Remove unnecessary imports.
#
#
@ -248,7 +249,6 @@ class IscMosaic:
## Logging methods ##
def __initLogger(self):
import logging, siteConfig
self.__logger=iscUtil.getLogger("iscMosaic",self.__logFile)
def __init__(self, args):

3
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/utility/common_static/base/stats/ffmpStats.xml
View file

@ -1,7 +1,8 @@
<statisticsConfig>
<!-- Event Type should be fully qualified name of stat event -->
<statisticsEvent type="com.raytheon.uf.common.stats.LoadEvent"
displayName="Load Time" category="FFMP Load Times">
displayName="Load Time" category="FFMP Load Times"
rawOfflineRetentionDays="90" aggregateOfflineRetentionDays="90">
<statisticsGroup name="type" displayName="Type" />
<!-- Processing time available display units:
ms, Seconds, Minutes, Hours -->

84
edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/request/GetLatestDbTimeRequest.java Normal file
View file

@ -0,0 +1,84 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.dataplugin.gfe.request;
import com.raytheon.uf.common.dataplugin.gfe.db.objects.DatabaseID;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
/**
* Request object for getting the latest insert time for a given database ID.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Aug 16, 2010 6349 bphillip Initial creation
* May 22, 2013 2025 dgilling Add DynamicSerialize support.
*
* </pre>
*
* @author bphillip
* @version 1.0
*/
@DynamicSerialize
public class GetLatestDbTimeRequest extends AbstractGfeRequest {
@DynamicSerializeElement
/** The database ID to get the latest insert time for */
private DatabaseID dbId;
public GetLatestDbTimeRequest() {
// no-op
}
/**
* Creates a new GetLatestDbTimeRequest
*
* @param dbId
* The database ID to get the latest insert time for
*/
public GetLatestDbTimeRequest(DatabaseID dbId) {
super();
this.dbId = dbId;
}
/**
* Creates a new GetLatestDbTimeRequest
*
* @param dbId
* The database ID to get the latest insert time for
*/
public GetLatestDbTimeRequest(String dbId) {
super();
this.dbId = new DatabaseID(dbId);
}
public DatabaseID getDbId() {
return dbId;
}
public void setDbId(DatabaseID dbId) {
this.dbId = dbId;
}
}

87
edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/request/GetLatestModelDbIdRequest.java Normal file
View file

@ -0,0 +1,87 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.dataplugin.gfe.request;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
/**
* Request object for getting the latest database ID for a given model name and
* site ID.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Aug 17, 2010 dgilling Initial creation
* May 22, 2013 2025 dgilling Add DynamicSerialize support.
*
* </pre>
*
* @author dgilling
* @version 1.0
*/
@DynamicSerialize
public class GetLatestModelDbIdRequest extends AbstractGfeRequest {
/**
* The model name to perform the request for.
*/
@DynamicSerializeElement
private String modelName;
public GetLatestModelDbIdRequest() {
// no-op
}
/**
* Creates a new GetLatestModelDbIdRequest object given a model name and
* site identifier.
*
* @param siteId
* The site identifier to search for.
* @param modelName
* The name of the model to search for.
*/
public GetLatestModelDbIdRequest(String siteId, String modelName) {
super();
this.modelName = modelName;
this.siteID = siteId;
}
public String getSiteId() {
return getSiteID();
}
public void setSiteId(String siteId) {
setSiteID(siteId);
}
public String getModelName() {
return modelName;
}
public void setModelName(String modelName) {
this.modelName = modelName;
}
}

65
edexOsgi/com.raytheon.uf.common.serialization/src/com/raytheon/uf/common/serialization/JAXBManager.java
View file

@ -49,7 +49,7 @@ import com.raytheon.uf.common.status.UFStatus.Priority;
* ------------ ---------- ----------- --------------------------
* Sep 24, 2008 chammack Initial creation
* Nov 13, 2008 njensen Added thrift methods
*
* May 22, 2013 1917 rjpeter Added non-pretty print option to jaxb serialize methods.
* </pre>
*
* @author chammack
@ -81,7 +81,7 @@ public class JAXBManager {
private static class MaintainEventsValidationHandler implements
ValidationEventHandler {
private ArrayList<ValidationEvent> events = new ArrayList<ValidationEvent>(
private final ArrayList<ValidationEvent> events = new ArrayList<ValidationEvent>(
0);
@Override
@ -105,9 +105,9 @@ public class JAXBManager {
private final JAXBContext jaxbContext;
private Queue<Unmarshaller> unmarshallers = new ConcurrentLinkedQueue<Unmarshaller>();
private final Queue<Unmarshaller> unmarshallers = new ConcurrentLinkedQueue<Unmarshaller>();
private Queue<Marshaller> marshallers = new ConcurrentLinkedQueue<Marshaller>();
private final Queue<Marshaller> marshallers = new ConcurrentLinkedQueue<Marshaller>();
public JAXBManager(Class<?>... clazz) throws JAXBException {
jaxbContext = JAXBContext.newInstance(clazz);
@ -164,7 +164,7 @@ public class JAXBManager {
return obj;
} finally {
handleEvents(msh, null);
if (msh != null && unmarshallers.size() < QUEUE_SIZE) {
if ((msh != null) && (unmarshallers.size() < QUEUE_SIZE)) {
unmarshallers.add(msh);
}
}
@ -221,8 +221,8 @@ public class JAXBManager {
}
/**
* Convert an instance of a class to an XML representation in a string. Uses
* JAXB.
* Convert an instance of a class to an XML pretty print representation in a
* string. Uses JAXB.
*
* @param obj
* Object being marshalled
@ -230,19 +230,51 @@ public class JAXBManager {
* @throws JAXBException
*/
public String marshalToXml(Object obj) throws JAXBException {
return marshalToXml(obj, true);
}
/**
* Convert an instance of a class to an XML representation in a string. Uses
* JAXB.
*
* @param obj
* Object being marshalled
* @param formattedOutput
* True if the output should be xml pretty print.
* @return XML string representation of the object
* @throws JAXBException
*/
public String marshalToXml(Object obj, boolean formatedOutput)
throws JAXBException {
Marshaller msh = getMarshaller();
try {
StringWriter writer = new StringWriter();
msh.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, new Boolean(true));
msh.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, new Boolean(
formatedOutput));
msh.marshal(obj, writer);
return writer.toString();
} finally {
if (msh != null && marshallers.size() < QUEUE_SIZE) {
if ((msh != null) && (marshallers.size() < QUEUE_SIZE)) {
marshallers.add(msh);
}
}
}
/**
* Convert an instance of a class to an XML representation and writes pretty
* print formatted XML to file. Uses JAXB.
*
* @param obj
* Object to be marshaled
* @param filePath
* Path to the output file
* @throws SerializationException
*/
public void jaxbMarshalToXmlFile(Object obj, String filePath)
throws SerializationException {
jaxbMarshalToXmlFile(obj, filePath, true);
}
/**
* Convert an instance of a class to an XML representation and write XML to
* file. Uses JAXB.
@ -251,21 +283,24 @@ public class JAXBManager {
* Object to be marshaled
* @param filePath
* Path to the output file
* @param formattedOutput
* True if the output should be xml pretty print.
* @throws SerializationException
*/
public void jaxbMarshalToXmlFile(Object obj, String filePath)
throws SerializationException {
public void jaxbMarshalToXmlFile(Object obj, String filePath,
boolean formattedOutput) throws SerializationException {
FileWriter writer = null;
Marshaller msh = null;
try {
msh = getMarshaller();
msh.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, new Boolean(true));
msh.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, new Boolean(
formattedOutput));
writer = new FileWriter(new File(filePath));
msh.marshal(obj, writer);
} catch (Exception e) {
throw new SerializationException(e);
} finally {
if (msh != null && marshallers.size() < QUEUE_SIZE) {
if ((msh != null) && (marshallers.size() < QUEUE_SIZE)) {
marshallers.add(msh);
}
if (writer != null) {
@ -315,7 +350,7 @@ public class JAXBManager {
if (msh != null) {
handleEvents(msh, file.getName());
}
if (msh != null && unmarshallers.size() < QUEUE_SIZE) {
if ((msh != null) && (unmarshallers.size() < QUEUE_SIZE)) {
unmarshallers.add(msh);
}
if (reader != null) {
@ -350,7 +385,7 @@ public class JAXBManager {
if (msh != null) {
handleEvents(msh, null);
}
if (msh != null && unmarshallers.size() < QUEUE_SIZE) {
if ((msh != null) && (unmarshallers.size() < QUEUE_SIZE)) {
unmarshallers.add(msh);
}
if (is != null) {

3
edexOsgi/com.raytheon.uf.common.stats/META-INF/MANIFEST.MF
View file

@ -20,4 +20,5 @@ Require-Bundle: com.raytheon.uf.common.time;bundle-version="1.12.1174",
com.raytheon.uf.common.status;bundle-version="1.12.1174",
javax.measure;bundle-version="1.0.0",
com.raytheon.uf.common.units;bundle-version="1.0.0",
org.apache.commons.lang;bundle-version="2.3.0"
org.apache.commons.lang;bundle-version="2.3.0",
org.hibernate

41
edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/StatsGrouping.java
View file

@ -33,8 +33,8 @@ import javax.xml.bind.annotation.XmlRootElement;
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 15, 2013 1487 djohnson Initial creation
*
* Jan 15, 2013 1487 djohnson Initial creation
* May 22, 2013 1917 rjpeter Added hashCode and equals.
* </pre>
*
* @author djohnson
@ -98,4 +98,41 @@ public class StatsGrouping {
this.value = value;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((name == null) ? 0 : name.hashCode());
result = prime * result + ((value == null) ? 0 : value.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
StatsGrouping other = (StatsGrouping) obj;
if (name == null) {
if (other.name != null) {
return false;
}
} else if (!name.equals(other.name)) {
return false;
}
if (value == null) {
if (other.value != null) {
return false;
}
} else if (!value.equals(other.value)) {
return false;
}
return true;
}
}

34
edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/StatsGroupingColumn.java
View file

@ -37,8 +37,8 @@ import com.google.common.collect.Lists;
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 15, 2013 1487 djohnson Initial creation
*
* Jan 15, 2013 1487 djohnson Initial creation
* May 22, 2013 1917 rjpeter Added hashCode and equals.
* </pre>
*
* @author djohnson
@ -84,4 +84,34 @@ public class StatsGroupingColumn {
return column;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((group == null) ? 0 : group.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
StatsGroupingColumn other = (StatsGroupingColumn) obj;
if (group == null) {
if (other.group != null) {
return false;
}
} else if (!group.equals(other.group)) {
return false;
}
return true;
}
}

8
edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/StatsRecord.java
View file

@ -31,6 +31,8 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import org.hibernate.annotations.BatchSize;
import com.raytheon.uf.common.dataplugin.persist.PersistableDataObject;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
@ -43,14 +45,14 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* SOFTWARE HISTORY
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Aug 21, 2012 jsanchez Initial creation
*
* Aug 21, 2012 jsanchez Initial creation
* May 22, 2013 1917 rjpeter Added BatchSize annotation.
* </pre>
*
* @author jsanchez
*
*/
@Entity
@BatchSize(size = 500)
@Table(name = "stats", schema = "events")
@XmlRootElement
@XmlAccessorType(XmlAccessType.NONE)

14
edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/xml/StatisticsConfig.java
View file

@ -43,8 +43,8 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Nov 6, 2012 728 mpduff Initial creation.
*
* Nov 6, 2012 728 mpduff Initial creation.
* May 22, 2013 1917 rjpeter Renamed StatisticsEvent to StatisticsEventConfig.
* </pre>
*
* @author mpduff
@ -54,14 +54,14 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
@XmlRootElement(name = "statisticsConfig")
@XmlAccessorType(XmlAccessType.NONE)
public class StatisticsConfig implements ISerializableObject {
@XmlElements({ @XmlElement(name = "statisticsEvent", type = StatisticsEvent.class) })
@XmlElements({ @XmlElement(name = "statisticsEvent", type = StatisticsEventConfig.class) })
@DynamicSerializeElement
private List<StatisticsEvent> events;
private List<StatisticsEventConfig> events;
/**
* @return the events
*/
public List<StatisticsEvent> getEvents() {
public List<StatisticsEventConfig> getEvents() {
return events;
}
@ -69,7 +69,7 @@ public class StatisticsConfig implements ISerializableObject {
* @param events
* the events to set
*/
public void setEvents(List<StatisticsEvent> events) {
public void setEvents(List<StatisticsEventConfig> events) {
this.events = events;
}
@ -81,7 +81,7 @@ public class StatisticsConfig implements ISerializableObject {
public List<String> getCategories() {
Set<String> categories = new HashSet<String>();
if (events != null && events.size() > 0) {
for (StatisticsEvent event : events) {
for (StatisticsEventConfig event : events) {
categories.add(event.getCategory());
}
}

39
edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/xml/StatisticsEvent.java → edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/xml/StatisticsEventConfig.java
View file

@ -42,8 +42,9 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Nov 6, 2012 728 mpduff Initial creation.
*
* Nov 6, 2012 728 mpduff Initial creation.
* May 22, 2013 1917 rjpeter Renamed to StatisticsEventConfig and
* added offline retention settings.
* </pre>
*
* @author mpduff
@ -52,7 +53,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
@DynamicSerialize
@XmlRootElement(name = "event")
@XmlAccessorType(XmlAccessType.NONE)
public class StatisticsEvent {
public class StatisticsEventConfig {
@XmlAttribute
@DynamicSerializeElement
@ -66,6 +67,22 @@ public class StatisticsEvent {
@DynamicSerializeElement
private String category;
/**
* Retention period for the raw offline statistic to be saved. Value < 0 do
* not retain, value = 0 retain all, value > 0 retain for value days.
*/
@XmlAttribute
@DynamicSerializeElement
private int rawOfflineRetentionDays = -1;
/**
* Retention period for the aggregate offline statistic to be saved. Value <
* 0 do not retain, value = 0 retain all, value > 0 retain for value days.
*/
@XmlAttribute
@DynamicSerializeElement
private int aggregateOfflineRetentionDays;
@XmlElements({ @XmlElement(name = "statisticsGroup", type = StatisticsGroup.class) })
@DynamicSerializeElement
private List<StatisticsGroup> groupList;
@ -179,4 +196,20 @@ public class StatisticsEvent {
this.aggregateMethods = aggregateMethods;
}
public int getRawOfflineRetentionDays() {
return rawOfflineRetentionDays;
}
public void setRawOfflineRetentionDays(int rawOfflineRetentionDays) {
this.rawOfflineRetentionDays = rawOfflineRetentionDays;
}
public int getAggregateOfflineRetentionDays() {
return aggregateOfflineRetentionDays;
}
public void setAggregateOfflineRetentionDays(
int aggregateOfflineRetentionDays) {
this.aggregateOfflineRetentionDays = aggregateOfflineRetentionDays;
}
}

4
edexOsgi/com.raytheon.uf.edex.datadelivery.event/resources/stats.properties
View file

@ -1,4 +0,0 @@
# scan interval of stats table in minutes
stats.scanInterval=15
# bucket interval or period of when to aggregate in minutes
stats.period=5

3
edexOsgi/com.raytheon.uf.edex.datadelivery.retrieval/utility/common_static/base/stats/retrievalProcessStats.xml
View file

@ -1,7 +1,8 @@
<statisticsConfig>
<!-- Event Type should be fully qualified name of stat event -->
<statisticsEvent type="com.raytheon.uf.common.datadelivery.event.retrieval.SubscriptionRetrievalEvent"
displayName="Subscription Retrieval" category="Data Delivery">
displayName="Subscription Retrieval" category="Data Delivery"
rawOfflineRetentionDays="-1" aggregateOfflineRetentionDays="90">
<statisticsGroup name="plugin" displayName="Data Type" />
<statisticsGroup name="provider" displayName="Data Provider" />
<statisticsGroup name="owner" displayName="Owner" />

3
edexOsgi/com.raytheon.uf.edex.registry.ebxml/utility/common_static/base/stats/registryProcessStats.xml
View file

@ -1,7 +1,8 @@
<statisticsConfig>
<!-- Event Type should be fully qualified name of stat event -->
<statisticsEvent type="com.raytheon.uf.common.registry.event.RegistryStatisticsEvent"
displayName="Registry Statistics" category="Registry">
displayName="Registry Statistics" category="Registry"
rawOfflineRetentionDays="-1" aggregateOfflineRetentionDays="90">
<statisticsGroup name="owner" displayName="Transaction Owner" />
<statisticsGroup name="status" displayName="Transaction Status" />
<statisticsGroup name="type" displayName="Transaction Type" />

1
edexOsgi/com.raytheon.uf.edex.stats/META-INF/MANIFEST.MF
View file

@ -10,6 +10,7 @@ Require-Bundle: com.raytheon.uf.common.serialization;bundle-version="1.12.1174",
com.raytheon.uf.common.event;bundle-version="1.0.0",
com.google.guava;bundle-version="1.0.0",
com.raytheon.uf.edex.database;bundle-version="1.0.0",
com.raytheon.edex.common,
com.raytheon.uf.common.localization;bundle-version="1.12.1174",
com.raytheon.uf.common.dataquery;bundle-version="1.0.0",
com.raytheon.uf.common.time;bundle-version="1.12.1174",

41
edexOsgi/com.raytheon.uf.edex.stats/res/spring/edex-process-stats.xml
View file

@ -1,41 +0,0 @@
<beans
xmlns="http://www.springframework.org/schema/beans"
xmlns:amq="http://activemq.apache.org/schema/core"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd">
<bean id="statsPurge"
class="com.raytheon.uf.edex.stats.StatsPurge"
depends-on="statsRegister"/>
<bean id="aggregateManager" class="com.raytheon.uf.edex.stats.AggregateManager">
<constructor-arg value="${stats.period}"/>
</bean>
<bean id="edexStatsRegistered" factory-bean="clusteredCamelContextMgr"
factory-method="register" depends-on="persistCamelRegistered">
<constructor-arg ref="edexStats-camel"/>
</bean>
<camelContext id="edexStats-camel"
xmlns="http://camel.apache.org/schema/spring"
errorHandlerRef="errorHandler"
autoStartup="false">
<endpoint id="statsScanTimer" uri="timer://scanStats?period=${stats.scanInterval}m"/>
<route id="statsTableScan">
<from ref="statsScanTimer" />
<doTry>
<bean ref="statsPurge" method="purgeAggregates"/>
<bean ref="aggregateManager" method="scan"/>
<doCatch>
<exception>java.lang.Throwable</exception>
<to uri="log:stats?level=ERROR&amp;showBody=false&amp;showCaughtException=true&amp;showStackTrace=true"/>
</doCatch>
</doTry>
</route>
</camelContext>
</beans>

13
edexOsgi/com.raytheon.uf.edex.stats/res/spring/stats-graph-request.xml
View file

@ -1,13 +0,0 @@
<beans
xmlns="http://www.springframework.org/schema/beans"
xmlns:amq="http://activemq.apache.org/schema/core"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd">
<bean id="statsGraphDataHandler" class="com.raytheon.uf.edex.stats.handler.GraphDataHandler"/>
<bean factory-bean="handlerRegistry" factory-method="register">
<constructor-arg value="com.raytheon.uf.common.stats.GraphDataRequest"/>
<constructor-arg ref="statsGraphDataHandler"/>
</bean>
</beans>

66
edexOsgi/com.raytheon.uf.edex.stats/res/spring/stats-ingest.xml Normal file
View file

@ -0,0 +1,66 @@
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd">
<bean id="statsPurge" class="com.raytheon.uf.edex.stats.StatsPurge"
depends-on="statsRegister"/>
<bean id="aggregateManager" class="com.raytheon.uf.edex.stats.AggregateManager">
<!-- Not directly exposing at this time, due to performance concerns from
improper values -->
<!-- Bucket interval in minutes for aggregation -->
<constructor-arg value="5"/>
</bean>
<bean id="edexStatsRegistered" factory-bean="clusteredCamelContextMgr"
factory-method="register" depends-on="persistCamelRegistered">
<constructor-arg ref="edexStats-camel"/>
</bean>
<camelContext id="edexStats-camel" xmlns="http://camel.apache.org/schema/spring"
errorHandlerRef="errorHandler" autoStartup="false">
<endpoint id="statsScanTimer" uri="timer://scanStats?period=${stats.scanInterval}m"/>
<endpoint id="aggrToCsvTimer"
uri="quartz://stats/aggrToCsv/?cron=${stats.aggregateToCsv.cron}"/>
<endpoint id="statsPurgeTimer" uri="quartz://stats/purge/?cron=${stats.purge.cron}"/>
<route id="statsTableScan">
<from ref="statsScanTimer"/>
<doTry>
<bean ref="aggregateManager" method="scan"/>
<doCatch>
<exception>java.lang.Throwable</exception>
<to
uri="log:stats?level=ERROR&amp;showBody=false&amp;showCaughtException=true&amp;showStackTrace=true"/>
</doCatch>
</doTry>
</route>
<route id="statsAggrToCsv">
<from ref="aggrToCsvTimer"/>
<doTry>
<bean ref="aggregateManager" method="offlineAggregates"/>
<doCatch>
<exception>java.lang.Throwable</exception>
<to
uri="log:stats?level=ERROR&amp;showBody=false&amp;showCaughtException=true&amp;showStackTrace=true"/>
</doCatch>
</doTry>
</route>
<route id="statsPurgeRoute">
<from ref="statsPurgeTimer"/>
<doTry>
<bean ref="statsPurge" method="purge"/>
<doCatch>
<exception>java.lang.Throwable</exception>
<to
uri="log:stats?level=ERROR&amp;showBody=false&amp;showCaughtException=true&amp;showStackTrace=true"/>
</doCatch>
</doTry>
</route>
</camelContext>
</beans>

26
edexOsgi/com.raytheon.uf.edex.stats/res/spring/stats-request.xml
View file

@ -1,19 +1,17 @@
<beans
xmlns="http://www.springframework.org/schema/beans"
xmlns:amq="http://activemq.apache.org/schema/core"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd">
<!-- Need to set up connect between cave and edex
1) The possible combinations to populate drop downs etc
2) Bucketizing so that Cave requests data in 15 minute buckets,
you would need to do the aggregation (still undecided on if this is a cave or edex feature).
-->
<bean id="aggregatedStatsHandler" class="com.raytheon.uf.edex.stats.handler.AggregatedStatsHandler"/>
<bean id="aggregatedStatsHandler" class="com.raytheon.uf.edex.stats.handler.AggregatedStatsHandler"/>
<bean factory-bean="handlerRegistry" factory-method="register">
<constructor-arg value="com.raytheon.uf.common.stats.AggregatedStatsRequest"/>
<constructor-arg ref="aggregatedStatsHandler"/>
<constructor-arg value="com.raytheon.uf.common.stats.AggregatedStatsRequest"/>
<constructor-arg ref="aggregatedStatsHandler"/>
</bean>
<bean id="statsGraphDataHandler" class="com.raytheon.uf.edex.stats.handler.GraphDataHandler"/>
<bean factory-bean="handlerRegistry" factory-method="register">
<constructor-arg value="com.raytheon.uf.common.stats.GraphDataRequest"/>
<constructor-arg ref="statsGraphDataHandler"/>
</bean>
</beans>

4
edexOsgi/com.raytheon.uf.edex.stats/resources/edexProcessStats.properties
View file

@ -1,4 +0,0 @@
# scan interval of stats table in minutes
stats.scanInterval=2
# bucket interval or period of when to aggregate in minutes
stats.period=5

8
edexOsgi/com.raytheon.uf.edex.stats/resources/stats.properties Normal file
View file

@ -0,0 +1,8 @@
# scan interval of stats table in minutes
stats.scanInterval=2
# When to save off aggregate data to csv format
stats.aggregateToCsv.cron=0+10+*+*+*+?
# When to run purge of aggregate tables and csv files
stats.purge.cron=0+15+*+*+*+?

138
edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/AggregateManager.java
View file

@ -24,6 +24,7 @@ import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@ -35,15 +36,15 @@ import javax.xml.bind.JAXBException;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
import com.raytheon.uf.common.event.Event;
import com.raytheon.uf.common.serialization.JAXBManager;
import com.raytheon.uf.common.serialization.SerializationUtil;
import com.raytheon.uf.common.stats.AggregateRecord;
import com.raytheon.uf.common.stats.StatisticsEvent;
import com.raytheon.uf.common.stats.StatsGrouping;
import com.raytheon.uf.common.stats.StatsGroupingColumn;
import com.raytheon.uf.common.stats.StatsRecord;
import com.raytheon.uf.common.stats.xml.StatisticsAggregate;
import com.raytheon.uf.common.stats.xml.StatisticsEvent;
import com.raytheon.uf.common.stats.xml.StatisticsEventConfig;
import com.raytheon.uf.common.stats.xml.StatisticsGroup;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
@ -66,10 +67,12 @@ import com.raytheon.uf.edex.stats.util.ConfigLoader;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Aug 21, 2012 jsanchez Stored the aggregate buckets in the db.
* Nov 07, 2012 1317 mpduff Updated Configuration Files.
* Nov 28, 2012 1350 rjpeter Simplied aggregation and added aggregation with current db aggregate records.
* Nov 07, 2012 1317 mpduff Updated Configuration Files.
* Nov 28, 2012 1350 rjpeter Simplied aggregation and added aggregation with current db aggregate records.
* Jan 07, 2013 1451 djohnson Use newGmtCalendar().
* Jan 15, 2013 1487 djohnson Use xml for the grouping information on an {@link AggregateRecord}.
* May 22, 2013 1917 rjpeter Added ability to save raw and aggregate stats, to reclaimSpace every scan call,
* and to not pretty print xml grouping information.
* </pre>
*
* @author jsanchez
@ -96,9 +99,6 @@ public class AggregateManager {
/** default value */
private static final int defaultBucketInterval = 5;
/** default value */
private static final int defaultScanInterval = 15;
public AggregateManager(String bucketInterval) {
validateIntervals(bucketInterval);
}
@ -112,8 +112,10 @@ public class AggregateManager {
* @param timeRange
* @param groupedEvents
*/
private void aggregate(AggregateRecordDao dao, StatisticsEvent statsEvent,
TimeRange timeRange, Multimap<String, Event> groupedEvents) {
private void aggregate(AggregateRecordDao dao,
StatisticsEventConfig statsEvent, TimeRange timeRange,
Multimap<StatsGroupingColumn, StatisticsEvent> groupedEvents)
throws JAXBException {
Calendar start = TimeUtil.newGmtCalendar();
start.setTime(timeRange.getStart());
@ -121,8 +123,10 @@ public class AggregateManager {
end.setTime(timeRange.getEnd());
// perform aggregate functions on the grouped data
for (String groupKey : groupedEvents.keySet()) {
Collection<Event> groupData = groupedEvents.get(groupKey);
for (StatsGroupingColumn group : groupedEvents.keySet()) {
Collection<StatisticsEvent> groupData = groupedEvents.get(group);
String groupKey = JAXB_MANAGER.marshalToXml(group, false);
Iterator<Method> aggrMethodIter = statsEvent.getAggregateMethods()
.iterator();
Iterator<StatisticsAggregate> statAggrIter = statsEvent
@ -138,7 +142,7 @@ public class AggregateManager {
double min = Double.MAX_VALUE;
double sum = 0;
for (Event event : groupData) {
for (StatisticsEvent event : groupData) {
Number number = (Number) m.invoke(event, new Object[0]);
double value = number.doubleValue();
sum += value;
@ -217,9 +221,10 @@ public class AggregateManager {
long t0 = System.currentTimeMillis();
ConfigLoader configLoader = ConfigLoader.getInstance();
StatsDao statsRecordDao = new StatsDao();
OfflineStatsManager offline = new OfflineStatsManager();
AggregateRecordDao aggrRecordDao = new AggregateRecordDao();
Map<String, StatisticsEvent> statsMap = configLoader.getTypeView();
Map<String, StatisticsEventConfig> statsMap = configLoader
.getTypeView();
// latest time to pull
Calendar timeToProcess = Calendar.getInstance(TimeZone
@ -227,9 +232,10 @@ public class AggregateManager {
int count = 0;
// process the events by type
for (Map.Entry<String, StatisticsEvent> entry : statsMap.entrySet()) {
for (Map.Entry<String, StatisticsEventConfig> entry : statsMap
.entrySet()) {
String type = entry.getKey();
StatisticsEvent event = entry.getValue();
StatisticsEventConfig event = entry.getValue();
List<StatsRecord> records = null;
do {
@ -239,10 +245,10 @@ public class AggregateManager {
if (!CollectionUtil.isNullOrEmpty(records)) {
// sort events into time buckets
Map<TimeRange, Multimap<String, Event>> timeMap = sort(
Map<TimeRange, Multimap<StatsGroupingColumn, StatisticsEvent>> timeMap = sort(
event, records);
for (Map.Entry<TimeRange, Multimap<String, Event>> timeMapEntry : timeMap
for (Map.Entry<TimeRange, Multimap<StatsGroupingColumn, StatisticsEvent>> timeMapEntry : timeMap
.entrySet()) {
aggregate(aggrRecordDao, event, timeMapEntry.getKey(),
timeMapEntry.getValue());
@ -255,10 +261,14 @@ public class AggregateManager {
}
count += records.size();
if (event.getRawOfflineRetentionDays() >= 0) {
offline.writeStatsToDisk(event, timeMap);
}
}
} while (!CollectionUtil.isNullOrEmpty(records));
}
statsRecordDao.reclaimSpace();
long t1 = System.currentTimeMillis();
statusHandler.info("Aggregated " + count + " stat events in "
+ (t1 - t0) + " ms");
@ -270,11 +280,11 @@ public class AggregateManager {
* @param records
* @return
*/
private Map<TimeRange, Multimap<String, Event>> sort(
StatisticsEvent statEvent, List<StatsRecord> records) {
Map<TimeRange, Multimap<String, Event>> rval = new HashMap<TimeRange, Multimap<String, Event>>();
private Map<TimeRange, Multimap<StatsGroupingColumn, StatisticsEvent>> sort(
StatisticsEventConfig statEvent, List<StatsRecord> records) {
Map<TimeRange, Multimap<StatsGroupingColumn, StatisticsEvent>> rval = new HashMap<TimeRange, Multimap<StatsGroupingColumn, StatisticsEvent>>();
TimeRange timeRange = null;
Multimap<String, Event> eventsByGroup = null;
Multimap<StatsGroupingColumn, StatisticsEvent> eventsByGroup = null;
for (StatsRecord record : records) {
if ((timeRange == null)
@ -290,13 +300,13 @@ public class AggregateManager {
try {
// get underlying event
Event event = SerializationUtil.transformFromThrift(
Event.class, record.getEvent());
StatisticsEvent event = SerializationUtil.transformFromThrift(
StatisticsEvent.class, record.getEvent());
String groupAsString = determineGroupRepresentationForEvent(
StatsGroupingColumn group = determineGroupRepresentationForEvent(
statEvent, event);
if (groupAsString != null) {
eventsByGroup.put(groupAsString, event);
if (group != null) {
eventsByGroup.put(group, event);
}
} catch (Exception e) {
statusHandler
@ -309,10 +319,9 @@ public class AggregateManager {
}
@VisibleForTesting
static String determineGroupRepresentationForEvent(
StatisticsEvent statEvent, Event event)
throws IllegalAccessException, InvocationTargetException,
JAXBException {
static StatsGroupingColumn determineGroupRepresentationForEvent(
StatisticsEventConfig statEvent, StatisticsEvent event)
throws IllegalAccessException, InvocationTargetException {
Iterator<Method> gMethodIter = statEvent.getGroupByMethods().iterator();
Iterator<StatisticsGroup> gFieldNameIter = statEvent.getGroupList()
.iterator();
@ -322,14 +331,13 @@ public class AggregateManager {
Method m = gMethodIter.next();
String field = gFieldNameIter.next().getName();
String gVal = String.valueOf(m.invoke(event, EMPTY_OBJ_ARR));
groupings.add(new StatsGrouping(field, gVal));
}
StatsGroupingColumn column = new StatsGroupingColumn();
column.setGroup(groupings);
return JAXB_MANAGER.marshalToXml(column);
return column;
}
/**
@ -361,4 +369,68 @@ public class AggregateManager {
+ bucketInterval + "'");
}
}
/**
* Scans the aggregate table for aggregate statistics to offline. It doesn't
* process any aggregate from within the 12 hours.
*/
public void offlineAggregates() {
ConfigLoader configLoader = ConfigLoader.getInstance();
OfflineStatsManager offline = new OfflineStatsManager();
AggregateRecordDao aggrRecordDao = new AggregateRecordDao();
Map<String, StatisticsEventConfig> statsMap = configLoader
.getTypeView();
// offline aggregate data older than 6 hours
long maxTime = (System.currentTimeMillis() / TimeUtil.MILLIS_PER_HOUR - 6)
* TimeUtil.MILLIS_PER_HOUR;
for (StatisticsEventConfig conf : statsMap.values()) {
if (conf.getAggregateOfflineRetentionDays() >= 0) {
String eventType = conf.getType();
try {
Date oldestAggregateDate = aggrRecordDao
.getOldestAggregateDate(eventType);
if (oldestAggregateDate != null) {
Date mostRecentOfflineDate = offline
.getMostRecentOfflinedAggregate(conf);
long startHour = oldestAggregateDate.getTime()
/ TimeUtil.MILLIS_PER_HOUR;
if (mostRecentOfflineDate != null) {
// move ahead one hour from most recent time on disk
long offlineHour = mostRecentOfflineDate.getTime()
/ TimeUtil.MILLIS_PER_HOUR + 1;
if (offlineHour > startHour) {
startHour = offlineHour;
}
}
Date startDate = new Date(startHour
* TimeUtil.MILLIS_PER_HOUR);
// process an hour at a time
Date endDate = new Date(startDate.getTime()
+ TimeUtil.MILLIS_PER_HOUR);
while (endDate.getTime() <= maxTime) {
List<AggregateRecord> records = aggrRecordDao
.getAggregates(eventType, startDate,
endDate);
offline.writeAggregatesToDisk(conf, records);
startDate = endDate;
endDate = new Date(startDate.getTime()
+ TimeUtil.MILLIS_PER_HOUR);
}
}
} catch (Exception e) {
statusHandler.error(
"Error occured generating offline aggregates for event "
+ conf.getType(), e);
}
}
}
// zip up old data?
}
}

599
edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/OfflineStatsManager.java Normal file
View file

@ -0,0 +1,599 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.stats;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.lang.reflect.Method;
import java.text.DecimalFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
import java.util.Map;
import java.util.TimeZone;
import javax.xml.bind.JAXBException;
import com.google.common.collect.Multimap;
import com.raytheon.edex.util.Util;
import com.raytheon.uf.common.localization.IPathManager;
import com.raytheon.uf.common.localization.LocalizationContext;
import com.raytheon.uf.common.localization.LocalizationContext.LocalizationLevel;
import com.raytheon.uf.common.localization.LocalizationContext.LocalizationType;
import com.raytheon.uf.common.localization.LocalizationFile;
import com.raytheon.uf.common.localization.PathManagerFactory;
import com.raytheon.uf.common.localization.exception.LocalizationException;
import com.raytheon.uf.common.stats.AggregateRecord;
import com.raytheon.uf.common.stats.StatisticsEvent;
import com.raytheon.uf.common.stats.StatsGrouping;
import com.raytheon.uf.common.stats.StatsGroupingColumn;
import com.raytheon.uf.common.stats.xml.StatisticsAggregate;
import com.raytheon.uf.common.stats.xml.StatisticsEventConfig;
import com.raytheon.uf.common.stats.xml.StatisticsGroup;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.time.TimeRange;
import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.common.util.FileUtil;
import com.raytheon.uf.edex.stats.data.StatsDataAccumulator;
/**
* Offlines data to csv format for long term comparison.
*
* <pre>
*
* SOFTWARE HISTORY
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Aug 21, 2012 jsanchez Initial creation.
* Nov 09, 2012 dhladky Changed to CSV output
* Jan 24, 2013 1357 mpduff Fix comma output and paths.
* May 22, 2013 1917 rjpeter Renamed from Archiver, added generation of raw statistics,
* added method to purge statistics, moved saving of statistics
* to configured instead of site level.
* </pre>
*
* @author jsanchez
*
*/
public class OfflineStatsManager {
private class StatisticsKey {
private final long epochHours;
public StatisticsKey(Date time) {
this.epochHours = time.getTime() / TimeUtil.MILLIS_PER_HOUR;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (int) (epochHours ^ (epochHours >>> 32));
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
StatisticsKey other = (StatisticsKey) obj;
if (!getOuterType().equals(other.getOuterType())) {
return false;
}
if (epochHours != other.epochHours) {
return false;
}
return true;
}
private OfflineStatsManager getOuterType() {
return OfflineStatsManager.this;
}
}
private static final String COMMA = ",";
private static final IUFStatusHandler statusHandler = UFStatus
.getHandler(OfflineStatsManager.class);
private final IPathManager pm = PathManagerFactory.getPathManager();
private final LocalizationContext configuredContext = pm.getContext(
LocalizationType.COMMON_STATIC, LocalizationLevel.CONFIGURED);
private final SimpleDateFormat fieldSdf;
private final SimpleDateFormat directorySdf;
private final SimpleDateFormat fileSdf;
private final DecimalFormat avgFormatter = new DecimalFormat("0.######");
public OfflineStatsManager() {
TimeZone gmt = TimeZone.getTimeZone("GMT");
fieldSdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
fieldSdf.setTimeZone(gmt);
directorySdf = new SimpleDateFormat("yyyyMMdd");
directorySdf.setTimeZone(gmt);
fileSdf = new SimpleDateFormat("yyyyMMddHH");
fileSdf.setTimeZone(gmt);
}
/**
* Gets a directory name in the format stats/[rawStats|aggregates]/StatType
*
* @param conf
* @param isAggregate
* @return
*/
private String getBaseDirectory(StatisticsEventConfig conf,
boolean isAggregate) {
StringBuffer sb = new StringBuffer(40);
sb.append("stats").append(File.separatorChar);
if (isAggregate) {
sb.append("aggregates");
} else {
sb.append("rawStats");
}
sb.append(File.separatorChar).append(conf.getTypeClass().getName());
return sb.toString();
}
/**
* Creates a filename in the format
* stats/[rawStats|aggregates]/StatType/yyyyMMdd/StatType_yyyyMMddHH.csv
*
* @param conf
* @param isAggregate
* @param epochHours
* @return
*/
private String getStatFilename(StatisticsEventConfig conf,
boolean isAggregate, long epochHours) {
String baseName = getBaseDirectory(conf, isAggregate);
StringBuilder sb = new StringBuilder(baseName.length() + 40);
Date time = new Date(epochHours * TimeUtil.MILLIS_PER_HOUR);
sb.append(baseName).append(File.separatorChar)
.append(directorySdf.format(time)).append(File.separatorChar)
.append(conf.getTypeClass().getSimpleName()).append("_")
.append(fileSdf.format(time)).append(".csv");
return sb.toString();
}
/**
* Writes a raw statistic in CSV format to the passed BufferedWriter.
*
* @param bw
* @param conf
* @param grouping
* @param event
* @throws IOException
*/
private void writeCSVOutput(BufferedWriter bw, StatisticsEventConfig conf,
StatsGroupingColumn grouping, StatisticsEvent event)
throws IOException {
Calendar time = event.getDate();
if (time != null) {
bw.write(fieldSdf.format(time.getTime()));
}
for (StatsGrouping group : grouping.getGroup()) {
bw.write(COMMA);
bw.write(group.getValue());
}
for (Method m : conf.getAggregateMethods()) {
try {
bw.write(COMMA);
Number number = (Number) m.invoke(event, new Object[0]);
bw.write(number.toString());
} catch (Exception e) {
statusHandler.error(
"Unable to aggregate '" + m.getName() + "'", e);
}
}
bw.newLine();
}
/**
* Writes the aggregate statistic to the passed BufferedWriter.
*
* @param bw
* @param conf
* @param agg
* @throws IOException
*/
private void writeCSVOutput(BufferedWriter bw, StatisticsEventConfig conf,
AggregateRecord agg) throws IOException {
Calendar startDate = agg.getStartDate();
Calendar endDate = agg.getEndDate();
double sum = agg.getSum();
double count = agg.getCount();
if (startDate != null) {
bw.write(fieldSdf.format(startDate.getTime()));
}
bw.write(COMMA);
if (endDate != null) {
bw.write(fieldSdf.format(endDate.getTime()));
}
StatsGroupingColumn grouping = StatsDataAccumulator
.unmarshalGroupingColumnFromRecord(agg);
for (StatsGrouping group : grouping.getGroup()) {
bw.write(COMMA);
bw.write(group.getValue());
}
bw.write(COMMA);
bw.write(agg.getField());
bw.write(COMMA);
if (count > 0) {
bw.write(avgFormatter.format(sum / count));
} else {
bw.write("0");
}
bw.write(COMMA);
bw.write(String.valueOf(agg.getMin()));
bw.write(COMMA);
bw.write(String.valueOf(agg.getMax()));
bw.write(COMMA);
bw.write(String.valueOf(sum));
bw.write(COMMA);
bw.write(String.valueOf(count));
bw.newLine();
}
/**
* Opens a buffered writer for the given StatisticsKey and
* StatisticsEventConfig. If its a new CSV file a header is also added to
* the file.
*
* @param key
* @param conf
* @return
* @throws IOException
*/
private BufferedWriter getStatEventBufferedWriter(StatisticsKey key,
StatisticsEventConfig conf) throws IOException {
BufferedWriter bw = null;
LocalizationFile siteLocalization = pm
.getLocalizationFile(configuredContext,
getStatFilename(conf, false, key.epochHours));
File outFile = siteLocalization.getFile();
boolean addHeader = outFile.length() == 0;
if (addHeader) {
// pre-create directories if necessary
outFile.getParentFile().mkdirs();
}
bw = new BufferedWriter(new FileWriter(outFile, true));
if (addHeader) {
bw.write("Time");
for (StatisticsGroup group : conf.getGroupList()) {
bw.write(COMMA);
bw.write(group.getDisplayName());
}
for (StatisticsAggregate aggr : conf.getAggregateList()) {
bw.write(COMMA);
bw.write(aggr.getDisplayName());
}
bw.newLine();
}
return bw;
}
/**
* Opens a buffered writer for the given StatisticsKey and
* StatisticsEventConfig. If its a new CSV file a header is also added to
* the file.
*
* @param key
* @param conf
* @return
* @throws IOException
*/
private BufferedWriter getAggregateBufferedWriter(StatisticsKey key,
StatisticsEventConfig conf) throws IOException {
BufferedWriter bw = null;
LocalizationFile siteLocalization = pm.getLocalizationFile(
configuredContext, getStatFilename(conf, true, key.epochHours));
File outFile = siteLocalization.getFile();
boolean addHeader = outFile.length() == 0;
if (addHeader) {
// pre-create directories if necessary
outFile.getParentFile().mkdirs();
}
bw = new BufferedWriter(new FileWriter(outFile, true));
if (addHeader) {
bw.write("Start,End,");
for (StatisticsGroup group : conf.getGroupList()) {
bw.write(group.getDisplayName());
bw.write(COMMA);
}
bw.write("Field,Avg,Min,Max,Sum,Count");
bw.newLine();
}
return bw;
}
/**
* Writes the raw statistics to disk in CSV format.
*
* @param conf
* @param timeMap
*/
public void writeStatsToDisk(
StatisticsEventConfig conf,
Map<TimeRange, Multimap<StatsGroupingColumn, StatisticsEvent>> timeMap) {
if (!timeMap.isEmpty()) {
String outfilePath = null;
BufferedWriter bw = null;
try {
for (Multimap<StatsGroupingColumn, StatisticsEvent> groupedEvents : timeMap
.values()) {
for (StatsGroupingColumn group : groupedEvents.keySet()) {
Iterator<StatisticsEvent> iter = groupedEvents.get(
group).iterator();
StatisticsKey prevKey = null;
while (iter.hasNext()) {
StatisticsEvent event = iter.next();
StatisticsKey curKey = new StatisticsKey(event
.getDate().getTime());
if (!curKey.equals(prevKey)) {
Util.close(bw);
bw = getStatEventBufferedWriter(curKey, conf);
}
writeCSVOutput(bw, conf, group, event);
}
}
}
} catch (IOException e) {
statusHandler.handle(Priority.ERROR, "Failed to write File: "
+ outfilePath, e);
} finally {
Util.close(bw);
}
}
}
/**
* Writes the aggregate records to disk in CSV format.
*
* @param conf
* The StatisticsEventConfig the aggregates belong to
* @param aggregateRecords
* The aggregate records
* @throws JAXBException
*/
public void writeAggregatesToDisk(StatisticsEventConfig conf,
Collection<AggregateRecord> aggregateRecords) {
if (!aggregateRecords.isEmpty()) {
String outfilePath = null;
BufferedWriter bw = null;
try {
Iterator<AggregateRecord> iter = aggregateRecords.iterator();
StatisticsKey prevKey = null;
while (iter.hasNext()) {
AggregateRecord agg = iter.next();
StatisticsKey curKey = new StatisticsKey(agg.getStartDate()
.getTime());
if (!curKey.equals(prevKey)) {
Util.close(bw);
bw = getAggregateBufferedWriter(curKey, conf);
}
writeCSVOutput(bw, conf, agg);
}
} catch (IOException e) {
statusHandler.handle(Priority.ERROR, "Failed to write File: "
+ outfilePath, e);
} finally {
Util.close(bw);
}
}
}
/**
* Returns the most recent offlined date for the given
* StatisticsEventConfig.
*
* @param conf
* @return
* @throws LocalizationException
* @throws IOException
*/
public Date getMostRecentOfflinedAggregate(StatisticsEventConfig conf)
throws LocalizationException, IOException {
Date rval = null;
LocalizationFile siteLocalization = pm.getLocalizationFile(
configuredContext, getBaseDirectory(conf, true));
File eventDir = siteLocalization.getFile(true);
if (eventDir.exists() && eventDir.isDirectory()) {
File latestDir = null;
for (File handle : eventDir.listFiles()) {
if (handle.isDirectory()) {
try {
Date handleDate = directorySdf.parse(handle.getName());
if ((rval == null) || rval.before(handleDate)) {
rval = handleDate;
latestDir = handle;
}
} catch (ParseException e) {
statusHandler.handle(Priority.WARN, "Directory ["
+ handle.getAbsolutePath()
+ "] is not in expected date format ["
+ directorySdf.toPattern() + "]");
}
}
}
// found latest directory date
if (latestDir != null) {
for (File csv : latestDir.listFiles()) {
String name = csv.getName();
if (csv.isFile() && name.endsWith(".csv")) {
// StatType_yyyyMMddHH.csv
int index = name.indexOf('_');
if (index >= 0) {
try {
Date handleDate = fileSdf.parse(name.substring(
index + 1, index + 11));
if ((rval == null) || rval.before(handleDate)) {
rval = handleDate;
}
} catch (ParseException e) {
statusHandler.handle(Priority.WARN, "File ["
+ csv.getAbsolutePath()
+ "] is not in expected date format ["
+ fileSdf.toPattern() + "]");
}
}
}
}
}
}
return rval;
}
/**
* Handle retention day rules, -1 keep nothing, 0 keep everything, any
* positive number keep that many full days.
*
* @param retentionDays
* @return
*/
private long getMinTime(int retentionDays) {
long currentDay = System.currentTimeMillis() / TimeUtil.MILLIS_PER_DAY;
if (retentionDays == 0) {
return 0;
} else if (retentionDays < 0) {
return currentDay * TimeUtil.MILLIS_PER_DAY;
} else {
// add 1 day to not include current day
return (currentDay - (retentionDays + 1)) * TimeUtil.MILLIS_PER_DAY;
}
}
/**
* Purges offline statistics directories for the given
* StatisticsEventConfig.
*
* @param conf
* @return
*/
public void purgeOffline(StatisticsEventConfig conf) {
// purge aggregates
long minTime = getMinTime(conf.getAggregateOfflineRetentionDays());
if (minTime > 0) {
purgeDir(getBaseDirectory(conf, true), minTime);
}
// purge raw
minTime = getMinTime(conf.getRawOfflineRetentionDays());
if (minTime > 0) {
purgeDir(getBaseDirectory(conf, false), minTime);
}
}
/**
* Purges a given stat event dir keeping any directories newer than minTime.
*
* @param dir
* @param minTime
*/
private void purgeDir(String dir, long minTime) {
LocalizationFile siteLocalization = pm.getLocalizationFile(
configuredContext, dir);
File eventDir = siteLocalization.getFile();
if (eventDir.exists() && eventDir.isDirectory()) {
try {
for (File handle : eventDir.listFiles()) {
if (handle.isDirectory()) {
try {
Date handleDate = directorySdf.parse(handle
.getName());
if (handleDate.getTime() <= minTime) {
FileUtil.deleteDir(handle);
}
} catch (ParseException e) {
statusHandler.warn("Directory ["
+ handle.getAbsolutePath()
+ "] is not in expected date format ["
+ directorySdf.toPattern() + "]");
}
}
}
} catch (Exception e) {
statusHandler.error(
"Error occurred purging " + eventDir.getAbsolutePath(),
e);
}
}
}
}

116
edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/StatsPurge.java
View file

@ -25,14 +25,13 @@ import java.util.Calendar;
import java.util.List;
import java.util.TimeZone;
import javax.xml.bind.JAXBException;
import com.raytheon.uf.common.dataquery.db.QueryParam.QueryOperand;
import com.raytheon.uf.common.localization.PathManagerFactory;
import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.common.serialization.SerializationUtil;
import com.raytheon.uf.common.stats.AggregateRecord;
import com.raytheon.uf.common.stats.StatsRecord;
import com.raytheon.uf.common.stats.xml.StatisticsEventConfig;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.edex.database.DataAccessLayerException;
@ -41,21 +40,18 @@ import com.raytheon.uf.edex.database.dao.DaoConfig;
import com.raytheon.uf.edex.database.purge.PurgeRule;
import com.raytheon.uf.edex.database.purge.PurgeRuleSet;
import com.raytheon.uf.edex.database.query.DatabaseQuery;
import com.raytheon.uf.edex.stats.util.Archiver;
import com.raytheon.uf.edex.stats.util.ConfigLoader;
/**
* Purges the stats table of expired/unused stat records. Purges the aggregate
* table and write it to disk.
*
* *
* Purges the stats table of expired/unused stat records.
*
* <pre>
*
* SOFTWARE HISTORY
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Aug 21, 2012 jsanchez Initial creation.
*
* Aug 21, 2012 jsanchez Initial creation.
* May 22, 2013 1917 rjpeter Added purging off offline statistics.
* </pre>
*
* @author jsanchez
@ -66,8 +62,6 @@ public class StatsPurge {
private static final transient IUFStatusHandler statusHandler = UFStatus
.getHandler(StatsPurge.class);
private Archiver archiver;
private final CoreDao aggregateRecordDao = new CoreDao(DaoConfig.forClass(
"metadata", AggregateRecord.class));
@ -81,57 +75,53 @@ public class StatsPurge {
public StatsPurge() {
aggregatePurgeRules = readPurgeRules("aggregatePurgeRules.xml");
statsPurgeRules = readPurgeRules("statsPurgeRules.xml");
try {
archiver = new Archiver();
purgeStats();
} catch (DataAccessLayerException e) {
statusHandler
.error("Error purging stats on start up. Stats will not be purged. ",
e);
}
public void purge() {
purgeAggregates();
purgeStats();
// purge offline stats
OfflineStatsManager offlineStats = new OfflineStatsManager();
ConfigLoader loader = ConfigLoader.getInstance();
for (StatisticsEventConfig conf : loader.getTypeView().values()) {
offlineStats.purgeOffline(conf);
}
}
/**
* Purges records from the aggregate table and writes them to disk.
*/
public void purgeAggregates() throws JAXBException,
DataAccessLayerException {
public void purgeAggregates() {
if (aggregatePurgeRules != null) {
Calendar expiration = Calendar.getInstance(TimeZone
.getTimeZone("GMT"));
DatabaseQuery query = new DatabaseQuery(AggregateRecord.class);
List<PurgeRule> allRules = new ArrayList<PurgeRule>();
try {
Calendar expiration = Calendar.getInstance(TimeZone
.getTimeZone("GMT"));
DatabaseQuery deleteStmt = new DatabaseQuery(
AggregateRecord.class);
List<PurgeRule> allRules = new ArrayList<PurgeRule>();
// check for specific rules, if none, apply defaults
if (!aggregatePurgeRules.getRules().isEmpty()) {
allRules.addAll(aggregatePurgeRules.getRules());
} else if (!aggregatePurgeRules.getDefaultRules().isEmpty()) {
allRules.addAll(aggregatePurgeRules.getDefaultRules());
}
// check for specific rules, if none, apply defaults
if (!aggregatePurgeRules.getRules().isEmpty()) {
allRules.addAll(aggregatePurgeRules.getRules());
} else if (!aggregatePurgeRules.getDefaultRules().isEmpty()) {
allRules.addAll(aggregatePurgeRules.getDefaultRules());
}
for (PurgeRule rule : allRules) {
if (rule.isPeriodSpecified()) {
long ms = rule.getPeriodInMillis();
int minutes = new Long(ms / (1000 * 60)).intValue();
expiration.add(Calendar.MINUTE, -minutes);
for (PurgeRule rule : allRules) {
if (rule.isPeriodSpecified()) {
long ms = rule.getPeriodInMillis();
int minutes = new Long(ms / (1000 * 60)).intValue();
expiration.add(Calendar.MINUTE, -minutes);
query.addQueryParam("endDate", expiration,
QueryOperand.LESSTHAN);
deleteStmt.addQueryParam("endDate", expiration,
QueryOperand.LESSTHAN);
List<?> objects = aggregateRecordDao.queryByCriteria(query);
if (!objects.isEmpty()) {
AggregateRecord[] aggregateRecords = new AggregateRecord[objects
.size()];
for (int i = 0; i < aggregateRecords.length; i++) {
aggregateRecords[i] = (AggregateRecord) objects
.get(i);
}
archiver.writeToDisk(aggregateRecords);
aggregateRecordDao.deleteAll(objects);
aggregateRecordDao.deleteByCriteria(deleteStmt);
}
}
} catch (DataAccessLayerException e) {
statusHandler.error("Error purging stats aggregates", e);
}
}
}
@ -140,21 +130,25 @@ public class StatsPurge {
* Purges records from the stats table if they are older than the expiration
* time.
*/
private void purgeStats() throws DataAccessLayerException {
private void purgeStats() {
if (statsPurgeRules != null) {
Calendar expiration = Calendar.getInstance(TimeZone
.getTimeZone("GMT"));
DatabaseQuery deleteStmt = new DatabaseQuery(StatsRecord.class);
try {
Calendar expiration = Calendar.getInstance(TimeZone
.getTimeZone("GMT"));
DatabaseQuery deleteStmt = new DatabaseQuery(StatsRecord.class);
for (PurgeRule rule : statsPurgeRules.getRules()) {
if (rule.isPeriodSpecified()) {
long ms = rule.getPeriodInMillis();
int minutes = new Long(ms / (1000 * 60)).intValue();
expiration.add(Calendar.MINUTE, -minutes);
deleteStmt.addQueryParam("date", expiration,
QueryOperand.LESSTHAN);
statsRecordDao.deleteByCriteria(deleteStmt);
for (PurgeRule rule : statsPurgeRules.getRules()) {
if (rule.isPeriodSpecified()) {
long ms = rule.getPeriodInMillis();
int minutes = new Long(ms / (1000 * 60)).intValue();
expiration.add(Calendar.MINUTE, -minutes);
deleteStmt.addQueryParam("date", expiration,
QueryOperand.LESSTHAN);
statsRecordDao.deleteByCriteria(deleteStmt);
}
}
} catch (DataAccessLayerException e) {
statusHandler.error("Error purging stats aggregates", e);
}
}
}

149
edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/dao/AggregateRecordDao.java
View file

@ -1,16 +1,60 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.stats.dao;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import org.hibernate.Query;
import org.hibernate.Session;
import org.hibernate.Transaction;
import com.raytheon.uf.common.dataquery.db.QueryParam.QueryOperand;
import com.raytheon.uf.common.stats.AggregateRecord;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.util.CollectionUtil;
import com.raytheon.uf.edex.database.DataAccessLayerException;
import com.raytheon.uf.edex.database.dao.CoreDao;
import com.raytheon.uf.edex.database.dao.DaoConfig;
import com.raytheon.uf.edex.database.query.DatabaseQuery;
/**
* Record class for stats waiting to be stored in the appropriate bucket.
*
* <pre>
*
* SOFTWARE HISTORY
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Aug 21, 2012 jsanchez Initial creation
* May 22, 2013 1917 rjpeter Added query methods for retrieving data about aggregates.
* </pre>
*
* @author jsanchez
*/
public class AggregateRecordDao extends CoreDao {
private static final IUFStatusHandler statusHandler = UFStatus
.getHandler(AggregateRecordDao.class);
/**
* Creates a new data access object
*/
@ -61,4 +105,109 @@ public class AggregateRecordDao extends CoreDao {
persist(newRecord);
}
}
/**
* Returns the oldest start date for a given aggregate eventType.
*
* @param eventType
* @return
* @throws DataAccessLayerException
*/
public Date getOldestAggregateDate(final String eventType)
throws DataAccessLayerException {
Session sess = null;
Transaction tx = null;
try {
sess = getHibernateTemplate().getSessionFactory().openSession();
tx = sess.beginTransaction();
Query query = sess
.createQuery("SELECT MIN(startDate) FROM AggregateRecord WHERE eventType = ?");
query.setString(0, eventType);
Calendar rval = (Calendar) query.uniqueResult();
tx.commit();
if (rval != null) {
return rval.getTime();
}
return null;
} catch (Exception e) {
if (tx != null) {
try {
tx.rollback();
} catch (Exception e1) {
statusHandler.error(
"Error occurred rolling back transaction", e1);
}
}
throw new DataAccessLayerException(
"Unable to look up min start date for event [" + eventType
+ "]", e);
} finally {
if (sess != null) {
try {
sess.close();
} catch (Exception e) {
statusHandler.error(
"Error occurred closing database session", e);
}
}
}
}
/**
* Returns all aggregates of a given type and such that startDate >=
* event.startDate < endDate.
*
* @param eventType
* @param startDate
* @param endDate
* @return
* @throws DataAccessLayerException
*/
public List<AggregateRecord> getAggregates(final String eventType,
final Date startDate, final Date endDate)
throws DataAccessLayerException {
Session sess = null;
Transaction tx = null;
try {
sess = getHibernateTemplate().getSessionFactory().openSession();
tx = sess.beginTransaction();
Query query = sess
.createQuery("FROM AggregateRecord WHERE eventType = ? AND startDate >= ? AND startDate < ? ORDER BY startDate");
query.setString(0, eventType);
query.setTimestamp(1, startDate);
query.setTimestamp(2, endDate);
@SuppressWarnings("unchecked")
List<AggregateRecord> rval = query.list();
tx.commit();
return rval;
} catch (Exception e) {
if (tx != null) {
try {
tx.rollback();
} catch (Exception e1) {
statusHandler.error(
"Error occurred rolling back transaction", e1);
}
}
throw new DataAccessLayerException(
"Unable to look up aggregates for event [" + eventType
+ "]", e);
} finally {
if (sess != null) {
try {
sess.close();
} catch (Exception e) {
statusHandler.error(
"Error occurred closing database session", e);
}
}
}
}
}

67
edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/dao/StatsDao.java
View file

@ -1,8 +1,30 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.stats.dao;
import java.util.Calendar;
import java.util.List;
import org.hibernate.Query;
import org.hibernate.StatelessSession;
import com.raytheon.uf.common.dataquery.db.QueryParam.QueryOperand;
import com.raytheon.uf.common.stats.StatsRecord;
import com.raytheon.uf.edex.database.DataAccessLayerException;
@ -10,6 +32,20 @@ import com.raytheon.uf.edex.database.dao.CoreDao;
import com.raytheon.uf.edex.database.dao.DaoConfig;
import com.raytheon.uf.edex.database.query.DatabaseQuery;
/**
* Data access object for raw statistics.
*
* <pre>
*
* SOFTWARE HISTORY
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Aug 21, 2012 jsanchez Initial creation
* May 22, 2013 1917 rjpeter Added reclaimSpace.
* </pre>
*
* @author jsanchez
*/
public class StatsDao extends CoreDao {
/**
* Creates a new data access object
@ -43,4 +79,35 @@ public class StatsDao extends CoreDao {
return (List<StatsRecord>) queryByCriteria(query);
}
/**
* Manually runs vacuum due to large numbers of inserts and deletes to keep
* table size to a minimum.
*/
public void reclaimSpace() {
StatelessSession sess = null;
try {
sess = getHibernateTemplate().getSessionFactory()
.openStatelessSession();
// vacuum can't run within a transaction, hack to allow vacuum to
// run from within hibernate
Query query = sess
.createSQLQuery("rollback; VACUUM ANALYZE events.stats");
query.executeUpdate();
statusHandler.info("stats vacuumed");
} catch (Exception e) {
statusHandler.error(
"Error occurred running VACUUM on events.stats", e);
} finally {
if (sess != null) {
try {
sess.close();
} catch (Exception e) {
statusHandler.error(
"Error occurred closing database session", e);
}
}
}
}
}

8
edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/data/StatsDataAccumulator.java
View file

@ -56,10 +56,10 @@ import com.raytheon.uf.common.util.CollectionUtil;
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Nov 15, 2012 728 mpduff Initial creation
* Nov 15, 2012 728 mpduff Initial creation
* Jan 15, 2013 1487 djohnson Use xml for the grouping information on an {@link AggregateRecord}.
* Jan 17, 2013 1357 mpduff Remove unit conversions, add time step, other cleanup.
*
* Jan 17, 2013 1357 mpduff Remove unit conversions, add time step, other cleanup.
* May 22, 2013 1917 rjpeter Made unmarshalGroupingColumnFromRecord public.
* </pre>
*
* @author mpduff
@ -268,7 +268,7 @@ public class StatsDataAccumulator {
* @return the unmarshalled column, or an empty column if unable to
* unmarshal
*/
private static StatsGroupingColumn unmarshalGroupingColumnFromRecord(
public static StatsGroupingColumn unmarshalGroupingColumnFromRecord(
AggregateRecord record) {
String groupingXmlAsString = record.getGrouping();
try {

8
edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/handler/GraphDataHandler.java
View file

@ -32,7 +32,7 @@ import com.raytheon.uf.common.stats.GraphDataResponse;
import com.raytheon.uf.common.stats.data.GraphData;
import com.raytheon.uf.common.stats.xml.StatisticsAggregate;
import com.raytheon.uf.common.stats.xml.StatisticsConfig;
import com.raytheon.uf.common.stats.xml.StatisticsEvent;
import com.raytheon.uf.common.stats.xml.StatisticsEventConfig;
import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.edex.database.dao.CoreDao;
import com.raytheon.uf.edex.database.dao.DaoConfig;
@ -49,9 +49,9 @@ import com.raytheon.uf.edex.stats.util.ConfigLoader;
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Sep 11, 2012 728 mpduff Initial creation
* Sep 11, 2012 728 mpduff Initial creation
* Jan 07, 2013 1451 djohnson Use newGmtCalendar().
*
* May 22, 2013 1917 rjpeter Renamed StatisticsEvent to StatisticsEventConfig.
* </pre>
*
* @author mpduff
@ -199,7 +199,7 @@ public class GraphDataHandler implements IRequestHandler<GraphDataRequest> {
for (StatisticsConfig config : configList) {
for (String cat : config.getCategories()) {
if (cat.equals(category)) {
for (StatisticsEvent event : config.getEvents()) {
for (StatisticsEventConfig event : config.getEvents()) {
if (event.getType().equals(type)) {
for (StatisticsAggregate agg : event
.getAggregateList()) {

6
edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/handler/StatsHandler.java
View file

@ -32,7 +32,7 @@ import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.common.serialization.SerializationUtil;
import com.raytheon.uf.common.stats.StatsRecord;
import com.raytheon.uf.common.stats.xml.StatisticsConfig;
import com.raytheon.uf.common.stats.xml.StatisticsEvent;
import com.raytheon.uf.common.stats.xml.StatisticsEventConfig;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.edex.database.dao.CoreDao;
@ -75,7 +75,7 @@ public class StatsHandler {
public static void setValidEventTypes(List<StatisticsConfig> configurations) {
validEventTypes = new HashSet<String>();
for (StatisticsConfig config : configurations) {
for (StatisticsEvent event : config.getEvents()) {
for (StatisticsEventConfig event : config.getEvents()) {
validEventTypes.add(event.getType());
}
}
@ -103,7 +103,7 @@ public class StatsHandler {
HashSet<String> myValidEventTypes = new HashSet<String>();
for (StatisticsConfig config : configLoader.getConfigurations()) {
for (StatisticsEvent event : config.getEvents()) {
for (StatisticsEventConfig event : config.getEvents()) {
myValidEventTypes.add(event.getType());
}
}

276
edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/util/Archiver.java
View file

@ -1,276 +0,0 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.stats.util;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import javax.xml.bind.JAXBException;
import com.raytheon.uf.common.localization.IPathManager;
import com.raytheon.uf.common.localization.LocalizationContext;
import com.raytheon.uf.common.localization.LocalizationContext.LocalizationLevel;
import com.raytheon.uf.common.localization.LocalizationContext.LocalizationType;
import com.raytheon.uf.common.localization.LocalizationFile;
import com.raytheon.uf.common.localization.PathManagerFactory;
import com.raytheon.uf.common.stats.AggregateRecord;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.time.TimeRange;
/**
* Archives the data in the aggregate_bucket table to an xml file.
*
* <pre>
*
* SOFTWARE HISTORY
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Aug 21, 2012 jsanchez Initial creation.
* Nov 09, 2012 dhladky Changed to CSV output
* Jan 24, 2013 1357 mpduff Fix comma output and paths.
*
* </pre>
*
* @author jsanchez
*
*/
public class Archiver {
private class StatisticsKey {
public String eventType;
public String grouping;
public TimeRange timeRange;
@Override
public boolean equals(Object o) {
if (o != null && o instanceof StatisticsKey) {
StatisticsKey other = (StatisticsKey) o;
return eventType.equals(other.eventType)
&& timeRange.getStart().equals(
other.timeRange.getStart())
&& timeRange.getEnd().equals(other.timeRange.getEnd());
}
return false;
}
@Override
public int hashCode() {
return 1;
}
}
private static final String COMMA = ",";
private static final Pattern NLPattern = Pattern.compile("[\\n\\r]+");
private static final IUFStatusHandler statusHandler = UFStatus
.getHandler(Archiver.class);
private final IPathManager pm = PathManagerFactory.getPathManager();
private final LocalizationContext context = pm.getContext(
LocalizationType.COMMON_STATIC, LocalizationLevel.SITE);
private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
private static final String FILE_DATE_FORMAT = "yyyyMMdd_HHmm";
private static final Pattern PERIOD_PATTERN = Pattern.compile("\\.");
public Archiver() {
}
/**
* Creates a filename in the format /stats/aggregates/group...
* /eventType.start-end.dat
*
* @param items
* @return
*/
private String createFilename(TimeRange tr, String eventType) {
SimpleDateFormat fileDateFormatter = new SimpleDateFormat(
FILE_DATE_FORMAT);
StringBuilder sb = new StringBuilder("stats/aggregates");
String[] chunks = PERIOD_PATTERN.split(eventType);
sb.append("/");
sb.append(chunks[chunks.length - 1]);
sb.append(".");
sb.append(fileDateFormatter.format(tr.getStart()));
sb.append("-");
sb.append(fileDateFormatter.format(tr.getEnd()));
sb.append(".csv");
return sb.toString();
}
/**
* Used for outputting the stats as CSV
*
* @return
*/
private String getCSVOutput(AggregateRecord agrec,
SimpleDateFormat dateFormat) {
StringBuilder sb = new StringBuilder();
String eventType = agrec.getEventType();
Calendar startDate = agrec.getStartDate();
Calendar endDate = agrec.getEndDate();
String grouping = agrec.getGrouping();
String field = agrec.getField();
double max = agrec.getMax();
double min = agrec.getMin();
double sum = agrec.getSum();
double count = agrec.getCount();
if (eventType != null) {
sb.append(eventType);
}
sb.append(COMMA);
if (startDate != null) {
sb.append(dateFormat.format(startDate.getTime()));
}
sb.append(COMMA);
if (endDate != null) {
sb.append(dateFormat.format(endDate.getTime()));
}
sb.append(COMMA);
if (grouping != null) {
sb.append(NLPattern.matcher(grouping).replaceAll(""));
}
sb.append(COMMA);
if (field != null) {
sb.append(field);
}
sb.append(COMMA);
sb.append(max).append(COMMA);
sb.append(min).append(COMMA);
sb.append(sum).append(COMMA);
sb.append(count);
return sb.toString();
}
/**
* Writes the aggregate records to disk.
*
* @param aggregateRecords
* @throws JAXBException
*/
public void writeToDisk(AggregateRecord[] aggregateRecords) {
Map<StatisticsKey, List<AggregateRecord>> statisticsMap = new HashMap<StatisticsKey, List<AggregateRecord>>();
for (AggregateRecord record : aggregateRecords) {
StatisticsKey key = new StatisticsKey();
key.eventType = record.getEventType();
key.grouping = record.getGrouping();
key.timeRange = new TimeRange(record.getStartDate(),
record.getEndDate());
List<AggregateRecord> aggregateRecordList = statisticsMap.get(key);
if (aggregateRecordList == null) {
aggregateRecordList = new ArrayList<AggregateRecord>();
statisticsMap.put(key, aggregateRecordList);
}
aggregateRecordList.add(record);
}
for (StatisticsKey key : statisticsMap.keySet()) {
String eventType = key.eventType;
List<AggregateRecord> records = statisticsMap.get(key);
String filename = createFilename(key.timeRange, eventType);
try {
writeToFile(filename, records);
} catch (JAXBException e) {
statusHandler.error("Unable to write statistics file "
+ filename, e);
}
}
}
/**
* Writes the statistics xml to disk.
*
* @param statistics
* @throws JAXBException
*/
public void writeToFile(String filename, List<AggregateRecord> records)
throws JAXBException {
BufferedWriter bw = null;
SimpleDateFormat dateFormatter = new SimpleDateFormat(DATE_FORMAT);
LocalizationFile siteLocalization = pm.getLocalizationFile(context,
filename);
String outputFilePath = siteLocalization.getFile().getAbsolutePath();
// pre-create directories if necessary
siteLocalization.getFile().getParentFile().mkdirs();
// Write this to output CSV
try {
bw = new BufferedWriter(new FileWriter(outputFilePath));
if (bw != null) {
for (AggregateRecord agrec : records) {
bw.write(getCSVOutput(agrec, dateFormatter));
bw.newLine();
}
}
} catch (IOException e) {
statusHandler.handle(Priority.ERROR, "Failed to write File: "
+ outputFilePath, e);
} finally {
if (bw != null) {
try {
bw.close();
} catch (IOException e) {
statusHandler.handle(Priority.PROBLEM,
"failed to close CSV output file stream. "
+ filename, e);
}
}
}
}
}

27
edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/util/ConfigLoader.java
View file

@ -41,7 +41,7 @@ import com.raytheon.uf.common.localization.exception.LocalizationException;
import com.raytheon.uf.common.serialization.JAXBManager;
import com.raytheon.uf.common.stats.xml.StatisticsAggregate;
import com.raytheon.uf.common.stats.xml.StatisticsConfig;
import com.raytheon.uf.common.stats.xml.StatisticsEvent;
import com.raytheon.uf.common.stats.xml.StatisticsEventConfig;
import com.raytheon.uf.common.stats.xml.StatisticsGroup;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
@ -58,11 +58,12 @@ import com.raytheon.uf.common.util.ReflectionUtil;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Aug 21, 2012 jsanchez Updated error handling and validated config files.
* Nov 07, 2012 1317 mpduff Update config files.
* Nov 29, 2012 1350 rjpeter Updated to static, fixed localization, increased validation.
* Jan 15, 2013 1487 djohnson Make validate() static and public, so it can be run independently.
* Mar 27, 2013 1834 mpduff Filter for xml files on localization file read, wrap unmarshall and
* Nov 07, 2012 1317 mpduff Update config files.
* Nov 29, 2012 1350 rjpeter Updated to static, fixed localization, increased validation.
* Jan 15, 2013 1487 djohnson Make validate() static and public, so it can be run independently.
* Mar 27, 2013 1834 mpduff Filter for xml files on localization file read, wrap unmarshall and
* log error if one occurs
* May 22, 2013 1917 rjpeter Updated validate to save typeClass back to StatisticsEventConfig.
* </pre>
*
* @author jsanchez
@ -81,7 +82,7 @@ public class ConfigLoader {
private List<StatisticsConfig> configurations = Collections.emptyList();
private Map<String, StatisticsEvent> classToEventMap = Collections
private Map<String, StatisticsEventConfig> classToEventMap = Collections
.emptyMap();
private static final String STATS_DIR = "stats";
@ -113,7 +114,7 @@ public class ConfigLoader {
*
* @return
*/
public Map<String, StatisticsEvent> getTypeView() {
public Map<String, StatisticsEventConfig> getTypeView() {
return classToEventMap;
}
@ -144,7 +145,7 @@ public class ConfigLoader {
if (!statConfs.isEmpty()) {
List<StatisticsConfig> myConfigurations = new ArrayList<StatisticsConfig>(
statConfs.size());
Map<String, StatisticsEvent> myEvents = new HashMap<String, StatisticsEvent>();
Map<String, StatisticsEventConfig> myEvents = new HashMap<String, StatisticsEventConfig>();
for (LocalizationFile lf : statConfs.values()) {
try {
@ -174,17 +175,17 @@ public class ConfigLoader {
* @param config
*/
@VisibleForTesting
public static void validate(Map<String, StatisticsEvent> eventMap,
public static void validate(Map<String, StatisticsEventConfig> eventMap,
StatisticsConfig config) {
for (Iterator<StatisticsEvent> iter = config.getEvents().iterator(); iter
.hasNext();) {
StatisticsEvent event = iter.next();
for (Iterator<StatisticsEventConfig> iter = config.getEvents()
.iterator(); iter.hasNext();) {
StatisticsEventConfig event = iter.next();
String eventType = event.getType();
if (!eventMap.containsKey(eventType)) {
try {
Class<?> clazz = Class.forName(eventType);
// verify the type is an Event
clazz.asSubclass(Event.class);
event.setTypeClass(clazz.asSubclass(Event.class));
// validate groupBy fields can be found
List<StatisticsGroup> groups = event.getGroupList();

4
edexOsgi/com.raytheon.uf.edex.stats/utility/common_static/base/stats/edexProcessStats.xml
View file

@ -1,7 +1,9 @@
<statisticsConfig>
<!-- Event Type should be fully qualified name of stat event -->
<!-- raw and aggregate OfflineRetentionDays: Value less than zero disables saving of raw statistic, zero is never purge -->
<statisticsEvent type="com.raytheon.uf.common.stats.ProcessEvent"
displayName="Processing Events" category="Data Ingest Events">
displayName="Processing Events" category="Data Ingest Events"
rawOfflineRetentionDays="-1" aggregateOfflineRetentionDays="90">
<statisticsGroup name="dataType" displayName="Data Type" />
<!-- Processing time available display units:
ms, Seconds, Minutes, Hours -->

1
pythonPackages/dynamicserialize/dstypes/__init__.py
View file

@ -22,6 +22,7 @@
__all__ = [
'com',
'gov',
'java'
]

70
pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/GetLatestDbTimeRequest.py Normal file
View file

@ -0,0 +1,70 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
# File auto-generated against equivalent DynamicSerialize Java class
# and then modified post-generation to use AbstractGfeRequest and
# implement str(), repr()
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 05/22/13 2025 dgilling Initial Creation.
#
#
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import AbstractGfeRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import DatabaseID
class GetLatestDbTimeRequest(AbstractGfeRequest):
def __init__(self, dbId=None):
super(GetLatestDbTimeRequest, self).__init__()
if dbId is not None and isinstance(dbId, DatabaseID):
self.dbId = dbId
self.siteID = dbId.getSiteId()
elif dbId is not None and not isinstance(dbId, DatabaseID):
raise TypeError(
"Attempt to construct GetLatestDbTimeRequest without providing a valid DatabaseID.")
def __str__(self):
retVal = "GetLatestDbTimeRequest["
retVal += "wokstationID: " + str(self.workstationID) + ", "
retVal += "siteID: " + str(self.siteID) + ", "
retVal += "dbId: " + str(self.dbId) + "]"
return retVal
def __repr__(self):
retVal = "ExecuteIfpNetCDFGridRequest("
retVal += "wokstationID=" + repr(self.workstationID) + ", "
retVal += "siteID=" + repr(self.siteID) + ", "
retVal += "dbId=" + repr(self.dbId) + ")"
return retVal
def getDbId(self):
return self.dbId
def setDbId(self, dbId):
if isinstance(dbId, DatabaseID):
self.dbId = dbId
else:
raise TypeError(
"Attempt to call GetLatestDbTimeRequest.setDbId() without providing a valid DatabaseID.")

63
pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/GetLatestModelDbIdRequest.py Normal file
View file

@ -0,0 +1,63 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
# File auto-generated against equivalent DynamicSerialize Java class
# and then modified post-generation to use AbstractGfeRequest and
# implement str(), repr()
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 05/22/13 2025 dgilling Initial Creation.
#
#
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import AbstractGfeRequest
class GetLatestModelDbIdRequest(AbstractGfeRequest):
def __init__(self, siteId=None, modelName=None):
super(GetLatestModelDbIdRequest, self).__init__()
if siteId is not None:
self.siteID = str(siteId)
if modelName is not None:
self.modelName = str(modelName)
def __str__(self):
retVal = "GetLatestModelDbIdRequest["
retVal += "wokstationID: " + str(self.workstationID) + ", "
retVal += "siteID: " + str(self.siteID) + ", "
retVal += "modelName: " + str(self.modelName) + "]"
return retVal
def __repr__(self):
retVal = "ExecuteIfpNetCDFGridRequest("
retVal += "wokstationID=" + repr(self.workstationID) + ", "
retVal += "siteID=" + repr(self.siteID) + ", "
retVal += "modelName=" + repr(self.modelName) + ")"
return retVal
def getModelName(self):
return self.modelName
def setModelName(self, modelName):
self.modelName = str(modelName)

4
pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/__init__.py
View file

@ -30,6 +30,8 @@ __all__ = [
'GetASCIIGridsRequest',
'GetGridDataRequest',
'GetGridInventoryRequest',
'GetLatestDbTimeRequest',
'GetLatestModelDbIdRequest',
'GetLockTablesRequest',
'GetOfficialDbNameRequest',
'GetParmListRequest',
@ -58,6 +60,8 @@ from ExecuteIscMosaicRequest import ExecuteIscMosaicRequest
from GetASCIIGridsRequest import GetASCIIGridsRequest
from GetGridDataRequest import GetGridDataRequest
from GetGridInventoryRequest import GetGridInventoryRequest
from GetLatestDbTimeRequest import GetLatestDbTimeRequest
from GetLatestModelDbIdRequest import GetLatestModelDbIdRequest
from GetLockTablesRequest import GetLockTablesRequest
from GetOfficialDbNameRequest import GetOfficialDbNameRequest
from GetParmListRequest import GetParmListRequest

6
rpms/awips2.core/Installer.edex-environment/edex/component.spec
View file

@ -19,9 +19,9 @@ AutoReq: no
provides: awips2-edex-environment
requires: awips2-edex-base
requires: awips2-postgresql
requires: qpid-java-broker
requires: qpid-java-client
requires: qpid-java-common
requires: awips2-qpid-java-broker
requires: awips2-qpid-java-client
requires: awips2-qpid-java-common
requires: awips2-python
requires: awips2-java
requires: awips2-psql

107
rpms/awips2.core/Installer.ldm/component.spec
View file

@ -21,6 +21,7 @@ Packager: Bryan Kowal
AutoReq: no
Requires: awips2-notification
Requires: qpid-cpp-client-devel
Requires: zlib-devel
provides: awips2-ldm
provides: awips2-base-component
@ -178,6 +179,52 @@ rm -f %{_ldm_src_tar}
if [ $? -ne 0 ]; then
exit 1
fi
# create .bash_profile
if [ ! -f /usr/local/ldm/.bash_profile ]; then
echo 'export PATH=$HOME/decoders:$HOME/util:$HOME/bin:$PATH' > \
/usr/local/ldm/.bash_profile
echo 'export MANPATH=$HOME/share/man:/usr/share/man' >> \
/usr/local/ldm/.bash_profile
/bin/chown ldm:fxalpha /usr/local/ldm/.bash_profile
fi
pushd . > /dev/null 2>&1
# build ldm
rm -f ~ldm/runtime
cd ${_ldm_root_dir}/src
if [ $? -ne 0 ]; then
exit 1
fi
export _current_dir=`pwd`
su ldm -lc "cd ${_current_dir}; ./configure --disable-max-size --with-noaaport --disable-root-actions --prefix=${_ldm_root_dir} CFLAGS='-g -O0'" \
> configure.log 2>&1
if [ $? -ne 0 ]; then
echo "FATAL: ldm configure has failed!"
exit 1
fi
export _current_dir=`pwd`
su ldm -lc "cd ${_current_dir}; make install" > install.log 2>&1
if [ $? -ne 0 ]; then
echo "FATAL: make install has failed!"
exit 1
fi
su ldm -lc "cd ${_current_dir}; /bin/bash my-install" > my-install.log 2>&1
if [ $? -ne 0 ]; then
echo "FATAL: my-install has failed!"
exit 1
fi
popd > /dev/null 2>&1
pushd . > /dev/null 2>&1
cd ${_ldm_root_dir}/src
make root-actions > root-actions.log 2>&1
if [ $? -ne 0 ]; then
echo "FATAL: root-actions has failed!"
exit 1
fi
popd > /dev/null 2>&1
# unpack bin, decoders, and etc.
_PATCH_DIRS=( 'bin' 'decoders' 'etc' )
for patchDir in ${_PATCH_DIRS[*]};
@ -191,21 +238,10 @@ do
exit 1
fi
done
/bin/chown -R ldm:fxalpha ${_ldm_dir}
if [ $? -ne 0 ]; then
exit 1
fi
/bin/chmod a+x ${_ldm_dir}/bin/*
/bin/chown -R ldm:fxalpha ${_ldm_dir}/etc ${_ldm_dir}/decoders
popd > /dev/null 2>&1
# create .bash_profile
if [ ! -f /usr/local/ldm/.bash_profile ]; then
echo 'export PATH=$HOME/decoders:$HOME/util:$HOME/bin:$PATH' > \
/usr/local/ldm/.bash_profile
echo 'export MANPATH=$HOME/share/man:/usr/share/man' >> \
/usr/local/ldm/.bash_profile
/bin/chown ldm:fxalpha /usr/local/ldm/.bash_profile
fi
# construct pqact
pushd . > /dev/null 2>&1
cd ${_ldm_dir}/etc
@ -235,47 +271,6 @@ if [ ${_myHost} != "cpsbn1" -a ${_myHost} != "cpsbn2" -a ${_myHost} != "dx1" -a
fi
popd > /dev/null 2>&1
pushd . > /dev/null 2>&1
# build ldm
cd ${_ldm_root_dir}/src
if [ $? -ne 0 ]; then
exit 1
fi
export _current_dir=`pwd`
su ldm -lc "cd ${_current_dir}; ./configure --disable-max-size --with-noaaport --disable-root-actions --prefix=${_ldm_dir}" \
> configure.log 2>&1
if [ $? -ne 0 ]; then
echo "FATAL: ldm configure has failed!"
exit 1
fi
export _current_dir=`pwd`
su ldm -lc "cd ${_current_dir}; make install" > install.log 2>&1
if [ $? -ne 0 ]; then
echo "FATAL: make install has failed!"
exit 1
fi
popd > /dev/null 2>&1
pushd . > /dev/null 2>&1
cd ${_ldm_root_dir}/src/noaaport
if [ $? -ne 0 ]; then
exit 1
fi
export _current_dir=`pwd`
su ldm -lc "cd ${_current_dir}; /bin/bash my-make" > my-make.log 2>&1
if [ $? -ne 0 ]; then
echo "FATAL: my-make has failed!"
exit 1
fi
popd > /dev/null 2>&1
pushd . > /dev/null 2>&1
cd ${_ldm_root_dir}/src
make root-actions > root-actions.log 2>&1
if [ $? -ne 0 ]; then
echo "FATAL: root-actions has failed!"
exit 1
fi
popd > /dev/null 2>&1
# build decrypt_file & edexBridge
pushd . > /dev/null 2>&1
cd ${_ldm_dir}/SOURCES
@ -349,7 +344,7 @@ fi
for _file in $( ls /tmp/ldm/etc/pqact.conf.* | grep -wE "pqact.conf.[a-z]{3,4}" | grep -v pqact.conf.dev | xargs ) ;
do
if [[ ! -f /usr/local/ldm/etc/${_file} ]]; then
scp -qp /tmp/ldm/etc/${_file} /usr/local/ldm/etc/
scp -qp ${_file} /usr/local/ldm/etc/
fi
done
#if a remote CP site, copy over the filtered data configuration
@ -432,5 +427,5 @@ rm -rf ${RPM_BUILD_ROOT}
%attr(755,root,root) /etc/profile.d/awipsLDM.csh
%attr(755,root,root) /etc/ld.so.conf.d/awips2-ldm-i386.conf
%attr(755,root,root) /etc/ld.so.conf.d/ldm.log
%attr(755,root,root) /etc/logrotate.d/ldm.log
%attr(755,root,root) /etc/init.d/ldmcp

BIN
rpms/awips2.core/Installer.ldm/src/ldm-6.11.5.tar.gz
View file

Binary file not shown.

22
tests/unit/com/raytheon/uf/edex/stats/AggregateManagerTest.java
View file

@ -44,7 +44,7 @@ import com.raytheon.uf.common.serialization.JAXBManager;
import com.raytheon.uf.common.stats.StatsGrouping;
import com.raytheon.uf.common.stats.StatsGroupingColumn;
import com.raytheon.uf.common.stats.xml.StatisticsConfig;
import com.raytheon.uf.common.stats.xml.StatisticsEvent;
import com.raytheon.uf.common.stats.xml.StatisticsEventConfig;
import com.raytheon.uf.common.util.FileUtil;
import com.raytheon.uf.edex.stats.util.ConfigLoader;
@ -70,8 +70,7 @@ public class AggregateManagerTest {
@BeforeClass
public static void classSetUp() throws JAXBException {
jaxbManager = new JAXBManager(StatisticsConfig.class,
StatsGroupingColumn.class);
jaxbManager = new JAXBManager(StatisticsConfig.class);
}
@Before
@ -90,7 +89,8 @@ public class AggregateManagerTest {
final StatisticsConfig statisticsConfig = lf.jaxbUnmarshal(
StatisticsConfig.class, jaxbManager);
ConfigLoader.validate(Maps.<String, StatisticsEvent> newHashMap(),
ConfigLoader.validate(
Maps.<String, StatisticsEventConfig> newHashMap(),
statisticsConfig);
MockEvent mockEvent = new MockEvent();
@ -102,15 +102,13 @@ public class AggregateManagerTest {
List<StatsGrouping> groupList = new ArrayList<StatsGrouping>();
groupList.add(new StatsGrouping("pluginName", "somePlugin"));
groupList.add(new StatsGrouping("fileName", "someFileName"));
StatsGroupingColumn column = new StatsGroupingColumn();
column.setGroup(groupList);
StatsGroupingColumn expectedGroupingColumn = new StatsGroupingColumn();
expectedGroupingColumn.setGroup(groupList);
final String expectedGroupRepresentation = jaxbManager
.marshalToXml(column);
final String actualGroupRepresentation = AggregateManager.determineGroupRepresentationForEvent(
statisticsConfig.getEvents().iterator().next(), mockEvent);
assertThat(actualGroupRepresentation,
is(equalTo(expectedGroupRepresentation)));
final StatsGroupingColumn actualGroupingColumn = AggregateManager
.determineGroupRepresentationForEvent(statisticsConfig
.getEvents().iterator().next(), mockEvent);
assertThat(actualGroupingColumn, is(equalTo(expectedGroupingColumn)));
}
}