xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #1487 Use XML to store stats grouping information in the database

Browse source 
Amend:
  Increase length of grouping column to 1024.
  Add deltaScript to convert the grouping column to xml.
  Change regex to replace any pluginName fields with dashes, not just satellite-mcidas.

Change-Id: I0d5081645c5f11909e5778c8e555d5da3cf77e1a

Former-commit-id: 38ecb1f36dce4717d0ba9066c8cd22116770d34b
This commit is contained in:
Dustin Johnson 2013-01-15 14:45:12 -06:00
parent 09fb2d0919
commit b627dfc054
18 changed files with 863 additions and 192 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
deltaScripts/13.2.1/aggregateRecordGroupingLength.sh Normal file
View file

@ -0,0 +1,23 @@
#!/bin/bash
SQL_SCRIPT="increaseAggregateRecordGroupingLength.sql"
# ensure that the sql script is present
if [ ! -f ${SQL_SCRIPT} ]; then
echo "ERROR: the required sql script - ${SQL_SCRIPT} was not found."
echo "FATAL: the update has failed!"
exit 1
fi
echo "INFO: update started - increasing the size of the aggregate.grouping column"
# run the update
/awips2/psql/bin/psql -U awips -d metadata -f ${SQL_SCRIPT}
if [ $? -ne 0 ]; then
echo "FATAL: the update has failed!"
exit 1
fi
echo "INFO: the update has completed successfully!"
exit 0

23
deltaScripts/13.2.1/convertAggregateRecordGroupToXml.sh Normal file
View file

@ -0,0 +1,23 @@
#!/bin/bash
SQL_SCRIPT="convertAggregateRecordGroupToXml.sql"
# ensure that the sql script is present
if [ ! -f ${SQL_SCRIPT} ]; then
echo "ERROR: the required sql script - ${SQL_SCRIPT} was not found."
echo "FATAL: the update has failed!"
exit 1
fi
echo "INFO: update started - converting the aggregate.grouping column to xml"
# run the update
/awips2/psql/bin/psql -U awips -d metadata -f ${SQL_SCRIPT}
if [ $? -ne 0 ]; then
echo "FATAL: the update has failed!"
exit 1
fi
echo "INFO: the update has completed successfully!"
exit 0

38
deltaScripts/13.2.1/convertAggregateRecordGroupToXml.sql Normal file
View file

@ -0,0 +1,38 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
\set ON_ERROR_STOP 1
\connect metadata;
-- Start a transaction
BEGIN;
-- Temporarily replace dashes in pluginName rows with @@@
update events.aggregate set grouping = regexp_replace(grouping, 'pluginName:(.*?)-(.*)', E'pluginName:\\1@@@\\2', 'g');
-- Convert to XML format
update events.aggregate set grouping = regexp_replace(grouping, ':', '" value="', 'g');
update events.aggregate set grouping = regexp_replace(grouping, '-', '"/><group name="', 'g');
update events.aggregate set grouping = '<?xml version="1.0" encoding="UTF-8" standalone="yes"?><stat><group name="' || grouping || '"/></stat>';
-- Restore dashes from @@@
update events.aggregate set grouping = regexp_replace(grouping, '<group name="(.*?)" value="(.*?)@@@(.*?)"', E'<group name="\\1" value="\\2-\\3"', 'g');
-- Commit the transaction
END;

22
deltaScripts/13.2.1/increaseAggregateRecordGroupingLength.sql Normal file
View file

@ -0,0 +1,22 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
\set ON_ERROR_STOP 1
\connect metadata;
ALTER TABLE events.aggregate ALTER COLUMN grouping TYPE varchar(1024);

12
edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/AggregateRecord.java
View file

@ -37,17 +37,18 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
/**
* Record class for an aggregate result.
*
*
* <pre>
*
*
* SOFTWARE HISTORY
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Aug 21, 2012 jsanchez Initial creation
* Nov 12, 2012 dhladky Updates some things for stats
*
* Jan 15, 2013 1487 djohnson Increase length of grouping to 1024.
*
* </pre>
*
*
* @author jsanchez
* @version 1.0
*/
@ -56,7 +57,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
@XmlRootElement
@XmlAccessorType(XmlAccessType.NONE)
@DynamicSerialize
public class AggregateRecord extends PersistableDataObject {
public class AggregateRecord extends PersistableDataObject<Integer> {
private static final long serialVersionUID = -4553588456131256014L;
@GeneratedValue(strategy = GenerationType.AUTO)
@ -77,6 +78,7 @@ public class AggregateRecord extends PersistableDataObject {
private String eventType;
@DynamicSerializeElement
@Column(length = 1024)
private String grouping;
@Column(nullable = false)

101
edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/StatsGrouping.java Normal file
View file

@ -0,0 +1,101 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.stats;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlRootElement;
/**
* Contains a grouping for statistics.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 15, 2013 1487 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
@XmlRootElement
@XmlAccessorType(XmlAccessType.NONE)
public class StatsGrouping {
@XmlAttribute(required = true)
private String name;
@XmlAttribute(required = true)
private String value;
/**
* Constructor.
*/
public StatsGrouping() {
this(null, null);
}
/**
* Constructor.
*
* @param name
* @param value
*/
public StatsGrouping(String name, String value) {
this.name = name;
this.value = value;
}
/**
* @return the name
*/
public String getName() {
return name;
}
/**
* @param name
* the name to set
*/
public void setName(String name) {
this.name = name;
}
/**
* @return the value
*/
public String getValue() {
return value;
}
/**
* @param value
* the value to set
*/
public void setValue(String value) {
this.value = value;
}
}

87
edexOsgi/com.raytheon.uf.common.stats/src/com/raytheon/uf/common/stats/StatsGroupingColumn.java Normal file
View file

@ -0,0 +1,87 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.stats;
import java.util.List;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import com.google.common.collect.Lists;
/**
* Contains a list of groupings for statistics.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 15, 2013 1487 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
@XmlRootElement(name = "stat")
@XmlAccessorType(XmlAccessType.NONE)
public class StatsGroupingColumn {
@XmlElement
private List<StatsGrouping> group = Lists.newArrayList();
/**
* @return the group
*/
public List<StatsGrouping> getGroup() {
return group;
}
/**
* @param group
* the group to set
*/
public void setGroup(List<StatsGrouping> group) {
this.group = group;
}
/**
* Create a {@link StatsGroupingColumn} to hold the specified
* {@link StatsGrouping} instances.
*
* @param statsGroupings
* the groupings
* @return the column
*/
public static StatsGroupingColumn withGroupings(
StatsGrouping... statsGroupings) {
StatsGroupingColumn column = new StatsGroupingColumn();
for (StatsGrouping grouping : statsGroupings) {
column.group.add(grouping);
}
return column;
}
}

73
edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/AggregateManager.java
View file

@ -19,7 +19,9 @@
**/
package com.raytheon.uf.edex.stats;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.HashMap;
@ -28,11 +30,17 @@ import java.util.List;
import java.util.Map;
import java.util.TimeZone;
import javax.xml.bind.JAXBException;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
import com.raytheon.uf.common.event.Event;
import com.raytheon.uf.common.serialization.JAXBManager;
import com.raytheon.uf.common.serialization.SerializationUtil;
import com.raytheon.uf.common.stats.AggregateRecord;
import com.raytheon.uf.common.stats.StatsGrouping;
import com.raytheon.uf.common.stats.StatsGroupingColumn;
import com.raytheon.uf.common.stats.StatsRecord;
import com.raytheon.uf.common.stats.xml.StatisticsAggregate;
import com.raytheon.uf.common.stats.xml.StatisticsEvent;
@ -61,6 +69,7 @@ import com.raytheon.uf.edex.stats.util.ConfigLoader;
* Nov 07, 2012 1317 mpduff Updated Configuration Files.
* Nov 28, 2012 1350 rjpeter Simplied aggregation and added aggregation with current db aggregate records.
* Jan 07, 2013 1451 djohnson Use newGmtCalendar().
* Jan 15, 2013 1487 djohnson Use xml for the grouping information on an {@link AggregateRecord}.
* </pre>
*
* @author jsanchez
@ -70,6 +79,17 @@ public class AggregateManager {
private static final IUFStatusHandler statusHandler = UFStatus
.getHandler(AggregateManager.class);
private static final Object[] EMPTY_OBJ_ARR = new Object[0];
private static final JAXBManager JAXB_MANAGER;
static {
try {
JAXB_MANAGER = new JAXBManager(StatsGroupingColumn.class);
} catch (JAXBException e) {
throw new ExceptionInInitializerError(e);
}
}
/** In minutes */
private int bucketInterval;
@ -255,8 +275,6 @@ public class AggregateManager {
Map<TimeRange, Multimap<String, Event>> rval = new HashMap<TimeRange, Multimap<String, Event>>();
TimeRange timeRange = null;
Multimap<String, Event> eventsByGroup = null;
final Object[] EMPTY_OBJ_ARR = new Object[0];
StringBuilder group = new StringBuilder();
for (StatsRecord record : records) {
if ((timeRange == null)
@ -275,30 +293,11 @@ public class AggregateManager {
Event event = SerializationUtil.transformFromThrift(
Event.class, record.getEvent());
// determine group
boolean addDelim = false;
Iterator<Method> gMethodIter = statEvent.getGroupByMethods()
.iterator();
Iterator<StatisticsGroup> gFieldNameIter = statEvent
.getGroupList().iterator();
group.setLength(0);
while (gMethodIter.hasNext() && gFieldNameIter.hasNext()) {
Method m = gMethodIter.next();
String field = gFieldNameIter.next().getName();
String gVal = String
.valueOf(m.invoke(event, EMPTY_OBJ_ARR));
if (addDelim) {
group.append('-');
} else {
addDelim = true;
}
group.append(field).append(':').append(gVal);
String groupAsString = determineGroupRepresentationForEvent(
statEvent, event);
if (groupAsString != null) {
eventsByGroup.put(groupAsString, event);
}
eventsByGroup.put(group.toString(), event);
} catch (Exception e) {
statusHandler
.error("Error processing event. Aggregation may be inaccurate. ",
@ -309,6 +308,30 @@ public class AggregateManager {
return rval;
}
@VisibleForTesting
static String determineGroupRepresentationForEvent(
StatisticsEvent statEvent, Event event)
throws IllegalAccessException, InvocationTargetException,
JAXBException {
Iterator<Method> gMethodIter = statEvent.getGroupByMethods().iterator();
Iterator<StatisticsGroup> gFieldNameIter = statEvent.getGroupList()
.iterator();
List<StatsGrouping> groupings = new ArrayList<StatsGrouping>();
while (gMethodIter.hasNext() && gFieldNameIter.hasNext()) {
Method m = gMethodIter.next();
String field = gFieldNameIter.next().getName();
String gVal = String.valueOf(m.invoke(event, EMPTY_OBJ_ARR));
groupings.add(new StatsGrouping(field, gVal));
}
StatsGroupingColumn column = new StatsGroupingColumn();
column.setGroup(groupings);
return JAXB_MANAGER.marshalToXml(column);
}
/**
* Tests if the bucket interval is a valid value. If value is invalid then
* value will be set to default value.

115
edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/data/StatsDataAccumulator.java
View file

@ -27,13 +27,18 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.regex.Pattern;
import javax.xml.bind.JAXBException;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.raytheon.uf.common.serialization.JAXBManager;
import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.common.serialization.SerializationUtil;
import com.raytheon.uf.common.stats.AggregateRecord;
import com.raytheon.uf.common.stats.StatsGrouping;
import com.raytheon.uf.common.stats.StatsGroupingColumn;
import com.raytheon.uf.common.stats.data.GraphData;
import com.raytheon.uf.common.stats.data.StatsBin;
import com.raytheon.uf.common.stats.data.StatsData;
@ -44,34 +49,42 @@ import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.time.TimeRange;
import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.common.util.CollectionUtil;
/**
* Accumulates the statistics data.
*
*
* <pre>
*
*
* SOFTWARE HISTORY
*
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Nov 15, 2012 728 mpduff Initial creation
*
* Jan 15, 2013 1487 djohnson Use xml for the grouping information on an {@link AggregateRecord}.
*
* </pre>
*
*
* @author mpduff
* @version 1.0
*/
public class StatsDataAccumulator {
private static final Pattern COLON_PATTERN = Pattern.compile(":");
private static final Pattern DASH_PATTERN = Pattern.compile("-");
private static final String COLON = ":";
private static final IUFStatusHandler statusHandler = UFStatus
.getHandler(StatsDataAccumulator.class);
private static final String COLON = ":";
private static final JAXBManager JAXB_MANAGER;
static {
try {
JAXB_MANAGER = new JAXBManager(StatsGroupingColumn.class);
} catch (JAXBException e) {
throw new ExceptionInInitializerError(e);
}
}
/** List of records */
private AggregateRecord[] records;
@ -108,7 +121,7 @@ public class StatsDataAccumulator {
/**
* Set the AggregateRecord[]
*
*
* @param records
* array of AggregateRecord objects
*/
@ -122,28 +135,27 @@ public class StatsDataAccumulator {
@VisibleForTesting
public void setupGroupings() {
for (AggregateRecord aggRec : records) {
String grouping = aggRec.getGrouping();
String[] groupString = DASH_PATTERN.split(grouping);
String group;
String member;
for (String grp : groupString) {
String[] parts = COLON_PATTERN.split(grp);
group = parts[0];
member = parts[1];
if (!groupMemberMap.containsKey(group)) {
groupMemberMap.put(group, new TreeSet<String>());
}
StatsGroupingColumn columnValue = unmarshalGroupingColumnFromRecord(aggRec);
groupMemberMap.get(group).add(member);
final List<StatsGrouping> groups = columnValue.getGroup();
if (CollectionUtil.isNullOrEmpty(groups)) {
continue;
}
for (StatsGrouping group : groups) {
final String groupName = group.getName();
if (!groupMemberMap.containsKey(groupName)) {
groupMemberMap.put(groupName, Sets.<String> newTreeSet());
}
groupMemberMap.get(groupName).add(group.getValue());
}
}
groups = new ArrayList<String>(groupMemberMap.keySet());
groups = Lists.newArrayList(groupMemberMap.keySet());
}
/**
* Get the GraphData object
*
*
* @param groups
* List of groups
* @return The GraphData object
@ -184,7 +196,7 @@ public class StatsDataAccumulator {
/**
* Create the StatsDataMap keys
*
*
* @param unitUtils
* UnitUtils object
* @param groups
@ -202,21 +214,25 @@ public class StatsDataAccumulator {
if (record.getEventType().equals(eventType)
&& record.getField().equals(dataType)) {
StatsGroupingColumn columnValue = unmarshalGroupingColumnFromRecord(record);
final List<StatsGrouping> columnValueGroups = columnValue
.getGroup();
if (CollectionUtil.isNullOrEmpty(columnValueGroups)) {
continue;
}
// Have a matching record
for (String key : keySequenceMap.keySet()) {
keySequenceMap.put(key, "");
}
String[] groupings = DASH_PATTERN.split(record.getGrouping());
for (String grouping : groupings) {
String[] parts = COLON_PATTERN.split(grouping);
String group = parts[0];
String groupMember = parts[1];
for (StatsGrouping group : columnValueGroups) {
for (String key : keySequenceMap.keySet()) {
if (group.equals(key)) {
if (group.getName().equals(key)) {
keySequenceMap.put(key, keySequenceMap.get(key)
.concat(groupMember + COLON));
.concat(group.getValue() + COLON));
break;
}
}
@ -242,9 +258,32 @@ public class StatsDataAccumulator {
}
}
/**
* Unmarshals the {@link StatsGroupingColumn} from the
* {@link AggregateRecord}.
*
* @param record
* the aggregate record
* @return the unmarshalled column, or an empty column if unable to
* unmarshal
*/
private static StatsGroupingColumn unmarshalGroupingColumnFromRecord(
AggregateRecord record) {
String groupingXmlAsString = record.getGrouping();
try {
return (StatsGroupingColumn) JAXB_MANAGER
.unmarshalFromXml(groupingXmlAsString);
} catch (JAXBException e) {
statusHandler.handle(Priority.PROBLEM,
"Unable to unmarshal stats grouping column, returning empty record, xml:\n"
+ groupingXmlAsString, e);
return new StatsGroupingColumn();
}
}
/**
* Gather the data together in each bin.
*
*
* @param unitUtils
* UnitUtils object
* @param groups
@ -323,7 +362,7 @@ public class StatsDataAccumulator {
/**
* Set the display units.
*
*
* @param displayUnit
* the displayUnit to set
*/
@ -333,7 +372,7 @@ public class StatsDataAccumulator {
/**
* TimeStep in minutes
*
*
* @param timeStep
*/
public void setTimeStep(int timeStep) {

5
edexOsgi/com.raytheon.uf.edex.stats/src/com/raytheon/uf/edex/stats/util/ConfigLoader.java
View file

@ -30,6 +30,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import com.google.common.annotations.VisibleForTesting;
import com.raytheon.uf.common.event.Event;
import com.raytheon.uf.common.localization.IPathManager;
import com.raytheon.uf.common.localization.LocalizationContext;
@ -57,6 +58,7 @@ import com.raytheon.uf.common.util.ReflectionUtil;
* Aug 21, 2012 jsanchez Updated error handling and validated config files.
* Nov 07, 2012 1317 mpduff Update config files.
* Nov 29, 2012 1350 rjpeter Updated to static, fixed localization, increased validation.
* Jan 15, 2013 1487 djohnson Make validate() static and public, so it can be run independently.
* </pre>
*
* @author jsanchez
@ -158,7 +160,8 @@ public class ConfigLoader {
*
* @param config
*/
private void validate(Map<String, StatisticsEvent> eventMap,
@VisibleForTesting
public static void validate(Map<String, StatisticsEvent> eventMap,
StatisticsConfig config) {
for (Iterator<StatisticsEvent> iter = config.getEvents().iterator(); iter
.hasNext();) {

1
tests/resources/META-INF/services/com.raytheon.uf.common.datadelivery.retrieval.util.LevelXmlWriter
View file

@ -1 +0,0 @@
com.raytheon.uf.common.datadelivery.retrieval.util.NullXmlWriter

1
tests/resources/META-INF/services/com.raytheon.uf.common.datadelivery.retrieval.util.ParameterXmlWriter
View file

@ -1 +0,0 @@
com.raytheon.uf.common.datadelivery.retrieval.util.NullXmlWriter

60
tests/unit/com/raytheon/uf/common/datadelivery/retrieval/util/NullXmlWriter.java
View file

@ -1,60 +0,0 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.datadelivery.retrieval.util;
import com.raytheon.uf.common.datadelivery.retrieval.xml.LevelLookup;
import com.raytheon.uf.common.datadelivery.retrieval.xml.ParameterLookup;
/**
* Makes parser created XML not write out in test
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 10, 2013 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public class NullXmlWriter implements LevelXmlWriter, ParameterXmlWriter {
/**
* {@inheritDoc}
*/
@Override
public void writeParameterXml(ParameterLookup pl, String modelName)
throws Exception {
}
/**
* {@inheritDoc}
*/
@Override
public void writeLevelXml(LevelLookup ll, String modelName)
throws Exception {
}
}

34
tests/unit/com/raytheon/uf/common/localization/TestPathManager.java
View file

@ -40,6 +40,7 @@ import com.raytheon.uf.common.util.FileUtil;
* ------------ ---------- ----------- --------------------------
* Jul 18, 2012 740 djohnson Initial creation
* Oct 23, 2012 1286 djohnson Change to find more localization files.
* Jan 16, 2013 1487 djohnson Avoid adding new localization files to baseline utility directories.
*
* </pre>
*
@ -131,21 +132,44 @@ public class TestPathManager extends PathManager {
}
if (foundFile == null
|| !foundFile.exists()
|| foundFile.getAbsolutePath().startsWith(
savedLocalizationFileDir.getAbsolutePath())) {
return foundFile;
}
// Make a copy in the savedFile folder, this way if any
// modifications are performed we don't mess with the file in
// the baseline
File savedFile = createTestIsolatedVersionOfLocalizationFile(
context, fileName, foundFile);
return savedFile;
}
/**
* Creates a test isolated version of the localization file. Allows the
* file to be written to, and changes to be read back, without affecting
* the baselined version of the file.
*
* @param context
* the context
* @param fileName
* the file path
* @param baselinedVersion
* the file reference
* @return
*/
private File createTestIsolatedVersionOfLocalizationFile(
LocalizationContext context, String fileName, File baselinedVersion) {
File savedFileBaseDir = new File(savedLocalizationFileDir,
context.toPath());
File savedFile = new File(savedFileBaseDir, fileName);
savedFile.getParentFile().mkdirs();
try {
FileUtil.copyFile(foundFile, savedFile);
if (baselinedVersion.exists()) {
if (baselinedVersion.isDirectory()) {
savedFile.mkdirs();
} else {
FileUtil.copyFile(baselinedVersion, savedFile);
}
}
} catch (IOException e) {
throw new RuntimeException(e);
}

116
tests/unit/com/raytheon/uf/edex/stats/AggregateManagerTest.java Normal file
View file

@ -0,0 +1,116 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.stats;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertThat;
import java.util.ArrayList;
import java.util.List;
import javax.xml.bind.JAXBException;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import com.google.common.collect.Maps;
import com.raytheon.uf.common.localization.IPathManager;
import com.raytheon.uf.common.localization.LocalizationContext;
import com.raytheon.uf.common.localization.LocalizationContext.LocalizationLevel;
import com.raytheon.uf.common.localization.LocalizationContext.LocalizationType;
import com.raytheon.uf.common.localization.LocalizationFile;
import com.raytheon.uf.common.localization.PathManagerFactory;
import com.raytheon.uf.common.localization.PathManagerFactoryTest;
import com.raytheon.uf.common.serialization.JAXBManager;
import com.raytheon.uf.common.stats.StatsGrouping;
import com.raytheon.uf.common.stats.StatsGroupingColumn;
import com.raytheon.uf.common.stats.xml.StatisticsConfig;
import com.raytheon.uf.common.stats.xml.StatisticsEvent;
import com.raytheon.uf.common.util.FileUtil;
import com.raytheon.uf.edex.stats.util.ConfigLoader;
/**
* Test {@link AggregateManager}.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 15, 2013 1487 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public class AggregateManagerTest {
private static JAXBManager jaxbManager;
@BeforeClass
public static void classSetUp() throws JAXBException {
jaxbManager = new JAXBManager(StatisticsConfig.class,
StatsGroupingColumn.class);
}
@Before
public void setUp() {
PathManagerFactoryTest.initLocalization();
}
@Test
public void testDeterminingGroupForEvent() throws Exception {
IPathManager pm = PathManagerFactory.getPathManager();
final LocalizationFile lf = pm.getLocalizationFile(
new LocalizationContext(LocalizationType.EDEX_STATIC,
LocalizationLevel.BASE), FileUtil.join("stats",
"mockStats.xml"));
final StatisticsConfig statisticsConfig = lf.jaxbUnmarshal(
StatisticsConfig.class, jaxbManager);
ConfigLoader.validate(Maps.<String, StatisticsEvent> newHashMap(),
statisticsConfig);
MockEvent mockEvent = new MockEvent();
mockEvent.setPluginName("somePlugin");
mockEvent.setFileName("someFileName");
mockEvent.setProcessingTime(1000L);
mockEvent.setProcessingLatency(500L);
List<StatsGrouping> groupList = new ArrayList<StatsGrouping>();
groupList.add(new StatsGrouping("pluginName", "somePlugin"));
groupList.add(new StatsGrouping("fileName", "someFileName"));
StatsGroupingColumn column = new StatsGroupingColumn();
column.setGroup(groupList);
final String expectedGroupRepresentation = jaxbManager
.marshalToXml(column);
final String actualGroupRepresentation = AggregateManager.determineGroupRepresentationForEvent(
statisticsConfig.getEvents().iterator().next(), mockEvent);
assertThat(actualGroupRepresentation,
is(equalTo(expectedGroupRepresentation)));
}
}

169
tests/unit/com/raytheon/uf/edex/stats/MockEvent.java Normal file
View file

@ -0,0 +1,169 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.stats;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
import com.raytheon.uf.common.stats.ProcessEvent;
import com.raytheon.uf.common.stats.StatisticsEvent;
/**
* Mock event based from {@link ProcessEvent}.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 15, 2013 1487 djohnson Initial creation
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
@DynamicSerialize
public class MockEvent extends StatisticsEvent {
private static final long serialVersionUID = 1L;
private static final Map<String, String> FIELD_UNIT_MAP;
static {
Map<String, String> m = new HashMap<String, String>();
m.put("processingLatency", "ms");
m.put("processingTime", "ms");
FIELD_UNIT_MAP = Collections.unmodifiableMap(m);
}
@DynamicSerializeElement
private String message;
@DynamicSerializeElement
private String pluginName;
@DynamicSerializeElement
private String fileName;
/*
* Processing time in milliseconds
*/
@DynamicSerializeElement
private long processingTime;
/*
* Processing latency in milliseconds
*/
@DynamicSerializeElement
private long processingLatency;
public MockEvent() {
}
@Override
protected Map<String, String> getFieldUnitMap() {
return FIELD_UNIT_MAP;
}
/**
* @return the fileName
*/
public String getFileName() {
return fileName;
}
/**
* @return the message
*/
public String getMessage() {
return message;
}
/**
* @return the pluginName
*/
public String getPluginName() {
return pluginName;
}
/**
* @return the processingLatency in milliseconds
*/
public long getProcessingLatency() {
return processingLatency;
}
/**
* @return the processingTime in milliseconds
*/
public long getProcessingTime() {
return processingTime;
}
/**
* @param fileName
* the fileName to set
*/
public void setFileName(String fileName) {
this.fileName = fileName;
}
/**
* @param message
* the message to set
*/
public void setMessage(String message) {
this.message = message;
}
/**
* @param pluginName
* the pluginName to set
*/
public void setPluginName(String pluginName) {
this.pluginName = pluginName;
}
/**
* @param processingLatency
* the processingLatency in milliseconds to set
*/
public void setProcessingLatency(long processingLatency) {
this.processingLatency = processingLatency;
}
/**
* @param processingTime
* the processingTime in milliseconds to set
*/
public void setProcessingTime(long processingTime) {
this.processingTime = processingTime;
}
@Override
public String toString() {
return super.toString() + " : " + getMessage();
}
}

163
tests/unit/com/raytheon/uf/edex/stats/data/StatsDataAccumulatorTest.java
View file

@ -1,47 +1,92 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.stats.data;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TimeZone;
import javax.xml.bind.JAXBException;
import org.junit.Test;
import com.google.common.collect.Maps;
import com.raytheon.uf.common.datadelivery.event.retrieval.DataRetrievalEvent;
import com.raytheon.uf.common.datadelivery.event.retrieval.SubscriptionRetrievalEvent;
import com.raytheon.uf.common.serialization.JAXBManager;
import com.raytheon.uf.common.stats.AggregateRecord;
import com.raytheon.uf.common.stats.StatsGrouping;
import com.raytheon.uf.common.stats.StatsGroupingColumn;
import com.raytheon.uf.common.stats.data.StatsData;
import com.raytheon.uf.common.stats.util.UnitUtils;
import com.raytheon.uf.common.time.TimeRange;
import com.raytheon.uf.common.time.util.TimeUtil;
/**
*
* Test {@link StatsDataAccumulator}.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 15, 2013 1487 djohnson Use XML for grouping column.
*
* </pre>
*
* @author djohnson
* @version 1.0
*/
public class StatsDataAccumulatorTest {
private static final JAXBManager JAXB_MANAGER;
static {
try {
JAXB_MANAGER = new JAXBManager(StatsGroupingColumn.class);
} catch (JAXBException e) {
throw new ExceptionInInitializerError(e);
}
}
@Test
public void testCalculateBinsCalculatesCorrectly() {
Calendar c = Calendar.getInstance(TimeZone.getTimeZone("GMT"));
c.set(Calendar.MILLISECOND, 0);
c.set(Calendar.SECOND, 0);
c.set(Calendar.MINUTE, 0);
c.set(Calendar.HOUR_OF_DAY, 0);
c.set(Calendar.DAY_OF_MONTH, 1);
c.set(Calendar.MONTH, 0);
long start = c.getTimeInMillis();
c.add(Calendar.DAY_OF_MONTH, 1);
long end = c.getTimeInMillis();
TimeRange tr = new TimeRange(start, end);
TimeRange tr = new TimeRange(0L, TimeUtil.MILLIS_PER_DAY);
StatsDataAccumulator acc = new StatsDataAccumulator();
acc.setTimeRange(tr);
acc.setTimeStep(5);
acc.calculateBins();
int expectedBinCount = 288; // 5 minute bins 12 per hour, * 24
int expectedBinCount = 288; // 5 minute bins 12 per hour, * 24
int actualBinCount = acc.bins.keySet().size();
assertEquals("Bin Counts do not match", expectedBinCount, actualBinCount);
assertEquals("Bin Counts do not match", expectedBinCount,
actualBinCount);
int count = 0;
for (long bin : acc.bins.keySet()) {
@ -51,7 +96,7 @@ public class StatsDataAccumulatorTest {
}
@Test
public void testSetupGroupings() {
public void testSetupGroupings() throws JAXBException {
List<AggregateRecord> recordList = getTestRecords();
StatsDataAccumulator acc = new StatsDataAccumulator();
acc.setRecords(recordList.toArray(new AggregateRecord[recordList.size()]));
@ -62,33 +107,39 @@ public class StatsDataAccumulatorTest {
expectedGroups.add("provider");
expectedGroups.add("plugin");
List<String> expectedGroupMembers = new ArrayList<String>();
expectedGroupMembers.add("nomads");
expectedGroupMembers.add("madis");
expectedGroupMembers.add("owner0");
expectedGroupMembers.add("owner1");
expectedGroupMembers.add("owner2");
expectedGroupMembers.add("owner3");
expectedGroupMembers.add("owner4");
expectedGroupMembers.add("grid");
List<String> expectedPlugins = Arrays.asList("grid");
List<String> expectedProviders = Arrays.asList("nomads", "madis");
List<String> expectedOwners = Arrays.asList("owner0", "owner1",
"owner2", "owner3", "owner4");
Map<String, List<String>> expectedGroupsToValues = Maps.newHashMap();
expectedGroupsToValues.put("provider", expectedProviders);
expectedGroupsToValues.put("plugin", expectedPlugins);
expectedGroupsToValues.put("owner", expectedOwners);
// Check the groups
for (String group : acc.groups) {
assertTrue(expectedGroups.contains(group));
for (String expected : expectedGroups) {
assertTrue("Did not find group [" + expected
+ "] in the group collection!",
acc.groups.contains(expected));
}
// Check the group members
for (String key: acc.groupMemberMap.keySet()) {
for (String member: acc.groupMemberMap.get(key)) {
assertTrue(expectedGroupMembers.contains(member));
final Map<String, Set<String>> groupMemberMap = acc.groupMemberMap;
for (Entry<String, List<String>> entry : expectedGroupsToValues
.entrySet()) {
final String groupName = entry.getKey();
final Set<String> setToCheck = groupMemberMap.get(groupName);
for (String member : entry.getValue()) {
assertTrue("Did not find entry [" + member + "] for group ["
+ groupName + "]!", setToCheck.contains(member));
}
}
}
@Test
public void testCreateStatsDataMapCreation() {
String eventType = "com.raytheon.uf.common.datadelivery.event.retrieval.DataRetrievalEvent";
public void testCreateStatsDataMapCreation() throws JAXBException {
String eventType = DataRetrievalEvent.class.getName();
String dataType = "bytes";
String displayUnit = "MB";
@ -116,54 +167,54 @@ public class StatsDataAccumulatorTest {
expectedSet.add("owner3:madis");
expectedSet.add("owner4:madis");
for (String key : acc.statsDataMap.keySet()) {
assertTrue(expectedSet.contains(key));
final Map<String, StatsData> statsDataMap = acc.statsDataMap;
for (String expected : expectedSet) {
assertTrue("Did not find expected value (" + expected
+ "] as key in the statsDataMap!",
statsDataMap.containsKey(expected));
}
}
// Build the Aggregate records
private List<AggregateRecord> getTestRecords() {
private List<AggregateRecord> getTestRecords() throws JAXBException {
String plugin = "plugin";
String provider = "provider";
String nomads = "nomads";
String madis = "madis";
String owner = "owner";
String grid = "grid";
String dash = "-";
String colon = ":";
List<String> groupings = new ArrayList<String>();
for (int i = 0; i < 5; i++) {
groupings.add(plugin + colon + grid + dash + owner + colon + owner + i);
List<StatsGroupingColumn> groupingColumns = new ArrayList<StatsGroupingColumn>();
for (int i = 0; i < 15; i++) {
groupingColumns.add(StatsGroupingColumn.withGroupings(
new StatsGrouping(plugin, grid), new StatsGrouping(owner,
owner + i)));
}
for (int i = 0; i < 5; i++) {
groupings.add(plugin + colon + grid + dash + owner + colon + owner + i);
groupingColumns.add(StatsGroupingColumn.withGroupings(
new StatsGrouping(provider, nomads), new StatsGrouping(
owner, owner + i)));
}
for (int i = 0; i < 5; i++) {
groupings.add(plugin + colon + grid + dash + owner + colon + owner + i);
}
for (int i = 0; i < 5; i++) {
groupings.add(provider + colon + nomads + dash + owner + colon + owner + i);
}
for (int i = 0; i < 5; i++) {
groupings.add(provider + colon + madis + dash + owner + colon + owner + i);
groupingColumns.add(StatsGroupingColumn.withGroupings(
new StatsGrouping(provider, madis), new StatsGrouping(
owner, owner + i)));
}
List<AggregateRecord> records = new ArrayList<AggregateRecord>();
for (String group : groupings) {
for (StatsGroupingColumn group : groupingColumns) {
AggregateRecord r = new AggregateRecord();
if (group.contains("provider")) {
r.setEventType("com.raytheon.uf.common.datadelivery.event.retrieval.DataRetrievalEvent");
if ("provider".equals(group.getGroup().iterator().next().getName())) {
r.setEventType(DataRetrievalEvent.class.getName());
r.setField("bytes");
} else {
r.setEventType("com.raytheon.uf.common.datadelivery.event.retrieval.SubscriptionRetrievalEvent");
r.setEventType(SubscriptionRetrievalEvent.class.getName());
r.setField("numRecords");
}
r.setGrouping(group);
r.setGrouping(JAXB_MANAGER.marshalToXml(group));
records.add(r);
}

12
tests/utility/edex_static/base/stats/mockStats.xml Normal file
View file

@ -0,0 +1,12 @@
<statisticsConfig>
<!-- Event Type should be fully qualified name of stat event -->
<statisticsEvent type="com.raytheon.uf.edex.stats.MockEvent"
displayName="Mock Events" category="Mock Events">
<statisticsGroup name="pluginName" displayName="Data Type" />
<statisticsGroup name="fileName" displayName="File Name" />
<statisticsAggregate field="processingTime"
displayName="Processing Time" displayUnit="ms" />
<statisticsAggregate field="processingLatency"
displayName="Processing Latency" displayUnit="ms" />
</statisticsEvent>
</statisticsConfig>