2970 lines
126 KiB
Python
2970 lines
126 KiB
Python
##
|
|
# This software was developed and / or modified by Raytheon Company,
|
|
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
|
|
#
|
|
# U.S. EXPORT CONTROLLED TECHNICAL DATA
|
|
# This software product contains export-restricted data whose
|
|
# export/transfer/disclosure is restricted by U.S. law. Dissemination
|
|
# to non-U.S. persons whether in the United States or abroad requires
|
|
# an export license or other authorization.
|
|
#
|
|
# Contractor Name: Raytheon Company
|
|
# Contractor Address: 6825 Pine Street, Suite 340
|
|
# Mail Stop B8
|
|
# Omaha, NE 68106
|
|
# 402.291.0100
|
|
#
|
|
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
|
|
# further licensing information.
|
|
##
|
|
# ----------------------------------------------------------------------------
|
|
# This software is in the public domain, furnished "as is", without technical
|
|
# support, and with no warranty, express or implied, as to its usefulness for
|
|
# any purpose.
|
|
#
|
|
# SampleAnalysis.py
|
|
# Class for producing summary statistics from Sampler data.
|
|
# Typically used for Text Product generation.
|
|
#
|
|
# Author: hansen
|
|
# ----------------------------------------------------------------------------
|
|
#
|
|
# SOFTWARE HISTORY
|
|
#
|
|
# Date Ticket# Engineer Description
|
|
# ------------- -------- --------- --------------------------------------------
|
|
# Feb 13, 2018 7494 randerso Fixed issue in temporalCoverage_flag for
|
|
# period combining
|
|
# Feb 09, 2021 8357 randerso Replace use of built-in round function
|
|
#
|
|
##
|
|
|
|
##
|
|
# This is a base file that is not intended to be overridden.
|
|
##
|
|
|
|
import logging
|
|
from math import *
|
|
|
|
import AbsTime
|
|
import CommonUtils
|
|
import TextUtils
|
|
import TimeRange
|
|
import WeatherSubKey
|
|
from com.raytheon.viz.gfe.sampler import HistValue, HistPair
|
|
from com.raytheon.viz.gfe.textformatter import FormatterUtil
|
|
|
|
|
|
## For complete documentation on the analysis methods available in this class,
|
|
## refer to the Text Product User Guide.
|
|
## Utility Methods:
|
|
## The following methods return an integer that indicated the expected
|
|
## return type....
|
|
## SCALAR
|
|
## MAGNITUDE
|
|
## DIRECTION
|
|
## VECTOR
|
|
## WEATHER
|
|
##
|
|
## getModeratedLimits
|
|
## returns the min and max percentages allowed for all moderated methods
|
|
## getStdDevLimits
|
|
## returns the min and max standard deviation limits for all stdDev methods
|
|
## getAccumSum
|
|
## returns the accumulated sum for the specified time period
|
|
## getAccumMinMax
|
|
## return the min and max for the specified time period
|
|
## getAverage
|
|
## returns the absolute average
|
|
## getMinMax
|
|
## returns the absolute minimum and maximum
|
|
## getStdDevAvg
|
|
## returns the average after filtering data based on standard deviation
|
|
## getStdDevMinMax
|
|
## returns the min and max values after filtering based in standard deviation
|
|
## getModeratedAvg
|
|
## returns the average after filtering based on percentage
|
|
## getModeratedMinMax
|
|
## returns the min and max after filtering based on percentage
|
|
## getVectorAvg
|
|
## returns the absolute vector average with no filtering
|
|
## getDominantDirection
|
|
## returns either the Average or Most Frequent direction based on:
|
|
## vectorDirection_algorithm
|
|
## getAverageDirection
|
|
## returns the average direction over the specified time period
|
|
## getMostFrequentDirection
|
|
## returns the most frequent direction over the specified time period
|
|
## temporalCoverage_flag
|
|
## returns 1 if the specified grid sufficiently overlaps the sample time period
|
|
## getDominantWx
|
|
## returns the dominant weather over the sample period
|
|
## getDominantDiscreteValue
|
|
## returns the dominant discrete values over the sample period
|
|
## getDominant -- handles both getDominantWx and getDominantDiscreteValue
|
|
## getSubkey_percentages -- gather all the Weather or Discrete SubKeys and percentages
|
|
## dirList
|
|
## converts a numerical direction into a string (e.g., N, SE)
|
|
## determineGridWeight
|
|
## returns the time wieght for a particular grid ( 1.0 = full weight)
|
|
## createStats
|
|
## reports statistics based on the method specified
|
|
## splitRange(timeRange, numPeriods)
|
|
## splits a timerange into the specified number of periods and returns a
|
|
## timeRange list
|
|
## getGridTimeRanges
|
|
## returns the list of timeRanges after splitting along sample time periods
|
|
## divideRange(timeRange, hours)
|
|
## splits a timeRange into sub periods each with the duration specified
|
|
## getMode
|
|
## returns a range around the median indicating the most frequent values
|
|
## getMedian
|
|
## returns median value over given timeRange
|
|
## getMedianRange
|
|
## returns 2-value range chosen around median, uses getRange
|
|
## getMedianHistPair
|
|
## returns median histPair over given timeRange
|
|
## getHistPairMean
|
|
## given two HistPairs returns a HistPair representing the mean of the two
|
|
## getModeRange
|
|
## range chosen around mode, uses getRange
|
|
## getModeHistPair
|
|
## returns most common HistPair over given timeRange
|
|
## getRange
|
|
## range chosen around given histPair
|
|
## returns: scalar: (min, max)
|
|
## vector: (minMag, maxMag, avgDir)
|
|
## getDeviation
|
|
## returns a deviation around median to include in range
|
|
## getBinnedPercent
|
|
## returns a list of tuples representing "bins" and corresponding
|
|
## percentages of values in each bin
|
|
##
|
|
## Conversion methods
|
|
## UVToMagDir
|
|
## MagDirToUV
|
|
## convertAnalysisList
|
|
## Breakdown of Sampler data:
|
|
## HistoSampler : Parms, TimeRanges, EditAreas
|
|
## Contains SeqOf<ParmHisto>
|
|
## ParmHisto: Parm, TimeRange, EditArea
|
|
## Contains SeqOf<HistSample> : (one for each grid overlapping timeRange)
|
|
## HistSample is a histogram for a Parm(implicit), Grid, Area
|
|
## Contains SeqOf<histPairs>
|
|
## HistPair : Parm(implicit), Grid(implicit), Area(implicit)
|
|
## count, HistValue (value)
|
|
## count = how many times that value occurred within the grid
|
|
## HistValue value: scalar, Vector (magnitude, direction), weather
|
|
class SampleAnalysis(CommonUtils.CommonUtils):
|
|
def __init__(self):
|
|
CommonUtils.CommonUtils.__init__(self)
|
|
self.log = logging.getLogger("FormatterRunner.SampleAnalysis.SampleAnalysis")
|
|
self._textUtils = TextUtils.TextUtils()
|
|
|
|
### CONSTANTS -- Do not override
|
|
def SCALAR(self):
|
|
return 0
|
|
def MAGNITUDE(self):
|
|
return 1
|
|
def DIRECTION(self):
|
|
return 2
|
|
def VECTOR(self):
|
|
return 3
|
|
def WEATHER(self):
|
|
return 4
|
|
def DISCRETE(self):
|
|
return 5
|
|
|
|
### GLOBAL THRESHOLDS AND VARIABLES
|
|
### To override, override the associated method in your text product class.
|
|
# To be included in the analysis, a grid must either:
|
|
# 1. Be completely contained in the time range OR
|
|
# 2. Meet BOTH the temporalCoverage_percentage and temporalCoverage_hours
|
|
# requirements.
|
|
# The temporalCoverage_percentage is:
|
|
# The percentage of the TIMERANGE covered by the
|
|
# grid in order to include it in the analysis.
|
|
# The temporalCoverage_hours is:
|
|
# The required hours of overlap of a grid with the TIMERANGE
|
|
# in order to include it in the analysis.
|
|
# In addition, if the temporalCoverage_hours is greater than or equal to the
|
|
# TIMERANGE duration and the grid covers the entire TIMERANGE,
|
|
# it will be included.
|
|
def temporalCoverage_percentage(self, parmHisto, timeRange, componentName):
|
|
# This is the percentage of the TIMERANGE covered by the
|
|
# grid in order to include it in the analysis.
|
|
# Percentage of temporal coverage default value (if not found in temporalCoverage_dict)
|
|
# Used by temporalCoverage_flag
|
|
return 20
|
|
|
|
def temporalCoverage_dict(self, parmHisto, timeRange, componentName):
|
|
# This is temporalCoverage percentage by weather element
|
|
# Used by temporalCoverage_flag
|
|
return {
|
|
"LAL": 0,
|
|
"MinRH": 0,
|
|
"MaxRH": 0,
|
|
"MinT": 1,
|
|
"MaxT": 1,
|
|
"Haines": 0,
|
|
"PoP" : 0,
|
|
"Hazards" : 0,
|
|
}
|
|
|
|
def temporalCoverage_hours(self, parmHisto, timeRange, componentName):
|
|
# This is the required hours of overlap of a grid with the TIMERANGE
|
|
# in order to include it in the analysis.
|
|
# In addition, if the temporalCoverage_hours is greater than or equal to the
|
|
# TIMERANGE duration and the grid covers the entire TIMERANGE,
|
|
# it will be included.
|
|
# Temporal coverage hours default value
|
|
# (if not found in temporalCoverage_hours_dict)
|
|
# Used by temporalCoverage_flag
|
|
return 0
|
|
|
|
def temporalCoverage_hours_dict(self, parmHisto, timeRange, componentName):
|
|
# This is the temporalCoverage_hours specified per weather element.
|
|
# Used by temporalCoverage_flag
|
|
return {
|
|
#"MinRH": 0,
|
|
#"MaxRH": 0,
|
|
"MinT": 5,
|
|
"MaxT": 5,
|
|
#"Haines":0,
|
|
#"PoP" : 0,
|
|
"pws34": 4,
|
|
"pws64": 4,
|
|
"pwsD34": 4,
|
|
"pwsN34": 4,
|
|
"pwsD64": 4,
|
|
"pwsN64": 4,
|
|
}
|
|
|
|
def moderated_dict(self, parmHisto, timeRange, componentName):
|
|
# This dictionary defines the low and high limit at which
|
|
# outliers will be removed when calculating moderated stats.
|
|
# By convention the first value listed is the percentage
|
|
# allowed for low values and second the percentage allowed
|
|
# for high values.
|
|
return {
|
|
"T" : (10, 10),
|
|
"Wind": (0, 20),
|
|
"LAL": (10, 10),
|
|
"MinRH": (10, 10),
|
|
"MaxRH": (10, 10),
|
|
"MinT": (10, 10),
|
|
"MaxT": (10, 10),
|
|
"Haines": (10, 10),
|
|
"PoP" : (10, 10),
|
|
}
|
|
|
|
def getModeratedLimits(self, parmHisto, timeRange, componentName):
|
|
compositeNameUI = parmHisto.getCompositeNameUI()
|
|
# get the stdDict min and max values
|
|
modMin = self.moderatedDefault(parmHisto, timeRange, componentName)
|
|
modMax = self.moderatedDefault(parmHisto, timeRange, componentName)
|
|
modDict = self.moderated_dict(parmHisto, timeRange, componentName)
|
|
if compositeNameUI in modDict:
|
|
modMin, modMax = modDict[compositeNameUI]
|
|
|
|
return modMin, modMax
|
|
|
|
def moderatedDefault(self, parmHisto, timeRange, componentName):
|
|
"Value used by moderated functions if not explicitly defined in moderated_dict"
|
|
return 5
|
|
|
|
def maxMode_increment_dict(self, parmHisto, timeRange, componentName):
|
|
return {
|
|
"PoP" : 10,
|
|
}
|
|
|
|
def stdDev_dict(self, parmHisto, timeRange, componentName):
|
|
# This dictionary defines the low and high limit at which
|
|
# outliers will be removed when calculating stdDev stats.
|
|
# These tuples represent the (low, high) number of standard
|
|
# deviations. Any values falling outside this range will
|
|
# not be included in the calculated statistic.
|
|
return {
|
|
"LAL": (1.0, 1.0),
|
|
"MinRH": (1.0, 1.0),
|
|
"MaxRH": (1.0, 1.0),
|
|
"MinT": (1.0, 1.0),
|
|
"MaxT": (1.0, 1.0),
|
|
"Haines": (1.0, 1.0),
|
|
"PoP" : (1.0, 1.0),
|
|
"T" : (1.0, 1.0),
|
|
"Wind" : (1.0, 1.0),
|
|
}
|
|
|
|
def getStdDevLimits(self, parmHisto, timeRange, componentName):
|
|
compositeNameUI = parmHisto.getCompositeNameUI()
|
|
# get the stdDict min and max values
|
|
stdDevDict = self.stdDev_dict(parmHisto, timeRange, componentName)
|
|
minStdDev = self.stdDevDefault(parmHisto, timeRange, componentName)
|
|
maxStdDev = self.stdDevDefault(parmHisto, timeRange, componentName)
|
|
if compositeNameUI in stdDevDict:
|
|
minStdDev, maxStdDev = stdDevDict[compositeNameUI]
|
|
|
|
return minStdDev, maxStdDev
|
|
|
|
def stdDevDefault(self, parmHisto, timeRange, componentName):
|
|
"Value used by all moderated functions if not explicitly defined in stdDev_dict"
|
|
return 1.0
|
|
|
|
def vectorDirection_algorithm(self, parmHisto, timeRange, componentName):
|
|
# Algorithm to use for computing vector direction for vector analysis methods.
|
|
# Can be "Average" or "MostFrequent"
|
|
return "Average"
|
|
|
|
# Variables for converting Wind Direction from degrees to letters
|
|
def dirList(self):
|
|
dirSpan = 45 # 45 degrees per direction
|
|
base = 22.5 # start with N
|
|
return [
|
|
('N', 360-base, 361),
|
|
('N', 0, base),
|
|
('NE',base , base + 1*dirSpan),
|
|
('E', base + 1*dirSpan, base + 2*dirSpan),
|
|
('SE',base + 2*dirSpan, base + 3*dirSpan),
|
|
('S', base + 3*dirSpan, base + 4*dirSpan),
|
|
('SW',base + 4*dirSpan, base + 5*dirSpan),
|
|
('W', base + 5*dirSpan, base + 6*dirSpan),
|
|
('NW',base + 6*dirSpan, base + 7*dirSpan)
|
|
]
|
|
|
|
|
|
########################################
|
|
## SCALAR
|
|
|
|
def avg(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getAverage(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return result
|
|
|
|
def minMax(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getMinMax(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
results = self.createStats(parmHisto,timeRange, componentName, args, primaryMethod)
|
|
return results
|
|
|
|
def minimum(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getMinMax(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return self.extractMinMax(result, "Min")
|
|
|
|
def maximum(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getMinMax(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return self.extractMinMax(result, "Max")
|
|
|
|
def accumMinMax(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getAccumMinMax(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
return self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
|
|
def accumSum(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getAccumSum(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
return self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
|
|
def moderatedAccumMinMax(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getModAccumMinMax(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
return self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
|
|
def moderatedAccumSum(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getModAccumSum(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
return self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
|
|
def stdDevAccumMinMax(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getStdDevAccumMinMax(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
return self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
|
|
def stdDevAccumSum(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getStdDevAccumSum(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
return self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
|
|
def median(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getMedian(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
return self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
|
|
def medianRange(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getMedianRange(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
return self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
|
|
def mode(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getMode(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
return self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
|
|
def modeRange(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getModeRange(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
return self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
|
|
def maxMode(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getMaxMode(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
return self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
|
|
def stdDevAvg(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getStdDevAvg(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return result
|
|
|
|
def stdDevMin(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getStdDevMinMax(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return self.extractMinMax(result, "Min")
|
|
|
|
def stdDevMax(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getStdDevMinMax(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return self.extractMinMax(result, "Max")
|
|
|
|
def stdDevMinMax(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getStdDevMinMax(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return result
|
|
|
|
def stdDevFirstAvg(self, parmHisto, timeRange, componentName, args=None):
|
|
return self.getStdDevAvg(self.SCALAR(), parmHisto, timeRange, componentName, 1)
|
|
|
|
def stdDevFirstMinMax(self, parmHisto, timeRange, componentName, args=None):
|
|
return self.getStdDevMinMax(self.SCALAR(), parmHisto, timeRange, componentName, 1)
|
|
|
|
def moderatedAvg(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getModeratedAvg(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return result
|
|
|
|
def moderatedMin(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getModeratedMinMax(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return self.extractMinMax(result, "Min")
|
|
|
|
def moderatedMax(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getModeratedMinMax(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return self.extractMinMax(result, "Max")
|
|
|
|
def moderatedMinMax(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getModeratedMinMax(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return result
|
|
|
|
def binnedPercent(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getBinnedPercent(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return result
|
|
|
|
def moderatedFirstAvg(self, parmHisto, timeRange, componentName, args=None):
|
|
return self.getModeratedAvg(self.SCALAR(), parmHisto, timeRange, componentName, 1)
|
|
|
|
def moderatedFirstMinMax(self, parmHisto, timeRange, componentName, args=None):
|
|
return self.getModeratedMinMax(self.SCALAR(), parmHisto, timeRange, componentName, 1)
|
|
|
|
def minMaxAvg(self, parmHisto, timeRange, componentName, args=None):
|
|
# Find Min and Max values
|
|
minMax = self.minMax( parmHisto, timeRange, componentName)
|
|
avg = self.avg(parmHisto, timeRange, componentName)
|
|
if minMax is None or avg is None:
|
|
return None
|
|
else:
|
|
min, max = minMax
|
|
return (min, max, avg)
|
|
|
|
def minMaxSum(self, parmHisto, timeRange, componentName, args=None):
|
|
values = parmHisto.minMaxSum()
|
|
if values is None:
|
|
return None
|
|
else:
|
|
minV, maxV, sumV = values
|
|
return minV, maxV, sumV
|
|
|
|
def maxAvg(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getMaxAvg(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return result
|
|
|
|
def stdDevMaxAvg(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getStdDevMaxAvg(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return result
|
|
|
|
def moderatedMaxAvg(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getModeratedMaxAvg(self.SCALAR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return result
|
|
|
|
def firstAvg(self, parmHisto, timeRange, componentName, args=None):
|
|
return self.getAverage(self.SCALAR(), parmHisto, timeRange, componentName, 1)
|
|
|
|
def firstMinMax(self, parmHisto, timeRange, componentName, args=None):
|
|
return self.getMinMax(self.SCALAR(), parmHisto, timeRange, componentName, 1)
|
|
|
|
def hourlyTemp(self, parmHisto, timeRange, componentName, args=None):
|
|
"Create hourly temperature stats"
|
|
# Produces a list of hourly temperature values in tuples
|
|
# Each tuple has an average temperature value and
|
|
# its hour of occurrence
|
|
# Assumptions:
|
|
# If there is no data for an hour, None is
|
|
# given instead of a temp value
|
|
|
|
if parmHisto.getSampleLen() == 0:
|
|
return None
|
|
|
|
start = timeRange.startTime()
|
|
start = AbsTime.absTimeYMD(start.year, start.month,
|
|
start.day, start.hour, 0, 0)
|
|
stats = []
|
|
while start < timeRange.endTime():
|
|
# Create Time Range for current hour
|
|
end = start + 3600 # 1 hour in seconds
|
|
hour = start.hour
|
|
tr = TimeRange.TimeRange(start, end)
|
|
#Get the Average T for current hour
|
|
value = self.getAverage(self.SCALAR(), parmHisto, tr, componentName)
|
|
|
|
# Append Value and Hour to Stat List
|
|
stats.append((value, hour))
|
|
start = end
|
|
return stats
|
|
|
|
########################################
|
|
## VECTOR
|
|
|
|
def vectorAvg(self, parmHisto, timeRange, componentName, args=None):
|
|
"Create a Vector ((minMag, maxMag), TextDir) Stats"
|
|
primaryMethod = "self.getAverage(self.VECTOR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return result
|
|
|
|
def vectorMinMax(self, parmHisto, timeRange, componentName, args=None):
|
|
"Create a Vector ((minMag, maxMag), TextDir) Stats"
|
|
primaryMethod = "self.getMinMax(self.VECTOR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return result
|
|
|
|
def vectorMin(self, parmHisto, timeRange, componentName, args=None):
|
|
"Create a Vector ((minMag, maxMag), TextDir) Stats"
|
|
primaryMethod = "self.getMinMax(self.VECTOR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return self.extractVectorMinMax(result, "Min")
|
|
|
|
def vectorMax(self, parmHisto, timeRange, componentName, args=None):
|
|
"Create a Vector ((minMag, maxMag), TextDir) Stats"
|
|
primaryMethod = "self.getMinMax(self.VECTOR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return self.extractVectorMinMax(result, "Max")
|
|
|
|
def vectorMedian(self, parmHisto, timeRange, componentName, args=None):
|
|
"Create a Vector (median, TextDir) Stats"
|
|
primaryMethod = "self.getMedian(self.VECTOR(), parmHisto, timeRange, componentName)"
|
|
return self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
|
|
def vectorMode(self, parmHisto, timeRange, componentName, args=None):
|
|
"Create a Vector (mode, TextDir) Stats"
|
|
primaryMethod = "self.getMode(self.VECTOR(), parmHisto, timeRange, componentName)"
|
|
return self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
|
|
def vectorMedianRange(self, parmHisto, timeRange, componentName, args=None):
|
|
"Create a Vector (medianRange, TextDir) Stats"
|
|
primaryMethod = "self.getMedianRange(self.VECTOR(), parmHisto, timeRange, componentName)"
|
|
return self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
|
|
def vectorModeRange(self, parmHisto, timeRange, componentName, args=None):
|
|
"Create a Vector (modeRange, TextDir) Stats"
|
|
primaryMethod = "self.getModeRange(self.VECTOR(), parmHisto, timeRange, componentName)"
|
|
return self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
|
|
def vectorStdDevAvg(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getStdDevAvg(self.VECTOR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod,)
|
|
return result
|
|
|
|
def vectorStdDevMinMax(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getStdDevMinMax(self.VECTOR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return result
|
|
|
|
def vectorStdDevMin(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getStdDevMinMax(self.VECTOR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return self.extractVectorMinMax(result, "Min")
|
|
|
|
def vectorStdDevMax(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getStdDevMinMax(self.VECTOR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return self.extractVectorMinMax(result, "Max")
|
|
|
|
def vectorModeratedAvg(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getModeratedAvg(self.VECTOR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod,)
|
|
return result
|
|
|
|
def vectorModeratedMinMax(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getModeratedMinMax(self.VECTOR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return result
|
|
|
|
def vectorBinnedPercent(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getBinnedPercent(self.VECTOR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return result
|
|
|
|
def vectorModeratedMin(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getModeratedMinMax(self.VECTOR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return self.extractVectorMinMax(result, "Min")
|
|
|
|
def vectorModeratedMax(self, parmHisto, timeRange, componentName, args=None):
|
|
primaryMethod = "self.getModeratedMinMax(self.VECTOR(), parmHisto, timeRange, componentName)"
|
|
result = self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
return self.extractVectorMinMax(result, "Max")
|
|
|
|
def vectorMagMinMax(self, parmHisto, timeRange, componentName, args=None):
|
|
"Create a Vector min/max Stats"
|
|
return self.getMinMax(self.MAGNITUDE(), parmHisto, timeRange, componentName)
|
|
|
|
def vectorMagMin(self, parmHisto, timeRange, componentName, args=None):
|
|
"Create a Vector min Stats"
|
|
minResult, maxResult = self.getMinMax(self.MAGNITUDE(), parmHisto, timeRange,
|
|
componentName)
|
|
return minResult
|
|
|
|
def vectorMagMax(self, parmHisto, timeRange, componentName, args=None):
|
|
"Create a Vector max Stats"
|
|
minResult, maxResult = self.getMinMax(self.MAGNITUDE(), parmHisto, timeRange,
|
|
componentName)
|
|
return maxResult
|
|
|
|
## This method is being kept for "table" type products
|
|
def vectorRange(self, parmHisto, timeRange, componentName=None):
|
|
"Create a Vector Stats"
|
|
# Split Time Period in half
|
|
# For each half find average values:
|
|
# mag1 = min Period 1, mag2 = max Period 1,
|
|
# mag3 = min Period 2, mag4 = max Period 2
|
|
periods = self.splitRange(timeRange)
|
|
period1 = periods[0]
|
|
period2 = periods[1]
|
|
result1 = self.getAverage(self.VECTOR(), parmHisto, period1, componentName)
|
|
result2 = self.getAverage(self.VECTOR(), parmHisto, period2, componentName)
|
|
|
|
if result1 is None or result2 is None:
|
|
return None
|
|
mag1, dir1 = result1
|
|
mag2, dir2 = result2
|
|
return (mag1, mag2, dir1, dir2)
|
|
|
|
########################################
|
|
## WEATHER
|
|
##
|
|
## dominantWx
|
|
##
|
|
## Thresholds and variables:
|
|
|
|
def coverage_weights_dict(self):
|
|
# Weight (between 0 and 1) for the coverage terms
|
|
return {
|
|
"<NoCov>": 0,
|
|
"Iso": .15,
|
|
"SChc": .15,
|
|
"Patchy": .15,
|
|
"Areas": .4,
|
|
"Chc": .4,
|
|
"Sct": .4,
|
|
"Lkly": .7,
|
|
"Num": .7,
|
|
"Brf": 1.0,
|
|
"Frq": 1.0,
|
|
"Ocnl": 1.0,
|
|
"Pds": 1.0,
|
|
"Inter": 1.0,
|
|
"Def": 1.0,
|
|
"Wide": 1.0,
|
|
}
|
|
|
|
def wxkey_coverage_weight(self, parmHisto, timeRange, componentName, wxkey):
|
|
# Return a weight (between 0 and 1) for the wxkey coverage term
|
|
cov = wxkey.coverage()
|
|
return self.coverage_weights_dict()[cov]
|
|
|
|
def wxkey_coverage_percentage(self, parmHisto, timeRange, componentName, wxkey):
|
|
# Return the required coverage percentage for the given wxkey which will be
|
|
# compared to its "rank" i.e. the percentage of areal coverage over the time period.
|
|
wxType = wxkey.wxType()
|
|
wxCov = wxkey.coverage()
|
|
inten = wxkey.intensity()
|
|
# These rules were from the workshop
|
|
if wxType == "T" and inten == "+":
|
|
return 0
|
|
if wxType in ["ZR", "ZL"]:
|
|
return 0
|
|
# Large Hail
|
|
attrList = wxkey.attributes()
|
|
if "LgA" in attrList:
|
|
return 0
|
|
# Heavy Fog
|
|
if wxType == "F" and inten == "+":
|
|
return 0
|
|
# Low visibility
|
|
if wxType in ["F", "H", "BS", "K", "BD"]:
|
|
vis = wxkey.visibility()
|
|
if vis == "1/4SM" or vis == "0SM":
|
|
return 0
|
|
if wxType in ["T", "R", "RW", "S", "SW", "L", "IP"]:
|
|
return 15
|
|
# For the rest: ["F", "H", "BS", "K", "BD", "SA", "LC", "FR", "WG", "VA"]
|
|
return 15
|
|
|
|
def checkPercentages(self, parmHisto, timeRange, componentName, wxKey, keyRankDict):
|
|
# If a wxKey does not pass the wxkey_coverage_percentage, this method will be called
|
|
# to give another chance.
|
|
# You can use the keyRankDict:
|
|
# subkey : (rank, percent coverage)
|
|
# to allow the wxKey to pass based on other values in the grid.
|
|
# For example: If I have 10% RW 10% SW, neither RW or SW will be reported
|
|
# Using the keyRankDict, I can allow them to pass when I learn
|
|
# that 20% of my area is covered with precip.
|
|
# Here's how this might be done:
|
|
#
|
|
#precip = ["SW", "RW", "R", "S"]
|
|
#totalPrecip = 0
|
|
#if wxKey.wxType() in precip:
|
|
# for subkey in keyRankDict.keys():
|
|
# if subkey.wxType() in precip:
|
|
# rank, percent = keyRankDict[subkey]
|
|
# totalPrecip += percent
|
|
#if totalPrecip > 15:
|
|
# return 1
|
|
#else:
|
|
# return 0
|
|
return 0
|
|
|
|
def attribute_coverage_percentage(self, parmHisto, timeRange, componentName, wxType, attr):
|
|
# Return the required coverage percentage for the given attribute.
|
|
# May be based on the wxType and attr if desired.
|
|
if attr == "Dry":
|
|
return 20
|
|
else:
|
|
return 0
|
|
|
|
def dominantKeys_threshold(self, parmHisto, timeRange, componentName):
|
|
# This is the maximum number of weather keys desired from the rankedWx method
|
|
return 10
|
|
|
|
def cleanOutEmptyValues(self, parmHisto, timeRange, componentName, dataType):
|
|
return 0
|
|
|
|
def noWx_percentage(self, parmHisto, timeRange, componentName):
|
|
# If the raw rank (areal and temporal coverage) of NoWx exceeds this value,
|
|
# NoWx will be reported (all other weather keys will be ignored).
|
|
return 100
|
|
|
|
def dominantWx(self, parmHisto, timeRange, componentName, args=None):
|
|
"Return a list of dominant wx subkeys in order by ranking"
|
|
primaryMethod = "self.getDominantWx(parmHisto, timeRange, componentName, withRank=0)"
|
|
return self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
|
|
def rankedWx(self, parmHisto, timeRange, componentName, args=None):
|
|
"Return a list of ranked (subkey, ranking) tuples"
|
|
primaryMethod = "self.getDominantWx(parmHisto, timeRange, componentName, withRank=1)"
|
|
return self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
|
|
def getDominantWx(self, parmHisto, timeRange, componentName, withRank=0):
|
|
# Determine the dominant Wx considering the following:
|
|
# areal coverage over time
|
|
return self.getDominantValues(parmHisto, timeRange, componentName,
|
|
dataType="WEATHER", withRank=withRank)
|
|
|
|
|
|
def getDominantValues(self, parmHisto, timeRange, componentName,
|
|
dataType="WEATHER", withRank=0, withAux=0):
|
|
# Determine the dominant Wx subkey OR Discrete subkey
|
|
# considering the following:
|
|
# areal coverage over time, called the "rank"
|
|
# Sub-methods:
|
|
# temporalCoverage_flag
|
|
# wxKey_coverage_percentage and dominantKeys_threshold OR
|
|
# discreteKey_coverage_percentage and dominantDiscreteKeys_threshold
|
|
# The algorithm:
|
|
# Temporal coverage by grid: Any individual grid must
|
|
# cover enough time in the timerange to set the temporalCoverage_flag.
|
|
# Loop over all samples, which are for all grids and
|
|
# all weather or discrete keys on each of those grids.
|
|
# For WEATHER,
|
|
# We aggregate weather types i.e.
|
|
# for each weather type (S, RW, R, etc.) we compute
|
|
# an aggregate subkey such that the resulting rankList
|
|
# will have just one entry per weather type.
|
|
# Aggregate Subkey:
|
|
# coverage: time-weighted average of coverages
|
|
# intensity: highest ranking OR if ranks are close, dominant
|
|
# visibility: lowest visibility
|
|
# attributes: aggregate attributes
|
|
# Rank: percentage of areal/temporal coverage over time
|
|
# For WEATHER, we weight this by the coverage term
|
|
# If a subkey does not meet the "wxkey_coverage_percentage" threshold
|
|
# or "discreteKey_coverage_percentage", it is removed.
|
|
# For WEATHER, in all cases, if a subkey has a Primary or Mention attribute,
|
|
# it automatically "wins" and is used.
|
|
# Finally, the highest ranking dominantKeys_threshold OR
|
|
# dominantDiscreteKeys_threshold number of keys are returned.
|
|
# If withRank == 1: return the ranked list of (subkey, rank) tuples
|
|
# Else: return the list of subkeys
|
|
#
|
|
totalHours = 0
|
|
totalPoints = parmHisto.numberOfGridPoints()
|
|
compositeNameUI = parmHisto.getCompositeNameUI()
|
|
|
|
# Loop over all samples, which are for all grids and
|
|
# all keys on each of those grids.
|
|
# In this process, we aggregate subkey types e.g.
|
|
# for each weather type (S, RW, R, etc.) we compute
|
|
# an aggregate subkey such that the resulting rankList
|
|
# will have just one entry per weather type.
|
|
# For discrete, we will have just one entry per
|
|
# discrete subkey (with or without Aux value).
|
|
|
|
#print "\n\nIn getDominantValues: DataType, TimeRange", dataType, timeRange
|
|
#print "STEP 1 -- Aggregate per grid"
|
|
|
|
subkeyTypeDict = {}
|
|
# STEP 1:
|
|
# For each wxType found in the grids,
|
|
# gather its 'hours' of coverage and 'count' of points covered.
|
|
for histSample in parmHisto.histoSamples():
|
|
validTime = TimeRange.TimeRange(histSample.validTime())
|
|
if self.temporalCoverage_flag(
|
|
parmHisto, timeRange, componentName, histSample) == 0:
|
|
continue
|
|
# Get the number of hours inside the timeRange that this
|
|
# sample comes from (i.e., we can get a sample that lasts
|
|
# for 3 weeks - but only 1 hour of it is inside the
|
|
# timeRange - and we only want to rank it by the 1 hour
|
|
# inside the range)
|
|
#
|
|
hours = validTime.intersection(timeRange).duration() // 3600
|
|
if hours < 1:
|
|
continue
|
|
|
|
totalHours += hours
|
|
|
|
# Gather the subkey Types for this grid in subkeyTypeDict
|
|
# Each entry ends up being a list of tuples: (subkey, hours, count)
|
|
self.gatherSubkeyTypes(
|
|
parmHisto, timeRange, componentName, histSample, dataType, hours,
|
|
subkeyTypeDict, withAux)
|
|
|
|
# STEP 2: For each subkeyType,
|
|
# --determine an aggregate subkey and rank i.e.
|
|
# aggregate areal coverage over time percentage
|
|
# --compare the rank to coverage threshold for the wxType.
|
|
rankList = []
|
|
#print "subkeyTypeDict", subkeyTypeDict
|
|
subkeyTypePointsDict = {}
|
|
noWxRawRank = 0
|
|
keyRankDict = {} # Holds: aggregateKey: (rank, rawRank)
|
|
for subkeyType in subkeyTypeDict:
|
|
#print "\nsubkeyType", subkeyType
|
|
subkeyList = subkeyTypeDict[subkeyType]
|
|
if dataType == "WEATHER":
|
|
covDict = {}
|
|
intenDict = {}
|
|
visDict = {}
|
|
attrList = []
|
|
attrDict = {}
|
|
primaryKey = None
|
|
mentionKey = None
|
|
subkeyTypeRank = 0
|
|
# IF WEATHER:
|
|
# Gather the coverages, intensities, visibilities and attributes
|
|
# for this weather type
|
|
# If Primary or Mention, use the subkey as the aggregate key
|
|
# Determine a subkeyType rank
|
|
subkeyTotalPoints = 0
|
|
for subkey, hours, count in subkeyList:
|
|
#print " subkey, hours, count", subkey, hours, count
|
|
subkeyTotalPoints += count
|
|
if dataType == "WEATHER":
|
|
attrs = subkey.attributes()
|
|
attrList = attrList + attrs
|
|
if "Primary" in attrs:
|
|
primaryKey = subkey
|
|
continue
|
|
if "Mention" in attrs:
|
|
mentionKey = subkey
|
|
continue
|
|
self.addToDict(covDict, subkey.coverage(), hours, count, 1)
|
|
self.addToDict(intenDict, subkey.intensity(), hours, count)
|
|
self.addToDict(visDict, subkey.visibility(), hours, count)
|
|
for attr in attrs:
|
|
self.addToDict(attrDict, attr, hours, count)
|
|
subkeyTypeRank += hours * count
|
|
subkeyTypePointsDict[subkeyType] = subkeyTotalPoints
|
|
|
|
# Determine aggregate key
|
|
#print " subkeyTypeRank", subkeyTypeRank, subkeyTotalPoints
|
|
#print " totalHours, totalPoints", totalHours, totalPoints
|
|
subkeyPoints = subkeyTypePointsDict[subkeyType]
|
|
if dataType == "WEATHER":
|
|
aggregateKey = self.getWxAggregateKey(
|
|
parmHisto, timeRange, componentName,
|
|
primaryKey, mentionKey, subkeyType, covDict, intenDict, visDict,
|
|
attrList, attrDict, totalHours, totalPoints, subkeyPoints)
|
|
else:
|
|
aggregateKey = subkeyType
|
|
|
|
# Determine rawRank and rank for the aggregateKey
|
|
if dataType == "WEATHER" \
|
|
and "Primary" in aggregateKey.attributes():
|
|
rank = 200
|
|
rawRank = 200
|
|
else:
|
|
rawRank = int(self._textUtils.round(float(subkeyTypeRank)/(totalHours*totalPoints)*100.0, "Nearest", 1))
|
|
if dataType == "WEATHER":
|
|
# Save the raw rank for NoWx
|
|
if aggregateKey.wxType() == "<NoWx>":
|
|
noWxRawRank = rawRank
|
|
# Multiply by the coverage weight
|
|
rank = int(rawRank * self.wxkey_coverage_weight(
|
|
parmHisto, timeRange, componentName, aggregateKey))
|
|
else:
|
|
rank = rawRank
|
|
#print " aggregateKey, rank", aggregateKey, rank, rawRank
|
|
keyRankDict[aggregateKey] = (rank, rawRank)
|
|
|
|
# Check to see if each aggregateKey meets the required coverage percentage
|
|
for (aggregateKey, (rank, rawRank)) in keyRankDict.items():
|
|
if dataType == "WEATHER" \
|
|
and ("Mention" in aggregateKey.attributes() \
|
|
or "Primary" in aggregateKey.attributes()):
|
|
rankList.append((aggregateKey, rank))
|
|
else:
|
|
if dataType == "WEATHER":
|
|
# Use rawRank which is the percentage of areal/temporal coverage
|
|
threshold = self.wxkey_coverage_percentage(
|
|
parmHisto, timeRange, componentName, aggregateKey)
|
|
flag = rawRank >= threshold
|
|
else:
|
|
threshold = self.discreteKey_coverage_percentage(
|
|
parmHisto, timeRange, componentName, aggregateKey)
|
|
flag = rawRank >= threshold
|
|
if not flag:
|
|
# Get another chance to pass
|
|
flag = self.checkPercentages(
|
|
parmHisto, timeRange, componentName, aggregateKey, keyRankDict)
|
|
if flag:
|
|
rankList.append((aggregateKey, rank))
|
|
else:
|
|
pass
|
|
#print "didn't make the cut", rank, aggregateKey
|
|
|
|
#print " rankList", rankList
|
|
if len(rankList) == 0:
|
|
return None
|
|
|
|
# Check the NoWx Threshold
|
|
if noWxRawRank > self.noWx_percentage(parmHisto, timeRange, componentName):
|
|
# Report NoWx
|
|
newList = []
|
|
for key, rank in rankList:
|
|
if key.wxType() == "<NoWx>":
|
|
newList.append((key, rank))
|
|
rankList = newList
|
|
|
|
# Clean out NoWx and None (Discrete)
|
|
if self.cleanOutEmptyValues(parmHisto, timeRange, componentName, dataType):
|
|
newList = []
|
|
for subkey, rank in rankList:
|
|
if dataType == "WEATHER":
|
|
if subkey.wxType() == "<NoWx>":
|
|
continue
|
|
else: # DISCRETE
|
|
if subkey == "<None>":
|
|
continue
|
|
newList.append((subkey, rank))
|
|
rankList = newList
|
|
|
|
# Sort into ranked order
|
|
# Limit the number of keys returned
|
|
if dataType == "WEATHER":
|
|
rankList.sort(key=self.rankedSortOrder)
|
|
rankList = [
|
|
(WeatherSubKey.weatherSubKey(self._argDict["site"], subkey.coverage(), subkey.wxType(), subkey.intensity(),
|
|
subkey.visibility(),
|
|
self.removeSimilarAttrs(subkey.attributes())),
|
|
rank) for subkey, rank in rankList
|
|
]
|
|
dominantKeys = self.dominantKeys_threshold(parmHisto, timeRange, componentName)
|
|
else: # DISCRETE
|
|
rankList.sort()
|
|
dominantKeys = self.dominantDiscreteKeys_threshold(parmHisto, timeRange, componentName)
|
|
|
|
if len(rankList) > dominantKeys:
|
|
rankList = rankList[0:dominantKeys]
|
|
if self._debug:
|
|
print("\nSampleAnalysis::ranked", dataType, " \n TimeRange ", timeRange)
|
|
print(" Area", parmHisto.area().getId())
|
|
if withRank:
|
|
if self._debug:
|
|
print(" returning with rank %s" % (rankList))
|
|
return rankList
|
|
else:
|
|
newList = []
|
|
for subkey, rank in rankList:
|
|
newList.append(subkey)
|
|
if self._debug:
|
|
print(" returning %s" % (newList))
|
|
return newList
|
|
|
|
def gatherSubkeyTypes(self, parmHisto, timeRange, componentName,
|
|
histSample, dataType, hours, subkeyTypeDict, withAux):
|
|
for histPair in histSample.histogram():
|
|
count = float(histPair.count())
|
|
if dataType == "WEATHER":
|
|
subkey = WeatherSubKey.WeatherSubKey(histPair.value().weather().get(0))
|
|
subkeyType = subkey.wxType()
|
|
if subkeyType == "RW" or subkeyType == "SW":
|
|
if subkey.intensity() == "--":
|
|
subkeyType = subkeyType + "--"
|
|
else: # DISCRETE
|
|
subkeyType = histPair.value().discrete().get(0)
|
|
if withAux == 0:
|
|
subkeyType = histPair.value().discrete().baseData(histPair.value().discrete().getSiteId(), subkeyType)
|
|
subkey = subkeyType
|
|
if subkeyType in subkeyTypeDict:
|
|
subkeyTypeDict[subkeyType].append((subkey, hours, count))
|
|
# Make new entry
|
|
else:
|
|
subkeyTypeDict[subkeyType] = [(subkey, hours, count)]
|
|
|
|
def getWxAggregateKey(self, parmHisto, timeRange, componentName,
|
|
primaryKey, mentionKey, subkeyType, covDict, intenDict, visDict,
|
|
attrList, attrDict, totalHours, totalPoints, subkeyPoints):
|
|
# Compute the aggregate key
|
|
# If Primary was an attribute in any subkey, take it as the aggregate
|
|
# plus all the other attributes
|
|
# Otherwise, if Mention was an attribute in any subkey, take it as the
|
|
# aggregate plus all the other attributes
|
|
# Otherwise, compute the aggregate subkey from the coverage, intensity,
|
|
# visibilities for this subkeyType weighted by temporal and areal coverages
|
|
#print "covDict", covDict
|
|
#print "intenDict", intenDict
|
|
#print "visDict", visDict
|
|
#print "attrDict", attrDict
|
|
if subkeyType in ["RW--", "SW--"]:
|
|
subkeyType = subkeyType.replace("--","")
|
|
aggregateKey = None
|
|
if primaryKey is not None:
|
|
aggregateKey = primaryKey
|
|
elif mentionKey is not None:
|
|
aggregateKey = mentionKey
|
|
|
|
if aggregateKey is None:
|
|
algorithm = self.aggregateCov_algorithm(parmHisto, timeRange, componentName)
|
|
aggCov = algorithm(parmHisto, timeRange, componentName,
|
|
subkeyType, "coverage", covDict, totalHours, totalPoints, subkeyPoints)
|
|
aggInten = self.getAggregate(parmHisto, timeRange, componentName,
|
|
subkeyType, "intensity", intenDict, totalHours, totalPoints, subkeyPoints)
|
|
aggVis = self.getAggregate(parmHisto, timeRange, componentName,
|
|
subkeyType, "visibility", visDict, totalHours, totalPoints, subkeyPoints)
|
|
aggAttrs = self.getAggregateAttributes(
|
|
parmHisto, timeRange, componentName,
|
|
subkeyType, attrDict, totalHours, totalPoints, subkeyPoints)
|
|
else:
|
|
aggCov = aggregateKey.coverage()
|
|
aggInten = aggregateKey.intensity()
|
|
aggVis = aggregateKey.visibility()
|
|
aggAttrs = aggregateKey.attributes()
|
|
attrList = self.removeDups(attrList)
|
|
aggregateKey = WeatherSubKey.weatherSubKey(self._argDict["site"], aggCov, subkeyType, aggInten, aggVis, aggAttrs)
|
|
#print "aggregateKey", aggregateKey
|
|
return aggregateKey
|
|
|
|
def addToDict(self, dict, key, hours, count, tuple=0):
|
|
# Add to a dictionary whose values are lists
|
|
|
|
if key in dict:
|
|
|
|
if tuple:
|
|
(curValue, curMaxCov) = dict[key]
|
|
|
|
if count > curMaxCov:
|
|
curMaxCov = count
|
|
|
|
dict[key] = ((curValue + hours*count), curMaxCov)
|
|
|
|
else:
|
|
curValue = dict[key]
|
|
|
|
dict[key] = curValue + hours * count
|
|
|
|
# Make new entry
|
|
else:
|
|
if tuple:
|
|
dict[key] = ((hours * count), count)
|
|
else:
|
|
dict[key] = hours * count
|
|
|
|
def aggregateCov_algorithm(self, parmHisto, timeRange, componentName):
|
|
# The algorithm for choosing the coverage term for multiple
|
|
# instances of a weather type.
|
|
# "getAggregateCov" chooses the coverage with the highest rank
|
|
# (in terms of areal and temporal coverage.)
|
|
# "getExistingWeightedAggregateCov" chooses the coverage with the
|
|
# highest WEIGHTED rank that exists in the grids.
|
|
# "getWeightedAggregateCov" computes a weighted average coverage
|
|
# If the resulting coverage is not in the grids,
|
|
# "creates" an appropriate coverage.
|
|
# "getHighestWeightedAggregateCov" returns the coverage with the
|
|
# highest weight in "coverage_weights_dict"
|
|
#
|
|
# For example,
|
|
# If you have
|
|
# Iso T(covers 50% of the zone)
|
|
# Sct T(covers 25% of the zone)
|
|
# Num T(covers 25% of the zone)
|
|
# "getAggregateCov" returns "Iso"
|
|
# "getWeightedAggregateCov" returns "Sct"
|
|
# "getExistingWeightedAggregateCov" returns "Num"
|
|
# "getHighestWeightedAggregateCov" returns "Num"
|
|
#
|
|
# If you have
|
|
# Iso T(covers 60% of the zone)
|
|
# Num T(covers 40% of the zone)
|
|
# "getAggregateCov" returns "Iso"
|
|
# "getWeightedAggregateCov" returns "Sct"
|
|
# "getExistingWeightedAggregateCov" returns "Num"
|
|
# "getHighestWeightedAggregateCov" returns "Num"
|
|
#
|
|
return self.getAggregateCov
|
|
# return self.getWeightedAggregateCov
|
|
# return self.getExistingWeightedAggregateCov
|
|
# return self.getHighestWeightedAggregateCov
|
|
|
|
# Submitted by Matt Belk 8/04
|
|
def getAggregateCov(self, parmHisto, timeRange, componentName,
|
|
wxType, wxPart, dict, totalHours, totalPoints,
|
|
subkeyPoints):
|
|
# From the entries in the dictionary,
|
|
# find the aggregate coverage
|
|
# Return coverage in one of two ways:
|
|
#
|
|
# 1) Coverage covers >= 90% of zone, or
|
|
# 2) Coverage has the highest subkey rank
|
|
|
|
# If there is only one coverage
|
|
if len(dict) == 1:
|
|
return next(iter(dict))
|
|
|
|
# Get ready to track properties of all coverage terms
|
|
maxRank = 0.0
|
|
aggCov = None
|
|
sameRank = []
|
|
highRank = []
|
|
|
|
# For each coverage
|
|
for key in dict:
|
|
# If this is a tuple
|
|
if type(dict[key]) is tuple:
|
|
(sum, max) = dict[key] # get the point sum and max coverage
|
|
# Otherwise, get the point sum and assume a max coverage
|
|
else:
|
|
sum = dict[key]
|
|
max = 0
|
|
|
|
# Compute subkey rank
|
|
subkeyRank = float(sum)/(totalHours * totalPoints) * 100.0
|
|
# If this is the highest subkey rank we have so far
|
|
if subkeyRank > maxRank:
|
|
# Store this as the highest subkey rank
|
|
maxRank = subkeyRank
|
|
aggCov = key
|
|
sameRank = []
|
|
sameRank.append(key)
|
|
# Otherwise, if this ties as the highest subkey rank
|
|
elif subkeyRank == maxRank:
|
|
sameRank.append(key)
|
|
|
|
# If the areal coverage of this subkey coverage is >= 90%
|
|
arealThreshold = self.__dict__.get("_aggregateCov_arealThreshold", 90.0)
|
|
if (max * 100.0)/float(totalPoints) >= arealThreshold:
|
|
# Store this as a candidate for highest coverage key
|
|
highRank.append(key)
|
|
|
|
# Get ready to process sameRank list (if needed)
|
|
maxVal = 0
|
|
# If there is more than one key in the highRank list
|
|
if len(highRank) > 0:
|
|
# Use this list to find the aggregate coverage
|
|
testRank = highRank
|
|
# Otherwise, if there are items in the sameRank list
|
|
elif len(sameRank) > 0:
|
|
testRank = sameRank
|
|
|
|
# Grab the most significant coverage
|
|
for cov in testRank:
|
|
# Compute the PoP range and a test value for this coverage
|
|
(lowVal, highVal) = self.coveragePoP_value(cov)
|
|
avgVal = (lowVal + highVal)/2.0
|
|
# If this is the most significant value
|
|
if avgVal >= maxVal:
|
|
# Use this coverage
|
|
aggCov = cov
|
|
maxVal = avgVal
|
|
|
|
# if the aggregate coverage is still not found
|
|
if aggCov is None:
|
|
aggCov = self.processNoAggCov(dict, wxType)
|
|
self.debug_print('in getAggregateCov -> returning %s' % (aggCov), 1)
|
|
return aggCov
|
|
|
|
def aggregateCov_weights_dict(self):
|
|
# Weight (between 0 and 1) for the coverage terms
|
|
return {
|
|
"<NoCov>": 0,
|
|
"Iso": .1,
|
|
"SChc": .1,
|
|
"Patchy": .1,
|
|
"Areas": .4,
|
|
"Chc": .4,
|
|
"Sct": .4,
|
|
"Lkly": .7,
|
|
"Num": .7,
|
|
"Brf": .9,
|
|
"Frq": .9,
|
|
"Ocnl": .9,
|
|
"Pds": .9,
|
|
"Inter": .9,
|
|
"Def": .9,
|
|
"Wide": .9,
|
|
}
|
|
|
|
def aggregateCov_weight(self, parmHisto, timeRange, componentName, cov):
|
|
# Return a weight (between 0 and 1) for the coverage term
|
|
return self.aggregateCov_weights_dict()[cov]
|
|
|
|
def getExistingWeightedAggregateCov(self, parmHisto, timeRange, componentName,
|
|
wxType, wxPart, dict, totalHours, totalPoints,
|
|
subkeyPoints):
|
|
# From the entries in the dictionary, find the aggregate coverage by
|
|
# using a weighting scheme.
|
|
# If the resulting coverage is not in the grids, use the coverage
|
|
# with the greatest weight.
|
|
if len(dict) == 1:
|
|
return next(iter(dict))
|
|
aggCov, wtSum = self.getAggCov_and_WtSum(parmHisto, timeRange, componentName,
|
|
wxType, wxPart, dict, totalHours, totalPoints,
|
|
subkeyPoints)
|
|
if aggCov is None:
|
|
aggCov = self.processNoAggCov(dict, wxType)
|
|
return aggCov
|
|
|
|
def getWeightedAggregateCov(self, parmHisto, timeRange, componentName,
|
|
wxType, wxPart, dict, totalHours, totalPoints,
|
|
subkeyPoints):
|
|
# From the entries in the dictionary, find the aggregate coverage by
|
|
# using a weighting scheme.
|
|
# If the resulting coverage is not in the grids, "create" an appropriate
|
|
# coverage.
|
|
if len(dict) == 1:
|
|
return next(iter(dict))
|
|
aggCov, wtSum = self.getAggCov_and_WtSum(parmHisto, timeRange, componentName,
|
|
wxType, wxPart, dict, totalHours, totalPoints,
|
|
subkeyPoints)
|
|
# Assign the new coverage
|
|
popValue = self.coveragePoP_table()
|
|
inGrids = 0
|
|
candidates = []
|
|
aggCov = None
|
|
for key in popValue:
|
|
lowVal, highVal = popValue[key]
|
|
#print "key, low, high", key, lowVal, highVal
|
|
# Ranges are inclusive and not contiguous,
|
|
# so we have to adjust
|
|
lowVal = lowVal - 10
|
|
if wtSum > lowVal and wtSum <= highVal:
|
|
# If this coverage was in the grids,
|
|
# choose it and we're done
|
|
#print "dict", dict
|
|
if key in dict:
|
|
aggCov = key
|
|
inGrids = 1
|
|
break
|
|
else:
|
|
candidates.append(key)
|
|
|
|
#print "inGrids", inGrids
|
|
if not inGrids:
|
|
# If the weighted average was not in the grids,
|
|
# we need to choose a coverage or prob term from
|
|
# the candidates
|
|
|
|
# Determine coverage or probability based on
|
|
# first dictionary key
|
|
arealCovs = self.arealCoverages()
|
|
for key in dict:
|
|
if key in arealCovs:
|
|
areal = 1
|
|
else:
|
|
areal = 0
|
|
break
|
|
for cov in candidates:
|
|
if cov in arealCovs:
|
|
covAreal = 1
|
|
else:
|
|
covAreal = 0
|
|
if covAreal == areal:
|
|
# Make sure this cov can be used with
|
|
# the wxType
|
|
availableCoverages = WeatherSubKey.availableCoverages(self._argDict["site"], wxType)
|
|
if cov in availableCoverages:
|
|
aggCov = cov
|
|
break
|
|
if aggCov is None:
|
|
aggCov = self.processNoAggCov(dict, wxType)
|
|
return aggCov
|
|
|
|
def getAggCov_and_WtSum(self, parmHisto, timeRange, componentName,
|
|
wxType, wxPart, dict, totalHours, totalPoints,
|
|
subkeyPoints):
|
|
if len(dict) == 1:
|
|
return (next(iter(dict)), 1)
|
|
# Compute weighted Sum
|
|
wtSum = 0.0
|
|
maxContrib=0
|
|
aggCov="<NoCov>"
|
|
for key in dict:
|
|
if type(dict[key]) is tuple:
|
|
(sum, max) = dict[key]
|
|
else:
|
|
sum = dict[key]
|
|
subkeyRank = float(sum)/(totalHours * totalPoints) * 100.0
|
|
#print "key", key
|
|
covLowVal, covHighVal = self.coveragePoP_value(key)
|
|
covWt = self.aggregateCov_weight(
|
|
parmHisto, timeRange, componentName,key)
|
|
#print " covWt, subkeyRank", covWt, subkeyRank
|
|
#print " contribution", covWt * subkeyRank
|
|
contrib = covWt * subkeyRank
|
|
wtSum += contrib
|
|
if contrib > maxContrib:
|
|
aggCov = key
|
|
maxContrib=contrib
|
|
#print "weighted value", aggCov, wtSum
|
|
return aggCov, wtSum
|
|
|
|
def getHighestWeightedAggregateCov(
|
|
self, parmHisto, timeRange, componentName, wxType, wxPart, dict,
|
|
totalHours, totalPoints, subkeyPoints):
|
|
# Return the Coverage with the highest weight in coverage_weights_dict
|
|
# Throw out Coverages that do not meet the wxkey_coverage_percentage
|
|
# EXCEPT if no Coverages meet that threshold, return the Coverage
|
|
# with the highest percentage.
|
|
#
|
|
# NOTE: In cases where the total percentages, e.g. 5% Sct, 7% Num,
|
|
# do not meet the threshold, it's ok for this method to return Num.
|
|
# because the total percentage will be checked in a later step.
|
|
#
|
|
# Handle case of only one coverage
|
|
if len(dict) == 1:
|
|
return next(iter(dict))
|
|
|
|
# Aggregate Coverage
|
|
aggCov = None
|
|
# Coverage Weight for aggregate Coverage
|
|
maxWt = -1
|
|
# Aggregate Cov for those that do not meet threshold
|
|
aggCovReject = None
|
|
# Max Percentage for those Coverages that do not meet threshold
|
|
maxPercentReject = -1
|
|
|
|
for cov in dict:
|
|
covWt = self.aggregateCov_weight(
|
|
parmHisto, timeRange, componentName, cov)
|
|
sum, maxCov = dict[cov]
|
|
percentCov = float(sum)/(totalHours*totalPoints)*100.0
|
|
percentCov = int(self._textUtils.round(percentCov, "Nearest", 1))
|
|
# Check to see if it meets threshold
|
|
# Make a temporary wxKey to check wxkey_coverage_percentage
|
|
wxKey = WeatherSubKey.weatherSubKey(self._argDict["site"], cov, wxType, "<NoInten>", "<NoVis>", [])
|
|
#print "wxKey", wxKey, sum, percentCov
|
|
if percentCov >= self.wxkey_coverage_percentage(
|
|
parmHisto, timeRange, componentName, wxKey):
|
|
if covWt > maxWt:
|
|
aggCov = cov
|
|
maxWt = covWt
|
|
else:
|
|
if percentCov > maxPercentReject:
|
|
aggCovReject = cov
|
|
maxPercentReject = percentCov
|
|
#print "aggCov, wt",aggCov, maxWt
|
|
#print "aggCovReject, %", aggCovReject, maxPercentReject
|
|
if aggCov is None:
|
|
aggCov = aggCovReject
|
|
#print "Returning", aggCov
|
|
return aggCov
|
|
|
|
def processNoAggCov(self, dict, wxType):
|
|
msg = "WARNING -- SampleAnalysis cannot aggregate coverages for " + wxType
|
|
log.warning(msg)
|
|
# There was no appropriate coverage for the wxType and given weight
|
|
# So take any coverage that exists in the grid
|
|
aggCov = "<NoCov>"
|
|
for key in dict:
|
|
aggCov = key
|
|
break
|
|
return aggCov
|
|
|
|
def getAggregate(self, parmHisto, timeRange, componentName,
|
|
wxType, wxPart, dict, totalHours, totalPoints, subkeyPoints):
|
|
# From the entries in the dictionary,
|
|
# find the aggregate wxPart (coverage, intensity, visibility)
|
|
# Do it 2 at a time and aggregate the ranks as you go
|
|
if len(dict) == 1:
|
|
return next(iter(dict))
|
|
|
|
firstTime = 1
|
|
for key, sum in dict.items():
|
|
subkeyRank = float(sum)/(totalHours * totalPoints) * 100.0
|
|
if wxPart == "coverage":
|
|
subkey = WeatherSubKey.weatherSubKey(self._argDict["site"], key, wxType, "<NoInten>", "<NoVis>", [])
|
|
elif wxPart == "intensity":
|
|
subkey = WeatherSubKey.weatherSubKey(self._argDict["site"], "<NoCov>", wxType, key, "<NoVis>", [])
|
|
elif wxPart == "visibility":
|
|
subkey = WeatherSubKey.weatherSubKey(self._argDict["site"], "<NoCov>", wxType,"<NoInten>" , key, [])
|
|
if firstTime:
|
|
aggRank = subkeyRank
|
|
curKey = subkey
|
|
firstTime = 0
|
|
else:
|
|
curKey = self.makeAggregateSubkey(curKey, aggRank, subkey, subkeyRank)
|
|
aggRank = int((aggRank + subkeyRank)/2.0)
|
|
method = getattr(curKey, wxPart)
|
|
aggValue = method()
|
|
return aggValue
|
|
|
|
def getAggregateAttributes(self, parmHisto, timeRange, componentName,
|
|
wxType, dict, totalHours, totalPoints, subkeyPoints):
|
|
# Take only attributes that meet the threshold
|
|
attrList = []
|
|
for key, sum in dict.items():
|
|
attrRank = float(sum)/(totalHours * totalPoints) * 100.0
|
|
threshold = self.attribute_coverage_percentage(
|
|
parmHisto, timeRange, componentName, wxType, key)
|
|
if attrRank > threshold:
|
|
attrList.append(key)
|
|
return attrList
|
|
|
|
def makeSubkeyList(self, weatherKey):
|
|
# Make sure subkeyList is a true list
|
|
length = len(weatherKey)
|
|
newList = []
|
|
index = 0
|
|
for subkey in weatherKey:
|
|
newList.append(subkey)
|
|
index = index + 1
|
|
if index >= length:
|
|
break
|
|
return newList
|
|
|
|
def weather_percentages(self, parmHisto, timeRange, componentName, args=None):
|
|
# Return a list of tuples:
|
|
# weather subkey, percentage of coverage
|
|
# All WeatherSubKeys are included
|
|
return self.getSubkey_percentages(parmHisto, timeRange, componentName,
|
|
dataType="WEATHER")
|
|
|
|
def getSubkey_percentages(self, parmHisto, timeRange, componentName,
|
|
dataType="WEATHER", withAux=1):
|
|
# Gather all the Weather or Discrete SubKeys and percentages
|
|
numPoints = parmHisto.numberOfGridPoints()
|
|
percentageList = []
|
|
|
|
# Each histSample represents a grid
|
|
# To determine percentage for a weather value, we need
|
|
# to aggregate it over grids
|
|
for histSample in parmHisto.histoSamples():
|
|
timeWeight = self.determineGridWeight(histSample, timeRange)
|
|
for histPair in histSample.histogram():
|
|
count = float(histPair.count())
|
|
gridPercentage = int((count/numPoints) * 100.0)
|
|
timeRangePercentage = gridPercentage * timeWeight
|
|
if dataType == "WEATHER":
|
|
subkey = WeatherSubKey.WeatherSubKey(histPair.value().weather().get(0))
|
|
else: # DISCRETE
|
|
subkey = histPair.value().discrete().get(0)
|
|
if withAux == 0:
|
|
subkey = histPair.value().discrete().baseData(histPair.value().discrete().getSiteId(), subkey)
|
|
# See if subkey is already in list
|
|
found = 0
|
|
for value, percentage in percentageList:
|
|
# If so, add it's percentage
|
|
if value == subkey:
|
|
found = 1
|
|
index = percentageList.index((value,percentage))
|
|
newPercentage = percentage + timeRangePercentage
|
|
percentageList[index] = (subkey, newPercentage)
|
|
if found == 0:
|
|
percentageList.append((subkey,timeRangePercentage))
|
|
#print "percentage list", dataType, percentageList
|
|
return percentageList
|
|
|
|
########################################
|
|
## DISCRETE
|
|
|
|
def discreteKey_coverage_percentage(self, parmHisto, timeRange, componentName, keyStr):
|
|
# Return the required coverage percentage for the given wxkey which will be
|
|
# compared to its "rank" i.e. the percentage of areal coverage over the time period.
|
|
return 1
|
|
|
|
def dominantDiscreteKeys_threshold(self, parmHisto, timeRange, componentName):
|
|
# This is the maximum number of discrete keys desired from the
|
|
# getDominantDiscreteKey method.
|
|
return 10
|
|
|
|
def dominantDiscreteValue(self, parmHisto, timeRange, componentName, args=None):
|
|
"Return the most common discrete value over the given timeRange"
|
|
primaryMethod = "self.getDominantDiscreteValue(parmHisto, timeRange, componentName, withAux=0)"
|
|
return self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
|
|
def dominantDiscreteValue_withAux(self, parmHisto, timeRange, componentName, args=None):
|
|
"Return the most common discrete value over the given timeRange"
|
|
primaryMethod = "self.getDominantDiscreteValue(parmHisto, timeRange, componentName, withAux=1)"
|
|
return self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
|
|
def rankedDiscreteValue(self, parmHisto, timeRange, componentName, args=None):
|
|
"Return the most common discrete value over the given timeRange"
|
|
primaryMethod = "self.getDominantDiscreteValue(parmHisto, timeRange, componentName, withAux=0, withRank=1)"
|
|
return self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
|
|
def rankedDiscreteValue_withAux(self, parmHisto, timeRange, componentName, args=None):
|
|
"Return the most common discrete value over the given timeRange"
|
|
primaryMethod = "self.getDominantDiscreteValue(parmHisto, timeRange, componentName, withAux=1, withAux=1)"
|
|
return self.createStats(parmHisto, timeRange, componentName, args, primaryMethod)
|
|
|
|
def getDominantDiscreteValue(self, parmHisto, timeRange, componentName, withRank=0, withAux=0):
|
|
# Return a list of dominant discrete subkeys
|
|
# If withRank, return (key, rank) pairs
|
|
# If withAux, include the auxillary field as part of the key
|
|
return self.getDominantValues(parmHisto, timeRange, componentName,
|
|
dataType="DISCRETE", withRank=withRank, withAux=withAux)
|
|
|
|
def discrete_percentages(self, parmHisto, timeRange, componentName, args=None):
|
|
return self.getSubkey_percentages(parmHisto, timeRange, componentName, dataType="DISCRETE",
|
|
withAux=0)
|
|
|
|
def discrete_percentages_withAux(self, parmHisto, timeRange, componentName, args=None):
|
|
return self.getSubkey_percentages(parmHisto, timeRange, componentName, dataType="DISCRETE",
|
|
withAux=1)
|
|
|
|
def discreteTimeRangesByKey(self, parmHisto, timeRange, componentName, args=None):
|
|
return self.getDiscreteTimeRangesByKey(
|
|
parmHisto, timeRange, componentName, args=None, withAux=0)
|
|
|
|
def discreteTimeRangesByKey_withAux(self, parmHisto, timeRange, componentName, args=None):
|
|
return self.getDiscreteTimeRangesByKey(
|
|
parmHisto, timeRange, componentName, args=None, withAux=1)
|
|
|
|
def getDiscreteTimeRangesByKey(self, parmHisto, timeRange, componentName, args=None,
|
|
withAux=0):
|
|
# This method returns a list of (discreteSubkey, timeRange) pairs ordered in ascending
|
|
# order by timeRange and then by priority of discrete keys as defined in the
|
|
# serverConfig files.
|
|
|
|
keyDict = {}
|
|
covDict = {}
|
|
totalHours = timeRange.duration() // 3600
|
|
totalPoints = parmHisto.numberOfGridPoints()
|
|
|
|
for histSample in parmHisto.histoSamples():
|
|
validTime = TimeRange.TimeRange(histSample.validTime())
|
|
if self.temporalCoverage_flag(
|
|
parmHisto, timeRange, componentName, histSample) == 0:
|
|
continue
|
|
|
|
hours = validTime.intersection(timeRange).duration() // 3600
|
|
if hours < 1:
|
|
continue
|
|
|
|
for histPair in histSample.histogram():
|
|
keyStr = histPair.value().discrete().get(0) # discrete value
|
|
if withAux == 0:
|
|
keyStr = histPair.value().discrete().baseData(histPair.value().discrete().getSiteId(), keyStr)
|
|
|
|
if keyStr in keyDict:
|
|
keyDict[keyStr].append(validTime)
|
|
else:
|
|
keyDict[keyStr] = [validTime]
|
|
|
|
# Keep a running total of the temporal and areal percentages
|
|
count = float(histPair.count())
|
|
if keyStr in covDict:
|
|
keyRank = covDict[keyStr]
|
|
newRank = keyRank + hours*count
|
|
covDict[keyStr] = newRank
|
|
else: # new entry
|
|
covDict[keyStr] = hours*count
|
|
|
|
## keyList = covDict.keys()
|
|
## for k in keyList:
|
|
## t, a, n = covDict[k]
|
|
## print "key:", k, "time%:", t, "area#:", a, "totalPoints:", n
|
|
|
|
keyList = list(covDict.keys())
|
|
for keyStr in keyList:
|
|
# get the temporal and areal thresholds
|
|
keyRank = covDict[keyStr]
|
|
rank = int(self._textUtils.round(float(keyRank)/(totalHours*totalPoints)*100.0, "Nearest", 1))
|
|
if rank < self.discreteKey_coverage_percentage(
|
|
parmHisto, timeRange, componentName, keyStr):
|
|
# remove the dict entry
|
|
del keyDict[keyStr]
|
|
|
|
# glue the timeranges that share a common end/start time
|
|
keyList = []
|
|
for (k, trList) in keyDict.items():
|
|
trList = sorted(trList)
|
|
tr = trList[0]
|
|
for i in range(1, len(trList)):
|
|
if tr.endTime() == trList[i].startTime():
|
|
# keep extending the time, if TRs are contiguous
|
|
tr = TimeRange.TimeRange(tr.startTime(), trList[i].endTime())
|
|
else:
|
|
# no match, append the tuple
|
|
keyList.append((k, tr))
|
|
tr = trList[i]
|
|
|
|
# Don't forget the last one
|
|
keyList.append((k, tr))
|
|
|
|
#print "discreteTimeRangesByKey keyList", keyList
|
|
return keyList
|
|
|
|
def mostSignificantDiscreteValue(self, parmHisto, timeRange, componentName, withAux=0):
|
|
"""Using mostSignificantDiscrete_keyOrder_dict and mostSignificantDiscrete_coveragePercentage_dict,
|
|
report the most significant discrete value for the given timeRange. If there is a tie,
|
|
report the most significant value.
|
|
"""
|
|
totalHours = 0
|
|
totalPoints = parmHisto.numberOfGridPoints()
|
|
compositeNameUI = parmHisto.parmID().compositeNameUI()
|
|
|
|
# Loop over all samples, which are for all grids and
|
|
# all keys on each of those grids.
|
|
# We will have just one entry per
|
|
# discrete key (with or without Aux value).
|
|
|
|
#print "\n\nIn mostSignificantDiscreteValue: DataType, TimeRange", "DISCRETE", timeRange
|
|
#print "STEP 1 -- Aggregate per grid"
|
|
|
|
subkeyTypeDict = {}
|
|
# STEP 1:
|
|
# For each discrete key found in the grids,
|
|
# gather its 'hours' of coverage and 'count' of points covered.
|
|
for histSample in parmHisto.histoSamples():
|
|
validTime = TimeRange.TimeRange(histSample.validTime())
|
|
if self.temporalCoverage_flag(
|
|
parmHisto, timeRange, componentName, histSample) == 0:
|
|
continue
|
|
# Get the number of hours inside the timeRange that this
|
|
# sample comes from (i.e., we can get a sample that lasts
|
|
# for 3 weeks - but only 1 hour of it is inside the
|
|
# timeRange - and we only want to rank it by the 1 hour
|
|
# inside the range)
|
|
#
|
|
hours = validTime.intersection(timeRange).duration() // 3600
|
|
if hours < 1:
|
|
continue
|
|
|
|
totalHours += hours
|
|
|
|
# Gather the subkey Types for this grid in subkeyTypeDict
|
|
# Each entry ends up being a list of tuples: (discreteKey, hours, count)
|
|
self.gatherSubkeyTypes(
|
|
parmHisto, timeRange, componentName, histSample, 'DISCRETE', hours,
|
|
subkeyTypeDict, withAux)
|
|
|
|
# STEP 2: For each subkeyType,
|
|
# --determine an aggregate subkey and rank i.e.
|
|
# aggregate areal coverage over time percentage
|
|
# --compare the rank to coverage threshold.
|
|
#print "subkeyTypeDict", subkeyTypeDict
|
|
keyRankDict = {} # Holds: subkeyType: rank
|
|
for subkeyType, subkeyList in subkeyTypeDict.items():
|
|
#print "\nsubkeyType", subkeyType
|
|
subkeyTypeRank = 0
|
|
# Determine a subkeyType rank
|
|
subkeyTotalPoints = 0
|
|
for subkey, hours, count in subkeyList:
|
|
#print " subkey, hours, count", subkey, hours, count
|
|
subkeyTotalPoints += count
|
|
subkeyTypeRank += hours * count
|
|
#print "total points =", subkeyTotalPoints
|
|
|
|
#print "subkeyTypeRank =", subkeyTypeRank
|
|
#print "totalHours =", totalHours
|
|
#print "totalPoints =", totalPoints
|
|
# Determine rank for the subkeyType
|
|
rank = int(self._textUtils.round(float(subkeyTypeRank)/(totalHours*totalPoints)*100.0, "Nearest", 1))
|
|
keyRankDict[subkeyType] = rank
|
|
#print "rank =", rank
|
|
|
|
# Check to see if each subkeyType meets the required coverage percentage
|
|
keyOrderDict = self.mostSignificantDiscrete_keyOrder_dict(parmHisto, timeRange, compositeNameUI)
|
|
keyOrder = keyOrderDict[compositeNameUI]
|
|
mostSignificantSubkey = None
|
|
highestOrderIndex = None
|
|
for subkeyType, rank in keyRankDict.items():
|
|
thresholdDict = self.mostSignificantDiscrete_coveragePercentage_dict(
|
|
parmHisto, timeRange, componentName, subkeyType)
|
|
threshold = thresholdDict.get(compositeNameUI, 0)
|
|
#print "threshold =", threshold
|
|
flag = rank >= threshold
|
|
if not flag:
|
|
# Get another chance to pass
|
|
flag = self.checkPercentages(
|
|
parmHisto, timeRange, componentName, subkeyType, keyRankDict)
|
|
if flag: # This type meets the threshold criteria
|
|
if self.cleanOutEmptyValues(parmHisto, timeRange, componentName, "DISCRETE"):
|
|
# Don't save empty values
|
|
#print "Ignoring", subkeyType
|
|
continue
|
|
try:
|
|
orderIndex = keyOrder.index(subkeyType)
|
|
if highestOrderIndex is None or orderIndex > highestOrderIndex:
|
|
highestOrderIndex = orderIndex
|
|
mostSignificantSubkey = subkeyType
|
|
#print "Found higher significance key =", subkeyType
|
|
except:
|
|
pass
|
|
else:
|
|
#print "didn't make the cut", rank, subkeyType
|
|
pass
|
|
|
|
#print "mostSignificantSubkey =", mostSignificantSubkey
|
|
return mostSignificantSubkey
|
|
|
|
def mostSignificantDiscrete_coveragePercentage_dict(self, parmHisto, timeRange, componentName, keyStr):
|
|
""" Return the required coverage percentage for the given key which will be
|
|
compared to its "rank" i.e. the percentage of areal coverage over the time period.
|
|
"""
|
|
return {
|
|
"WindThreat": 5,
|
|
"FloodingRainThreat": 5,
|
|
"StormSurgeThreat": 5,
|
|
"TornadoThreat": 5,
|
|
}
|
|
|
|
def mostSignificantDiscrete_keyOrder_dict(self, parmHisto, timeRange, componentName):
|
|
""" Returns a list of keys from least to most significant for a discrete type (componentName). """
|
|
threatKeyOrder = [None, "None", "Elevated", "Mod", "High", "Extreme"]
|
|
return {
|
|
"WindThreat": threatKeyOrder,
|
|
"FloodingRainThreat": threatKeyOrder,
|
|
"StormSurgeThreat": threatKeyOrder,
|
|
"TornadoThreat": threatKeyOrder,
|
|
}
|
|
|
|
|
|
########################################
|
|
## UTILITIES
|
|
|
|
|
|
def determineGridWeight(self, histSample, timeRange):
|
|
# Returns the ratio: histSample overlap duration / timeRange duration
|
|
validTime = TimeRange.TimeRange(histSample.validTime())
|
|
if validTime.contains(timeRange):
|
|
gridWeight = 1.0
|
|
# Determine time histSample intersects timeRange
|
|
else:
|
|
intersect = validTime.intersection(timeRange).duration()
|
|
try:
|
|
gridWeight = float(intersect)/timeRange.duration()
|
|
except:
|
|
gridWeight = 0.0
|
|
return gridWeight
|
|
|
|
def createStats(self, parmHisto, timeRange, componentName, args, primaryMethod):
|
|
# Call appropriate methods to produce statistics based on args which tell us
|
|
# how to report the statistics with respect to the time range.
|
|
#
|
|
if args is None:
|
|
result = eval(primaryMethod)
|
|
return result
|
|
period = args[0]
|
|
if period == 0:
|
|
subRanges = self.getGridTimeRanges(parmHisto, timeRange)
|
|
else:
|
|
subRanges = self.divideRange(timeRange, period)
|
|
statsByRange = []
|
|
for subRange in subRanges:
|
|
timeRange = subRange
|
|
result = eval(primaryMethod)
|
|
# Handle no data
|
|
# If a subRange has no data continue
|
|
if result is None:
|
|
continue
|
|
statsByRange.append((result, subRange))
|
|
return statsByRange
|
|
|
|
def temporalCoverage_flag(self, parmHisto, timeRange, componentName,
|
|
histSample):
|
|
# Return 1 if the histSample time range is completely included in the timeRange
|
|
# OR the histSample time range sufficiently covers the timeRange
|
|
# i.e. meets BOTH the temporalCoverage_percentage and temporalCoverage_hours
|
|
# requirements.
|
|
|
|
# Sub-methods:
|
|
# temporalCoverage_dict
|
|
# temporalCoverage_percentage
|
|
# temporalCoverage_hours
|
|
# temporalCoverage_hours_dict
|
|
#
|
|
|
|
# njensen: I changed this to act directly on the java time ranges since they aren't
|
|
# part of the return value and it's faster to skip creating python TimeRanges
|
|
javaValidTime = histSample.validTime()
|
|
javaTimeRange = timeRange.toJavaObj()
|
|
compositeNameUI = parmHisto.getCompositeNameUI()
|
|
|
|
# Is the histSample time range completely included in the timeRange?
|
|
#if timeRange.contains(validTime):
|
|
if javaTimeRange.contains(javaValidTime):
|
|
result = 1
|
|
# Look at intersection of histSample and timeRange
|
|
else:
|
|
covDict = self.temporalCoverage_dict(parmHisto, timeRange, componentName)
|
|
if compositeNameUI in covDict:
|
|
percentage = covDict[compositeNameUI]
|
|
else:
|
|
percentage = self.temporalCoverage_percentage(
|
|
parmHisto, timeRange, componentName)
|
|
hoursDict = self.temporalCoverage_hours_dict(
|
|
parmHisto, timeRange, componentName)
|
|
if compositeNameUI in hoursDict:
|
|
coverageHours = hoursDict[compositeNameUI]
|
|
else:
|
|
coverageHours = self.temporalCoverage_hours(
|
|
parmHisto, timeRange, componentName)
|
|
#intersect = javaValidTime.intersection(javaTimeRange).getDuration()
|
|
intersect = FormatterUtil.getTimeRangeIntersectionDuration(javaValidTime, javaTimeRange) // 1000
|
|
# The intersection should be at least the percentage of the timeRange
|
|
# AND at least the number of coverageHours
|
|
fullPeriod = javaTimeRange.getDuration() // 1000
|
|
try:
|
|
if fullPeriod > 0:
|
|
percentIn = float(intersect)/fullPeriod
|
|
else:
|
|
percentIn = 0.0
|
|
if percentIn > 0 and percentIn >= percentage/100.0:
|
|
result = 1
|
|
else: # saying no - not enough is inside timeRange"
|
|
result = 0
|
|
except: # saying no - could not figure percentIn"
|
|
result = 0
|
|
# If temporal coverage percentage requirement met,
|
|
# check temporal coverage hours requirement
|
|
if result == 1:
|
|
intersectHours = intersect // 3600
|
|
trHours = fullPeriod // 3600
|
|
if intersectHours >= coverageHours:
|
|
result = 1
|
|
elif coverageHours >= trHours and intersectHours == trHours:
|
|
result = 1
|
|
else:
|
|
result = 0
|
|
return result
|
|
|
|
def getAccumSum(self, dataType, parmHisto, timeRange, componentName, firstOnly=0):
|
|
"Return the cummulative sum over the given time period"
|
|
minVal, maxVal, sumVal = parmHisto.minMaxSum()
|
|
return sumVal
|
|
|
|
def getAccumMinMax(self, dataType, parmHisto, timeRange, componentName, firstOnly=0):
|
|
"Return the cummulative min/max over the given time period"
|
|
minVal, maxVal, sumVal = parmHisto.minMaxSum()
|
|
return minVal, maxVal
|
|
|
|
def getModAccumSum(self, dataType, parmHisto, timeRange, componentName, firstOnly=0):
|
|
"Return the moderated cummulative sum over the given time period"
|
|
minLimit, maxLimit = self.getModeratedLimits(parmHisto, timeRange, componentName)
|
|
minVal, maxVal, sumVal = parmHisto.moderatedMinMaxSum(minLimit, maxLimit)
|
|
return sumVal
|
|
|
|
def getModAccumMinMax(self, dataType, parmHisto, timeRange, componentName, firstOnly=0):
|
|
"Return the modereted cummulative min/max over the given time period"
|
|
minLimit, maxLimit = self.getModeratedLimits(parmHisto, timeRange, componentName)
|
|
minVal, maxVal, sumVal = parmHisto.moderatedMinMaxSum(minLimit, maxLimit)
|
|
return minVal, maxVal
|
|
|
|
def getStdDevAccumSum(self, dataType, parmHisto, timeRange, componentName, firstOnly=0):
|
|
"Return the standard deviation sum over the given time period"
|
|
minLimit, maxLimit = self.getStdDevLimits(parmHisto, timeRange, componentName)
|
|
minVal, maxVal, sumVal = parmHisto.stdDevMinMaxSum(minLimit, maxLimit)
|
|
return sumVal
|
|
|
|
def getStdDevAccumMinMax(self, dataType, parmHisto, timeRange, componentName, firstOnly=0):
|
|
"Return the standard deviation min/max over the given time period"
|
|
minLimit, maxLimit = self.getStdDevLimits(parmHisto, timeRange, componentName)
|
|
minVal, maxVal, sumVal = parmHisto.stdDevMinMaxSum(minLimit, maxLimit)
|
|
return minVal, maxVal
|
|
|
|
def getAverage(self, dataType, parmHisto, timeRange, componentName, firstOnly = 0):
|
|
"Return the time weighted average values over the given time period"
|
|
totValue = 0.0
|
|
totWeight = 0.0
|
|
for histSample in parmHisto.histoSamples():
|
|
if self.temporalCoverage_flag(parmHisto, timeRange, componentName,
|
|
histSample) == 0:
|
|
continue
|
|
# return None if no histSample pairs
|
|
if histSample.numOfPoints() == 0:
|
|
return None
|
|
|
|
avg = histSample.average(True)
|
|
|
|
# njensen: faster to do this without wrapping java objects
|
|
validTime = histSample.validTime()
|
|
weight = FormatterUtil.getTimeRangeIntersectionDuration(validTime, timeRange.toJavaObj())
|
|
|
|
if dataType == self.SCALAR():
|
|
value = avg.scalar()
|
|
elif dataType == self.VECTOR():
|
|
value = avg.magnitude()
|
|
|
|
# sum weighted averages
|
|
totValue = totValue + weight * value
|
|
totWeight = totWeight + weight
|
|
|
|
if firstOnly == 1:
|
|
break
|
|
|
|
if totWeight > 0.0:
|
|
result = totValue / totWeight
|
|
else:
|
|
return None
|
|
|
|
if dataType == self.VECTOR():
|
|
dir = self.getDominantDirection(dataType, parmHisto, timeRange,
|
|
componentName)
|
|
return result, dir
|
|
|
|
return result
|
|
|
|
def getMinMax(self, dataType, parmHisto, timeRange, componentName,
|
|
firstOnly = 0):
|
|
"Return the minimum and maximum values over the given time period"
|
|
firstTime = 1
|
|
minValue = 0.0
|
|
maxValue = 0.0
|
|
minResult = 0.0
|
|
maxResult = 0.0
|
|
noData = 1
|
|
for histSample in parmHisto.histoSamples():
|
|
if self.temporalCoverage_flag(parmHisto, timeRange, componentName,
|
|
histSample) == 0:
|
|
continue
|
|
# return None if no histSample pairs
|
|
if histSample.numOfPoints() == 0:
|
|
return None
|
|
noData = 0
|
|
|
|
min = histSample.absoluteMin()
|
|
max = histSample.absoluteMax()
|
|
if dataType == self.SCALAR():
|
|
minValue = min.scalar()
|
|
maxValue = max.scalar()
|
|
elif dataType == self.VECTOR() or dataType == self.MAGNITUDE():
|
|
minValue = min.magnitude()
|
|
maxValue = max.magnitude()
|
|
if firstTime == 1:
|
|
firstTime = 0
|
|
minResult = minValue
|
|
maxResult = maxValue
|
|
else:
|
|
if minValue < minResult:
|
|
minResult = minValue
|
|
if maxValue > maxResult:
|
|
maxResult = maxValue
|
|
if firstOnly == 1:
|
|
break
|
|
|
|
if noData == 1:
|
|
return None
|
|
if dataType == self.VECTOR():
|
|
dir = self.getDominantDirection(dataType, parmHisto,
|
|
timeRange, componentName)
|
|
return (minResult, maxResult), dir
|
|
|
|
return minResult, maxResult
|
|
|
|
def getStdDevAvg(self, dataType, parmHisto, timeRange, componentName,
|
|
firstOnly = 0):
|
|
"Return the time wieghted average values over the given time period"
|
|
# get the stdDev limits from the stdDev dictionary
|
|
minStd, maxStd = self.getStdDevLimits(parmHisto, timeRange, componentName)
|
|
totValue = 0.0
|
|
totWeight = 0.0
|
|
for histSample in parmHisto.histoSamples():
|
|
if self.temporalCoverage_flag(parmHisto, timeRange, componentName,
|
|
histSample) == 0:
|
|
continue
|
|
# return None if no histSample pairs
|
|
if histSample.numOfPoints() == 0:
|
|
return None
|
|
|
|
# In AWIPS1, stdDevAvg utilized a default value of True for
|
|
# separateMagDir argument
|
|
avg = histSample.stdDevAvg(minStd, maxStd, True)
|
|
validTime = TimeRange.TimeRange(histSample.validTime())
|
|
weight = validTime.intersection(timeRange).duration()
|
|
|
|
if dataType == self.SCALAR():
|
|
value = avg.scalar()
|
|
elif dataType == self.VECTOR():
|
|
value = avg.magnitude()
|
|
|
|
# sum weighted averages
|
|
totValue = totValue + weight * value
|
|
totWeight = totWeight + weight
|
|
|
|
if firstOnly == 1:
|
|
break
|
|
|
|
if totWeight > 0.0:
|
|
result = totValue / totWeight
|
|
else:
|
|
return None
|
|
|
|
if dataType == self.VECTOR():
|
|
dir = self.getDominantDirection(dataType, parmHisto, timeRange,
|
|
componentName)
|
|
return result, dir
|
|
|
|
return result
|
|
|
|
def getStdDevMinMax(self, dataType, parmHisto, timeRange, componentName,
|
|
firstOnly = 0):
|
|
"Return the minimum and maximum values over the given time period"
|
|
firstTime = 1
|
|
minValue = 0.0
|
|
maxValue = 0.0
|
|
minResult = 0.0
|
|
maxResult = 0.0
|
|
noData = 1
|
|
# get the stdDev limits from the stdDev dictionary
|
|
minStd, maxStd = self.getStdDevLimits(parmHisto, timeRange, componentName)
|
|
|
|
for histSample in parmHisto.histoSamples():
|
|
if self.temporalCoverage_flag(parmHisto, timeRange, componentName,
|
|
histSample) == 0:
|
|
continue
|
|
# return None if no histSample pairs
|
|
if histSample.numOfPoints() == 0:
|
|
return None
|
|
noData = 0
|
|
|
|
min = histSample.stdDevMin(minStd)
|
|
max = histSample.stdDevMax(maxStd)
|
|
if dataType == self.SCALAR():
|
|
minValue = min.scalar()
|
|
maxValue = max.scalar()
|
|
elif dataType == self.VECTOR():
|
|
minValue = min.magnitude()
|
|
maxValue = max.magnitude()
|
|
if firstTime == 1:
|
|
firstTime = 0
|
|
minResult = minValue
|
|
maxResult = maxValue
|
|
else:
|
|
if minValue < minResult:
|
|
minResult = minValue
|
|
if maxValue > maxResult:
|
|
maxResult = maxValue
|
|
if firstOnly == 1:
|
|
break
|
|
|
|
if noData == 1:
|
|
return None
|
|
if dataType == self.VECTOR():
|
|
dir = self.getDominantDirection(dataType, parmHisto,
|
|
timeRange, componentName)
|
|
return (minResult, maxResult), dir
|
|
|
|
return minResult, maxResult
|
|
|
|
def getModeratedAvg(self, dataType, parmHisto, timeRange, componentName,
|
|
firstOnly = 0):
|
|
"Return the time weighted average values over the given time period"
|
|
# get the stdDev limits from the stdDev dictionary
|
|
minMod, maxMod = self.getModeratedLimits(parmHisto, timeRange, componentName)
|
|
totValue = 0.0
|
|
totWeight = 0.0
|
|
noData = 1
|
|
for histSample in parmHisto.histoSamples():
|
|
if self.temporalCoverage_flag(parmHisto, timeRange, componentName,
|
|
histSample) == 0:
|
|
continue
|
|
# return None if no histSample pairs
|
|
if histSample.numOfPoints() == 0:
|
|
return None
|
|
noData = 0
|
|
|
|
avg = histSample.moderatedAverage(minMod, maxMod, True)
|
|
validTime = TimeRange.TimeRange(histSample.validTime())
|
|
weight = validTime.intersection(timeRange).duration()
|
|
|
|
if dataType == self.SCALAR():
|
|
value = avg.scalar()
|
|
elif dataType == self.VECTOR():
|
|
value = avg.magnitude()
|
|
|
|
# sum weighted averages
|
|
totValue = totValue + weight * value
|
|
totWeight = totWeight + weight
|
|
|
|
if firstOnly == 1:
|
|
break
|
|
|
|
if noData == 1:
|
|
return None
|
|
if totWeight > 0.0:
|
|
result = totValue / totWeight
|
|
else:
|
|
return None
|
|
|
|
if dataType == self.VECTOR():
|
|
dir = self.getDominantDirection(dataType, parmHisto, timeRange,
|
|
componentName)
|
|
return result, dir
|
|
|
|
return result
|
|
|
|
def getModeratedMinMax(self, dataType, parmHisto, timeRange, componentName,
|
|
firstOnly = 0):
|
|
"Return the minimum and maximum values over the given time period"
|
|
firstTime = 1
|
|
minValue = 0.0
|
|
maxValue = 0.0
|
|
minResult = 0.0
|
|
maxResult = 0.0
|
|
noData = 1
|
|
# get the stdDev limits from the stdDev dictionary
|
|
minMod, maxMod = self.getModeratedLimits(parmHisto, timeRange, componentName)
|
|
|
|
for histSample in parmHisto.histoSamples():
|
|
if self.temporalCoverage_flag(parmHisto, timeRange, componentName,
|
|
histSample) == 0:
|
|
continue
|
|
# return None if no histSample pairs
|
|
if histSample.numOfPoints() == 0:
|
|
return None
|
|
noData = 0
|
|
|
|
min = histSample.moderatedMin(minMod)
|
|
max = histSample.moderatedMax(maxMod)
|
|
if dataType == self.SCALAR():
|
|
minValue = min.scalar()
|
|
maxValue = max.scalar()
|
|
elif dataType == self.VECTOR():
|
|
minValue = min.magnitude()
|
|
maxValue = max.magnitude()
|
|
if firstTime == 1:
|
|
firstTime = 0
|
|
minResult = minValue
|
|
maxResult = maxValue
|
|
else:
|
|
if minValue < minResult:
|
|
minResult = minValue
|
|
if maxValue > maxResult:
|
|
maxResult = maxValue
|
|
if firstOnly == 1:
|
|
break
|
|
|
|
if noData == 1:
|
|
return None
|
|
if dataType == self.VECTOR():
|
|
dir = self.getDominantDirection(dataType, parmHisto, timeRange,
|
|
componentName)
|
|
return (minResult, maxResult), dir
|
|
|
|
return minResult, maxResult
|
|
|
|
def getMaxAvg(self, dataType, parmHisto, timeRange, componentName):
|
|
"Return the maximum average value over the given time period"
|
|
firstTime = 1
|
|
maxValue = 0.0
|
|
maxResult = 0.0
|
|
noData = 1
|
|
|
|
for histSample in parmHisto.histoSamples():
|
|
if self.temporalCoverage_flag(parmHisto, timeRange, componentName,
|
|
histSample) == 0:
|
|
continue
|
|
# return None if no histSample pairs
|
|
if histSample.numOfPoints() == 0:
|
|
return None
|
|
noData = 0
|
|
|
|
maxV = histSample.average()
|
|
if dataType == self.SCALAR():
|
|
maxValue = maxV.scalar()
|
|
elif dataType == self.VECTOR():
|
|
maxValue = maxV.magnitude()
|
|
if firstTime == 1:
|
|
firstTime = 0
|
|
maxResult = maxValue
|
|
else:
|
|
if maxValue > maxResult:
|
|
maxResult = maxValue
|
|
|
|
if noData == 1:
|
|
return None
|
|
if dataType == self.VECTOR():
|
|
dir = self.getDominantDirection(dataType, parmHisto, timeRange,
|
|
componentName)
|
|
return (maxResult, dir)
|
|
|
|
return maxResult
|
|
|
|
def getStdDevMaxAvg(self, dataType, parmHisto, timeRange, componentName):
|
|
"Return the maximum average value filtering by standard deviation"
|
|
firstTime = 1
|
|
maxValue = 0.0
|
|
maxResult = 0.0
|
|
noData = 1
|
|
# get the stdDev limits from the stdDev dictionary
|
|
minStd, maxStd = self.getStdDevLimits(parmHisto, timeRange, componentName)
|
|
|
|
for histSample in parmHisto.histoSamples():
|
|
if self.temporalCoverage_flag(parmHisto, timeRange, componentName,
|
|
histSample) == 0:
|
|
continue
|
|
# return None if no histSample pairs
|
|
if histSample.numOfPoints() == 0:
|
|
return None
|
|
noData = 0
|
|
|
|
maxV = histSample.stdDevAvg(minStd, maxStd, True)
|
|
if dataType == self.SCALAR():
|
|
maxValue = maxV.scalar()
|
|
elif dataType == self.VECTOR():
|
|
maxValue = maxV.magnitude()
|
|
if firstTime == 1:
|
|
firstTime = 0
|
|
maxResult = maxValue
|
|
else:
|
|
if maxValue > maxResult:
|
|
maxResult = maxValue
|
|
|
|
if noData == 1:
|
|
return None
|
|
if dataType == self.VECTOR():
|
|
dir = self.getDominantDirection(dataType, parmHisto, timeRange,
|
|
componentName)
|
|
return (maxResult, dir)
|
|
|
|
return maxResult
|
|
|
|
def getModeratedMaxAvg(self, dataType, parmHisto, timeRange, componentName):
|
|
"Return the maximum average value filtering by percentage"
|
|
firstTime = 1
|
|
maxValue = 0.0
|
|
maxResult = 0.0
|
|
noData = 1
|
|
# get the moderated limits from the stdDev dictionary
|
|
minMod, maxMod = self.getModeratedLimits(parmHisto, timeRange, componentName)
|
|
|
|
for histSample in parmHisto.histoSamples():
|
|
if self.temporalCoverage_flag(parmHisto, timeRange, componentName,
|
|
histSample) == 0:
|
|
continue
|
|
# return None if no histSample pairs
|
|
if histSample.numOfPoints() == 0:
|
|
return None
|
|
noData = 0
|
|
|
|
maxV = histSample.moderatedAverage(minMod, maxMod, True)
|
|
if dataType == self.SCALAR():
|
|
maxValue = maxV.scalar()
|
|
elif dataType == self.VECTOR():
|
|
maxValue = maxV.magnitude()
|
|
if firstTime == 1:
|
|
firstTime = 0
|
|
maxResult = maxValue
|
|
else:
|
|
if maxValue > maxResult:
|
|
maxResult = maxValue
|
|
|
|
if noData == 1:
|
|
return None
|
|
if dataType == self.VECTOR():
|
|
dir = self.getDominantDirection(dataType, parmHisto, timeRange,
|
|
componentName)
|
|
return (maxResult, dir)
|
|
|
|
return maxResult
|
|
|
|
def getVectorAvg(self, histPairs):
|
|
# Temporary method to emulate new HistSample.average(0)
|
|
# to be supplied in next release
|
|
uSum = 0.0
|
|
vSum = 0.0
|
|
totCount = 0
|
|
for histPair in histPairs:
|
|
count = histPair.count()
|
|
totCount = totCount + count
|
|
val = histPair.value()
|
|
uw, vw = self.MagDirToUV(val.magnitude(), val.direction())
|
|
uSum = uSum + uw * count
|
|
vSum = vSum + vw * count
|
|
|
|
# calculate the average wind vector
|
|
if totCount > 0:
|
|
u = uSum / float(totCount)
|
|
v = vSum / float(totCount)
|
|
mag, dir = self.UVToMagDir(u, v)
|
|
mag = int(mag + 0.5)
|
|
dir = int(dir + 0.5)
|
|
return HistValue(float(mag), float(dir))
|
|
else:
|
|
return HistValue()
|
|
|
|
def extractMinMax(self, minMaxList, minOrMax, dataType=None):
|
|
# returns the min or max value in the list depending on minOrMax
|
|
# minMaxList is a list returned from createStats
|
|
# minOrMax can have the values "Min" or "Max" and nothing else
|
|
|
|
if dataType == self.VECTOR():
|
|
return self.extractVectorMinMax(minMaxList, minOrMax)
|
|
|
|
# sanity checks - must be a list or tuple
|
|
if type(minMaxList) is not list and \
|
|
type(minMaxList) is not tuple:
|
|
return None
|
|
|
|
# minOrMax must be "Min" or "Max"
|
|
if not (minOrMax == "Min" or minOrMax == "Max"):
|
|
return None
|
|
|
|
if type(minMaxList) is tuple:
|
|
if minOrMax == "Min":
|
|
return minMaxList[0] # return min value
|
|
elif minOrMax == "Max":
|
|
return minMaxList[1] # return max value
|
|
else:
|
|
print("extractMinMax error - Bad min/max string:", minOrMax)
|
|
print("Must be: 'Min' or 'Max'. ")
|
|
return None
|
|
|
|
# check for empty list
|
|
if len(minMaxList) <= 0:
|
|
return None
|
|
|
|
newList = []
|
|
# loop through and find the min and max
|
|
for (vMin, vMax), timeRange in minMaxList:
|
|
if minOrMax == "Min":
|
|
value = vMin
|
|
else:
|
|
value = vMax
|
|
newList.append((value, timeRange))
|
|
return newList
|
|
|
|
def extractVectorMinMax(self, minMaxList, minOrMax):
|
|
# returns the min or max value in the list depending on minOrMax
|
|
# minMaxList is a list returned from createStats
|
|
# minOrMax can have the values "Min" or "Max" and nothing else
|
|
|
|
# sanity checks - must be a list or tuple
|
|
if type(minMaxList) is not list and \
|
|
type(minMaxList) is not tuple:
|
|
return None
|
|
|
|
# minOrMax must be "Min" or "Max"
|
|
if not (minOrMax == "Min" or minOrMax == "Max"):
|
|
return None
|
|
|
|
if type(minMaxList) is tuple:
|
|
(minMag, maxMag), dir = minMaxList
|
|
if minOrMax == "Min":
|
|
mag = minMag # return min value
|
|
elif minOrMax == "Max":
|
|
mag = maxMag # return max value
|
|
else:
|
|
print("extractMinMax error - Bad min/max string:", minOrMax)
|
|
print("Must be: 'Min' or 'Max'. ")
|
|
return None
|
|
return (mag, dir)
|
|
|
|
# check for empty list
|
|
if len(minMaxList) <= 0:
|
|
return None
|
|
|
|
newList = []
|
|
# loop through and find the min and max
|
|
for ((vMin, vMax), dir), timeRange in minMaxList:
|
|
if minOrMax == "Min":
|
|
value = vMin
|
|
else:
|
|
value = vMax
|
|
newList.append(((value, dir), timeRange))
|
|
return newList
|
|
|
|
def getDominantDirection(self, dataType, parmHisto, timeRange, componentName):
|
|
# returns the dominant direction according to "vectorDirection_algorithm"
|
|
# which can be "Average" or "MostFrequent"
|
|
|
|
if not dataType == self.VECTOR():
|
|
return None
|
|
if self.vectorDirection_algorithm(parmHisto, timeRange, componentName) == "Average":
|
|
return self.getAverageDirection(parmHisto, timeRange, componentName)
|
|
else: #Most Frequent
|
|
return self.getMostFrequentDirection(parmHisto, timeRange, componentName)
|
|
|
|
def getAverageDirection(self, parmHisto, timeRange, componentName):
|
|
# returns the dominant direction calculated by assuming a mag of 1 always
|
|
uSum = 0.0
|
|
vSum = 0.0
|
|
totCount = 0
|
|
weight = 0
|
|
totWeight = 0
|
|
for histSample in parmHisto.histoSamples():
|
|
if self.temporalCoverage_flag(parmHisto, timeRange, componentName, histSample) == 0:
|
|
continue
|
|
# sum u and v components assigning a magnitude one 1 always
|
|
histPairs = histSample.histogram()
|
|
for histPair in histPairs:
|
|
validTime = TimeRange.TimeRange(histSample.validTime())
|
|
weight = validTime.intersection(timeRange).duration()
|
|
totWeight = totWeight + weight
|
|
count = histPair.count()
|
|
totCount = totCount + count
|
|
uw, vw = self.MagDirToUV(1.0, histPair.value().direction())
|
|
uSum = uSum + (uw * count) * weight
|
|
vSum = vSum + (vw * count) * weight
|
|
|
|
# calculate the average wind vector
|
|
if totCount > 0:
|
|
u = uSum / (float(totCount) * totWeight)
|
|
v = vSum / (float(totCount) * totWeight)
|
|
mag, dir = self.UVToMagDir(u, v)
|
|
return dir
|
|
else:
|
|
return None
|
|
|
|
def getMostFrequentDirection(self, parmHisto, timeRange, componentName):
|
|
# returns the most frequent direction binned to 8-point numerical direction
|
|
binDict = {}
|
|
totWeight = 0.0
|
|
#print "\nGetting most frequent", timeRange
|
|
for histSample in parmHisto.histoSamples():
|
|
if self.temporalCoverage_flag(parmHisto, timeRange, componentName, histSample) == 0:
|
|
continue
|
|
numOfPoints = histSample.numOfPoints()
|
|
if numOfPoints == 0:
|
|
return None
|
|
histPairs = histSample.histogram()
|
|
for histPair in histPairs:
|
|
validTime = TimeRange.TimeRange(histSample.validTime())
|
|
weight = validTime.intersection(timeRange).duration()
|
|
weight = weight/float(timeRange.duration()) * 100.0
|
|
totWeight += weight
|
|
count = float(histPair.count())
|
|
binnedDir = self.binDir(histPair.value().direction())
|
|
#print "dir, binnedDir", histPair.value().direction(), binnedDir
|
|
percent = count/numOfPoints * weight
|
|
if binnedDir in binDict:
|
|
binDict[binnedDir] += percent
|
|
else:
|
|
binDict[binnedDir] = percent
|
|
|
|
if totWeight == 0.0:
|
|
return None
|
|
|
|
# Pick the most frequent direction
|
|
maxFreq = 0
|
|
mostFreqDir = None
|
|
for direction, freq in binDict.items():
|
|
# print "direction, freq", direction, freq
|
|
if freq > maxFreq:
|
|
maxFreq = freq
|
|
mostFreqDir = direction
|
|
#print "returning", mostFreqDir
|
|
return mostFreqDir
|
|
|
|
def binDir(self, dir):
|
|
# Return the "bin" direction value for the given direction
|
|
for textDir, low, high in self.dirList():
|
|
if dir >= low and dir < high:
|
|
# Handle N
|
|
if textDir == "N":
|
|
return 0
|
|
else:
|
|
return int(low+high)/2.0
|
|
|
|
|
|
def splitRange(self, timeRange, numPeriods=2):
|
|
"Split the timeRange into the given number of periods and return the resulting list of time ranges"
|
|
|
|
periods = []
|
|
# Changed this division to integer division to maintain the existing
|
|
# behavior, as part of the Python 3 upgrade. This behavior seems odd--if
|
|
# numPeriods does not evenly divide the timeRange, the resulting
|
|
# periods will not cover the entire timeRange. But it's what the method
|
|
# did under Python 2, so we are keeping it the same under Python 3.
|
|
duration = (timeRange.endTime() - timeRange.startTime()) // numPeriods
|
|
startTime = timeRange.startTime()
|
|
for i in range(numPeriods):
|
|
endTime = startTime + duration
|
|
newRange = TimeRange.TimeRange(startTime, endTime)
|
|
periods.append(newRange)
|
|
startTime = endTime
|
|
return periods
|
|
|
|
def getGridTimeRanges(self, parmHisto, timeRange):
|
|
# Return the set of timeRanges that overlap the specified timeRange
|
|
# If a histSample partially overlaps, trim the timeRange to the
|
|
# specified timeRange's startTime() or endTime()
|
|
subRanges = []
|
|
for histSample in parmHisto.histoSamples():
|
|
tr = TimeRange.TimeRange(histSample.validTime()) # get the histSample timeRange
|
|
overlap = timeRange.intersection(tr).duration() # calc overlap
|
|
if overlap == 0: # no overlap -> skip to next grid
|
|
continue
|
|
if overlap == tr.duration(): # the whole grid is included
|
|
subRanges.append(tr)
|
|
elif timeRange.startTime() > tr.startTime():
|
|
newTR = TimeRange.TimeRange(timeRange.startTime(), tr.endTime())
|
|
subRanges.append(newTR)
|
|
elif timeRange.endTime() < tr.endTime():
|
|
newTR = TimeRange.TimeRange(tr.startTime(), timeRange.endTime())
|
|
subRanges.append(newTR)
|
|
|
|
return subRanges
|
|
|
|
def getMedianHistPair(self, dataType, parmHisto, timeRange, componentName):
|
|
# Return the median HistPair over the timeRange
|
|
# Note: There could be multiple grids (histSamples) in this timeRange
|
|
# over which we are sampling.
|
|
#
|
|
# we can't figure a median if there are no samples
|
|
#
|
|
if len(parmHisto.histoSamples()) == 0:
|
|
return None
|
|
#
|
|
# we can only figure the median based on the scalar value,
|
|
# or, for vectors, the magnitude or direction. Other types
|
|
# are invalid, and we have to return None.
|
|
#
|
|
if ((dataType!=self.SCALAR()) and (dataType!=self.MAGNITUDE())
|
|
and (dataType!=self.DIRECTION())):
|
|
return None
|
|
#
|
|
# Get the samples inside the time range, keeping track
|
|
# of the values along the way (to sort later). Since
|
|
# there may be several grids, each with the same values
|
|
# that cross the desired timeRange, we need to add to
|
|
# the saved histogram counts when we encounter such
|
|
# values. Make a key with consistent floating point
|
|
# numbers so that the sorting works right later.
|
|
#
|
|
|
|
totalCount=0
|
|
pairDict = {}
|
|
compositeNameUI = parmHisto.getCompositeNameUI()
|
|
for histSample in parmHisto.histoSamples():
|
|
if self.temporalCoverage_flag(
|
|
parmHisto, timeRange, componentName, histSample) == 0:
|
|
continue
|
|
# calc the time weight
|
|
validTime = TimeRange.TimeRange(histSample.validTime())
|
|
weight = validTime.intersection(timeRange).duration()
|
|
for histPair in histSample.histogram():
|
|
tempCount=histPair.count()
|
|
tempValue=histPair.value()
|
|
tempKey=0.0
|
|
if dataType == self.SCALAR():
|
|
tempKey = tempValue.scalar()
|
|
elif dataType == self.MAGNITUDE():
|
|
tempKey = tempValue.magnitude()
|
|
elif dataType == self.DIRECTION():
|
|
tempKey = tempValue.direction()
|
|
valuestring="%020.10f" % float(tempKey)
|
|
totalCount = totalCount + tempCount * weight
|
|
if valuestring in pairDict:
|
|
pairDict[valuestring].incrementCount(int(tempCount * weight))
|
|
else:
|
|
# njensen: I added the clone(), because otherwise we are incrementing
|
|
# the original histpair reference, which corrupts the statistics when
|
|
# the same method is called against a different subTimeRange from
|
|
# createStats
|
|
pairDict[valuestring] = histPair.clone()
|
|
pairDict[valuestring].incrementCount(int((tempCount * weight) - 1))
|
|
#
|
|
# if no samples landed within the timeRange then we have
|
|
# to return a median of None
|
|
#
|
|
if totalCount == 0:
|
|
return None
|
|
#
|
|
# now we know the total number of pairs in the timeRange
|
|
# so we figure out the middle pair number and then
|
|
# go through the pairs in numerical order until we get
|
|
# to that count value
|
|
#
|
|
medianNumber=int(totalCount/2.0)
|
|
odd = 0
|
|
if medianNumber%2 == 1:
|
|
medianNumber == medianNumber + 1
|
|
odd = 1
|
|
count=0
|
|
names=sorted(pairDict.keys())
|
|
for valueStr in names:
|
|
addCount=pairDict[valueStr].count()
|
|
if ((count+addCount)>=medianNumber):
|
|
if odd == 1:
|
|
return pairDict[valueStr]
|
|
elif count + addCount == medianNumber:
|
|
# need to take mean of this value and the next
|
|
histPair1 = pairDict[valueStr]
|
|
index1 = names.index(valueStr)
|
|
if index1 < len(names)-1:
|
|
valueStr2 = names[index1+1]
|
|
histPair2 = pairDict[valueStr2]
|
|
return self.getHistPairMean(dataType, histPair1, histPair2)
|
|
else:
|
|
return histPair1
|
|
else:
|
|
return pairDict[valueStr]
|
|
count+=addCount
|
|
return None
|
|
|
|
def getHistPairMean(self, dataType, histPair1, histPair2):
|
|
# Return a HistPair that represents the mean of the two histPair values
|
|
# for the given dataType which can be "SCALAR", "MAGNITUDE", "DIRECTION"
|
|
if dataType == self.SCALAR():
|
|
val1 = histPair1.value().scalar()
|
|
val2 = histPair2.value().scalar()
|
|
avg = float(val1 + val2)/2.0
|
|
value = HistValue(avg)
|
|
return HistPair(value)
|
|
|
|
elif dataType == self.MAGNITUDE():
|
|
val1 = histPair1.value().magnitude()
|
|
val2 = histPair2.value().magnitude()
|
|
avg = float(val1 + val2)/2.0
|
|
value = HistValue(avg, 0.0)
|
|
return HistPair(value)
|
|
|
|
else: #dataType == self.DIRECTION():
|
|
dir1 = histPair1.value().direction()
|
|
dir2 = histPair2.value().direction()
|
|
u1, v1 = self.MagDirToUV(1.0, dir1)
|
|
u2, v2 = self.MagDirToUV(1.0, dir2)
|
|
u = (u1 + u2)/2
|
|
v = (v1 + v2)/2
|
|
mag, dir = self.UVToMagDir(u,v)
|
|
value = HistValue(mag, dir)
|
|
return HistPair(value)
|
|
|
|
def getModeHistPair(self, dataType, parmHisto, timeRange, componentName):
|
|
# Return the most common HistPair over the timeRange
|
|
# Note: There could be multiple grids (histSamples) in this timeRange
|
|
# over which we are sampling.
|
|
|
|
if len(parmHisto.histoSamples()) == 0:
|
|
return None
|
|
maxCount = 0
|
|
modePair = None
|
|
compositeNameUI = parmHisto.getCompositeNameUI()
|
|
|
|
for histSample in parmHisto.histoSamples():
|
|
if self.temporalCoverage_flag(parmHisto, timeRange, componentName,
|
|
histSample) == 0:
|
|
continue
|
|
|
|
# calc the time weight
|
|
validTime = TimeRange.TimeRange(histSample.validTime())
|
|
weight = validTime.intersection(timeRange).duration()
|
|
for histPair in histSample.histogram():
|
|
if (histPair.count() * weight) > maxCount:
|
|
maxCount = histPair.count() * weight
|
|
modePair = histPair
|
|
|
|
return modePair
|
|
|
|
def getMedian(self, dataType, parmHisto, timeRange, componentName):
|
|
# Return a range around the median
|
|
# For vector also return an average direction over that range.
|
|
|
|
if len(parmHisto.histoSamples()) == 0:
|
|
return None
|
|
|
|
if dataType == self.VECTOR():
|
|
# For VECTOR, base the median on the magnitude
|
|
pairType = self.MAGNITUDE()
|
|
else:
|
|
pairType = dataType
|
|
|
|
# Determine the median
|
|
medianPair = self.getMedianHistPair(pairType, parmHisto, timeRange, componentName)
|
|
if medianPair is None:
|
|
return None
|
|
if dataType == self.VECTOR():
|
|
return (medianPair.value().magnitude(), medianPair.value().direction())
|
|
else:
|
|
return medianPair.value().scalar()
|
|
|
|
def getMedianRange(self, dataType, parmHisto, timeRange, componentName):
|
|
# Return a range around the median
|
|
# For vector also return an average direction over that range.
|
|
|
|
if len(parmHisto.histoSamples()) == 0:
|
|
return None
|
|
|
|
if dataType == self.VECTOR():
|
|
# For VECTOR, base the median on the magnitude
|
|
pairType = self.MAGNITUDE()
|
|
else:
|
|
pairType = dataType
|
|
|
|
# Determine the median
|
|
medianPair = self.getMedianHistPair(pairType, parmHisto, timeRange, componentName)
|
|
if medianPair is None:
|
|
return None
|
|
# print "\nGetting Median Range"
|
|
return self.getRange(dataType, medianPair, parmHisto, timeRange, componentName, "Median")
|
|
|
|
def getMode(self, dataType, parmHisto, timeRange, componentName):
|
|
# Return a range around the median
|
|
# For vector also return an average direction over that range.
|
|
|
|
if len(parmHisto.histoSamples()) == 0:
|
|
return None
|
|
|
|
if dataType == self.VECTOR():
|
|
# For VECTOR, base the median on the magnitude
|
|
pairType = self.MAGNITUDE()
|
|
else:
|
|
pairType = dataType
|
|
|
|
# Determine the median
|
|
modePair = self.getModeHistPair(pairType, parmHisto, timeRange, componentName)
|
|
if modePair is None:
|
|
return None
|
|
# print "\nGetting Median Range"
|
|
if dataType == self.VECTOR():
|
|
return modePair.value().magnitude(), modePair.value().direction()
|
|
else:
|
|
return modePair.value().scalar()
|
|
|
|
def getModeRange(self, dataType, parmHisto, timeRange, componentName):
|
|
# Return a range around the mode.
|
|
# For vector also return an average direction over that range.
|
|
|
|
if len(parmHisto.histoSamples()) == 0:
|
|
return None
|
|
|
|
if dataType == self.VECTOR():
|
|
pairType = self.MAGNITUDE()
|
|
else:
|
|
pairType = dataType
|
|
|
|
# Determine the median
|
|
modePair = self.getModeHistPair(pairType, parmHisto, timeRange, componentName)
|
|
if modePair is None:
|
|
return None
|
|
return self.getRange(dataType, modePair, parmHisto, timeRange, componentName, "Mode")
|
|
|
|
def getMaxMode(self, dataType, parmHisto, timeRange, componentName):
|
|
# Return the maximum mode over all grids
|
|
|
|
if len(parmHisto.histoSamples()) == 0:
|
|
return None
|
|
|
|
compositeNameUI = parmHisto.getCompositeNameUI()
|
|
incDict = self.maxMode_increment_dict(parmHisto, timeRange, componentName)
|
|
if compositeNameUI in incDict:
|
|
binRes = incDict[compositeNameUI]
|
|
else:
|
|
binRes = 10 # default value
|
|
|
|
maxMode = -99999
|
|
for histSample in parmHisto.histoSamples():
|
|
# Ignore samples that are less than the temporal threshold
|
|
if self.temporalCoverage_flag(
|
|
parmHisto, timeRange, componentName, histSample) == 0:
|
|
continue
|
|
mode = histSample.mostCommonValueBinned(float(binRes)).scalar()
|
|
if mode > maxMode:
|
|
maxMode = mode
|
|
|
|
return maxMode
|
|
|
|
def getRange(self, dataType, basePair, parmHisto, timeRange,
|
|
componentName, rangeBase="Median"):
|
|
# Return a range around the basePair.
|
|
# For vector also return an average direction over that range.
|
|
|
|
compositeNameUI = parmHisto.getCompositeNameUI()
|
|
# Determine deviation
|
|
deviation = self.getDeviation(dataType, compositeNameUI, basePair)
|
|
# Take pairs that are within deviation of basePair
|
|
# and determine min, max and number of points covered by range
|
|
if dataType == self.SCALAR():
|
|
scalarflag = 1
|
|
baseValue = basePair.value().scalar()
|
|
else:
|
|
scalarflag = 0
|
|
baseValue = basePair.value().magnitude()
|
|
|
|
min = baseValue
|
|
max = baseValue
|
|
pairs = []
|
|
samplesInRange = 0
|
|
totalCount = 0
|
|
|
|
for histSample in parmHisto.histoSamples():
|
|
if self.temporalCoverage_flag(parmHisto, timeRange, componentName,
|
|
histSample) == 0:
|
|
continue
|
|
samplesInRange = samplesInRange + 1
|
|
for histPair in histSample.histogram():
|
|
if scalarflag:
|
|
histValue = histPair.value().scalar()
|
|
else:
|
|
histValue = histPair.value().magnitude()
|
|
#print "histvalue, baseValue, deviation", \
|
|
# histValue, baseValue, deviation
|
|
if histValue >= baseValue - deviation and \
|
|
histValue <= baseValue + deviation:
|
|
pairs.append(histPair)
|
|
if histValue < min:
|
|
min = histValue
|
|
if histValue > max:
|
|
max = histValue
|
|
totalCount = totalCount + histPair.count()
|
|
|
|
# If vector, find average direction for pairs in range
|
|
if dataType == self.VECTOR():
|
|
avgValue = self.getVectorAvg(pairs)
|
|
direction = avgValue.direction()
|
|
return ((min, max), direction)
|
|
else:
|
|
return (min, max)
|
|
|
|
def getDeviation(self, dataType, compositeNameUI, histPair):
|
|
# Returns a deviation around the median to include in range
|
|
if dataType == self.VECTOR():
|
|
mag = histPair.value().magnitude()
|
|
if mag < 15:
|
|
return 3
|
|
elif mag < 30:
|
|
return 5
|
|
else:
|
|
return 8
|
|
else:
|
|
return 10
|
|
|
|
def UVToMagDir(self, u, v):
|
|
# Converts u, v to magnitude, direction
|
|
RAD_TO_DEG = 57.296083
|
|
speed = sqrt(u * u + v * v)
|
|
dir = atan2(u, v) * RAD_TO_DEG
|
|
while dir < 0.0:
|
|
dir = dir + 360.0
|
|
while dir >= 360.0:
|
|
dir = dir - 360.0
|
|
#print "Speed, dir ", speed, dir
|
|
return (speed, dir)
|
|
|
|
def MagDirToUV(self, mag, dir):
|
|
#Converts magnitude, direction to u, v
|
|
DEG_TO_RAD = 0.017453292
|
|
uw = sin(dir * DEG_TO_RAD) * mag
|
|
vw = cos(dir * DEG_TO_RAD) * mag
|
|
return (uw, vw)
|
|
|
|
def convertAnalysisList(self, analysisList):
|
|
# Replace text string methods with SampleAnalysis methods
|
|
newList = []
|
|
for entry in analysisList:
|
|
if len(entry) == 2:
|
|
element, method = entry
|
|
if type(method) is str:
|
|
method = getattr(self, method)
|
|
newList.append((element,method))
|
|
if len(entry) == 3:
|
|
element, method, args = entry
|
|
if type(method) is str:
|
|
method = getattr(self, method)
|
|
newList.append((element,method, args))
|
|
return newList
|
|
|
|
def bin_dict(self, parmHisto, timeRange, componentName):
|
|
# Bins for binnedPercent. Bins are inclusive.
|
|
return {
|
|
"Sky": [(0,89),(90, 100)],
|
|
"PoP": [(0,4), (5,14), (15,24), (25,34), (35,44), (45,54),
|
|
(55,64), (65,74), (75,84), (85,94), (95,100)],
|
|
"LAL": [(1,1), (2,2), (3,3), (4,4), (5,5), (6,6)],
|
|
}
|
|
|
|
def getBinnedPercent(self, dataType, parmHisto, timeRange, componentName, firstOnly = 0):
|
|
"Returns a list of tuples representing bins and corresponding percentages of values in each bin"
|
|
binsDict = self.bin_dict(parmHisto, timeRange, componentName)
|
|
try:
|
|
bins = binsDict[parmHisto.getCompositeNameUI()]
|
|
except:
|
|
return None
|
|
|
|
# Returns a list of tuples
|
|
# Each tuple is of the form:
|
|
# (lowBin_value, highBin_value, percent)
|
|
# lowBin_value and highBin_value are the inclusive values of the bin
|
|
# percent is the percentage of data values in that bin
|
|
|
|
percents = []
|
|
for bin in bins:
|
|
percents.append(0.0)
|
|
numBins = len(bins)
|
|
|
|
totWeight = 0.0
|
|
for histSample in parmHisto.histoSamples():
|
|
if self.temporalCoverage_flag(parmHisto, timeRange, componentName,
|
|
histSample) == 0:
|
|
continue
|
|
# return None if no histSample pairs
|
|
numOfPoints = histSample.numOfPoints()
|
|
if numOfPoints == 0:
|
|
return None
|
|
|
|
validTime = TimeRange.TimeRange(histSample.validTime())
|
|
weight = validTime.intersection(timeRange).duration()
|
|
weight = weight/float(timeRange.duration()) * 100.0
|
|
totWeight = totWeight + weight
|
|
|
|
for histPair in histSample.histogram():
|
|
if dataType == self.SCALAR():
|
|
value = histPair.value().scalar()
|
|
elif dataType == self.VECTOR():
|
|
value = histPair.value().magnitude()
|
|
count = float(histPair.count())
|
|
percent = count/numOfPoints * weight
|
|
|
|
# Find the bin for this histPair value
|
|
for i in range(numBins):
|
|
low,high = bins[i]
|
|
if value >= low and value <= high:
|
|
# add to percentage for this bin
|
|
percents[i] += percent
|
|
|
|
if totWeight == 0.0:
|
|
return None
|
|
|
|
# Glue the bins and the percents together
|
|
newBins = []
|
|
for i in range(numBins):
|
|
low, high = bins[i]
|
|
newBins.append((low, high, percents[i]))
|
|
#print "returning bins", newBins, timeRange
|
|
return newBins
|