From 685338d7810997f1cd683610707b0d75816b7a2d Mon Sep 17 00:00:00 2001 From: mjames-upc Date: Thu, 24 Mar 2016 16:43:44 -0500 Subject: [PATCH] awips/ufpy ncep_15.1.1-n -> ncep_16.1.4-n --- awips/DateTimeConverter.py | 107 ++++++++++ awips/dataaccess/DataAccessLayer.py | 3 +- awips/dataaccess/SoundingsSupport.py | 283 +++++++++++++++++++++++++ awips/dataaccess/ThriftClientRouter.py | 12 +- 4 files changed, 400 insertions(+), 5 deletions(-) create mode 100644 awips/DateTimeConverter.py create mode 100644 awips/dataaccess/SoundingsSupport.py diff --git a/awips/DateTimeConverter.py b/awips/DateTimeConverter.py new file mode 100644 index 0000000..db9d2a1 --- /dev/null +++ b/awips/DateTimeConverter.py @@ -0,0 +1,107 @@ +# # +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +# # + +# +# Functions for converting between the various "Java" dynamic serialize types +# used by EDEX to the native python time datetime. +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 06/24/15 #4480 dgilling Initial Creation. +# + +import datetime +import time + +from dynamicserialize.dstypes.java.util import Date +from dynamicserialize.dstypes.java.sql import Timestamp +from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange + + +MAX_TIME = pow(2, 31) - 1 +MICROS_IN_SECOND = 1000000 + + +def convertToDateTime(timeArg): + """ + Converts the given object to a python datetime object. Supports native + python representations like datetime and struct_time, but also + the dynamicserialize types like Date and Timestamp. Raises TypeError + if no conversion can be performed. + + Args: + timeArg: a python object representing a date and time. Supported + types include datetime, struct_time, float, int, long and the + dynamicserialize types Date and Timestamp. + + Returns: + A datetime that represents the same date/time as the passed in object. + """ + if isinstance(timeArg, datetime.datetime): + return timeArg + elif isinstance(timeArg, time.struct_time): + return datetime.datetime(*timeArg[:6]) + elif isinstance(timeArg, float): + # seconds as float, should be avoided due to floating point errors + totalSecs = long(timeArg) + micros = int((timeArg - totalSecs) * MICROS_IN_SECOND) + return _convertSecsAndMicros(totalSecs, micros) + elif isinstance(timeArg, (int, long)): + # seconds as integer + totalSecs = timeArg + return _convertSecsAndMicros(totalSecs, 0) + elif isinstance(timeArg, (Date, Timestamp)): + totalSecs = timeArg.getTime() + return _convertSecsAndMicros(totalSecs, 0) + else: + objType = str(type(timeArg)) + raise TypeError("Cannot convert object of type " + objType + " to datetime.") + +def _convertSecsAndMicros(seconds, micros): + if seconds < MAX_TIME: + rval = datetime.datetime.utcfromtimestamp(seconds) + else: + extraTime = datetime.timedelta(seconds=(seconds - MAX_TIME)) + rval = datetime.datetime.utcfromtimestamp(MAX_TIME) + extraTime + return rval.replace(microsecond=micros) + +def constructTimeRange(*args): + """ + Builds a python dynamicserialize TimeRange object from the given + arguments. + + Args: + args*: must be a TimeRange or a pair of objects that can be + converted to a datetime via convertToDateTime(). + + Returns: + A TimeRange. + """ + + if len(args) == 1 and isinstance(args[0], TimeRange): + return args[0] + if len(args) != 2: + raise TypeError("constructTimeRange takes exactly 2 arguments, " + str(len(args)) + " provided.") + startTime = convertToDateTime(args[0]) + endTime = convertToDateTime(args[1]) + return TimeRange(startTime, endTime) diff --git a/awips/dataaccess/DataAccessLayer.py b/awips/dataaccess/DataAccessLayer.py index 2284948..4ab02b7 100644 --- a/awips/dataaccess/DataAccessLayer.py +++ b/awips/dataaccess/DataAccessLayer.py @@ -35,6 +35,7 @@ # 03/03/14 2673 bsteffen Add ability to query only ref times. # 07/22/14 3185 njensen Added optional/default args to newDataRequest # 07/30/14 3185 njensen Renamed valid identifiers to optional +# Apr 26, 2015 4259 njensen Updated for new JEP API # # # @@ -46,7 +47,7 @@ import subprocess THRIFT_HOST = "edex" USING_NATIVE_THRIFT = False -if sys.modules.has_key('JavaImporter'): +if sys.modules.has_key('jep'): # intentionally do not catch if this fails to import, we want it to # be obvious that something is configured wrong when running from within # Java instead of allowing false confidence and fallback behavior diff --git a/awips/dataaccess/SoundingsSupport.py b/awips/dataaccess/SoundingsSupport.py new file mode 100644 index 0000000..7e043c5 --- /dev/null +++ b/awips/dataaccess/SoundingsSupport.py @@ -0,0 +1,283 @@ +# # +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +# # + +# +# Classes for retrieving soundings based on gridded data from the Data Access +# Framework +# +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 06/24/15 #4480 dgilling Initial Creation. +# + +from collections import defaultdict +from shapely.geometry import Point + +from awips import DateTimeConverter +from awips.dataaccess import DataAccessLayer + +from dynamicserialize.dstypes.com.raytheon.uf.common.time import DataTime +from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.level import Level + + +def getSounding(modelName, weatherElements, levels, samplePoint, refTime=None, timeRange=None): + """" + Performs a series of Data Access Framework requests to retrieve a sounding object + based on the specified request parameters. + + Args: + modelName: the grid model datasetid to use as the basis of the sounding. + weatherElements: a list of parameters to return in the sounding. + levels: a list of levels to sample the given weather elements at + samplePoint: a lat/lon pair to perform the sampling of data at. + refTime: (optional) the grid model reference time to use for the sounding. + If not specified, the latest ref time in the system will be used. + timeRange: (optional) a TimeRange to specify which forecast hours to use. + If not specified, will default to all forecast hours. + + Returns: + A _SoundingCube instance, which acts a 3-tiered dictionary, keyed + by DataTime, then by level and finally by weather element. If no + data is available for the given request parameters, None is returned. + """ + + (locationNames, parameters, levels, envelope, refTime, timeRange) = \ + __sanitizeInputs(modelName, weatherElements, levels, samplePoint, refTime, timeRange) + + requestArgs = { 'datatype' : 'grid', + 'locationNames' : locationNames, + 'parameters' : parameters, + 'levels' : levels, + 'envelope' : envelope, + } + + req = DataAccessLayer.newDataRequest(**requestArgs) + + forecastHours = __determineForecastHours(req, refTime, timeRange) + if not forecastHours: + return None + + response = DataAccessLayer.getGeometryData(req, forecastHours) + soundingObject = _SoundingCube(response) + + return soundingObject + +def setEDEXHost(host): + """ + Changes the EDEX host the Data Access Framework is communicating with. + + Args: + host: the EDEX host to connect to + """ + + if host: + DataAccessLayer.changeEDEXHost(str(host)) + +def __sanitizeInputs(modelName, weatherElements, levels, samplePoint, refTime, timeRange): + locationNames = [str(modelName)] + parameters = __buildStringList(weatherElements) + levels = __buildStringList(levels) + envelope = Point(samplePoint) + if refTime is not None: + refTime = DataTime(refTime=DateTimeConverter.convertToDateTime(refTime)) + if timeRange is not None: + timeRange = DateTimeConverter.constructTimeRange(*timeRange) + return (locationNames, parameters, levels, envelope, refTime, timeRange) + +def __determineForecastHours(request, refTime, timeRange): + dataTimes = DataAccessLayer.getAvailableTimes(request, False) + timesGen = [(DataTime(refTime=dataTime.getRefTime()), dataTime) for dataTime in dataTimes] + dataTimesMap = defaultdict(list) + for baseTime, dataTime in timesGen: + dataTimesMap[baseTime].append(dataTime) + + if refTime is None: + refTime = max(dataTimesMap.keys()) + + forecastHours = dataTimesMap[refTime] + if timeRange is None: + return forecastHours + else: + return [forecastHour for forecastHour in forecastHours if timeRange.contains(forecastHour.getValidPeriod())] + +def __buildStringList(param): + if __notStringIter(param): + return [str(item) for item in param] + else: + return [str(param)] + +def __notStringIter(iterable): + if not isinstance(iterable, basestring): + try: + iter(iterable) + return True + except TypeError: + return False + + + +class _SoundingCube(object): + """ + The top-level sounding object returned when calling SoundingsSupport.getSounding. + + This object acts as a 3-tiered dict which is keyed by time then level + then parameter name. Calling times() will return all valid keys into this + object. + """ + + def __init__(self, geometryDataObjects): + self._dataDict = {} + self._sortedTimes = [] + if geometryDataObjects: + for geometryData in geometryDataObjects: + dataTime = geometryData.getDataTime() + level = geometryData.getLevel() + for parameter in geometryData.getParameters(): + self.__addItem(parameter, dataTime, level, geometryData.getNumber(parameter)) + + def __addItem(self, parameter, dataTime, level, value): + timeLayer = self._dataDict.get(dataTime, _SoundingTimeLayer(dataTime)) + self._dataDict[dataTime] = timeLayer + timeLayer._addItem(parameter, level, value) + if dataTime not in self._sortedTimes: + self._sortedTimes.append(dataTime) + self._sortedTimes.sort() + + def __getitem__(self, key): + return self._dataDict[key] + + def __len__(self): + return len(self._dataDict) + + def times(self): + """ + Returns the valid times for this sounding. + + Returns: + A list containing the valid DataTimes for this sounding in order. + """ + return self._sortedTimes + + +class _SoundingTimeLayer(object): + """ + The second-level sounding object returned when calling SoundingsSupport.getSounding. + + This object acts as a 2-tiered dict which is keyed by level then parameter + name. Calling levels() will return all valid keys into this + object. Calling time() will return the DataTime for this particular layer. + """ + + def __init__(self, dataTime): + self._dataTime = dataTime + self._dataDict = {} + + def _addItem(self, parameter, level, value): + asString = str(level) + levelLayer = self._dataDict.get(asString, _SoundingTimeAndLevelLayer(self._dataTime, asString)) + levelLayer._addItem(parameter, value) + self._dataDict[asString] = levelLayer + + def __getitem__(self, key): + asString = str(key) + if asString in self._dataDict: + return self._dataDict[asString] + else: + raise KeyError("Level " + str(key) + " is not a valid level for this sounding.") + + def __len__(self): + return len(self._dataDict) + + def time(self): + """ + Returns the DataTime for this sounding cube layer. + + Returns: + The DataTime for this sounding layer. + """ + return self._dataTime + + def levels(self): + """ + Returns the valid levels for this sounding. + + Returns: + A list containing the valid levels for this sounding in order of + closest to surface to highest from surface. + """ + sortedLevels = [Level(level) for level in self._dataDict.keys()] + sortedLevels.sort() + return [str(level) for level in sortedLevels] + + +class _SoundingTimeAndLevelLayer(object): + """ + The bottom-level sounding object returned when calling SoundingsSupport.getSounding. + + This object acts as a dict which is keyed by parameter name. Calling + parameters() will return all valid keys into this object. Calling time() + will return the DataTime for this particular layer. Calling level() will + return the level for this layer. + """ + + def __init__(self, time, level): + self._time = time + self._level = level + self._parameters = {} + + def _addItem(self, parameter, value): + self._parameters[parameter] = value + + def __getitem__(self, key): + return self._parameters[key] + + def __len__(self): + return len(self._parameters) + + def level(self): + """ + Returns the level for this sounding cube layer. + + Returns: + The level for this sounding layer. + """ + return self._level + + def parameters(self): + """ + Returns the valid parameters for this sounding. + + Returns: + A list containing the valid parameter names. + """ + return list(self._parameters.keys()) + + def time(self): + """ + Returns the DataTime for this sounding cube layer. + + Returns: + The DataTime for this sounding layer. + """ + return self._time diff --git a/awips/dataaccess/ThriftClientRouter.py b/awips/dataaccess/ThriftClientRouter.py index 545c5c5..b568b56 100644 --- a/awips/dataaccess/ThriftClientRouter.py +++ b/awips/dataaccess/ThriftClientRouter.py @@ -33,11 +33,12 @@ # 07/22/14 #3185 njensen Added optional/default args to newDataRequest # 07/23/14 #3185 njensen Added new methods # 07/30/14 #3185 njensen Renamed valid identifiers to optional +# 06/30/15 #4569 nabowle Use hex WKB for geometries. # import numpy -import shapely.wkt +import shapely.wkb from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.impl import DefaultDataRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableLocationNamesRequest @@ -111,12 +112,15 @@ class ThriftClientRouter(object): geoDataRequest.setRequestedPeriod(times) response = self._client.sendRequest(geoDataRequest) geometries = [] - for wkt in response.getGeometryWKTs(): - geometries.append(shapely.wkt.loads(wkt)) + for wkb in response.getGeometryWKBs(): + # convert the wkb to a bytearray with only positive values + byteArrWKB = bytearray(map(lambda x: x % 256,wkb.tolist())) + # convert the bytearray to a byte string and load it. + geometries.append(shapely.wkb.loads(str(byteArrWKB))) retVal = [] for geoDataRecord in response.getGeoData(): - geom = geometries[geoDataRecord.getGeometryWKTindex()] + geom = geometries[geoDataRecord.getGeometryWKBindex()] retVal.append(PyGeometryData.PyGeometryData(geoDataRecord, geom)) return retVal