awips/ufpy ncep_15.1.1-n -> ncep_16.1.4-n

awips/DateTimeConverter.py

@@ -0,0 +1,107 @@
+# #
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+# #
+
+#
+# Functions for converting between the various "Java" dynamic serialize types
+# used by EDEX to the native python time datetime.
+#
+#
+#     SOFTWARE HISTORY
+#
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    06/24/15         #4480        dgilling       Initial Creation.
+#
+
+import datetime
+import time
+
+from dynamicserialize.dstypes.java.util import Date
+from dynamicserialize.dstypes.java.sql import Timestamp
+from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange
+
+
+MAX_TIME = pow(2, 31) - 1
+MICROS_IN_SECOND = 1000000
+
+
+def convertToDateTime(timeArg):
+    """
+    Converts the given object to a python datetime object. Supports native
+    python representations like datetime and struct_time, but also
+    the dynamicserialize types like Date and Timestamp. Raises TypeError
+    if no conversion can be performed.
+
+    Args:
+            timeArg: a python object representing a date and time. Supported
+            types include datetime, struct_time, float, int, long and the
+            dynamicserialize types Date and Timestamp.
+
+    Returns:
+        A datetime that represents the same date/time as the passed in object.
+    """
+    if isinstance(timeArg, datetime.datetime):
+        return timeArg
+    elif isinstance(timeArg, time.struct_time):
+        return datetime.datetime(*timeArg[:6])
+    elif isinstance(timeArg, float):
+        # seconds as float, should be avoided due to floating point errors
+        totalSecs = long(timeArg)
+        micros = int((timeArg - totalSecs) * MICROS_IN_SECOND)
+        return _convertSecsAndMicros(totalSecs, micros)
+    elif isinstance(timeArg, (int, long)):
+        # seconds as integer
+        totalSecs = timeArg
+        return _convertSecsAndMicros(totalSecs, 0)
+    elif isinstance(timeArg, (Date, Timestamp)):
+        totalSecs = timeArg.getTime()
+        return _convertSecsAndMicros(totalSecs, 0)
+    else:
+        objType = str(type(timeArg))
+        raise TypeError("Cannot convert object of type " + objType + " to datetime.")
+
+def _convertSecsAndMicros(seconds, micros):
+    if seconds < MAX_TIME:
+        rval = datetime.datetime.utcfromtimestamp(seconds)
+    else:
+        extraTime = datetime.timedelta(seconds=(seconds - MAX_TIME))
+        rval = datetime.datetime.utcfromtimestamp(MAX_TIME) + extraTime
+    return rval.replace(microsecond=micros)
+
+def constructTimeRange(*args):
+    """
+    Builds a python dynamicserialize TimeRange object from the given
+    arguments.
+
+    Args:
+            args*: must be a TimeRange or a pair of objects that can be
+                   converted to a datetime via convertToDateTime().
+
+    Returns:
+        A TimeRange.
+    """
+
+    if len(args) == 1 and isinstance(args[0], TimeRange):
+        return args[0]
+    if len(args) != 2:
+        raise TypeError("constructTimeRange takes exactly 2 arguments, " + str(len(args)) + " provided.")
+    startTime = convertToDateTime(args[0])
+    endTime = convertToDateTime(args[1])
+    return TimeRange(startTime, endTime)

awips/dataaccess/DataAccessLayer.py

@@ -35,6 +35,7 @@
 #    03/03/14         2673         bsteffen       Add ability to query only ref times.
 #    07/22/14         3185         njensen        Added optional/default args to newDataRequest
 #    07/30/14         3185         njensen        Renamed valid identifiers to optional
+#    Apr 26, 2015     4259         njensen        Updated for new JEP API
 #
 #
 #
@@ -46,7 +47,7 @@ import subprocess
 THRIFT_HOST = "edex"
 USING_NATIVE_THRIFT = False
 
-if sys.modules.has_key('JavaImporter'):
+if sys.modules.has_key('jep'):
     # intentionally do not catch if this fails to import, we want it to
     # be obvious that something is configured wrong when running from within
     # Java instead of allowing false confidence and fallback behavior

awips/dataaccess/SoundingsSupport.py

@@ -0,0 +1,283 @@
+# #
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+# #
+
+#
+# Classes for retrieving soundings based on gridded data from the Data Access
+# Framework
+#
+#
+#
+#     SOFTWARE HISTORY
+#
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    06/24/15         #4480        dgilling       Initial Creation.
+#
+
+from collections import defaultdict
+from shapely.geometry import Point
+
+from awips import DateTimeConverter
+from awips.dataaccess import DataAccessLayer
+
+from dynamicserialize.dstypes.com.raytheon.uf.common.time import DataTime
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.level import Level
+
+
+def getSounding(modelName, weatherElements, levels, samplePoint, refTime=None, timeRange=None):
+    """"
+    Performs a series of Data Access Framework requests to retrieve a sounding object
+    based on the specified request parameters.
+
+    Args:
+            modelName: the grid model datasetid to use as the basis of the sounding.
+            weatherElements: a list of parameters to return in the sounding.
+            levels: a list of levels to sample the given weather elements at
+            samplePoint: a lat/lon pair to perform the sampling of data at.
+            refTime: (optional) the grid model reference time to use for the sounding.
+                     If not specified, the latest ref time in the system will be used.
+            timeRange: (optional) a TimeRange to specify which forecast hours to use.
+                     If not specified, will default to all forecast hours.
+
+    Returns:
+            A _SoundingCube instance, which acts a 3-tiered dictionary, keyed
+            by DataTime, then by level and finally by weather element. If no
+            data is available for the given request parameters, None is returned.
+    """
+
+    (locationNames, parameters, levels, envelope, refTime, timeRange) = \
+        __sanitizeInputs(modelName, weatherElements, levels, samplePoint, refTime, timeRange)
+
+    requestArgs = { 'datatype'      : 'grid',
+                    'locationNames' : locationNames,
+                    'parameters'    : parameters,
+                    'levels'        : levels,
+                    'envelope'      : envelope,
+                   }
+
+    req = DataAccessLayer.newDataRequest(**requestArgs)
+
+    forecastHours = __determineForecastHours(req, refTime, timeRange)
+    if not forecastHours:
+        return None
+
+    response = DataAccessLayer.getGeometryData(req, forecastHours)
+    soundingObject = _SoundingCube(response)
+
+    return soundingObject
+
+def setEDEXHost(host):
+    """
+    Changes the EDEX host the Data Access Framework is communicating with.
+
+    Args:
+            host: the EDEX host to connect to
+    """
+
+    if host:
+        DataAccessLayer.changeEDEXHost(str(host))
+
+def __sanitizeInputs(modelName, weatherElements, levels, samplePoint, refTime, timeRange):
+    locationNames = [str(modelName)]
+    parameters = __buildStringList(weatherElements)
+    levels = __buildStringList(levels)
+    envelope = Point(samplePoint)
+    if refTime is not None:
+        refTime = DataTime(refTime=DateTimeConverter.convertToDateTime(refTime))
+    if timeRange is not None:
+        timeRange = DateTimeConverter.constructTimeRange(*timeRange)
+    return (locationNames, parameters, levels, envelope, refTime, timeRange)
+
+def __determineForecastHours(request, refTime, timeRange):
+    dataTimes = DataAccessLayer.getAvailableTimes(request, False)
+    timesGen = [(DataTime(refTime=dataTime.getRefTime()), dataTime) for dataTime in dataTimes]
+    dataTimesMap = defaultdict(list)
+    for baseTime, dataTime in timesGen:
+        dataTimesMap[baseTime].append(dataTime)
+
+    if refTime is None:
+        refTime = max(dataTimesMap.keys())
+
+    forecastHours = dataTimesMap[refTime]
+    if timeRange is None:
+        return forecastHours
+    else:
+        return [forecastHour for forecastHour in forecastHours if timeRange.contains(forecastHour.getValidPeriod())]
+
+def __buildStringList(param):
+    if __notStringIter(param):
+        return [str(item) for item in param]
+    else:
+        return [str(param)]
+
+def __notStringIter(iterable):
+    if not isinstance(iterable, basestring):
+        try:
+            iter(iterable)
+            return True
+        except TypeError:
+            return False
+
+
+
+class _SoundingCube(object):
+    """
+    The top-level sounding object returned when calling SoundingsSupport.getSounding.
+
+    This object acts as a 3-tiered dict which is keyed by time then level
+    then parameter name. Calling times() will return all valid keys into this
+    object.
+    """
+
+    def __init__(self, geometryDataObjects):
+        self._dataDict = {}
+        self._sortedTimes = []
+        if geometryDataObjects:
+            for geometryData in geometryDataObjects:
+                dataTime = geometryData.getDataTime()
+                level = geometryData.getLevel()
+                for parameter in geometryData.getParameters():
+                    self.__addItem(parameter, dataTime, level, geometryData.getNumber(parameter))
+
+    def __addItem(self, parameter, dataTime, level, value):
+        timeLayer = self._dataDict.get(dataTime, _SoundingTimeLayer(dataTime))
+        self._dataDict[dataTime] = timeLayer
+        timeLayer._addItem(parameter, level, value)
+        if dataTime not in self._sortedTimes:
+            self._sortedTimes.append(dataTime)
+            self._sortedTimes.sort()
+
+    def __getitem__(self, key):
+        return self._dataDict[key]
+
+    def __len__(self):
+        return len(self._dataDict)
+
+    def times(self):
+        """
+        Returns the valid times for this sounding.
+
+        Returns:
+            A list containing the valid DataTimes for this sounding in order.
+        """
+        return self._sortedTimes
+
+
+class _SoundingTimeLayer(object):
+    """
+    The second-level sounding object returned when calling SoundingsSupport.getSounding.
+
+    This object acts as a 2-tiered dict which is keyed by level then parameter
+    name. Calling levels() will return all valid keys into this
+    object. Calling time() will return the DataTime for this particular layer.
+    """
+
+    def __init__(self, dataTime):
+        self._dataTime = dataTime
+        self._dataDict = {}
+
+    def _addItem(self, parameter, level, value):
+        asString = str(level)
+        levelLayer = self._dataDict.get(asString, _SoundingTimeAndLevelLayer(self._dataTime, asString))
+        levelLayer._addItem(parameter, value)
+        self._dataDict[asString] = levelLayer
+
+    def __getitem__(self, key):
+        asString = str(key)
+        if asString in self._dataDict:
+            return self._dataDict[asString]
+        else:
+            raise KeyError("Level " + str(key) + " is not a valid level for this sounding.")
+
+    def __len__(self):
+        return len(self._dataDict)
+
+    def time(self):
+        """
+        Returns the DataTime for this sounding cube layer.
+
+        Returns:
+            The DataTime for this sounding layer.
+        """
+        return self._dataTime
+
+    def levels(self):
+        """
+        Returns the valid levels for this sounding.
+
+        Returns:
+            A list containing the valid levels for this sounding in order of
+            closest to surface to highest from surface.
+        """
+        sortedLevels = [Level(level) for level in self._dataDict.keys()]
+        sortedLevels.sort()
+        return [str(level) for level in sortedLevels]
+
+
+class _SoundingTimeAndLevelLayer(object):
+    """
+    The bottom-level sounding object returned when calling SoundingsSupport.getSounding.
+
+    This object acts as a dict which is keyed by parameter name. Calling
+    parameters() will return all valid keys into this object. Calling time()
+    will return the DataTime for this particular layer. Calling level() will
+    return the level for this layer.
+    """
+
+    def __init__(self, time, level):
+        self._time = time
+        self._level = level
+        self._parameters = {}
+
+    def _addItem(self, parameter, value):
+        self._parameters[parameter] = value
+
+    def __getitem__(self, key):
+        return self._parameters[key]
+
+    def __len__(self):
+        return len(self._parameters)
+
+    def level(self):
+        """
+        Returns the level for this sounding cube layer.
+
+        Returns:
+            The level for this sounding layer.
+        """
+        return self._level
+
+    def parameters(self):
+        """
+        Returns the valid parameters for this sounding.
+
+        Returns:
+            A list containing the valid parameter names.
+        """
+        return list(self._parameters.keys())
+
+    def time(self):
+        """
+        Returns the DataTime for this sounding cube layer.
+
+        Returns:
+            The DataTime for this sounding layer.
+        """
+        return self._time

awips/dataaccess/ThriftClientRouter.py

@@ -33,11 +33,12 @@
 #    07/22/14         #3185        njensen        Added optional/default args to newDataRequest
 #    07/23/14         #3185        njensen        Added new methods
 #    07/30/14         #3185        njensen        Renamed valid identifiers to optional
+#    06/30/15         #4569        nabowle        Use hex WKB for geometries.
 #
 
 
 import numpy
-import shapely.wkt
+import shapely.wkb
 
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.impl import DefaultDataRequest
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableLocationNamesRequest
@@ -111,12 +112,15 @@ class ThriftClientRouter(object):
             geoDataRequest.setRequestedPeriod(times)
         response = self._client.sendRequest(geoDataRequest)
         geometries = []
-        for wkt in response.getGeometryWKTs():
-            geometries.append(shapely.wkt.loads(wkt))
+        for wkb in response.getGeometryWKBs():
+            # convert the wkb to a bytearray with only positive values
+            byteArrWKB = bytearray(map(lambda x: x % 256,wkb.tolist()))
+            # convert the bytearray to a byte string and load it.
+            geometries.append(shapely.wkb.loads(str(byteArrWKB)))
 
         retVal = []
         for geoDataRecord in response.getGeoData():
-            geom = geometries[geoDataRecord.getGeometryWKTindex()]
+            geom = geometries[geoDataRecord.getGeometryWKBindex()]
             retVal.append(PyGeometryData.PyGeometryData(geoDataRecord, geom))
         return retVal