From dce5a7a25ab751cd69813fe0994f80afee64f1c1 Mon Sep 17 00:00:00 2001 From: Michael James Date: Tue, 9 Oct 2018 14:24:43 -0600 Subject: [PATCH] six.PY conditionals for utf8 encoding/decoding --- awips/dataaccess/PyData.py | 5 ++ awips/dataaccess/PyGeometryData.py | 24 +++++- awips/dataaccess/PyGridData.py | 8 +- awips/dataaccess/ThriftClientRouter.py | 35 +++++++-- awips/test/dafTests/baseDafTestCase.py | 2 +- awips/test/testQpidTimeToLive.py | 1 + .../ThriftSerializationContext.py | 78 +++++++++++++------ .../response/AbstractResponseData.py | 17 ++-- .../dataaccess/response/GridResponseData.py | 14 +++- .../uf/common/dataplugin/level/MasterLevel.py | 46 ++++++++--- 10 files changed, 174 insertions(+), 56 deletions(-) diff --git a/awips/dataaccess/PyData.py b/awips/dataaccess/PyData.py index 6b09af5..eaa1953 100644 --- a/awips/dataaccess/PyData.py +++ b/awips/dataaccess/PyData.py @@ -13,6 +13,7 @@ # from awips.dataaccess import IData +import six class PyData(IData): @@ -33,6 +34,10 @@ class PyData(IData): return self.__time def getLevel(self): + if six.PY2: + return self.__level + if type(self.__level) is not str: + return self.__level.decode('utf-8') return self.__level def getLocationName(self): diff --git a/awips/dataaccess/PyGeometryData.py b/awips/dataaccess/PyGeometryData.py index 01575b4..9c7ec30 100644 --- a/awips/dataaccess/PyGeometryData.py +++ b/awips/dataaccess/PyGeometryData.py @@ -19,6 +19,7 @@ from awips.dataaccess import IGeometryData from awips.dataaccess import PyData +import six class PyGeometryData(IGeometryData, PyData.PyData): @@ -34,15 +35,22 @@ class PyGeometryData(IGeometryData, PyData.PyData): return self.__geometry def getParameters(self): - return list(self.__dataMap.keys()) + if six.PY2: + return list(self.__dataMap.keys()) + else: + return [x.decode('utf-8') for x in list(self.__dataMap.keys())] def getString(self, param): - value = self.__dataMap[param][0] + if six.PY2: + return self.__dataMap[param][0] + value = self.__dataMap[param.encode('utf-8')][0] + if value is not None: + return value.decode('utf-8') return value def getNumber(self, param): t = self.getType(param) - value = self.__dataMap[param][0] + value = self.__dataMap[param.encode('utf-8')][0] if t == 'INT' or t == 'SHORT' or t == 'LONG': return int(value) elif t == 'FLOAT': @@ -54,8 +62,16 @@ class PyGeometryData(IGeometryData, PyData.PyData): def getUnit(self, param): unit = self.__dataMap[param][2] + if six.PY2: + return unit + if unit is not None: + return unit.decode('utf-8') return unit def getType(self, param): - type = self.__dataMap[param][1] + if six.PY2: + return self.__dataMap[param][1] + type = self.__dataMap[param.encode('utf-8')][1] + if type is not None: + return type.decode('utf-8') return type diff --git a/awips/dataaccess/PyGridData.py b/awips/dataaccess/PyGridData.py index 0a7ac15..df6589b 100644 --- a/awips/dataaccess/PyGridData.py +++ b/awips/dataaccess/PyGridData.py @@ -1,6 +1,3 @@ -# # -# # - # # Implements IGridData for use by native Python clients to the Data Access # Framework. @@ -20,6 +17,7 @@ import numpy import warnings +import six from awips.dataaccess import IGridData from awips.dataaccess import PyData @@ -46,6 +44,10 @@ class PyGridData(IGridData, PyData.PyData): return self.__parameter def getUnit(self): + if six.PY2: + return self.__unit + if self.__unit is not None and type(self.__unit) is not str: + return self.__unit.decode('utf-8') return self.__unit def getRawData(self, unit=None): diff --git a/awips/dataaccess/ThriftClientRouter.py b/awips/dataaccess/ThriftClientRouter.py index d45f4dc..db52d6b 100644 --- a/awips/dataaccess/ThriftClientRouter.py +++ b/awips/dataaccess/ThriftClientRouter.py @@ -1,6 +1,3 @@ -# # -# # - # # Routes requests to the Data Access Framework through Python Thrift. # @@ -28,6 +25,7 @@ import numpy +import six import shapely.wkb from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.impl import DefaultDataRequest @@ -125,7 +123,10 @@ class ThriftClientRouter(object): retVal = [] for gridDataRecord in response.getGridData(): locationName = gridDataRecord.getLocationName() - locData = locSpecificData[locationName] + if locationName is not None: + locData = locSpecificData[locationName.encode('utf-8')] + else: + locData = locSpecificData[locationName] if self._lazyLoadGridLatLon: retVal.append(PyGridData.PyGridData(gridDataRecord, locData[ 0], locData[1], latLonDelegate=locData[2])) @@ -163,12 +164,20 @@ class ThriftClientRouter(object): locNamesRequest = GetAvailableLocationNamesRequest() locNamesRequest.setRequestParameters(request) response = self._client.sendRequest(locNamesRequest) - return [item.decode('utf8') for item in response] + if six.PY2: + return response + if response is not None: + return [x.decode('utf-8') for x in response] + return response def getAvailableParameters(self, request): paramReq = GetAvailableParametersRequest() paramReq.setRequestParameters(request) response = self._client.sendRequest(paramReq) + if six.PY2: + return response + if response is not None: + return [x.decode('utf-8') for x in response] return response def getAvailableLevels(self, request): @@ -184,6 +193,10 @@ class ThriftClientRouter(object): idReq = GetRequiredIdentifiersRequest() idReq.setRequest(request) response = self._client.sendRequest(idReq) + if six.PY2: + return response + if response is not None: + return [x.decode('utf-8') for x in response] return response def getOptionalIdentifiers(self, request): @@ -193,6 +206,10 @@ class ThriftClientRouter(object): idReq = GetOptionalIdentifiersRequest() idReq.setRequest(request) response = self._client.sendRequest(idReq) + if six.PY2: + return response + if response is not None: + return [x.decode('utf-8') for x in response] return response def getIdentifierValues(self, request, identifierKey): @@ -200,6 +217,10 @@ class ThriftClientRouter(object): idValReq.setIdentifierKey(identifierKey) idValReq.setRequestParameters(request) response = self._client.sendRequest(idValReq) + if six.PY2: + return response + if response is not None: + return [x.decode('utf-8') for x in response] return response def newDataRequest(self, datatype, parameters=[], levels=[], locationNames=[], envelope=None, **kwargs): @@ -221,6 +242,10 @@ class ThriftClientRouter(object): def getSupportedDatatypes(self): response = self._client.sendRequest(GetSupportedDatatypesRequest()) + if six.PY2: + return response + if response is not None: + return [x.decode('utf-8') for x in response] return response def getNotificationFilter(self, request): diff --git a/awips/test/dafTests/baseDafTestCase.py b/awips/test/dafTests/baseDafTestCase.py index 67aba91..9ed99ba 100644 --- a/awips/test/dafTests/baseDafTestCase.py +++ b/awips/test/dafTests/baseDafTestCase.py @@ -73,7 +73,7 @@ class DafTestCase(unittest.TestCase): return times def testDatatypeIsSupported(self): - allSupported = (item.lower().decode('utf-8') for item in DAL.getSupportedDatatypes()) + allSupported = DAL.getSupportedDatatypes() self.assertIn(self.datatype.lower(), allSupported) def testGetRequiredIdentifiers(self): diff --git a/awips/test/testQpidTimeToLive.py b/awips/test/testQpidTimeToLive.py index f91e2af..25f53ea 100644 --- a/awips/test/testQpidTimeToLive.py +++ b/awips/test/testQpidTimeToLive.py @@ -17,6 +17,7 @@ import time, sys import threading import dynamicserialize +from io import open TIME_TO_SLEEP = 300 diff --git a/dynamicserialize/ThriftSerializationContext.py b/dynamicserialize/ThriftSerializationContext.py index 303c0d9..1814325 100644 --- a/dynamicserialize/ThriftSerializationContext.py +++ b/dynamicserialize/ThriftSerializationContext.py @@ -24,6 +24,8 @@ from thrift.Thrift import TType import inspect import sys +import types +import six import numpy import dynamicserialize from dynamicserialize import dstypes, adapters @@ -49,29 +51,54 @@ def buildObjMap(module): buildObjMap(dstypes) -pythonToThriftMap = { - bytes: TType.STRING, - int: TType.I32, - int: TType.I64, - list: TType.LIST, - dict: TType.MAP, - type(set([])): TType.SET, - float: SelfDescribingBinaryProtocol.FLOAT, - # types.FloatType: TType.DOUBLE, - bool: TType.BOOL, - object: TType.STRUCT, - str: TType.STRING, - type(None): TType.VOID, - numpy.float32: SelfDescribingBinaryProtocol.FLOAT, - numpy.int32: TType.I32, - numpy.ndarray: TType.LIST, - numpy.object_: TType.STRING, # making an assumption here - numpy.string_: TType.STRING, - numpy.float64: TType.DOUBLE, - numpy.int16: TType.I16, - numpy.int8: TType.BYTE, - numpy.int64: TType.I64 -} +if six.PY2: + pythonToThriftMap = { + types.StringType: TType.STRING, + types.IntType: TType.I32, + types.LongType: TType.I64, + types.ListType: TType.LIST, + unicode: TType.STRING, + types.DictionaryType: TType.MAP, + type(set([])): TType.SET, + types.FloatType: SelfDescribingBinaryProtocol.FLOAT, + # types.FloatType: TType.DOUBLE, + types.BooleanType: TType.BOOL, + types.InstanceType: TType.STRUCT, + types.NoneType: TType.VOID, + numpy.float32: SelfDescribingBinaryProtocol.FLOAT, + numpy.int32: TType.I32, + numpy.ndarray: TType.LIST, + numpy.object_: TType.STRING, # making an assumption here + numpy.string_: TType.STRING, + numpy.float64: TType.DOUBLE, + numpy.int16: TType.I16, + numpy.int8: TType.BYTE, + numpy.int64: TType.I64 + } +else: + pythonToThriftMap = { + bytes: TType.STRING, + int: TType.I32, + int: TType.I64, + list: TType.LIST, + dict: TType.MAP, + type(set([])): TType.SET, + float: SelfDescribingBinaryProtocol.FLOAT, + # types.FloatType: TType.DOUBLE, + bool: TType.BOOL, + object: TType.STRUCT, + str: TType.STRING, + type(None): TType.VOID, + numpy.float32: SelfDescribingBinaryProtocol.FLOAT, + numpy.int32: TType.I32, + numpy.ndarray: TType.LIST, + numpy.object_: TType.STRING, # making an assumption here + numpy.string_: TType.STRING, + numpy.float64: TType.DOUBLE, + numpy.int16: TType.I16, + numpy.int8: TType.BYTE, + numpy.int64: TType.I64 + } primitiveSupport = (TType.BYTE, TType.I16, TType.I32, TType.I64, SelfDescribingBinaryProtocol.FLOAT, TType.DOUBLE) @@ -227,7 +254,10 @@ class ThriftSerializationContext(object): if pyt in pythonToThriftMap: return pythonToThriftMap[pyt] elif pyt.__module__[:DS_LEN - 1] == ('dynamicserialize.dstypes'): - return pythonToThriftMap[object] + if six.PY2: + return pythonToThriftMap[types.InstanceType] + else: + return pythonToThriftMap[object] else: raise dynamicserialize.SerializationException( "Don't know how to serialize object of type: " + str(pyt)) diff --git a/dynamicserialize/dstypes/com/raytheon/uf/common/dataaccess/response/AbstractResponseData.py b/dynamicserialize/dstypes/com/raytheon/uf/common/dataaccess/response/AbstractResponseData.py index 15275db..605033b 100644 --- a/dynamicserialize/dstypes/com/raytheon/uf/common/dataaccess/response/AbstractResponseData.py +++ b/dynamicserialize/dstypes/com/raytheon/uf/common/dataaccess/response/AbstractResponseData.py @@ -1,12 +1,10 @@ -## -## - # File auto-generated against equivalent DynamicSerialize Java class import abc -from six import with_metaclass +import six -class AbstractResponseData(with_metaclass(abc.ABCMeta, object)): + +class AbstractResponseData(six.with_metaclass(abc.ABCMeta, object)): @abc.abstractmethod def __init__(self): self.time = None @@ -27,13 +25,20 @@ class AbstractResponseData(with_metaclass(abc.ABCMeta, object)): self.level = level def getLocationName(self): + if six.PY2: + return self.locationName + if self.locationName is not None: + return self.locationName.decode('utf-8') return self.locationName def setLocationName(self, locationName): self.locationName = locationName def getAttributes(self): - return self.attributes + if six.PY2: + return self.attributes + else: + return [item.decode('utf-8') for item in self.attributes] def setAttributes(self, attributes): self.attributes = attributes diff --git a/dynamicserialize/dstypes/com/raytheon/uf/common/dataaccess/response/GridResponseData.py b/dynamicserialize/dstypes/com/raytheon/uf/common/dataaccess/response/GridResponseData.py index 9d2125a..1f90a3e 100644 --- a/dynamicserialize/dstypes/com/raytheon/uf/common/dataaccess/response/GridResponseData.py +++ b/dynamicserialize/dstypes/com/raytheon/uf/common/dataaccess/response/GridResponseData.py @@ -1,6 +1,3 @@ -## -## - # File auto-generated against equivalent DynamicSerialize Java class # and then modified post-generation to use AbstractResponseData. # @@ -12,8 +9,9 @@ # # - from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.response import AbstractResponseData +import six + class GridResponseData(AbstractResponseData): @@ -24,12 +22,20 @@ class GridResponseData(AbstractResponseData): self.gridData = None def getParameter(self): + if six.PY2: + return self.parameter + if self.parameter is not None: + return self.parameter.decode('utf-8') return self.parameter def setParameter(self, parameter): self.parameter = parameter def getUnit(self): + if six.PY2: + return self.unit + if self.unit is not None: + return self.unit.decode('utf-8') return self.unit def setUnit(self, unit): diff --git a/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/level/MasterLevel.py b/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/level/MasterLevel.py index 1237619..e2808df 100644 --- a/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/level/MasterLevel.py +++ b/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/level/MasterLevel.py @@ -1,6 +1,3 @@ -## -## - # File auto-generated against equivalent DynamicSerialize Java class # and then modified post-generation to add additional features to better # match Java implementation. @@ -15,6 +12,9 @@ # # +import six + + class MasterLevel(object): def __init__(self, name=None): @@ -37,39 +37,67 @@ class MasterLevel(object): return not self.__eq__(other) def __str__(self): - retVal = "MasterLevel[" - retVal += "name=" + str(self.name) + "," - retVal += "type=" + str(self.type) + "," - retVal += "unit=" + str(self.unitString) + "," - retVal += "description=" + str(self.description) - retVal += "]" + if six.PY2: + retVal = "MasterLevel[" + retVal += "name=" + str(self.name) + "," + retVal += "type=" + str(self.type) + "," + retVal += "unit=" + str(self.unitString) + "," + retVal += "description=" + str(self.description) + retVal += "]" + else: + retVal = "MasterLevel[" + retVal += "name=" + str(self.name.decode('utf-8')) + "," + retVal += "type=" + str(self.type.decode('utf-8')) + "," + retVal += "unit=" + str(self.unitString.decode('utf-8')) + "," + retVal += "description=" + str(self.description.decode('utf-8')) + retVal += "]" return retVal def getName(self): + if six.PY2: + return self.name + if self.name is not None and type(self.name) is not str: + return self.name.decode('utf-8') return self.name def setName(self, name): self.name = name def getDescription(self): + if six.PY2: + return self.description + if self.description is not None: + return self.description.decode('utf-8') return self.description def setDescription(self, description): self.description = description def getUnitString(self): + if six.PY2: + return self.unitString + if self.unitString is not None: + return self.unitString.decode('utf-8') return self.unitString def setUnitString(self, unitString): self.unitString = unitString def getType(self): + if six.PY2: + return self.type + if self.type is not None: + return self.type.decode('utf-8') return self.type def setType(self, type): self.type = type def getIdentifier(self): + if six.PY2: + return self.identifier + if self.identifier is not None: + return self.identifier.decode('utf-8') return self.identifier def setIdentifier(self, identifier):