From 5c51ad599e9bcbe41669345c606d9e25d6c413c6 Mon Sep 17 00:00:00 2001 From: mjames-upc Date: Mon, 6 Nov 2017 13:47:15 -0700 Subject: [PATCH] Revert "changes for 17.3.1" This reverts commit 8c3145b63ee5279b898750a527b2d5a0db15c1e4. --- awips/QpidSubscriber.py | 19 +- awips/UsageArgumentParser.py | 23 --- awips/dataaccess/DataAccessLayer.py | 28 +-- awips/dataaccess/PyGridData.py | 10 +- awips/dataaccess/ThriftClientRouter.py | 68 ++------ awips/qpidingest.py | 24 +-- .../ThriftSerializationContext.py | 162 +++++++++--------- dynamicserialize/adapters/__init__.py | 117 +++++-------- .../com/raytheon/uf/common/__init__.py | 17 +- .../raytheon/uf/common/dataplugin/__init__.py | 8 +- 10 files changed, 168 insertions(+), 308 deletions(-) diff --git a/awips/QpidSubscriber.py b/awips/QpidSubscriber.py index 25753f9..e55d967 100644 --- a/awips/QpidSubscriber.py +++ b/awips/QpidSubscriber.py @@ -30,12 +30,9 @@ # 11/17/10 njensen Initial Creation. # 08/15/13 2169 bkowal Optionally gzip decompress any data that is read. # 08/04/16 2416 tgurney Add queueStarted property -# 02/16/17 6084 bsteffen Support ssl connections # # -import os -import os.path import qpid import zlib @@ -44,24 +41,11 @@ from qpid.exceptions import Closed class QpidSubscriber: - def __init__(self, host='127.0.0.1', port=5672, decompress=False, ssl=None): + def __init__(self, host='127.0.0.1', port=5672, decompress=False): self.host = host self.port = port self.decompress = decompress; socket = qpid.util.connect(host, port) - if "QPID_SSL_CERT_DB" in os.environ: - certdb = os.environ["QPID_SSL_CERT_DB"] - else: - certdb = os.path.expanduser("~/.qpid/") - if "QPID_SSL_CERT_NAME" in os.environ: - certname = os.environ["QPID_SSL_CERT_NAME"] - else: - certname = "guest" - certfile = os.path.join(certdb, certname + ".crt") - if ssl or (ssl is None and os.path.exists(certfile)): - keyfile = os.path.join(certdb, certname + ".key") - trustfile = os.path.join(certdb, "root.crt") - socket = qpid.util.ssl(socket, keyfile=keyfile, certfile=certfile, ca_certs=trustfile) self.__connection = qpid.connection.Connection(sock=socket, username='guest', password='guest') self.__connection.start() self.__session = self.__connection.session(str(qpid.datatypes.uuid4())) @@ -119,4 +103,3 @@ class QpidSubscriber: @property def queueStarted(self): return self.__queueStarted - diff --git a/awips/UsageArgumentParser.py b/awips/UsageArgumentParser.py index 313d206..3722c25 100644 --- a/awips/UsageArgumentParser.py +++ b/awips/UsageArgumentParser.py @@ -17,23 +17,13 @@ # See the AWIPS II Master Rights File ("Master Rights File.pdf") for # further licensing information. ## -# -# SOFTWARE HISTORY -# -# Date Ticket# Engineer Description -# ------------- -------- --------- --------------------------------------------- -# Feb 13, 2017 6092 randerso Added StoreTimeAction -# -## import argparse import sys -import time from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import DatabaseID from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import ParmID -TIME_FORMAT = "%Y%m%d_%H%M" class UsageArgumentParser(argparse.ArgumentParser): """ @@ -66,16 +56,3 @@ class AppendParmNameAndLevelAction(argparse.Action): else: setattr(namespace, self.dest, [comp]) -class StoreTimeAction(argparse.Action): - """ - argparse.Action subclass to validate GFE formatted time strings - and parse them to time.struct_time - """ - def __call__(self, parser, namespace, values, option_string=None): - try: - timeStruct = time.strptime(values, TIME_FORMAT) - except: - parser.error(str(values) + " is not a valid time string of the format YYYYMMDD_hhmm") - - setattr(namespace, self.dest, timeStruct) - diff --git a/awips/dataaccess/DataAccessLayer.py b/awips/dataaccess/DataAccessLayer.py index 7f436b5..115305e 100644 --- a/awips/dataaccess/DataAccessLayer.py +++ b/awips/dataaccess/DataAccessLayer.py @@ -40,8 +40,8 @@ # Jun 01, 2016 5587 tgurney Add new signatures for # getRequiredIdentifiers() and # getOptionalIdentifiers() -# Oct 07, 2016 ---- mjames@ucar Added getForecastRun -# Oct 18, 2016 5916 bsteffen Add setLazyLoadGridLatLon +# 10/07/16 ---- mjames@ucar Added getForecastRun +# # # @@ -53,7 +53,6 @@ import warnings THRIFT_HOST = "edex" USING_NATIVE_THRIFT = False - if sys.modules.has_key('jep'): # intentionally do not catch if this fails to import, we want it to # be obvious that something is configured wrong when running from within @@ -253,26 +252,3 @@ def changeEDEXHost(newHostName): router = ThriftClientRouter.ThriftClientRouter(THRIFT_HOST) else: raise TypeError("Cannot call changeEDEXHost when using JepRouter.") - -def setLazyLoadGridLatLon(lazyLoadGridLatLon): - """ - Provide a hint to the Data Access Framework indicating whether to load the - lat/lon data for a grid immediately or wait until it is needed. This is - provided as a performance tuning hint and should not affect the way the - Data Access Framework is used. Depending on the internal implementation of - the Data Access Framework this hint might be ignored. Examples of when this - should be set to True are when the lat/lon information is not used or when - it is used only if certain conditions within the data are met. It could be - set to False if it is guaranteed that all lat/lon information is needed and - it would be better to get any performance overhead for generating the - lat/lon data out of the way during the initial request. - - - Args: - lazyLoadGridLatLon: Boolean value indicating whether to lazy load. - """ - try: - router.setLazyLoadGridLatLon(lazyLoadGridLatLon) - except AttributeError: - # The router is not required to support this capability. - pass diff --git a/awips/dataaccess/PyGridData.py b/awips/dataaccess/PyGridData.py index d85c7ea..87fffac 100644 --- a/awips/dataaccess/PyGridData.py +++ b/awips/dataaccess/PyGridData.py @@ -28,9 +28,7 @@ # Date Ticket# Engineer Description # ------------ ---------- ----------- -------------------------- # 06/03/13 #2023 dgilling Initial Creation. -# 10/13/16 #5916 bsteffen Correct grid shape, allow lat/lon # 11/10/16 #5900 bsteffen Correct grid shape -# to be requested by a delegate # # @@ -48,7 +46,7 @@ The ability to unit convert grid data is not currently available in this version class PyGridData(IGridData, PyData.PyData): - def __init__(self, gridDataRecord, nx, ny, latLonGrid = None, latLonDelegate = None): + def __init__(self, gridDataRecord, nx, ny, latLonGrid): PyData.PyData.__init__(self, gridDataRecord) nx = nx ny = ny @@ -56,8 +54,6 @@ class PyGridData(IGridData, PyData.PyData): self.__unit = gridDataRecord.getUnit() self.__gridData = numpy.reshape(numpy.array(gridDataRecord.getGridData()), (ny, nx)) self.__latLonGrid = latLonGrid - self.__latLonDelegate = latLonDelegate - def getParameter(self): return self.__parameter @@ -74,8 +70,4 @@ class PyGridData(IGridData, PyData.PyData): return self.__gridData def getLatLonCoords(self): - if self.__latLonGrid is not None: - return self.__latLonGrid - elif self.__latLonDelegate is not None: - return self.__latLonDelegate() return self.__latLonGrid diff --git a/awips/dataaccess/ThriftClientRouter.py b/awips/dataaccess/ThriftClientRouter.py index 14948fe..e1c0e1e 100644 --- a/awips/dataaccess/ThriftClientRouter.py +++ b/awips/dataaccess/ThriftClientRouter.py @@ -39,8 +39,7 @@ # getRequiredIdentifiers() and # getOptionalIdentifiers() # 08/01/16 2416 tgurney Add getNotificationFilter() -# 10/13/16 5916 bsteffen Correct grid shape, allow lazy grid lat/lon -# 10/26/16 5919 njensen Speed up geometry creation in getGeometryData() +# 11/10/16 5900 bsteffen Correct grid shape # @@ -52,7 +51,6 @@ from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import G from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableTimesRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetGeometryDataRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetGridDataRequest -from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetGridLatLonRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableParametersRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableLevelsRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetRequiredIdentifiersRequest @@ -66,39 +64,10 @@ from awips.dataaccess import PyGeometryData from awips.dataaccess import PyGridData -class LazyGridLatLon(object): - - def __init__(self, client, nx, ny, envelope, crsWkt): - self._latLonGrid = None - self._client = client - self._request = GetGridLatLonRequest() - self._request.setNx(nx) - self._request.setNy(ny) - self._request.setEnvelope(envelope) - self._request.setCrsWkt(crsWkt) - - def __call__(self): - # Its important that the data is cached internally so that if multiple - # GridData are sharing the same delegate then they can also share a - # single request for the LatLon information. - if self._latLonGrid is None: - response = self._client.sendRequest(self._request) - nx = response.getNx() - ny = response.getNy() - latData = numpy.reshape(numpy.array(response.getLats()), (ny, nx)) - lonData = numpy.reshape(numpy.array(response.getLons()), (ny, nx)) - self._latLonGrid = (lonData, latData) - return self._latLonGrid - - class ThriftClientRouter(object): def __init__(self, host='localhost'): self._client = ThriftClient.ThriftClient(host) - self._lazyLoadGridLatLon = False - - def setLazyLoadGridLatLon(self, lazyLoadGridLatLon): - self._lazyLoadGridLatLon = lazyLoadGridLatLon def getAvailableTimes(self, request, refTimeOnly): timesRequest = GetAvailableTimesRequest() @@ -109,7 +78,6 @@ class ThriftClientRouter(object): def getGridData(self, request, times): gridDataRequest = GetGridDataRequest() - gridDataRequest.setIncludeLatLonData(not self._lazyLoadGridLatLon) gridDataRequest.setRequestParameters(request) # if we have an iterable times instance, then the user must have asked # for grid data with the List of DataTime objects @@ -127,28 +95,15 @@ class ThriftClientRouter(object): for location in locNames: nx = response.getSiteNxValues()[location] ny = response.getSiteNyValues()[location] - if self._lazyLoadGridLatLon: - envelope = response.getSiteEnvelopes()[location] - crsWkt = response.getSiteCrsWkt()[location] - delegate = LazyGridLatLon( - self._client, nx, ny, envelope, crsWkt) - locSpecificData[location] = (nx, ny, delegate) - else: - latData = numpy.reshape(numpy.array( - response.getSiteLatGrids()[location]), (ny, nx)) - lonData = numpy.reshape(numpy.array( - response.getSiteLonGrids()[location]), (ny, nx)) - locSpecificData[location] = (nx, ny, (lonData, latData)) + latData = numpy.reshape(numpy.array(response.getSiteLatGrids()[location]), (ny, nx)) + lonData = numpy.reshape(numpy.array(response.getSiteLonGrids()[location]), (ny, nx)) + locSpecificData[location] = (nx, ny, (lonData, latData)) + retVal = [] for gridDataRecord in response.getGridData(): locationName = gridDataRecord.getLocationName() locData = locSpecificData[locationName] - if self._lazyLoadGridLatLon: - retVal.append(PyGridData.PyGridData(gridDataRecord, locData[ - 0], locData[1], latLonDelegate=locData[2])) - else: - retVal.append(PyGridData.PyGridData( - gridDataRecord, locData[0], locData[1], locData[2])) + retVal.append(PyGridData.PyGridData(gridDataRecord, locData[0], locData[1], locData[2])) return retVal def getGeometryData(self, request, times): @@ -166,9 +121,10 @@ class ThriftClientRouter(object): response = self._client.sendRequest(geoDataRequest) geometries = [] for wkb in response.getGeometryWKBs(): - # the wkb is a numpy.ndarray of dtype int8 - # convert the bytearray to a byte string and load it - geometries.append(shapely.wkb.loads(wkb.tostring())) + # convert the wkb to a bytearray with only positive values + byteArrWKB = bytearray(map(lambda x: x % 256,wkb.tolist())) + # convert the bytearray to a byte string and load it. + geometries.append(shapely.wkb.loads(str(byteArrWKB))) retVal = [] for geoDataRecord in response.getGeoData(): @@ -219,7 +175,7 @@ class ThriftClientRouter(object): response = self._client.sendRequest(idValReq) return response - def newDataRequest(self, datatype, parameters=[], levels=[], locationNames=[], envelope=None, **kwargs): + def newDataRequest(self, datatype, parameters=[], levels=[], locationNames = [], envelope=None, **kwargs): req = DefaultDataRequest() if datatype: req.setDatatype(datatype) @@ -244,4 +200,4 @@ class ThriftClientRouter(object): notifReq = GetNotificationFilterRequest() notifReq.setRequestParameters(request) response = self._client.sendRequest(notifReq) - return response + return response \ No newline at end of file diff --git a/awips/qpidingest.py b/awips/qpidingest.py index 0a9dc1b..f5aa83f 100644 --- a/awips/qpidingest.py +++ b/awips/qpidingest.py @@ -59,13 +59,9 @@ # .... # 06/13/2013 DR 16242 D. Friedman Add Qpid authentication info # 03/06/2014 DR 17907 D. Friedman Workaround for issue QPID-5569 -# 02/16/2017 DR 6084 bsteffen Support ssl connections # #=============================================================================== -import os -import os.path - import qpid from qpid.util import connect from qpid.connection import Connection @@ -75,31 +71,17 @@ QPID_USERNAME = 'guest' QPID_PASSWORD = 'guest' class IngestViaQPID: - def __init__(self, host='localhost', port=5672, ssl=None): + def __init__(self, host='localhost', port=5672): ''' Connect to QPID and make bindings to route message to external.dropbox queue @param host: string hostname of computer running EDEX and QPID (default localhost) @param port: integer port used to connect to QPID (default 5672) - @param ssl: boolean to determine whether ssl is used, default value of None will use ssl only if a client certificate is found. ''' try: # - socket = connect(host, port) - if "QPID_SSL_CERT_DB" in os.environ: - certdb = os.environ["QPID_SSL_CERT_DB"] - else: - certdb = os.path.expanduser("~/.qpid/") - if "QPID_SSL_CERT_NAME" in os.environ: - certname = os.environ["QPID_SSL_CERT_NAME"] - else: - certname = QPID_USERNAME - certfile = os.path.join(certdb, certname + ".crt") - if ssl or (ssl is None and os.path.exists(certfile)): - keyfile = os.path.join(certdb, certname + ".key") - trustfile = os.path.join(certdb, "root.crt") - socket = qpid.util.ssl(socket, keyfile=keyfile, certfile=certfile, ca_certs=trustfile) - self.connection = Connection (sock=socket, username=QPID_USERNAME, password=QPID_PASSWORD) + self.socket = connect(host, port) + self.connection = Connection (sock=self.socket, username=QPID_USERNAME, password=QPID_PASSWORD) self.connection.start() self.session = self.connection.session(str(uuid4())) self.session.exchange_bind(exchange='amq.direct', queue='external.dropbox', binding_key='external.dropbox') diff --git a/dynamicserialize/ThriftSerializationContext.py b/dynamicserialize/ThriftSerializationContext.py index 75aa08e..0ed39d2 100644 --- a/dynamicserialize/ThriftSerializationContext.py +++ b/dynamicserialize/ThriftSerializationContext.py @@ -29,7 +29,7 @@ # match what they should be in the destination language. # # -# SOFTWARE HISTORY +# SOFTWARE HISTORY # # Date Ticket# Engineer Description # ------------ ---------- ----------- -------------------------- @@ -37,27 +37,20 @@ # 06/12/13 #2099 dgilling Implement readObject() and # writeObject(). # Apr 24, 2015 4425 nabowle Add Double support -# Oct 17, 2016 5919 njensen Optimized for speed # # from thrift.Thrift import TType -import inspect -import sys -import types -import time +import inspect, sys, types import dynamicserialize from dynamicserialize import dstypes, adapters import SelfDescribingBinaryProtocol import numpy -DS_LEN = len('dynamicserialize.dstypes.') - dsObjTypes = {} - def buildObjMap(module): - if '__all__' in module.__dict__: + if module.__dict__.has_key('__all__'): for i in module.__all__: name = module.__name__ + '.' + i __import__(name) @@ -66,7 +59,7 @@ def buildObjMap(module): clzName = module.__name__[module.__name__.rfind('.') + 1:] clz = module.__dict__[clzName] tname = module.__name__ - tname = tname[DS_LEN:] + tname = tname.replace('dynamicserialize.dstypes.', '') dsObjTypes[tname] = clz buildObjMap(dstypes) @@ -79,7 +72,7 @@ pythonToThriftMap = { types.DictionaryType: TType.MAP, type(set([])): TType.SET, types.FloatType: SelfDescribingBinaryProtocol.FLOAT, - # types.FloatType: TType.DOUBLE, + #types.FloatType: TType.DOUBLE, types.BooleanType: TType.BOOL, types.InstanceType: TType.STRUCT, types.NoneType: TType.VOID, @@ -94,9 +87,7 @@ pythonToThriftMap = { numpy.int64: TType.I64 } -primitiveSupport = (TType.BYTE, TType.I16, TType.I32, TType.I64, - SelfDescribingBinaryProtocol.FLOAT, TType.DOUBLE) - +primitiveSupport = (TType.BYTE, TType.I16, TType.I32, TType.I64, SelfDescribingBinaryProtocol.FLOAT, TType.DOUBLE) class ThriftSerializationContext(object): @@ -104,51 +95,52 @@ class ThriftSerializationContext(object): self.serializationManager = serializationManager self.protocol = selfDescribingBinaryProtocol self.typeDeserializationMethod = { - TType.STRING: self.protocol.readString, - TType.I16: self.protocol.readI16, - TType.I32: self.protocol.readI32, - TType.LIST: self._deserializeArray, - TType.MAP: self._deserializeMap, - TType.SET: self._deserializeSet, - SelfDescribingBinaryProtocol.FLOAT: self.protocol.readFloat, - TType.BYTE: self.protocol.readByte, - TType.I64: self.protocol.readI64, - TType.DOUBLE: self.protocol.readDouble, - TType.BOOL: self.protocol.readBool, - TType.STRUCT: self.deserializeMessage, - TType.VOID: lambda: None - } + TType.STRING: self.protocol.readString, + TType.I16: self.protocol.readI16, + TType.I32: self.protocol.readI32, + TType.LIST: self._deserializeArray, + TType.MAP: self._deserializeMap, + TType.SET: self._deserializeSet, + SelfDescribingBinaryProtocol.FLOAT: self.protocol.readFloat, + TType.BYTE: self.protocol.readByte, + TType.I64: self.protocol.readI64, + TType.DOUBLE: self.protocol.readDouble, + TType.BOOL: self.protocol.readBool, + TType.STRUCT: self.deserializeMessage, + TType.VOID: lambda: None + } self.typeSerializationMethod = { - TType.STRING: self.protocol.writeString, - TType.I16: self.protocol.writeI16, - TType.I32: self.protocol.writeI32, - TType.LIST: self._serializeArray, - TType.MAP: self._serializeMap, - TType.SET: self._serializeSet, - SelfDescribingBinaryProtocol.FLOAT: self.protocol.writeFloat, - TType.BYTE: self.protocol.writeByte, - TType.I64: self.protocol.writeI64, - TType.DOUBLE: self.protocol.writeDouble, - TType.BOOL: self.protocol.writeBool, - TType.STRUCT: self.serializeMessage, - TType.VOID: lambda x: None - } + TType.STRING: self.protocol.writeString, + TType.I16: self.protocol.writeI16, + TType.I32: self.protocol.writeI32, + TType.LIST: self._serializeArray, + TType.MAP: self._serializeMap, + TType.SET: self._serializeSet, + SelfDescribingBinaryProtocol.FLOAT: self.protocol.writeFloat, + TType.BYTE: self.protocol.writeByte, + TType.I64: self.protocol.writeI64, + TType.DOUBLE: self.protocol.writeDouble, + TType.BOOL: self.protocol.writeBool, + TType.STRUCT: self.serializeMessage, + TType.VOID: lambda x: None + } self.listDeserializationMethod = { - TType.BYTE: self.protocol.readI8List, - TType.I16: self.protocol.readI16List, - TType.I32: self.protocol.readI32List, - TType.I64: self.protocol.readI64List, - SelfDescribingBinaryProtocol.FLOAT: self.protocol.readF32List, - TType.DOUBLE: self.protocol.readF64List - } + TType.BYTE: self.protocol.readI8List, + TType.I16: self.protocol.readI16List, + TType.I32: self.protocol.readI32List, + TType.I64: self.protocol.readI64List, + SelfDescribingBinaryProtocol.FLOAT: self.protocol.readF32List, + TType.DOUBLE: self.protocol.readF64List + } self.listSerializationMethod = { - TType.BYTE: self.protocol.writeI8List, - TType.I16: self.protocol.writeI16List, - TType.I32: self.protocol.writeI32List, - TType.I64: self.protocol.writeI64List, - SelfDescribingBinaryProtocol.FLOAT: self.protocol.writeF32List, - TType.DOUBLE: self.protocol.writeF64List - } + TType.BYTE: self.protocol.writeI8List, + TType.I16: self.protocol.writeI16List, + TType.I32: self.protocol.writeI32List, + TType.I64: self.protocol.writeI64List, + SelfDescribingBinaryProtocol.FLOAT: self.protocol.writeF32List, + TType.DOUBLE: self.protocol.writeF64List + } + def readMessageStart(self): msg = self.protocol.readMessageBegin() @@ -159,19 +151,17 @@ class ThriftSerializationContext(object): def deserializeMessage(self): name = self.protocol.readStructBegin() + name = name.replace('_', '.') if name.isdigit(): obj = self._deserializeType(int(name)) return obj - name = name.replace('_', '.') - if name in adapters.classAdapterRegistry: + elif adapters.classAdapterRegistry.has_key(name): return adapters.classAdapterRegistry[name].deserialize(self) - elif '$' in name: - # it's an inner class, we're going to hope it's an enum, treat it - # special + elif name.find('$') > -1: + # it's an inner class, we're going to hope it's an enum, treat it special fieldName, fieldType, fieldId = self.protocol.readFieldBegin() if fieldName != '__enumValue__': - raise dynamiceserialize.SerializationException( - "Expected to find enum payload. Found: " + fieldName) + raise dynamiceserialize.SerializationException("Expected to find enum payload. Found: " + fieldName) obj = self.protocol.readString() self.protocol.readFieldEnd() return obj @@ -186,30 +176,37 @@ class ThriftSerializationContext(object): return obj def _deserializeType(self, b): - try: + if self.typeDeserializationMethod.has_key(b): return self.typeDeserializationMethod[b]() - except KeyError: - raise dynamicserialize.SerializationException( - "Unsupported type value " + str(b)) + else: + raise dynamicserialize.SerializationException("Unsupported type value " + str(b)) + def _deserializeField(self, structname, obj): fieldName, fieldType, fieldId = self.protocol.readFieldBegin() if fieldType == TType.STOP: return False elif fieldType != TType.VOID: +# if adapters.fieldAdapterRegistry.has_key(structname) and adapters.fieldAdapterRegistry[structname].has_key(fieldName): +# result = adapters.fieldAdapterRegistry[structname][fieldName].deserialize(self) +# else: result = self._deserializeType(fieldType) lookingFor = "set" + fieldName[0].upper() + fieldName[1:] try: setMethod = getattr(obj, lookingFor) - setMethod(result) + + if callable(setMethod): + setMethod(result) + else: + raise dynamicserialize.SerializationException("Couldn't find setter method " + lookingFor) except: - raise dynamicserialize.SerializationException( - "Couldn't find setter method " + lookingFor) + raise dynamicserialize.SerializationException("Couldn't find setter method " + lookingFor) self.protocol.readFieldEnd() return True + def _deserializeArray(self): listType, size = self.protocol.readListBegin() result = [] @@ -244,20 +241,19 @@ class ThriftSerializationContext(object): def _lookupType(self, obj): pyt = type(obj) - if pyt in pythonToThriftMap: + if pythonToThriftMap.has_key(pyt): return pythonToThriftMap[pyt] - elif pyt.__module__[:DS_LEN - 1] == ('dynamicserialize.dstypes'): + elif pyt.__module__.startswith('dynamicserialize.dstypes'): return pythonToThriftMap[types.InstanceType] else: - raise dynamicserialize.SerializationException( - "Don't know how to serialize object of type: " + str(pyt)) + raise dynamicserialize.SerializationException("Don't know how to serialize object of type: " + str(pyt)) def serializeMessage(self, obj): tt = self._lookupType(obj) if tt == TType.STRUCT: - fqn = obj.__module__[DS_LEN:] - if fqn in adapters.classAdapterRegistry: + fqn = obj.__module__.replace('dynamicserialize.dstypes.', '') + if adapters.classAdapterRegistry.has_key(fqn): # get proper class name when writing class name to serialization stream # in case we have a special inner-class case m = sys.modules[adapters.classAdapterRegistry[fqn].__name__] @@ -277,7 +273,7 @@ class ThriftSerializationContext(object): val = m[1]() ft = self._lookupType(val) if ft == TType.STRUCT: - fc = val.__module__[DS_LEN:] + fc = val.__module__.replace('dynamicserialize.dstypes.', '') self._serializeField(fieldname, ft, fid, val) else: self._serializeField(fieldname, ft, fid, val) @@ -297,18 +293,17 @@ class ThriftSerializationContext(object): self.protocol.writeFieldEnd() def _serializeType(self, fieldValue, fieldType): - if fieldType in self.typeSerializationMethod: + if self.typeSerializationMethod.has_key(fieldType): return self.typeSerializationMethod[fieldType](fieldValue) else: - raise dynamicserialize.SerializationException( - "Unsupported type value " + str(fieldType)) + raise dynamicserialize.SerializationException("Unsupported type value " + str(fieldType)) def _serializeArray(self, obj): size = len(obj) if size: if type(obj) is numpy.ndarray: - t = pythonToThriftMap[obj.dtype.type] - size = obj.size + t = pythonToThriftMap[obj.dtype.type] + size = obj.size else: t = self._lookupType(obj[0]) else: @@ -336,6 +331,7 @@ class ThriftSerializationContext(object): self.listSerializationMethod[t](obj) self.protocol.writeListEnd() + def _serializeMap(self, obj): size = len(obj) self.protocol.writeMapBegin(TType.VOID, TType.VOID, size) diff --git a/dynamicserialize/adapters/__init__.py b/dynamicserialize/adapters/__init__.py index f72aaa8..d7e643f 100644 --- a/dynamicserialize/adapters/__init__.py +++ b/dynamicserialize/adapters/__init__.py @@ -1,19 +1,19 @@ ## # This software was developed and / or modified by Raytheon Company, # pursuant to Contract DG133W-05-CQ-1067 with the US Government. -# +# # U.S. EXPORT CONTROLLED TECHNICAL DATA # This software product contains export-restricted data whose # export/transfer/disclosure is restricted by U.S. law. Dissemination # to non-U.S. persons whether in the United States or abroad requires # an export license or other authorization. -# +# # Contractor Name: Raytheon Company # Contractor Address: 6825 Pine Street, Suite 340 # Mail Stop B8 # Omaha, NE 68106 # 402.291.0100 -# +# # See the AWIPS II Master Rights File ("Master Rights File.pdf") for # further licensing information. ## @@ -21,80 +21,56 @@ # # __init__.py for Dynamic Serialize adapters. -# -# Plugins can contribute to dynamicserialize.adapters by either including their -# classes directly in pythonPackages/dynamicserialize/adapters/ within their -# plugin. The plugin's adapter will automatically be added to __all__ at runtime -# and registered. -# Plugins should not include a custom __init__.py in -# pythonPackages/dynamicserialize/adapters/ because it will overwrite this file. -# If custom package initialization is needed, a subpackage should be created -# with an __init__.py that includes the following: -# -# __all__ = ['CustomAdapter1', 'CustomAdapter2'] -# from dynamicserialize.adapters import registerAdapters -# registerAdapters(__name__, __all__) -# -# +# +# # SOFTWARE HISTORY -# +# # Date Ticket# Engineer Description # ------------ ---------- ----------- -------------------------- -# 08/31/10 njensen Initial Creation. -# 03/20/13 #1774 randerso Added TimeConstraintsAdapter -# 04/22/13 #1949 rjpeter Added LockTableAdapter -# 02/06/14 #2672 bsteffen Added JTSEnvelopeAdapter -# 06/22/2015 #4573 randerso Added JobProgressAdapter -# 09/21/2015 #4486 rjpeter Added FormattedDateAdapter -# 06/23/2016 #5696 rjpeter Added CommutativeTimestampAdapter -# 10/17/2016 #5919 njensen Added GeomDataRespAdapter -# 01/09/2017 #5997 nabowle Allow contribution from plugins. +# 08/31/10 njensen Initial Creation. +# 03/20/13 #1774 randerso Added TimeConstraintsAdapter +# 04/22/13 #1949 rjpeter Added LockTableAdapter +# 02/06/14 #2672 bsteffen Added JTSEnvelopeAdapter +# 06/22/2015 #4573 randerso Added JobProgressAdapter +# 09/21/2015 #4486 rjpeter Added FormattedDateAdapter +# 06/23/2016 #5696 rjpeter Added CommutativeTimestampAdapter # __all__ = [ - 'PointAdapter', - 'StackTraceElementAdapter', - 'CalendarAdapter', - 'GregorianCalendarAdapter', - 'DateAdapter', - 'GeometryTypeAdapter', - 'CoordAdapter', - 'TimestampAdapter', - 'EnumSetAdapter', - 'FloatBufferAdapter', - 'ByteBufferAdapter', - 'JTSEnvelopeAdapter' -] - + 'PointAdapter', + 'StackTraceElementAdapter', + 'WsIdAdapter', + 'CalendarAdapter', + 'GregorianCalendarAdapter', + 'ActiveTableModeAdapter', + 'DateAdapter', + 'FormattedDateAdapter', + 'LocalizationLevelSerializationAdapter', + 'LocalizationTypeSerializationAdapter', + 'GeometryTypeAdapter', + 'CoordAdapter', + 'TimeRangeTypeAdapter', + 'ParmIDAdapter', + 'DatabaseIDAdapter', + 'TimestampAdapter', + 'CommutativeTimestampAdapter', + 'EnumSetAdapter', + 'FloatBufferAdapter', + 'ByteBufferAdapter', + 'TimeConstraintsAdapter', + 'LockTableAdapter', + 'JTSEnvelopeAdapter', + 'JobProgressAdapter', + ] + classAdapterRegistry = {} - + def getAdapterRegistry(): - import pkgutil - - discoveredPackages = [] - # allow other plugins to contribute to adapters by dropping their adapter or - # package into the dynamicserialize.adapters package - for _, modname, ispkg in pkgutil.iter_modules(__path__): - if ispkg: - discoveredPackages.append(modname) - else: - if modname not in __all__: - __all__.append(modname) - - registerAdapters(__name__, __all__) - - for pkg in discoveredPackages: - __import__(__name__ + '.' + pkg) - - -def registerAdapters(package, modules): - import sys - if not package.endswith('.'): - package += '.' - for x in modules: - exec 'import ' + package + x - m = sys.modules[package + x] + import sys + for x in __all__: + exec 'import ' + x + m = sys.modules['dynamicserialize.adapters.' + x] d = m.__dict__ if d.has_key('ClassAdapter'): if isinstance(m.ClassAdapter, list): @@ -104,8 +80,9 @@ def registerAdapters(package, modules): clzName = m.ClassAdapter classAdapterRegistry[clzName] = m else: - raise LookupError('Adapter class ' + x + ' has no ClassAdapter field ' + + raise LookupError('Adapter class ' + x + ' has no ClassAdapter field ' + \ 'and cannot be registered.') - + getAdapterRegistry() + diff --git a/dynamicserialize/dstypes/com/raytheon/uf/common/__init__.py b/dynamicserialize/dstypes/com/raytheon/uf/common/__init__.py index 40e4cff..dfd5b4a 100644 --- a/dynamicserialize/dstypes/com/raytheon/uf/common/__init__.py +++ b/dynamicserialize/dstypes/com/raytheon/uf/common/__init__.py @@ -21,7 +21,22 @@ # File auto-generated by PythonFileGenerator __all__ = [ - 'dataplugin' + 'activetable', + 'alertviz', + 'auth', + 'dataaccess', + 'dataplugin', + 'dataquery', + 'datastorage', + 'localization', + 'management', + 'message', + 'plugin', + 'pointdata', + 'pypies', + 'serialization', + 'site', + 'time' ] diff --git a/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/__init__.py b/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/__init__.py index acb7c9b..006c19d 100644 --- a/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/__init__.py +++ b/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/__init__.py @@ -21,7 +21,13 @@ # File auto-generated by PythonFileGenerator __all__ = [ - 'events' + 'events', + 'gfe', + 'grid', + 'level', + 'message', + 'radar', + 'text' ]