From 8c3145b63ee5279b898750a527b2d5a0db15c1e4 Mon Sep 17 00:00:00 2001 From: mjames-upc Date: Sat, 30 Sep 2017 21:47:32 -0600 Subject: [PATCH] changes for 17.3.1 --- awips/QpidSubscriber.py | 19 +- awips/UsageArgumentParser.py | 23 +++ awips/dataaccess/DataAccessLayer.py | 28 ++- awips/dataaccess/PyGridData.py | 10 +- awips/dataaccess/ThriftClientRouter.py | 68 ++++++-- awips/qpidingest.py | 24 ++- .../ThriftSerializationContext.py | 162 +++++++++--------- dynamicserialize/adapters/__init__.py | 117 ++++++++----- .../com/raytheon/uf/common/__init__.py | 17 +- .../raytheon/uf/common/dataplugin/__init__.py | 8 +- 10 files changed, 308 insertions(+), 168 deletions(-) diff --git a/awips/QpidSubscriber.py b/awips/QpidSubscriber.py index e55d967..25753f9 100644 --- a/awips/QpidSubscriber.py +++ b/awips/QpidSubscriber.py @@ -30,9 +30,12 @@ # 11/17/10 njensen Initial Creation. # 08/15/13 2169 bkowal Optionally gzip decompress any data that is read. # 08/04/16 2416 tgurney Add queueStarted property +# 02/16/17 6084 bsteffen Support ssl connections # # +import os +import os.path import qpid import zlib @@ -41,11 +44,24 @@ from qpid.exceptions import Closed class QpidSubscriber: - def __init__(self, host='127.0.0.1', port=5672, decompress=False): + def __init__(self, host='127.0.0.1', port=5672, decompress=False, ssl=None): self.host = host self.port = port self.decompress = decompress; socket = qpid.util.connect(host, port) + if "QPID_SSL_CERT_DB" in os.environ: + certdb = os.environ["QPID_SSL_CERT_DB"] + else: + certdb = os.path.expanduser("~/.qpid/") + if "QPID_SSL_CERT_NAME" in os.environ: + certname = os.environ["QPID_SSL_CERT_NAME"] + else: + certname = "guest" + certfile = os.path.join(certdb, certname + ".crt") + if ssl or (ssl is None and os.path.exists(certfile)): + keyfile = os.path.join(certdb, certname + ".key") + trustfile = os.path.join(certdb, "root.crt") + socket = qpid.util.ssl(socket, keyfile=keyfile, certfile=certfile, ca_certs=trustfile) self.__connection = qpid.connection.Connection(sock=socket, username='guest', password='guest') self.__connection.start() self.__session = self.__connection.session(str(qpid.datatypes.uuid4())) @@ -103,3 +119,4 @@ class QpidSubscriber: @property def queueStarted(self): return self.__queueStarted + diff --git a/awips/UsageArgumentParser.py b/awips/UsageArgumentParser.py index 3722c25..313d206 100644 --- a/awips/UsageArgumentParser.py +++ b/awips/UsageArgumentParser.py @@ -17,13 +17,23 @@ # See the AWIPS II Master Rights File ("Master Rights File.pdf") for # further licensing information. ## +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------- -------- --------- --------------------------------------------- +# Feb 13, 2017 6092 randerso Added StoreTimeAction +# +## import argparse import sys +import time from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import DatabaseID from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import ParmID +TIME_FORMAT = "%Y%m%d_%H%M" class UsageArgumentParser(argparse.ArgumentParser): """ @@ -56,3 +66,16 @@ class AppendParmNameAndLevelAction(argparse.Action): else: setattr(namespace, self.dest, [comp]) +class StoreTimeAction(argparse.Action): + """ + argparse.Action subclass to validate GFE formatted time strings + and parse them to time.struct_time + """ + def __call__(self, parser, namespace, values, option_string=None): + try: + timeStruct = time.strptime(values, TIME_FORMAT) + except: + parser.error(str(values) + " is not a valid time string of the format YYYYMMDD_hhmm") + + setattr(namespace, self.dest, timeStruct) + diff --git a/awips/dataaccess/DataAccessLayer.py b/awips/dataaccess/DataAccessLayer.py index 115305e..7f436b5 100644 --- a/awips/dataaccess/DataAccessLayer.py +++ b/awips/dataaccess/DataAccessLayer.py @@ -40,8 +40,8 @@ # Jun 01, 2016 5587 tgurney Add new signatures for # getRequiredIdentifiers() and # getOptionalIdentifiers() -# 10/07/16 ---- mjames@ucar Added getForecastRun -# +# Oct 07, 2016 ---- mjames@ucar Added getForecastRun +# Oct 18, 2016 5916 bsteffen Add setLazyLoadGridLatLon # # @@ -53,6 +53,7 @@ import warnings THRIFT_HOST = "edex" USING_NATIVE_THRIFT = False + if sys.modules.has_key('jep'): # intentionally do not catch if this fails to import, we want it to # be obvious that something is configured wrong when running from within @@ -252,3 +253,26 @@ def changeEDEXHost(newHostName): router = ThriftClientRouter.ThriftClientRouter(THRIFT_HOST) else: raise TypeError("Cannot call changeEDEXHost when using JepRouter.") + +def setLazyLoadGridLatLon(lazyLoadGridLatLon): + """ + Provide a hint to the Data Access Framework indicating whether to load the + lat/lon data for a grid immediately or wait until it is needed. This is + provided as a performance tuning hint and should not affect the way the + Data Access Framework is used. Depending on the internal implementation of + the Data Access Framework this hint might be ignored. Examples of when this + should be set to True are when the lat/lon information is not used or when + it is used only if certain conditions within the data are met. It could be + set to False if it is guaranteed that all lat/lon information is needed and + it would be better to get any performance overhead for generating the + lat/lon data out of the way during the initial request. + + + Args: + lazyLoadGridLatLon: Boolean value indicating whether to lazy load. + """ + try: + router.setLazyLoadGridLatLon(lazyLoadGridLatLon) + except AttributeError: + # The router is not required to support this capability. + pass diff --git a/awips/dataaccess/PyGridData.py b/awips/dataaccess/PyGridData.py index 87fffac..d85c7ea 100644 --- a/awips/dataaccess/PyGridData.py +++ b/awips/dataaccess/PyGridData.py @@ -28,7 +28,9 @@ # Date Ticket# Engineer Description # ------------ ---------- ----------- -------------------------- # 06/03/13 #2023 dgilling Initial Creation. +# 10/13/16 #5916 bsteffen Correct grid shape, allow lat/lon # 11/10/16 #5900 bsteffen Correct grid shape +# to be requested by a delegate # # @@ -46,7 +48,7 @@ The ability to unit convert grid data is not currently available in this version class PyGridData(IGridData, PyData.PyData): - def __init__(self, gridDataRecord, nx, ny, latLonGrid): + def __init__(self, gridDataRecord, nx, ny, latLonGrid = None, latLonDelegate = None): PyData.PyData.__init__(self, gridDataRecord) nx = nx ny = ny @@ -54,6 +56,8 @@ class PyGridData(IGridData, PyData.PyData): self.__unit = gridDataRecord.getUnit() self.__gridData = numpy.reshape(numpy.array(gridDataRecord.getGridData()), (ny, nx)) self.__latLonGrid = latLonGrid + self.__latLonDelegate = latLonDelegate + def getParameter(self): return self.__parameter @@ -70,4 +74,8 @@ class PyGridData(IGridData, PyData.PyData): return self.__gridData def getLatLonCoords(self): + if self.__latLonGrid is not None: + return self.__latLonGrid + elif self.__latLonDelegate is not None: + return self.__latLonDelegate() return self.__latLonGrid diff --git a/awips/dataaccess/ThriftClientRouter.py b/awips/dataaccess/ThriftClientRouter.py index e1c0e1e..14948fe 100644 --- a/awips/dataaccess/ThriftClientRouter.py +++ b/awips/dataaccess/ThriftClientRouter.py @@ -39,7 +39,8 @@ # getRequiredIdentifiers() and # getOptionalIdentifiers() # 08/01/16 2416 tgurney Add getNotificationFilter() -# 11/10/16 5900 bsteffen Correct grid shape +# 10/13/16 5916 bsteffen Correct grid shape, allow lazy grid lat/lon +# 10/26/16 5919 njensen Speed up geometry creation in getGeometryData() # @@ -51,6 +52,7 @@ from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import G from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableTimesRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetGeometryDataRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetGridDataRequest +from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetGridLatLonRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableParametersRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableLevelsRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetRequiredIdentifiersRequest @@ -64,10 +66,39 @@ from awips.dataaccess import PyGeometryData from awips.dataaccess import PyGridData +class LazyGridLatLon(object): + + def __init__(self, client, nx, ny, envelope, crsWkt): + self._latLonGrid = None + self._client = client + self._request = GetGridLatLonRequest() + self._request.setNx(nx) + self._request.setNy(ny) + self._request.setEnvelope(envelope) + self._request.setCrsWkt(crsWkt) + + def __call__(self): + # Its important that the data is cached internally so that if multiple + # GridData are sharing the same delegate then they can also share a + # single request for the LatLon information. + if self._latLonGrid is None: + response = self._client.sendRequest(self._request) + nx = response.getNx() + ny = response.getNy() + latData = numpy.reshape(numpy.array(response.getLats()), (ny, nx)) + lonData = numpy.reshape(numpy.array(response.getLons()), (ny, nx)) + self._latLonGrid = (lonData, latData) + return self._latLonGrid + + class ThriftClientRouter(object): def __init__(self, host='localhost'): self._client = ThriftClient.ThriftClient(host) + self._lazyLoadGridLatLon = False + + def setLazyLoadGridLatLon(self, lazyLoadGridLatLon): + self._lazyLoadGridLatLon = lazyLoadGridLatLon def getAvailableTimes(self, request, refTimeOnly): timesRequest = GetAvailableTimesRequest() @@ -78,6 +109,7 @@ class ThriftClientRouter(object): def getGridData(self, request, times): gridDataRequest = GetGridDataRequest() + gridDataRequest.setIncludeLatLonData(not self._lazyLoadGridLatLon) gridDataRequest.setRequestParameters(request) # if we have an iterable times instance, then the user must have asked # for grid data with the List of DataTime objects @@ -95,15 +127,28 @@ class ThriftClientRouter(object): for location in locNames: nx = response.getSiteNxValues()[location] ny = response.getSiteNyValues()[location] - latData = numpy.reshape(numpy.array(response.getSiteLatGrids()[location]), (ny, nx)) - lonData = numpy.reshape(numpy.array(response.getSiteLonGrids()[location]), (ny, nx)) - locSpecificData[location] = (nx, ny, (lonData, latData)) - + if self._lazyLoadGridLatLon: + envelope = response.getSiteEnvelopes()[location] + crsWkt = response.getSiteCrsWkt()[location] + delegate = LazyGridLatLon( + self._client, nx, ny, envelope, crsWkt) + locSpecificData[location] = (nx, ny, delegate) + else: + latData = numpy.reshape(numpy.array( + response.getSiteLatGrids()[location]), (ny, nx)) + lonData = numpy.reshape(numpy.array( + response.getSiteLonGrids()[location]), (ny, nx)) + locSpecificData[location] = (nx, ny, (lonData, latData)) retVal = [] for gridDataRecord in response.getGridData(): locationName = gridDataRecord.getLocationName() locData = locSpecificData[locationName] - retVal.append(PyGridData.PyGridData(gridDataRecord, locData[0], locData[1], locData[2])) + if self._lazyLoadGridLatLon: + retVal.append(PyGridData.PyGridData(gridDataRecord, locData[ + 0], locData[1], latLonDelegate=locData[2])) + else: + retVal.append(PyGridData.PyGridData( + gridDataRecord, locData[0], locData[1], locData[2])) return retVal def getGeometryData(self, request, times): @@ -121,10 +166,9 @@ class ThriftClientRouter(object): response = self._client.sendRequest(geoDataRequest) geometries = [] for wkb in response.getGeometryWKBs(): - # convert the wkb to a bytearray with only positive values - byteArrWKB = bytearray(map(lambda x: x % 256,wkb.tolist())) - # convert the bytearray to a byte string and load it. - geometries.append(shapely.wkb.loads(str(byteArrWKB))) + # the wkb is a numpy.ndarray of dtype int8 + # convert the bytearray to a byte string and load it + geometries.append(shapely.wkb.loads(wkb.tostring())) retVal = [] for geoDataRecord in response.getGeoData(): @@ -175,7 +219,7 @@ class ThriftClientRouter(object): response = self._client.sendRequest(idValReq) return response - def newDataRequest(self, datatype, parameters=[], levels=[], locationNames = [], envelope=None, **kwargs): + def newDataRequest(self, datatype, parameters=[], levels=[], locationNames=[], envelope=None, **kwargs): req = DefaultDataRequest() if datatype: req.setDatatype(datatype) @@ -200,4 +244,4 @@ class ThriftClientRouter(object): notifReq = GetNotificationFilterRequest() notifReq.setRequestParameters(request) response = self._client.sendRequest(notifReq) - return response \ No newline at end of file + return response diff --git a/awips/qpidingest.py b/awips/qpidingest.py index f5aa83f..0a9dc1b 100644 --- a/awips/qpidingest.py +++ b/awips/qpidingest.py @@ -59,9 +59,13 @@ # .... # 06/13/2013 DR 16242 D. Friedman Add Qpid authentication info # 03/06/2014 DR 17907 D. Friedman Workaround for issue QPID-5569 +# 02/16/2017 DR 6084 bsteffen Support ssl connections # #=============================================================================== +import os +import os.path + import qpid from qpid.util import connect from qpid.connection import Connection @@ -71,17 +75,31 @@ QPID_USERNAME = 'guest' QPID_PASSWORD = 'guest' class IngestViaQPID: - def __init__(self, host='localhost', port=5672): + def __init__(self, host='localhost', port=5672, ssl=None): ''' Connect to QPID and make bindings to route message to external.dropbox queue @param host: string hostname of computer running EDEX and QPID (default localhost) @param port: integer port used to connect to QPID (default 5672) + @param ssl: boolean to determine whether ssl is used, default value of None will use ssl only if a client certificate is found. ''' try: # - self.socket = connect(host, port) - self.connection = Connection (sock=self.socket, username=QPID_USERNAME, password=QPID_PASSWORD) + socket = connect(host, port) + if "QPID_SSL_CERT_DB" in os.environ: + certdb = os.environ["QPID_SSL_CERT_DB"] + else: + certdb = os.path.expanduser("~/.qpid/") + if "QPID_SSL_CERT_NAME" in os.environ: + certname = os.environ["QPID_SSL_CERT_NAME"] + else: + certname = QPID_USERNAME + certfile = os.path.join(certdb, certname + ".crt") + if ssl or (ssl is None and os.path.exists(certfile)): + keyfile = os.path.join(certdb, certname + ".key") + trustfile = os.path.join(certdb, "root.crt") + socket = qpid.util.ssl(socket, keyfile=keyfile, certfile=certfile, ca_certs=trustfile) + self.connection = Connection (sock=socket, username=QPID_USERNAME, password=QPID_PASSWORD) self.connection.start() self.session = self.connection.session(str(uuid4())) self.session.exchange_bind(exchange='amq.direct', queue='external.dropbox', binding_key='external.dropbox') diff --git a/dynamicserialize/ThriftSerializationContext.py b/dynamicserialize/ThriftSerializationContext.py index 0ed39d2..75aa08e 100644 --- a/dynamicserialize/ThriftSerializationContext.py +++ b/dynamicserialize/ThriftSerializationContext.py @@ -29,7 +29,7 @@ # match what they should be in the destination language. # # -# SOFTWARE HISTORY +# SOFTWARE HISTORY # # Date Ticket# Engineer Description # ------------ ---------- ----------- -------------------------- @@ -37,20 +37,27 @@ # 06/12/13 #2099 dgilling Implement readObject() and # writeObject(). # Apr 24, 2015 4425 nabowle Add Double support +# Oct 17, 2016 5919 njensen Optimized for speed # # from thrift.Thrift import TType -import inspect, sys, types +import inspect +import sys +import types +import time import dynamicserialize from dynamicserialize import dstypes, adapters import SelfDescribingBinaryProtocol import numpy +DS_LEN = len('dynamicserialize.dstypes.') + dsObjTypes = {} + def buildObjMap(module): - if module.__dict__.has_key('__all__'): + if '__all__' in module.__dict__: for i in module.__all__: name = module.__name__ + '.' + i __import__(name) @@ -59,7 +66,7 @@ def buildObjMap(module): clzName = module.__name__[module.__name__.rfind('.') + 1:] clz = module.__dict__[clzName] tname = module.__name__ - tname = tname.replace('dynamicserialize.dstypes.', '') + tname = tname[DS_LEN:] dsObjTypes[tname] = clz buildObjMap(dstypes) @@ -72,7 +79,7 @@ pythonToThriftMap = { types.DictionaryType: TType.MAP, type(set([])): TType.SET, types.FloatType: SelfDescribingBinaryProtocol.FLOAT, - #types.FloatType: TType.DOUBLE, + # types.FloatType: TType.DOUBLE, types.BooleanType: TType.BOOL, types.InstanceType: TType.STRUCT, types.NoneType: TType.VOID, @@ -87,7 +94,9 @@ pythonToThriftMap = { numpy.int64: TType.I64 } -primitiveSupport = (TType.BYTE, TType.I16, TType.I32, TType.I64, SelfDescribingBinaryProtocol.FLOAT, TType.DOUBLE) +primitiveSupport = (TType.BYTE, TType.I16, TType.I32, TType.I64, + SelfDescribingBinaryProtocol.FLOAT, TType.DOUBLE) + class ThriftSerializationContext(object): @@ -95,52 +104,51 @@ class ThriftSerializationContext(object): self.serializationManager = serializationManager self.protocol = selfDescribingBinaryProtocol self.typeDeserializationMethod = { - TType.STRING: self.protocol.readString, - TType.I16: self.protocol.readI16, - TType.I32: self.protocol.readI32, - TType.LIST: self._deserializeArray, - TType.MAP: self._deserializeMap, - TType.SET: self._deserializeSet, - SelfDescribingBinaryProtocol.FLOAT: self.protocol.readFloat, - TType.BYTE: self.protocol.readByte, - TType.I64: self.protocol.readI64, - TType.DOUBLE: self.protocol.readDouble, - TType.BOOL: self.protocol.readBool, - TType.STRUCT: self.deserializeMessage, - TType.VOID: lambda: None - } + TType.STRING: self.protocol.readString, + TType.I16: self.protocol.readI16, + TType.I32: self.protocol.readI32, + TType.LIST: self._deserializeArray, + TType.MAP: self._deserializeMap, + TType.SET: self._deserializeSet, + SelfDescribingBinaryProtocol.FLOAT: self.protocol.readFloat, + TType.BYTE: self.protocol.readByte, + TType.I64: self.protocol.readI64, + TType.DOUBLE: self.protocol.readDouble, + TType.BOOL: self.protocol.readBool, + TType.STRUCT: self.deserializeMessage, + TType.VOID: lambda: None + } self.typeSerializationMethod = { - TType.STRING: self.protocol.writeString, - TType.I16: self.protocol.writeI16, - TType.I32: self.protocol.writeI32, - TType.LIST: self._serializeArray, - TType.MAP: self._serializeMap, - TType.SET: self._serializeSet, - SelfDescribingBinaryProtocol.FLOAT: self.protocol.writeFloat, - TType.BYTE: self.protocol.writeByte, - TType.I64: self.protocol.writeI64, - TType.DOUBLE: self.protocol.writeDouble, - TType.BOOL: self.protocol.writeBool, - TType.STRUCT: self.serializeMessage, - TType.VOID: lambda x: None - } + TType.STRING: self.protocol.writeString, + TType.I16: self.protocol.writeI16, + TType.I32: self.protocol.writeI32, + TType.LIST: self._serializeArray, + TType.MAP: self._serializeMap, + TType.SET: self._serializeSet, + SelfDescribingBinaryProtocol.FLOAT: self.protocol.writeFloat, + TType.BYTE: self.protocol.writeByte, + TType.I64: self.protocol.writeI64, + TType.DOUBLE: self.protocol.writeDouble, + TType.BOOL: self.protocol.writeBool, + TType.STRUCT: self.serializeMessage, + TType.VOID: lambda x: None + } self.listDeserializationMethod = { - TType.BYTE: self.protocol.readI8List, - TType.I16: self.protocol.readI16List, - TType.I32: self.protocol.readI32List, - TType.I64: self.protocol.readI64List, - SelfDescribingBinaryProtocol.FLOAT: self.protocol.readF32List, - TType.DOUBLE: self.protocol.readF64List - } + TType.BYTE: self.protocol.readI8List, + TType.I16: self.protocol.readI16List, + TType.I32: self.protocol.readI32List, + TType.I64: self.protocol.readI64List, + SelfDescribingBinaryProtocol.FLOAT: self.protocol.readF32List, + TType.DOUBLE: self.protocol.readF64List + } self.listSerializationMethod = { - TType.BYTE: self.protocol.writeI8List, - TType.I16: self.protocol.writeI16List, - TType.I32: self.protocol.writeI32List, - TType.I64: self.protocol.writeI64List, - SelfDescribingBinaryProtocol.FLOAT: self.protocol.writeF32List, - TType.DOUBLE: self.protocol.writeF64List - } - + TType.BYTE: self.protocol.writeI8List, + TType.I16: self.protocol.writeI16List, + TType.I32: self.protocol.writeI32List, + TType.I64: self.protocol.writeI64List, + SelfDescribingBinaryProtocol.FLOAT: self.protocol.writeF32List, + TType.DOUBLE: self.protocol.writeF64List + } def readMessageStart(self): msg = self.protocol.readMessageBegin() @@ -151,17 +159,19 @@ class ThriftSerializationContext(object): def deserializeMessage(self): name = self.protocol.readStructBegin() - name = name.replace('_', '.') if name.isdigit(): obj = self._deserializeType(int(name)) return obj - elif adapters.classAdapterRegistry.has_key(name): + name = name.replace('_', '.') + if name in adapters.classAdapterRegistry: return adapters.classAdapterRegistry[name].deserialize(self) - elif name.find('$') > -1: - # it's an inner class, we're going to hope it's an enum, treat it special + elif '$' in name: + # it's an inner class, we're going to hope it's an enum, treat it + # special fieldName, fieldType, fieldId = self.protocol.readFieldBegin() if fieldName != '__enumValue__': - raise dynamiceserialize.SerializationException("Expected to find enum payload. Found: " + fieldName) + raise dynamiceserialize.SerializationException( + "Expected to find enum payload. Found: " + fieldName) obj = self.protocol.readString() self.protocol.readFieldEnd() return obj @@ -176,37 +186,30 @@ class ThriftSerializationContext(object): return obj def _deserializeType(self, b): - if self.typeDeserializationMethod.has_key(b): + try: return self.typeDeserializationMethod[b]() - else: - raise dynamicserialize.SerializationException("Unsupported type value " + str(b)) - + except KeyError: + raise dynamicserialize.SerializationException( + "Unsupported type value " + str(b)) def _deserializeField(self, structname, obj): fieldName, fieldType, fieldId = self.protocol.readFieldBegin() if fieldType == TType.STOP: return False elif fieldType != TType.VOID: -# if adapters.fieldAdapterRegistry.has_key(structname) and adapters.fieldAdapterRegistry[structname].has_key(fieldName): -# result = adapters.fieldAdapterRegistry[structname][fieldName].deserialize(self) -# else: result = self._deserializeType(fieldType) lookingFor = "set" + fieldName[0].upper() + fieldName[1:] try: setMethod = getattr(obj, lookingFor) - - if callable(setMethod): - setMethod(result) - else: - raise dynamicserialize.SerializationException("Couldn't find setter method " + lookingFor) + setMethod(result) except: - raise dynamicserialize.SerializationException("Couldn't find setter method " + lookingFor) + raise dynamicserialize.SerializationException( + "Couldn't find setter method " + lookingFor) self.protocol.readFieldEnd() return True - def _deserializeArray(self): listType, size = self.protocol.readListBegin() result = [] @@ -241,19 +244,20 @@ class ThriftSerializationContext(object): def _lookupType(self, obj): pyt = type(obj) - if pythonToThriftMap.has_key(pyt): + if pyt in pythonToThriftMap: return pythonToThriftMap[pyt] - elif pyt.__module__.startswith('dynamicserialize.dstypes'): + elif pyt.__module__[:DS_LEN - 1] == ('dynamicserialize.dstypes'): return pythonToThriftMap[types.InstanceType] else: - raise dynamicserialize.SerializationException("Don't know how to serialize object of type: " + str(pyt)) + raise dynamicserialize.SerializationException( + "Don't know how to serialize object of type: " + str(pyt)) def serializeMessage(self, obj): tt = self._lookupType(obj) if tt == TType.STRUCT: - fqn = obj.__module__.replace('dynamicserialize.dstypes.', '') - if adapters.classAdapterRegistry.has_key(fqn): + fqn = obj.__module__[DS_LEN:] + if fqn in adapters.classAdapterRegistry: # get proper class name when writing class name to serialization stream # in case we have a special inner-class case m = sys.modules[adapters.classAdapterRegistry[fqn].__name__] @@ -273,7 +277,7 @@ class ThriftSerializationContext(object): val = m[1]() ft = self._lookupType(val) if ft == TType.STRUCT: - fc = val.__module__.replace('dynamicserialize.dstypes.', '') + fc = val.__module__[DS_LEN:] self._serializeField(fieldname, ft, fid, val) else: self._serializeField(fieldname, ft, fid, val) @@ -293,17 +297,18 @@ class ThriftSerializationContext(object): self.protocol.writeFieldEnd() def _serializeType(self, fieldValue, fieldType): - if self.typeSerializationMethod.has_key(fieldType): + if fieldType in self.typeSerializationMethod: return self.typeSerializationMethod[fieldType](fieldValue) else: - raise dynamicserialize.SerializationException("Unsupported type value " + str(fieldType)) + raise dynamicserialize.SerializationException( + "Unsupported type value " + str(fieldType)) def _serializeArray(self, obj): size = len(obj) if size: if type(obj) is numpy.ndarray: - t = pythonToThriftMap[obj.dtype.type] - size = obj.size + t = pythonToThriftMap[obj.dtype.type] + size = obj.size else: t = self._lookupType(obj[0]) else: @@ -331,7 +336,6 @@ class ThriftSerializationContext(object): self.listSerializationMethod[t](obj) self.protocol.writeListEnd() - def _serializeMap(self, obj): size = len(obj) self.protocol.writeMapBegin(TType.VOID, TType.VOID, size) diff --git a/dynamicserialize/adapters/__init__.py b/dynamicserialize/adapters/__init__.py index d7e643f..f72aaa8 100644 --- a/dynamicserialize/adapters/__init__.py +++ b/dynamicserialize/adapters/__init__.py @@ -1,19 +1,19 @@ ## # This software was developed and / or modified by Raytheon Company, # pursuant to Contract DG133W-05-CQ-1067 with the US Government. -# +# # U.S. EXPORT CONTROLLED TECHNICAL DATA # This software product contains export-restricted data whose # export/transfer/disclosure is restricted by U.S. law. Dissemination # to non-U.S. persons whether in the United States or abroad requires # an export license or other authorization. -# +# # Contractor Name: Raytheon Company # Contractor Address: 6825 Pine Street, Suite 340 # Mail Stop B8 # Omaha, NE 68106 # 402.291.0100 -# +# # See the AWIPS II Master Rights File ("Master Rights File.pdf") for # further licensing information. ## @@ -21,56 +21,80 @@ # # __init__.py for Dynamic Serialize adapters. -# -# +# +# Plugins can contribute to dynamicserialize.adapters by either including their +# classes directly in pythonPackages/dynamicserialize/adapters/ within their +# plugin. The plugin's adapter will automatically be added to __all__ at runtime +# and registered. +# Plugins should not include a custom __init__.py in +# pythonPackages/dynamicserialize/adapters/ because it will overwrite this file. +# If custom package initialization is needed, a subpackage should be created +# with an __init__.py that includes the following: +# +# __all__ = ['CustomAdapter1', 'CustomAdapter2'] +# from dynamicserialize.adapters import registerAdapters +# registerAdapters(__name__, __all__) +# +# # SOFTWARE HISTORY -# +# # Date Ticket# Engineer Description # ------------ ---------- ----------- -------------------------- -# 08/31/10 njensen Initial Creation. -# 03/20/13 #1774 randerso Added TimeConstraintsAdapter -# 04/22/13 #1949 rjpeter Added LockTableAdapter -# 02/06/14 #2672 bsteffen Added JTSEnvelopeAdapter -# 06/22/2015 #4573 randerso Added JobProgressAdapter -# 09/21/2015 #4486 rjpeter Added FormattedDateAdapter -# 06/23/2016 #5696 rjpeter Added CommutativeTimestampAdapter +# 08/31/10 njensen Initial Creation. +# 03/20/13 #1774 randerso Added TimeConstraintsAdapter +# 04/22/13 #1949 rjpeter Added LockTableAdapter +# 02/06/14 #2672 bsteffen Added JTSEnvelopeAdapter +# 06/22/2015 #4573 randerso Added JobProgressAdapter +# 09/21/2015 #4486 rjpeter Added FormattedDateAdapter +# 06/23/2016 #5696 rjpeter Added CommutativeTimestampAdapter +# 10/17/2016 #5919 njensen Added GeomDataRespAdapter +# 01/09/2017 #5997 nabowle Allow contribution from plugins. # __all__ = [ - 'PointAdapter', - 'StackTraceElementAdapter', - 'WsIdAdapter', - 'CalendarAdapter', - 'GregorianCalendarAdapter', - 'ActiveTableModeAdapter', - 'DateAdapter', - 'FormattedDateAdapter', - 'LocalizationLevelSerializationAdapter', - 'LocalizationTypeSerializationAdapter', - 'GeometryTypeAdapter', - 'CoordAdapter', - 'TimeRangeTypeAdapter', - 'ParmIDAdapter', - 'DatabaseIDAdapter', - 'TimestampAdapter', - 'CommutativeTimestampAdapter', - 'EnumSetAdapter', - 'FloatBufferAdapter', - 'ByteBufferAdapter', - 'TimeConstraintsAdapter', - 'LockTableAdapter', - 'JTSEnvelopeAdapter', - 'JobProgressAdapter', - ] - + 'PointAdapter', + 'StackTraceElementAdapter', + 'CalendarAdapter', + 'GregorianCalendarAdapter', + 'DateAdapter', + 'GeometryTypeAdapter', + 'CoordAdapter', + 'TimestampAdapter', + 'EnumSetAdapter', + 'FloatBufferAdapter', + 'ByteBufferAdapter', + 'JTSEnvelopeAdapter' +] + classAdapterRegistry = {} - + def getAdapterRegistry(): - import sys - for x in __all__: - exec 'import ' + x - m = sys.modules['dynamicserialize.adapters.' + x] + import pkgutil + + discoveredPackages = [] + # allow other plugins to contribute to adapters by dropping their adapter or + # package into the dynamicserialize.adapters package + for _, modname, ispkg in pkgutil.iter_modules(__path__): + if ispkg: + discoveredPackages.append(modname) + else: + if modname not in __all__: + __all__.append(modname) + + registerAdapters(__name__, __all__) + + for pkg in discoveredPackages: + __import__(__name__ + '.' + pkg) + + +def registerAdapters(package, modules): + import sys + if not package.endswith('.'): + package += '.' + for x in modules: + exec 'import ' + package + x + m = sys.modules[package + x] d = m.__dict__ if d.has_key('ClassAdapter'): if isinstance(m.ClassAdapter, list): @@ -80,9 +104,8 @@ def getAdapterRegistry(): clzName = m.ClassAdapter classAdapterRegistry[clzName] = m else: - raise LookupError('Adapter class ' + x + ' has no ClassAdapter field ' + \ + raise LookupError('Adapter class ' + x + ' has no ClassAdapter field ' + 'and cannot be registered.') - + getAdapterRegistry() - diff --git a/dynamicserialize/dstypes/com/raytheon/uf/common/__init__.py b/dynamicserialize/dstypes/com/raytheon/uf/common/__init__.py index dfd5b4a..40e4cff 100644 --- a/dynamicserialize/dstypes/com/raytheon/uf/common/__init__.py +++ b/dynamicserialize/dstypes/com/raytheon/uf/common/__init__.py @@ -21,22 +21,7 @@ # File auto-generated by PythonFileGenerator __all__ = [ - 'activetable', - 'alertviz', - 'auth', - 'dataaccess', - 'dataplugin', - 'dataquery', - 'datastorage', - 'localization', - 'management', - 'message', - 'plugin', - 'pointdata', - 'pypies', - 'serialization', - 'site', - 'time' + 'dataplugin' ] diff --git a/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/__init__.py b/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/__init__.py index 006c19d..acb7c9b 100644 --- a/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/__init__.py +++ b/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/__init__.py @@ -21,13 +21,7 @@ # File auto-generated by PythonFileGenerator __all__ = [ - 'events', - 'gfe', - 'grid', - 'level', - 'message', - 'radar', - 'text' + 'events' ]