Revert "changes for 17.3.1"

This reverts commit 8c3145b63e.
This commit is contained in:
mjames-upc 2017-11-06 13:47:15 -07:00
parent 8c3145b63e
commit 5c51ad599e
10 changed files with 168 additions and 308 deletions

View file

@ -30,12 +30,9 @@
# 11/17/10 njensen Initial Creation. # 11/17/10 njensen Initial Creation.
# 08/15/13 2169 bkowal Optionally gzip decompress any data that is read. # 08/15/13 2169 bkowal Optionally gzip decompress any data that is read.
# 08/04/16 2416 tgurney Add queueStarted property # 08/04/16 2416 tgurney Add queueStarted property
# 02/16/17 6084 bsteffen Support ssl connections
# #
# #
import os
import os.path
import qpid import qpid
import zlib import zlib
@ -44,24 +41,11 @@ from qpid.exceptions import Closed
class QpidSubscriber: class QpidSubscriber:
def __init__(self, host='127.0.0.1', port=5672, decompress=False, ssl=None): def __init__(self, host='127.0.0.1', port=5672, decompress=False):
self.host = host self.host = host
self.port = port self.port = port
self.decompress = decompress; self.decompress = decompress;
socket = qpid.util.connect(host, port) socket = qpid.util.connect(host, port)
if "QPID_SSL_CERT_DB" in os.environ:
certdb = os.environ["QPID_SSL_CERT_DB"]
else:
certdb = os.path.expanduser("~/.qpid/")
if "QPID_SSL_CERT_NAME" in os.environ:
certname = os.environ["QPID_SSL_CERT_NAME"]
else:
certname = "guest"
certfile = os.path.join(certdb, certname + ".crt")
if ssl or (ssl is None and os.path.exists(certfile)):
keyfile = os.path.join(certdb, certname + ".key")
trustfile = os.path.join(certdb, "root.crt")
socket = qpid.util.ssl(socket, keyfile=keyfile, certfile=certfile, ca_certs=trustfile)
self.__connection = qpid.connection.Connection(sock=socket, username='guest', password='guest') self.__connection = qpid.connection.Connection(sock=socket, username='guest', password='guest')
self.__connection.start() self.__connection.start()
self.__session = self.__connection.session(str(qpid.datatypes.uuid4())) self.__session = self.__connection.session(str(qpid.datatypes.uuid4()))
@ -119,4 +103,3 @@ class QpidSubscriber:
@property @property
def queueStarted(self): def queueStarted(self):
return self.__queueStarted return self.__queueStarted

View file

@ -17,23 +17,13 @@
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for # See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information. # further licensing information.
## ##
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------- -------- --------- ---------------------------------------------
# Feb 13, 2017 6092 randerso Added StoreTimeAction
#
##
import argparse import argparse
import sys import sys
import time
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import DatabaseID from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import DatabaseID
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import ParmID from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import ParmID
TIME_FORMAT = "%Y%m%d_%H%M"
class UsageArgumentParser(argparse.ArgumentParser): class UsageArgumentParser(argparse.ArgumentParser):
""" """
@ -66,16 +56,3 @@ class AppendParmNameAndLevelAction(argparse.Action):
else: else:
setattr(namespace, self.dest, [comp]) setattr(namespace, self.dest, [comp])
class StoreTimeAction(argparse.Action):
"""
argparse.Action subclass to validate GFE formatted time strings
and parse them to time.struct_time
"""
def __call__(self, parser, namespace, values, option_string=None):
try:
timeStruct = time.strptime(values, TIME_FORMAT)
except:
parser.error(str(values) + " is not a valid time string of the format YYYYMMDD_hhmm")
setattr(namespace, self.dest, timeStruct)

View file

@ -40,8 +40,8 @@
# Jun 01, 2016 5587 tgurney Add new signatures for # Jun 01, 2016 5587 tgurney Add new signatures for
# getRequiredIdentifiers() and # getRequiredIdentifiers() and
# getOptionalIdentifiers() # getOptionalIdentifiers()
# Oct 07, 2016 ---- mjames@ucar Added getForecastRun # 10/07/16 ---- mjames@ucar Added getForecastRun
# Oct 18, 2016 5916 bsteffen Add setLazyLoadGridLatLon #
# #
# #
@ -53,7 +53,6 @@ import warnings
THRIFT_HOST = "edex" THRIFT_HOST = "edex"
USING_NATIVE_THRIFT = False USING_NATIVE_THRIFT = False
if sys.modules.has_key('jep'): if sys.modules.has_key('jep'):
# intentionally do not catch if this fails to import, we want it to # intentionally do not catch if this fails to import, we want it to
# be obvious that something is configured wrong when running from within # be obvious that something is configured wrong when running from within
@ -253,26 +252,3 @@ def changeEDEXHost(newHostName):
router = ThriftClientRouter.ThriftClientRouter(THRIFT_HOST) router = ThriftClientRouter.ThriftClientRouter(THRIFT_HOST)
else: else:
raise TypeError("Cannot call changeEDEXHost when using JepRouter.") raise TypeError("Cannot call changeEDEXHost when using JepRouter.")
def setLazyLoadGridLatLon(lazyLoadGridLatLon):
"""
Provide a hint to the Data Access Framework indicating whether to load the
lat/lon data for a grid immediately or wait until it is needed. This is
provided as a performance tuning hint and should not affect the way the
Data Access Framework is used. Depending on the internal implementation of
the Data Access Framework this hint might be ignored. Examples of when this
should be set to True are when the lat/lon information is not used or when
it is used only if certain conditions within the data are met. It could be
set to False if it is guaranteed that all lat/lon information is needed and
it would be better to get any performance overhead for generating the
lat/lon data out of the way during the initial request.
Args:
lazyLoadGridLatLon: Boolean value indicating whether to lazy load.
"""
try:
router.setLazyLoadGridLatLon(lazyLoadGridLatLon)
except AttributeError:
# The router is not required to support this capability.
pass

View file

@ -28,9 +28,7 @@
# Date Ticket# Engineer Description # Date Ticket# Engineer Description
# ------------ ---------- ----------- -------------------------- # ------------ ---------- ----------- --------------------------
# 06/03/13 #2023 dgilling Initial Creation. # 06/03/13 #2023 dgilling Initial Creation.
# 10/13/16 #5916 bsteffen Correct grid shape, allow lat/lon
# 11/10/16 #5900 bsteffen Correct grid shape # 11/10/16 #5900 bsteffen Correct grid shape
# to be requested by a delegate
# #
# #
@ -48,7 +46,7 @@ The ability to unit convert grid data is not currently available in this version
class PyGridData(IGridData, PyData.PyData): class PyGridData(IGridData, PyData.PyData):
def __init__(self, gridDataRecord, nx, ny, latLonGrid = None, latLonDelegate = None): def __init__(self, gridDataRecord, nx, ny, latLonGrid):
PyData.PyData.__init__(self, gridDataRecord) PyData.PyData.__init__(self, gridDataRecord)
nx = nx nx = nx
ny = ny ny = ny
@ -56,8 +54,6 @@ class PyGridData(IGridData, PyData.PyData):
self.__unit = gridDataRecord.getUnit() self.__unit = gridDataRecord.getUnit()
self.__gridData = numpy.reshape(numpy.array(gridDataRecord.getGridData()), (ny, nx)) self.__gridData = numpy.reshape(numpy.array(gridDataRecord.getGridData()), (ny, nx))
self.__latLonGrid = latLonGrid self.__latLonGrid = latLonGrid
self.__latLonDelegate = latLonDelegate
def getParameter(self): def getParameter(self):
return self.__parameter return self.__parameter
@ -74,8 +70,4 @@ class PyGridData(IGridData, PyData.PyData):
return self.__gridData return self.__gridData
def getLatLonCoords(self): def getLatLonCoords(self):
if self.__latLonGrid is not None:
return self.__latLonGrid
elif self.__latLonDelegate is not None:
return self.__latLonDelegate()
return self.__latLonGrid return self.__latLonGrid

View file

@ -39,8 +39,7 @@
# getRequiredIdentifiers() and # getRequiredIdentifiers() and
# getOptionalIdentifiers() # getOptionalIdentifiers()
# 08/01/16 2416 tgurney Add getNotificationFilter() # 08/01/16 2416 tgurney Add getNotificationFilter()
# 10/13/16 5916 bsteffen Correct grid shape, allow lazy grid lat/lon # 11/10/16 5900 bsteffen Correct grid shape
# 10/26/16 5919 njensen Speed up geometry creation in getGeometryData()
# #
@ -52,7 +51,6 @@ from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import G
from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableTimesRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableTimesRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetGeometryDataRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetGeometryDataRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetGridDataRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetGridDataRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetGridLatLonRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableParametersRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableParametersRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableLevelsRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableLevelsRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetRequiredIdentifiersRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetRequiredIdentifiersRequest
@ -66,39 +64,10 @@ from awips.dataaccess import PyGeometryData
from awips.dataaccess import PyGridData from awips.dataaccess import PyGridData
class LazyGridLatLon(object):
def __init__(self, client, nx, ny, envelope, crsWkt):
self._latLonGrid = None
self._client = client
self._request = GetGridLatLonRequest()
self._request.setNx(nx)
self._request.setNy(ny)
self._request.setEnvelope(envelope)
self._request.setCrsWkt(crsWkt)
def __call__(self):
# Its important that the data is cached internally so that if multiple
# GridData are sharing the same delegate then they can also share a
# single request for the LatLon information.
if self._latLonGrid is None:
response = self._client.sendRequest(self._request)
nx = response.getNx()
ny = response.getNy()
latData = numpy.reshape(numpy.array(response.getLats()), (ny, nx))
lonData = numpy.reshape(numpy.array(response.getLons()), (ny, nx))
self._latLonGrid = (lonData, latData)
return self._latLonGrid
class ThriftClientRouter(object): class ThriftClientRouter(object):
def __init__(self, host='localhost'): def __init__(self, host='localhost'):
self._client = ThriftClient.ThriftClient(host) self._client = ThriftClient.ThriftClient(host)
self._lazyLoadGridLatLon = False
def setLazyLoadGridLatLon(self, lazyLoadGridLatLon):
self._lazyLoadGridLatLon = lazyLoadGridLatLon
def getAvailableTimes(self, request, refTimeOnly): def getAvailableTimes(self, request, refTimeOnly):
timesRequest = GetAvailableTimesRequest() timesRequest = GetAvailableTimesRequest()
@ -109,7 +78,6 @@ class ThriftClientRouter(object):
def getGridData(self, request, times): def getGridData(self, request, times):
gridDataRequest = GetGridDataRequest() gridDataRequest = GetGridDataRequest()
gridDataRequest.setIncludeLatLonData(not self._lazyLoadGridLatLon)
gridDataRequest.setRequestParameters(request) gridDataRequest.setRequestParameters(request)
# if we have an iterable times instance, then the user must have asked # if we have an iterable times instance, then the user must have asked
# for grid data with the List of DataTime objects # for grid data with the List of DataTime objects
@ -127,28 +95,15 @@ class ThriftClientRouter(object):
for location in locNames: for location in locNames:
nx = response.getSiteNxValues()[location] nx = response.getSiteNxValues()[location]
ny = response.getSiteNyValues()[location] ny = response.getSiteNyValues()[location]
if self._lazyLoadGridLatLon: latData = numpy.reshape(numpy.array(response.getSiteLatGrids()[location]), (ny, nx))
envelope = response.getSiteEnvelopes()[location] lonData = numpy.reshape(numpy.array(response.getSiteLonGrids()[location]), (ny, nx))
crsWkt = response.getSiteCrsWkt()[location] locSpecificData[location] = (nx, ny, (lonData, latData))
delegate = LazyGridLatLon(
self._client, nx, ny, envelope, crsWkt)
locSpecificData[location] = (nx, ny, delegate)
else:
latData = numpy.reshape(numpy.array(
response.getSiteLatGrids()[location]), (ny, nx))
lonData = numpy.reshape(numpy.array(
response.getSiteLonGrids()[location]), (ny, nx))
locSpecificData[location] = (nx, ny, (lonData, latData))
retVal = [] retVal = []
for gridDataRecord in response.getGridData(): for gridDataRecord in response.getGridData():
locationName = gridDataRecord.getLocationName() locationName = gridDataRecord.getLocationName()
locData = locSpecificData[locationName] locData = locSpecificData[locationName]
if self._lazyLoadGridLatLon: retVal.append(PyGridData.PyGridData(gridDataRecord, locData[0], locData[1], locData[2]))
retVal.append(PyGridData.PyGridData(gridDataRecord, locData[
0], locData[1], latLonDelegate=locData[2]))
else:
retVal.append(PyGridData.PyGridData(
gridDataRecord, locData[0], locData[1], locData[2]))
return retVal return retVal
def getGeometryData(self, request, times): def getGeometryData(self, request, times):
@ -166,9 +121,10 @@ class ThriftClientRouter(object):
response = self._client.sendRequest(geoDataRequest) response = self._client.sendRequest(geoDataRequest)
geometries = [] geometries = []
for wkb in response.getGeometryWKBs(): for wkb in response.getGeometryWKBs():
# the wkb is a numpy.ndarray of dtype int8 # convert the wkb to a bytearray with only positive values
# convert the bytearray to a byte string and load it byteArrWKB = bytearray(map(lambda x: x % 256,wkb.tolist()))
geometries.append(shapely.wkb.loads(wkb.tostring())) # convert the bytearray to a byte string and load it.
geometries.append(shapely.wkb.loads(str(byteArrWKB)))
retVal = [] retVal = []
for geoDataRecord in response.getGeoData(): for geoDataRecord in response.getGeoData():
@ -219,7 +175,7 @@ class ThriftClientRouter(object):
response = self._client.sendRequest(idValReq) response = self._client.sendRequest(idValReq)
return response return response
def newDataRequest(self, datatype, parameters=[], levels=[], locationNames=[], envelope=None, **kwargs): def newDataRequest(self, datatype, parameters=[], levels=[], locationNames = [], envelope=None, **kwargs):
req = DefaultDataRequest() req = DefaultDataRequest()
if datatype: if datatype:
req.setDatatype(datatype) req.setDatatype(datatype)
@ -244,4 +200,4 @@ class ThriftClientRouter(object):
notifReq = GetNotificationFilterRequest() notifReq = GetNotificationFilterRequest()
notifReq.setRequestParameters(request) notifReq.setRequestParameters(request)
response = self._client.sendRequest(notifReq) response = self._client.sendRequest(notifReq)
return response return response

View file

@ -59,13 +59,9 @@
# .... # ....
# 06/13/2013 DR 16242 D. Friedman Add Qpid authentication info # 06/13/2013 DR 16242 D. Friedman Add Qpid authentication info
# 03/06/2014 DR 17907 D. Friedman Workaround for issue QPID-5569 # 03/06/2014 DR 17907 D. Friedman Workaround for issue QPID-5569
# 02/16/2017 DR 6084 bsteffen Support ssl connections
# #
#=============================================================================== #===============================================================================
import os
import os.path
import qpid import qpid
from qpid.util import connect from qpid.util import connect
from qpid.connection import Connection from qpid.connection import Connection
@ -75,31 +71,17 @@ QPID_USERNAME = 'guest'
QPID_PASSWORD = 'guest' QPID_PASSWORD = 'guest'
class IngestViaQPID: class IngestViaQPID:
def __init__(self, host='localhost', port=5672, ssl=None): def __init__(self, host='localhost', port=5672):
''' '''
Connect to QPID and make bindings to route message to external.dropbox queue Connect to QPID and make bindings to route message to external.dropbox queue
@param host: string hostname of computer running EDEX and QPID (default localhost) @param host: string hostname of computer running EDEX and QPID (default localhost)
@param port: integer port used to connect to QPID (default 5672) @param port: integer port used to connect to QPID (default 5672)
@param ssl: boolean to determine whether ssl is used, default value of None will use ssl only if a client certificate is found.
''' '''
try: try:
# #
socket = connect(host, port) self.socket = connect(host, port)
if "QPID_SSL_CERT_DB" in os.environ: self.connection = Connection (sock=self.socket, username=QPID_USERNAME, password=QPID_PASSWORD)
certdb = os.environ["QPID_SSL_CERT_DB"]
else:
certdb = os.path.expanduser("~/.qpid/")
if "QPID_SSL_CERT_NAME" in os.environ:
certname = os.environ["QPID_SSL_CERT_NAME"]
else:
certname = QPID_USERNAME
certfile = os.path.join(certdb, certname + ".crt")
if ssl or (ssl is None and os.path.exists(certfile)):
keyfile = os.path.join(certdb, certname + ".key")
trustfile = os.path.join(certdb, "root.crt")
socket = qpid.util.ssl(socket, keyfile=keyfile, certfile=certfile, ca_certs=trustfile)
self.connection = Connection (sock=socket, username=QPID_USERNAME, password=QPID_PASSWORD)
self.connection.start() self.connection.start()
self.session = self.connection.session(str(uuid4())) self.session = self.connection.session(str(uuid4()))
self.session.exchange_bind(exchange='amq.direct', queue='external.dropbox', binding_key='external.dropbox') self.session.exchange_bind(exchange='amq.direct', queue='external.dropbox', binding_key='external.dropbox')

View file

@ -29,7 +29,7 @@
# match what they should be in the destination language. # match what they should be in the destination language.
# #
# #
# SOFTWARE HISTORY # SOFTWARE HISTORY
# #
# Date Ticket# Engineer Description # Date Ticket# Engineer Description
# ------------ ---------- ----------- -------------------------- # ------------ ---------- ----------- --------------------------
@ -37,27 +37,20 @@
# 06/12/13 #2099 dgilling Implement readObject() and # 06/12/13 #2099 dgilling Implement readObject() and
# writeObject(). # writeObject().
# Apr 24, 2015 4425 nabowle Add Double support # Apr 24, 2015 4425 nabowle Add Double support
# Oct 17, 2016 5919 njensen Optimized for speed
# #
# #
from thrift.Thrift import TType from thrift.Thrift import TType
import inspect import inspect, sys, types
import sys
import types
import time
import dynamicserialize import dynamicserialize
from dynamicserialize import dstypes, adapters from dynamicserialize import dstypes, adapters
import SelfDescribingBinaryProtocol import SelfDescribingBinaryProtocol
import numpy import numpy
DS_LEN = len('dynamicserialize.dstypes.')
dsObjTypes = {} dsObjTypes = {}
def buildObjMap(module): def buildObjMap(module):
if '__all__' in module.__dict__: if module.__dict__.has_key('__all__'):
for i in module.__all__: for i in module.__all__:
name = module.__name__ + '.' + i name = module.__name__ + '.' + i
__import__(name) __import__(name)
@ -66,7 +59,7 @@ def buildObjMap(module):
clzName = module.__name__[module.__name__.rfind('.') + 1:] clzName = module.__name__[module.__name__.rfind('.') + 1:]
clz = module.__dict__[clzName] clz = module.__dict__[clzName]
tname = module.__name__ tname = module.__name__
tname = tname[DS_LEN:] tname = tname.replace('dynamicserialize.dstypes.', '')
dsObjTypes[tname] = clz dsObjTypes[tname] = clz
buildObjMap(dstypes) buildObjMap(dstypes)
@ -79,7 +72,7 @@ pythonToThriftMap = {
types.DictionaryType: TType.MAP, types.DictionaryType: TType.MAP,
type(set([])): TType.SET, type(set([])): TType.SET,
types.FloatType: SelfDescribingBinaryProtocol.FLOAT, types.FloatType: SelfDescribingBinaryProtocol.FLOAT,
# types.FloatType: TType.DOUBLE, #types.FloatType: TType.DOUBLE,
types.BooleanType: TType.BOOL, types.BooleanType: TType.BOOL,
types.InstanceType: TType.STRUCT, types.InstanceType: TType.STRUCT,
types.NoneType: TType.VOID, types.NoneType: TType.VOID,
@ -94,9 +87,7 @@ pythonToThriftMap = {
numpy.int64: TType.I64 numpy.int64: TType.I64
} }
primitiveSupport = (TType.BYTE, TType.I16, TType.I32, TType.I64, primitiveSupport = (TType.BYTE, TType.I16, TType.I32, TType.I64, SelfDescribingBinaryProtocol.FLOAT, TType.DOUBLE)
SelfDescribingBinaryProtocol.FLOAT, TType.DOUBLE)
class ThriftSerializationContext(object): class ThriftSerializationContext(object):
@ -104,51 +95,52 @@ class ThriftSerializationContext(object):
self.serializationManager = serializationManager self.serializationManager = serializationManager
self.protocol = selfDescribingBinaryProtocol self.protocol = selfDescribingBinaryProtocol
self.typeDeserializationMethod = { self.typeDeserializationMethod = {
TType.STRING: self.protocol.readString, TType.STRING: self.protocol.readString,
TType.I16: self.protocol.readI16, TType.I16: self.protocol.readI16,
TType.I32: self.protocol.readI32, TType.I32: self.protocol.readI32,
TType.LIST: self._deserializeArray, TType.LIST: self._deserializeArray,
TType.MAP: self._deserializeMap, TType.MAP: self._deserializeMap,
TType.SET: self._deserializeSet, TType.SET: self._deserializeSet,
SelfDescribingBinaryProtocol.FLOAT: self.protocol.readFloat, SelfDescribingBinaryProtocol.FLOAT: self.protocol.readFloat,
TType.BYTE: self.protocol.readByte, TType.BYTE: self.protocol.readByte,
TType.I64: self.protocol.readI64, TType.I64: self.protocol.readI64,
TType.DOUBLE: self.protocol.readDouble, TType.DOUBLE: self.protocol.readDouble,
TType.BOOL: self.protocol.readBool, TType.BOOL: self.protocol.readBool,
TType.STRUCT: self.deserializeMessage, TType.STRUCT: self.deserializeMessage,
TType.VOID: lambda: None TType.VOID: lambda: None
} }
self.typeSerializationMethod = { self.typeSerializationMethod = {
TType.STRING: self.protocol.writeString, TType.STRING: self.protocol.writeString,
TType.I16: self.protocol.writeI16, TType.I16: self.protocol.writeI16,
TType.I32: self.protocol.writeI32, TType.I32: self.protocol.writeI32,
TType.LIST: self._serializeArray, TType.LIST: self._serializeArray,
TType.MAP: self._serializeMap, TType.MAP: self._serializeMap,
TType.SET: self._serializeSet, TType.SET: self._serializeSet,
SelfDescribingBinaryProtocol.FLOAT: self.protocol.writeFloat, SelfDescribingBinaryProtocol.FLOAT: self.protocol.writeFloat,
TType.BYTE: self.protocol.writeByte, TType.BYTE: self.protocol.writeByte,
TType.I64: self.protocol.writeI64, TType.I64: self.protocol.writeI64,
TType.DOUBLE: self.protocol.writeDouble, TType.DOUBLE: self.protocol.writeDouble,
TType.BOOL: self.protocol.writeBool, TType.BOOL: self.protocol.writeBool,
TType.STRUCT: self.serializeMessage, TType.STRUCT: self.serializeMessage,
TType.VOID: lambda x: None TType.VOID: lambda x: None
} }
self.listDeserializationMethod = { self.listDeserializationMethod = {
TType.BYTE: self.protocol.readI8List, TType.BYTE: self.protocol.readI8List,
TType.I16: self.protocol.readI16List, TType.I16: self.protocol.readI16List,
TType.I32: self.protocol.readI32List, TType.I32: self.protocol.readI32List,
TType.I64: self.protocol.readI64List, TType.I64: self.protocol.readI64List,
SelfDescribingBinaryProtocol.FLOAT: self.protocol.readF32List, SelfDescribingBinaryProtocol.FLOAT: self.protocol.readF32List,
TType.DOUBLE: self.protocol.readF64List TType.DOUBLE: self.protocol.readF64List
} }
self.listSerializationMethod = { self.listSerializationMethod = {
TType.BYTE: self.protocol.writeI8List, TType.BYTE: self.protocol.writeI8List,
TType.I16: self.protocol.writeI16List, TType.I16: self.protocol.writeI16List,
TType.I32: self.protocol.writeI32List, TType.I32: self.protocol.writeI32List,
TType.I64: self.protocol.writeI64List, TType.I64: self.protocol.writeI64List,
SelfDescribingBinaryProtocol.FLOAT: self.protocol.writeF32List, SelfDescribingBinaryProtocol.FLOAT: self.protocol.writeF32List,
TType.DOUBLE: self.protocol.writeF64List TType.DOUBLE: self.protocol.writeF64List
} }
def readMessageStart(self): def readMessageStart(self):
msg = self.protocol.readMessageBegin() msg = self.protocol.readMessageBegin()
@ -159,19 +151,17 @@ class ThriftSerializationContext(object):
def deserializeMessage(self): def deserializeMessage(self):
name = self.protocol.readStructBegin() name = self.protocol.readStructBegin()
name = name.replace('_', '.')
if name.isdigit(): if name.isdigit():
obj = self._deserializeType(int(name)) obj = self._deserializeType(int(name))
return obj return obj
name = name.replace('_', '.') elif adapters.classAdapterRegistry.has_key(name):
if name in adapters.classAdapterRegistry:
return adapters.classAdapterRegistry[name].deserialize(self) return adapters.classAdapterRegistry[name].deserialize(self)
elif '$' in name: elif name.find('$') > -1:
# it's an inner class, we're going to hope it's an enum, treat it # it's an inner class, we're going to hope it's an enum, treat it special
# special
fieldName, fieldType, fieldId = self.protocol.readFieldBegin() fieldName, fieldType, fieldId = self.protocol.readFieldBegin()
if fieldName != '__enumValue__': if fieldName != '__enumValue__':
raise dynamiceserialize.SerializationException( raise dynamiceserialize.SerializationException("Expected to find enum payload. Found: " + fieldName)
"Expected to find enum payload. Found: " + fieldName)
obj = self.protocol.readString() obj = self.protocol.readString()
self.protocol.readFieldEnd() self.protocol.readFieldEnd()
return obj return obj
@ -186,30 +176,37 @@ class ThriftSerializationContext(object):
return obj return obj
def _deserializeType(self, b): def _deserializeType(self, b):
try: if self.typeDeserializationMethod.has_key(b):
return self.typeDeserializationMethod[b]() return self.typeDeserializationMethod[b]()
except KeyError: else:
raise dynamicserialize.SerializationException( raise dynamicserialize.SerializationException("Unsupported type value " + str(b))
"Unsupported type value " + str(b))
def _deserializeField(self, structname, obj): def _deserializeField(self, structname, obj):
fieldName, fieldType, fieldId = self.protocol.readFieldBegin() fieldName, fieldType, fieldId = self.protocol.readFieldBegin()
if fieldType == TType.STOP: if fieldType == TType.STOP:
return False return False
elif fieldType != TType.VOID: elif fieldType != TType.VOID:
# if adapters.fieldAdapterRegistry.has_key(structname) and adapters.fieldAdapterRegistry[structname].has_key(fieldName):
# result = adapters.fieldAdapterRegistry[structname][fieldName].deserialize(self)
# else:
result = self._deserializeType(fieldType) result = self._deserializeType(fieldType)
lookingFor = "set" + fieldName[0].upper() + fieldName[1:] lookingFor = "set" + fieldName[0].upper() + fieldName[1:]
try: try:
setMethod = getattr(obj, lookingFor) setMethod = getattr(obj, lookingFor)
setMethod(result)
if callable(setMethod):
setMethod(result)
else:
raise dynamicserialize.SerializationException("Couldn't find setter method " + lookingFor)
except: except:
raise dynamicserialize.SerializationException( raise dynamicserialize.SerializationException("Couldn't find setter method " + lookingFor)
"Couldn't find setter method " + lookingFor)
self.protocol.readFieldEnd() self.protocol.readFieldEnd()
return True return True
def _deserializeArray(self): def _deserializeArray(self):
listType, size = self.protocol.readListBegin() listType, size = self.protocol.readListBegin()
result = [] result = []
@ -244,20 +241,19 @@ class ThriftSerializationContext(object):
def _lookupType(self, obj): def _lookupType(self, obj):
pyt = type(obj) pyt = type(obj)
if pyt in pythonToThriftMap: if pythonToThriftMap.has_key(pyt):
return pythonToThriftMap[pyt] return pythonToThriftMap[pyt]
elif pyt.__module__[:DS_LEN - 1] == ('dynamicserialize.dstypes'): elif pyt.__module__.startswith('dynamicserialize.dstypes'):
return pythonToThriftMap[types.InstanceType] return pythonToThriftMap[types.InstanceType]
else: else:
raise dynamicserialize.SerializationException( raise dynamicserialize.SerializationException("Don't know how to serialize object of type: " + str(pyt))
"Don't know how to serialize object of type: " + str(pyt))
def serializeMessage(self, obj): def serializeMessage(self, obj):
tt = self._lookupType(obj) tt = self._lookupType(obj)
if tt == TType.STRUCT: if tt == TType.STRUCT:
fqn = obj.__module__[DS_LEN:] fqn = obj.__module__.replace('dynamicserialize.dstypes.', '')
if fqn in adapters.classAdapterRegistry: if adapters.classAdapterRegistry.has_key(fqn):
# get proper class name when writing class name to serialization stream # get proper class name when writing class name to serialization stream
# in case we have a special inner-class case # in case we have a special inner-class case
m = sys.modules[adapters.classAdapterRegistry[fqn].__name__] m = sys.modules[adapters.classAdapterRegistry[fqn].__name__]
@ -277,7 +273,7 @@ class ThriftSerializationContext(object):
val = m[1]() val = m[1]()
ft = self._lookupType(val) ft = self._lookupType(val)
if ft == TType.STRUCT: if ft == TType.STRUCT:
fc = val.__module__[DS_LEN:] fc = val.__module__.replace('dynamicserialize.dstypes.', '')
self._serializeField(fieldname, ft, fid, val) self._serializeField(fieldname, ft, fid, val)
else: else:
self._serializeField(fieldname, ft, fid, val) self._serializeField(fieldname, ft, fid, val)
@ -297,18 +293,17 @@ class ThriftSerializationContext(object):
self.protocol.writeFieldEnd() self.protocol.writeFieldEnd()
def _serializeType(self, fieldValue, fieldType): def _serializeType(self, fieldValue, fieldType):
if fieldType in self.typeSerializationMethod: if self.typeSerializationMethod.has_key(fieldType):
return self.typeSerializationMethod[fieldType](fieldValue) return self.typeSerializationMethod[fieldType](fieldValue)
else: else:
raise dynamicserialize.SerializationException( raise dynamicserialize.SerializationException("Unsupported type value " + str(fieldType))
"Unsupported type value " + str(fieldType))
def _serializeArray(self, obj): def _serializeArray(self, obj):
size = len(obj) size = len(obj)
if size: if size:
if type(obj) is numpy.ndarray: if type(obj) is numpy.ndarray:
t = pythonToThriftMap[obj.dtype.type] t = pythonToThriftMap[obj.dtype.type]
size = obj.size size = obj.size
else: else:
t = self._lookupType(obj[0]) t = self._lookupType(obj[0])
else: else:
@ -336,6 +331,7 @@ class ThriftSerializationContext(object):
self.listSerializationMethod[t](obj) self.listSerializationMethod[t](obj)
self.protocol.writeListEnd() self.protocol.writeListEnd()
def _serializeMap(self, obj): def _serializeMap(self, obj):
size = len(obj) size = len(obj)
self.protocol.writeMapBegin(TType.VOID, TType.VOID, size) self.protocol.writeMapBegin(TType.VOID, TType.VOID, size)

View file

@ -1,19 +1,19 @@
## ##
# This software was developed and / or modified by Raytheon Company, # This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government. # pursuant to Contract DG133W-05-CQ-1067 with the US Government.
# #
# U.S. EXPORT CONTROLLED TECHNICAL DATA # U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose # This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination # export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires # to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization. # an export license or other authorization.
# #
# Contractor Name: Raytheon Company # Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340 # Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8 # Mail Stop B8
# Omaha, NE 68106 # Omaha, NE 68106
# 402.291.0100 # 402.291.0100
# #
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for # See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information. # further licensing information.
## ##
@ -21,80 +21,56 @@
# #
# __init__.py for Dynamic Serialize adapters. # __init__.py for Dynamic Serialize adapters.
# #
# Plugins can contribute to dynamicserialize.adapters by either including their #
# classes directly in pythonPackages/dynamicserialize/adapters/ within their
# plugin. The plugin's adapter will automatically be added to __all__ at runtime
# and registered.
# Plugins should not include a custom __init__.py in
# pythonPackages/dynamicserialize/adapters/ because it will overwrite this file.
# If custom package initialization is needed, a subpackage should be created
# with an __init__.py that includes the following:
#
# __all__ = ['CustomAdapter1', 'CustomAdapter2']
# from dynamicserialize.adapters import registerAdapters
# registerAdapters(__name__, __all__)
#
#
# SOFTWARE HISTORY # SOFTWARE HISTORY
# #
# Date Ticket# Engineer Description # Date Ticket# Engineer Description
# ------------ ---------- ----------- -------------------------- # ------------ ---------- ----------- --------------------------
# 08/31/10 njensen Initial Creation. # 08/31/10 njensen Initial Creation.
# 03/20/13 #1774 randerso Added TimeConstraintsAdapter # 03/20/13 #1774 randerso Added TimeConstraintsAdapter
# 04/22/13 #1949 rjpeter Added LockTableAdapter # 04/22/13 #1949 rjpeter Added LockTableAdapter
# 02/06/14 #2672 bsteffen Added JTSEnvelopeAdapter # 02/06/14 #2672 bsteffen Added JTSEnvelopeAdapter
# 06/22/2015 #4573 randerso Added JobProgressAdapter # 06/22/2015 #4573 randerso Added JobProgressAdapter
# 09/21/2015 #4486 rjpeter Added FormattedDateAdapter # 09/21/2015 #4486 rjpeter Added FormattedDateAdapter
# 06/23/2016 #5696 rjpeter Added CommutativeTimestampAdapter # 06/23/2016 #5696 rjpeter Added CommutativeTimestampAdapter
# 10/17/2016 #5919 njensen Added GeomDataRespAdapter
# 01/09/2017 #5997 nabowle Allow contribution from plugins.
# #
__all__ = [ __all__ = [
'PointAdapter', 'PointAdapter',
'StackTraceElementAdapter', 'StackTraceElementAdapter',
'CalendarAdapter', 'WsIdAdapter',
'GregorianCalendarAdapter', 'CalendarAdapter',
'DateAdapter', 'GregorianCalendarAdapter',
'GeometryTypeAdapter', 'ActiveTableModeAdapter',
'CoordAdapter', 'DateAdapter',
'TimestampAdapter', 'FormattedDateAdapter',
'EnumSetAdapter', 'LocalizationLevelSerializationAdapter',
'FloatBufferAdapter', 'LocalizationTypeSerializationAdapter',
'ByteBufferAdapter', 'GeometryTypeAdapter',
'JTSEnvelopeAdapter' 'CoordAdapter',
] 'TimeRangeTypeAdapter',
'ParmIDAdapter',
'DatabaseIDAdapter',
'TimestampAdapter',
'CommutativeTimestampAdapter',
'EnumSetAdapter',
'FloatBufferAdapter',
'ByteBufferAdapter',
'TimeConstraintsAdapter',
'LockTableAdapter',
'JTSEnvelopeAdapter',
'JobProgressAdapter',
]
classAdapterRegistry = {} classAdapterRegistry = {}
def getAdapterRegistry(): def getAdapterRegistry():
import pkgutil import sys
for x in __all__:
discoveredPackages = [] exec 'import ' + x
# allow other plugins to contribute to adapters by dropping their adapter or m = sys.modules['dynamicserialize.adapters.' + x]
# package into the dynamicserialize.adapters package
for _, modname, ispkg in pkgutil.iter_modules(__path__):
if ispkg:
discoveredPackages.append(modname)
else:
if modname not in __all__:
__all__.append(modname)
registerAdapters(__name__, __all__)
for pkg in discoveredPackages:
__import__(__name__ + '.' + pkg)
def registerAdapters(package, modules):
import sys
if not package.endswith('.'):
package += '.'
for x in modules:
exec 'import ' + package + x
m = sys.modules[package + x]
d = m.__dict__ d = m.__dict__
if d.has_key('ClassAdapter'): if d.has_key('ClassAdapter'):
if isinstance(m.ClassAdapter, list): if isinstance(m.ClassAdapter, list):
@ -104,8 +80,9 @@ def registerAdapters(package, modules):
clzName = m.ClassAdapter clzName = m.ClassAdapter
classAdapterRegistry[clzName] = m classAdapterRegistry[clzName] = m
else: else:
raise LookupError('Adapter class ' + x + ' has no ClassAdapter field ' + raise LookupError('Adapter class ' + x + ' has no ClassAdapter field ' + \
'and cannot be registered.') 'and cannot be registered.')
getAdapterRegistry() getAdapterRegistry()

View file

@ -21,7 +21,22 @@
# File auto-generated by PythonFileGenerator # File auto-generated by PythonFileGenerator
__all__ = [ __all__ = [
'dataplugin' 'activetable',
'alertviz',
'auth',
'dataaccess',
'dataplugin',
'dataquery',
'datastorage',
'localization',
'management',
'message',
'plugin',
'pointdata',
'pypies',
'serialization',
'site',
'time'
] ]

View file

@ -21,7 +21,13 @@
# File auto-generated by PythonFileGenerator # File auto-generated by PythonFileGenerator
__all__ = [ __all__ = [
'events' 'events',
'gfe',
'grid',
'level',
'message',
'radar',
'text'
] ]