diff --git a/awips/NotificationMessage.py b/awips/NotificationMessage.py index 197f7c9..ec10820 100644 --- a/awips/NotificationMessage.py +++ b/awips/NotificationMessage.py @@ -1,19 +1,33 @@ ## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. ## -from string import Template +from __future__ import print_function -import ctypes -from . import stomp +import stomp import socket import sys import time -import threading import xml.etree.ElementTree as ET -from . import ThriftClient +import ThriftClient from dynamicserialize.dstypes.com.raytheon.uf.common.alertviz import AlertVizRequest -from dynamicserialize import DynamicSerializationManager # # Provides a capability of constructing notification messages and sending @@ -32,6 +46,7 @@ from dynamicserialize import DynamicSerializationManager # value # 07/27/15 4654 skorolev Added filters # 11/11/15 5120 rferrel Cannot serialize empty filters. +# 03/05/18 6899 dgilling Update to latest version of stomp.py API. # class NotificationMessage: @@ -75,7 +90,7 @@ class NotificationMessage: priorityInt = int(5) if (priorityInt < 0 or priorityInt > 5): - print("Error occurred, supplied an invalid Priority value: " + str(priorityInt)) + print("Error occurred, supplied an invalid Priority value:", str(priorityInt)) print("Priority values are 0, 1, 2, 3, 4 and 5.") sys.exit(1) @@ -84,16 +99,6 @@ class NotificationMessage: else: self.priority = priority - def connection_timeout(self, connection): - if (connection is not None and not connection.is_connected()): - print("Connection Retry Timeout") - for tid, tobj in list(threading._active.items()): - if tobj.name is "MainThread": - res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(SystemExit)) - if res != 0 and res != 1: - # problem, reset state - ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, 0) - def send(self): # depending on the value of the port number indicates the distribution # of the message to AlertViz @@ -101,32 +106,26 @@ class NotificationMessage: # 61999 is local distribution if (int(self.port) == 61999): # use stomp.py - conn = stomp.Connection(host_and_ports=[(self.host, 61999)]) - timeout = threading.Timer(5.0, self.connection_timeout, [conn]) + conn = stomp.Connection(host_and_ports=[(self.host, 61999)], + timeout=5.) try: - timeout.start(); conn.start() - finally: - timeout.cancel() + conn.connect() - conn.connect() + sm = ET.Element("statusMessage") + sm.set("machine", socket.gethostname()) + sm.set("priority", self.priority) + sm.set("category", self.category) + sm.set("sourceKey", self.source) + sm.set("audioFile", self.audioFile) + if self.filters: + sm.set("filters", self.filters) + msg = ET.SubElement(sm, "message") + msg.text = self.message + msg = ET.tostring(sm, "UTF-8") - sm = ET.Element("statusMessage") - sm.set("machine", socket.gethostname()) - sm.set("priority", self.priority) - sm.set("category", self.category) - sm.set("sourceKey", self.source) - sm.set("audioFile", self.audioFile) - if self.filters is not None and len(self.filters) > 0: - sm.set("filters", self.filters) - msg = ET.SubElement(sm, "message") - msg.text = self.message - details = ET.SubElement(sm, "details") - msg = ET.tostring(sm, "UTF-8") - - try : - conn.send(msg, destination='/queue/messages') + conn.send(destination='/queue/messages', body=msg, content_type='application/xml;charset=utf-8') time.sleep(2) finally: conn.stop() @@ -139,13 +138,13 @@ class NotificationMessage: try: serverResponse = thriftClient.sendRequest(alertVizRequest) except Exception as ex: - print("Caught exception submitting AlertVizRequest: ", str(ex)) + print("Caught exception submitting AlertVizRequest:", str(ex)) if (serverResponse != "None"): - print("Error occurred submitting Notification Message to AlertViz receiver: ", serverResponse) + print("Error occurred submitting Notification Message to AlertViz receiver:", serverResponse) sys.exit(1) else: - print("Response: " + str(serverResponse)) + print("Response:", str(serverResponse)) def createRequest(message, priority, source, category, audioFile, filters): obj = AlertVizRequest() diff --git a/awips/QpidSubscriber.py b/awips/QpidSubscriber.py index e55d967..b64a394 100644 --- a/awips/QpidSubscriber.py +++ b/awips/QpidSubscriber.py @@ -30,9 +30,13 @@ # 11/17/10 njensen Initial Creation. # 08/15/13 2169 bkowal Optionally gzip decompress any data that is read. # 08/04/16 2416 tgurney Add queueStarted property +# 02/16/17 6084 bsteffen Support ssl connections +# 09/07/17 6175 tgurney Remove "decompressing" log message # # +import os +import os.path import qpid import zlib @@ -41,11 +45,24 @@ from qpid.exceptions import Closed class QpidSubscriber: - def __init__(self, host='127.0.0.1', port=5672, decompress=False): + def __init__(self, host='127.0.0.1', port=5672, decompress=False, ssl=None): self.host = host self.port = port self.decompress = decompress; socket = qpid.util.connect(host, port) + if "QPID_SSL_CERT_DB" in os.environ: + certdb = os.environ["QPID_SSL_CERT_DB"] + else: + certdb = os.path.expanduser("~/.qpid/") + if "QPID_SSL_CERT_NAME" in os.environ: + certname = os.environ["QPID_SSL_CERT_NAME"] + else: + certname = "guest" + certfile = os.path.join(certdb, certname + ".crt") + if ssl or (ssl is None and os.path.exists(certfile)): + keyfile = os.path.join(certdb, certname + ".key") + trustfile = os.path.join(certdb, "root.crt") + socket = qpid.util.ssl(socket, keyfile=keyfile, certfile=certfile, ca_certs=trustfile) self.__connection = qpid.connection.Connection(sock=socket, username='guest', password='guest') self.__connection.start() self.__session = self.__connection.session(str(qpid.datatypes.uuid4())) @@ -78,7 +95,6 @@ class QpidSubscriber: content = message.body self.__session.message_accept(qpid.datatypes.RangedSet(message.id)) if (self.decompress): - print "Decompressing received content" try: # http://stackoverflow.com/questions/2423866/python-decompressing-gzip-chunk-by-chunk d = zlib.decompressobj(16+zlib.MAX_WBITS) @@ -103,3 +119,4 @@ class QpidSubscriber: @property def queueStarted(self): return self.__queueStarted + diff --git a/awips/UsageArgumentParser.py b/awips/UsageArgumentParser.py index 3722c25..313d206 100644 --- a/awips/UsageArgumentParser.py +++ b/awips/UsageArgumentParser.py @@ -17,13 +17,23 @@ # See the AWIPS II Master Rights File ("Master Rights File.pdf") for # further licensing information. ## +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------- -------- --------- --------------------------------------------- +# Feb 13, 2017 6092 randerso Added StoreTimeAction +# +## import argparse import sys +import time from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import DatabaseID from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import ParmID +TIME_FORMAT = "%Y%m%d_%H%M" class UsageArgumentParser(argparse.ArgumentParser): """ @@ -56,3 +66,16 @@ class AppendParmNameAndLevelAction(argparse.Action): else: setattr(namespace, self.dest, [comp]) +class StoreTimeAction(argparse.Action): + """ + argparse.Action subclass to validate GFE formatted time strings + and parse them to time.struct_time + """ + def __call__(self, parser, namespace, values, option_string=None): + try: + timeStruct = time.strptime(values, TIME_FORMAT) + except: + parser.error(str(values) + " is not a valid time string of the format YYYYMMDD_hhmm") + + setattr(namespace, self.dest, timeStruct) + diff --git a/awips/dataaccess/CombinedTimeQuery.py b/awips/dataaccess/CombinedTimeQuery.py index 8810e0e..269999e 100644 --- a/awips/dataaccess/CombinedTimeQuery.py +++ b/awips/dataaccess/CombinedTimeQuery.py @@ -90,10 +90,11 @@ def __getAvailableTimesForEachLocation(request, refTimeOnly=False): else: return DataAccessLayer.getAvailableTimes(request, refTimeOnly) + def __cloneRequest(request): return DataAccessLayer.newDataRequest(datatype = request.getDatatype(), parameters = request.getParameters(), levels = request.getLevels(), locationNames = request.getLocationNames(), envelope = request.getEnvelope(), - **request.getIdentifiers()) + **request.getIdentifiers()) \ No newline at end of file diff --git a/awips/dataaccess/DataAccessLayer.py b/awips/dataaccess/DataAccessLayer.py index 115305e..ac61cd5 100644 --- a/awips/dataaccess/DataAccessLayer.py +++ b/awips/dataaccess/DataAccessLayer.py @@ -41,7 +41,7 @@ # getRequiredIdentifiers() and # getOptionalIdentifiers() # 10/07/16 ---- mjames@ucar Added getForecastRun -# +# Oct 18, 2016 5916 bsteffen Add setLazyLoadGridLatLon # # @@ -252,3 +252,26 @@ def changeEDEXHost(newHostName): router = ThriftClientRouter.ThriftClientRouter(THRIFT_HOST) else: raise TypeError("Cannot call changeEDEXHost when using JepRouter.") + +def setLazyLoadGridLatLon(lazyLoadGridLatLon): + """ + Provide a hint to the Data Access Framework indicating whether to load the + lat/lon data for a grid immediately or wait until it is needed. This is + provided as a performance tuning hint and should not affect the way the + Data Access Framework is used. Depending on the internal implementation of + the Data Access Framework this hint might be ignored. Examples of when this + should be set to True are when the lat/lon information is not used or when + it is used only if certain conditions within the data are met. It could be + set to False if it is guaranteed that all lat/lon information is needed and + it would be better to get any performance overhead for generating the + lat/lon data out of the way during the initial request. + + + Args: + lazyLoadGridLatLon: Boolean value indicating whether to lazy load. + """ + try: + router.setLazyLoadGridLatLon(lazyLoadGridLatLon) + except AttributeError: + # The router is not required to support this capability. + pass diff --git a/awips/dataaccess/PyGeometryNotification.py b/awips/dataaccess/PyGeometryNotification.py index 7c3cc05..57947b2 100644 --- a/awips/dataaccess/PyGeometryNotification.py +++ b/awips/dataaccess/PyGeometryNotification.py @@ -27,12 +27,28 @@ # Date Ticket# Engineer Description # ------------ ---------- ----------- -------------------------- # 07/22/16 2416 tgurney Initial creation +# 09/07/17 6175 tgurney Override messageReceived # +import dynamicserialize from awips.dataaccess.PyNotification import PyNotification from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint class PyGeometryNotification(PyNotification): + def messageReceived(self, msg): + dataUriMsg = dynamicserialize.deserialize(msg) + dataUris = dataUriMsg.getDataURIs() + dataTimes = set() + for dataUri in dataUris: + if self.notificationFilter.accept(dataUri): + dataTimes.add(self.getDataTime(dataUri)) + if dataTimes: + try: + data = self.getData(self.request, list(dataTimes)) + self.callback(data) + except Exception as e: + traceback.print_exc() + def getData(self, request, dataTimes): return self.DAL.getGeometryData(request, dataTimes) diff --git a/awips/dataaccess/PyGridData.py b/awips/dataaccess/PyGridData.py index 87fffac..d85c7ea 100644 --- a/awips/dataaccess/PyGridData.py +++ b/awips/dataaccess/PyGridData.py @@ -28,7 +28,9 @@ # Date Ticket# Engineer Description # ------------ ---------- ----------- -------------------------- # 06/03/13 #2023 dgilling Initial Creation. +# 10/13/16 #5916 bsteffen Correct grid shape, allow lat/lon # 11/10/16 #5900 bsteffen Correct grid shape +# to be requested by a delegate # # @@ -46,7 +48,7 @@ The ability to unit convert grid data is not currently available in this version class PyGridData(IGridData, PyData.PyData): - def __init__(self, gridDataRecord, nx, ny, latLonGrid): + def __init__(self, gridDataRecord, nx, ny, latLonGrid = None, latLonDelegate = None): PyData.PyData.__init__(self, gridDataRecord) nx = nx ny = ny @@ -54,6 +56,8 @@ class PyGridData(IGridData, PyData.PyData): self.__unit = gridDataRecord.getUnit() self.__gridData = numpy.reshape(numpy.array(gridDataRecord.getGridData()), (ny, nx)) self.__latLonGrid = latLonGrid + self.__latLonDelegate = latLonDelegate + def getParameter(self): return self.__parameter @@ -70,4 +74,8 @@ class PyGridData(IGridData, PyData.PyData): return self.__gridData def getLatLonCoords(self): + if self.__latLonGrid is not None: + return self.__latLonGrid + elif self.__latLonDelegate is not None: + return self.__latLonDelegate() return self.__latLonGrid diff --git a/awips/dataaccess/PyGridNotification.py b/awips/dataaccess/PyGridNotification.py index 2bc425d..1a3be0e 100644 --- a/awips/dataaccess/PyGridNotification.py +++ b/awips/dataaccess/PyGridNotification.py @@ -26,13 +26,34 @@ # # Date Ticket# Engineer Description # ------------ ---------- ----------- -------------------------- -# 06/03/16 2416 rjpeter Initial Creation. +# 06/03/16 2416 rjpeter Initial Creation. +# 09/06/17 6175 tgurney Override messageReceived # +import dynamicserialize from awips.dataaccess.PyNotification import PyNotification from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint class PyGridNotification(PyNotification): + def messageReceived(self, msg): + dataUriMsg = dynamicserialize.deserialize(msg) + dataUris = dataUriMsg.getDataURIs() + for dataUri in dataUris: + if not self.notificationFilter.accept(dataUri): + continue + try: + # This improves performance over requesting by datatime since it requests only the + # parameter that the notification was received for (instead of this and all previous + # parameters for the same forecast hour) + # TODO: This utterly fails for derived requests + newReq = self.DAL.newDataRequest(self.request.getDatatype()) + newReq.addIdentifier("dataURI", dataUri) + newReq.setParameters(self.request.getParameters()) + data = self.getData(newReq, []) + self.callback(data) + except Exception as e: + traceback.print_exc() + def getData(self, request, dataTimes): return self.DAL.getGridData(request, dataTimes) diff --git a/awips/dataaccess/PyNotification.py b/awips/dataaccess/PyNotification.py index 79c25c5..76c12ab 100644 --- a/awips/dataaccess/PyNotification.py +++ b/awips/dataaccess/PyNotification.py @@ -29,6 +29,7 @@ # ------------ ---------- ----------- -------------------------- # Jun 22, 2016 2416 rjpeter Initial creation # Jul 22, 2016 2416 tgurney Finish implementation +# Sep 07, 2017 6175 tgurney Override messageReceived in subclasses # @@ -55,11 +56,11 @@ class PyNotification(INotificationSubscriber): def __init__(self, request, filter, host='localhost', port=5672, requestHost='localhost'): self.DAL = DataAccessLayer self.DAL.changeEDEXHost(requestHost) - self.__request = request - self.__notificationFilter = filter + self.request = request + self.notificationFilter = filter self.__topicSubscriber = QpidSubscriber(host, port, decompress=True) self.__topicName = "edex.alerts" - self.__callback = None + self.callback = None def subscribe(self, callback): """ @@ -70,48 +71,25 @@ class PyNotification(INotificationSubscriber): Will be called once for each request made for data. """ assert hasattr(callback, '__call__'), 'callback arg must be callable' - self.__callback = callback - self.__topicSubscriber.topicSubscribe(self.__topicName, self._messageReceived) + self.callback = callback + self.__topicSubscriber.topicSubscribe(self.__topicName, self.messageReceived) # Blocks here def close(self): if self.__topicSubscriber.subscribed: self.__topicSubscriber.close() - def _getDataTime(self, dataURI): + def getDataTime(self, dataURI): dataTimeStr = dataURI.split('/')[2] return DataTime(dataTimeStr) - def _messageReceived(self, msg): - dataUriMsg = dynamicserialize.deserialize(msg) - dataUris = dataUriMsg.getDataURIs() - dataTimes = [ - self._getDataTime(dataUri) - for dataUri in dataUris - if self.__notificationFilter.accept(dataUri) - ] - if dataTimes: - secondTry = False - while True: - try: - data = self.getData(self.__request, dataTimes) - break - except ThriftRequestException: - if secondTry: - try: - self.close() - except Exception: - pass - raise - else: - secondTry = True - time.sleep(5) - try: - self.__callback(data) - except Exception as e: - # don't want callback to blow up the notifier itself. - traceback.print_exc() - # TODO: This utterly fails for derived requests + @abc.abstractmethod + def messageReceived(self, msg): + """Called when a message is received from QpidSubscriber. + + This method must call self.callback once for each request made for data + """ + pass @abc.abstractmethod def getData(self, request, dataTimes): diff --git a/awips/dataaccess/ThriftClientRouter.py b/awips/dataaccess/ThriftClientRouter.py index e1c0e1e..14948fe 100644 --- a/awips/dataaccess/ThriftClientRouter.py +++ b/awips/dataaccess/ThriftClientRouter.py @@ -39,7 +39,8 @@ # getRequiredIdentifiers() and # getOptionalIdentifiers() # 08/01/16 2416 tgurney Add getNotificationFilter() -# 11/10/16 5900 bsteffen Correct grid shape +# 10/13/16 5916 bsteffen Correct grid shape, allow lazy grid lat/lon +# 10/26/16 5919 njensen Speed up geometry creation in getGeometryData() # @@ -51,6 +52,7 @@ from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import G from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableTimesRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetGeometryDataRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetGridDataRequest +from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetGridLatLonRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableParametersRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableLevelsRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetRequiredIdentifiersRequest @@ -64,10 +66,39 @@ from awips.dataaccess import PyGeometryData from awips.dataaccess import PyGridData +class LazyGridLatLon(object): + + def __init__(self, client, nx, ny, envelope, crsWkt): + self._latLonGrid = None + self._client = client + self._request = GetGridLatLonRequest() + self._request.setNx(nx) + self._request.setNy(ny) + self._request.setEnvelope(envelope) + self._request.setCrsWkt(crsWkt) + + def __call__(self): + # Its important that the data is cached internally so that if multiple + # GridData are sharing the same delegate then they can also share a + # single request for the LatLon information. + if self._latLonGrid is None: + response = self._client.sendRequest(self._request) + nx = response.getNx() + ny = response.getNy() + latData = numpy.reshape(numpy.array(response.getLats()), (ny, nx)) + lonData = numpy.reshape(numpy.array(response.getLons()), (ny, nx)) + self._latLonGrid = (lonData, latData) + return self._latLonGrid + + class ThriftClientRouter(object): def __init__(self, host='localhost'): self._client = ThriftClient.ThriftClient(host) + self._lazyLoadGridLatLon = False + + def setLazyLoadGridLatLon(self, lazyLoadGridLatLon): + self._lazyLoadGridLatLon = lazyLoadGridLatLon def getAvailableTimes(self, request, refTimeOnly): timesRequest = GetAvailableTimesRequest() @@ -78,6 +109,7 @@ class ThriftClientRouter(object): def getGridData(self, request, times): gridDataRequest = GetGridDataRequest() + gridDataRequest.setIncludeLatLonData(not self._lazyLoadGridLatLon) gridDataRequest.setRequestParameters(request) # if we have an iterable times instance, then the user must have asked # for grid data with the List of DataTime objects @@ -95,15 +127,28 @@ class ThriftClientRouter(object): for location in locNames: nx = response.getSiteNxValues()[location] ny = response.getSiteNyValues()[location] - latData = numpy.reshape(numpy.array(response.getSiteLatGrids()[location]), (ny, nx)) - lonData = numpy.reshape(numpy.array(response.getSiteLonGrids()[location]), (ny, nx)) - locSpecificData[location] = (nx, ny, (lonData, latData)) - + if self._lazyLoadGridLatLon: + envelope = response.getSiteEnvelopes()[location] + crsWkt = response.getSiteCrsWkt()[location] + delegate = LazyGridLatLon( + self._client, nx, ny, envelope, crsWkt) + locSpecificData[location] = (nx, ny, delegate) + else: + latData = numpy.reshape(numpy.array( + response.getSiteLatGrids()[location]), (ny, nx)) + lonData = numpy.reshape(numpy.array( + response.getSiteLonGrids()[location]), (ny, nx)) + locSpecificData[location] = (nx, ny, (lonData, latData)) retVal = [] for gridDataRecord in response.getGridData(): locationName = gridDataRecord.getLocationName() locData = locSpecificData[locationName] - retVal.append(PyGridData.PyGridData(gridDataRecord, locData[0], locData[1], locData[2])) + if self._lazyLoadGridLatLon: + retVal.append(PyGridData.PyGridData(gridDataRecord, locData[ + 0], locData[1], latLonDelegate=locData[2])) + else: + retVal.append(PyGridData.PyGridData( + gridDataRecord, locData[0], locData[1], locData[2])) return retVal def getGeometryData(self, request, times): @@ -121,10 +166,9 @@ class ThriftClientRouter(object): response = self._client.sendRequest(geoDataRequest) geometries = [] for wkb in response.getGeometryWKBs(): - # convert the wkb to a bytearray with only positive values - byteArrWKB = bytearray(map(lambda x: x % 256,wkb.tolist())) - # convert the bytearray to a byte string and load it. - geometries.append(shapely.wkb.loads(str(byteArrWKB))) + # the wkb is a numpy.ndarray of dtype int8 + # convert the bytearray to a byte string and load it + geometries.append(shapely.wkb.loads(wkb.tostring())) retVal = [] for geoDataRecord in response.getGeoData(): @@ -175,7 +219,7 @@ class ThriftClientRouter(object): response = self._client.sendRequest(idValReq) return response - def newDataRequest(self, datatype, parameters=[], levels=[], locationNames = [], envelope=None, **kwargs): + def newDataRequest(self, datatype, parameters=[], levels=[], locationNames=[], envelope=None, **kwargs): req = DefaultDataRequest() if datatype: req.setDatatype(datatype) @@ -200,4 +244,4 @@ class ThriftClientRouter(object): notifReq = GetNotificationFilterRequest() notifReq.setRequestParameters(request) response = self._client.sendRequest(notifReq) - return response \ No newline at end of file + return response diff --git a/awips/localization/LocalizationFileManager.py b/awips/localization/LocalizationFileManager.py new file mode 100644 index 0000000..e57c8a5 --- /dev/null +++ b/awips/localization/LocalizationFileManager.py @@ -0,0 +1,470 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# +# Library for accessing localization files from python. +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# --------- -------- --------- -------------------------- +# 08/09/17 5731 bsteffen Initial Creation. + + +import urllib2 +from json import load as loadjson +from xml.etree.ElementTree import parse as parseXml +from base64 import b64encode +from StringIO import StringIO +from getpass import getuser +import dateutil.parser +import contextlib +import os +from urlparse import urlunparse, urljoin + +NON_EXISTENT_CHECKSUM = 'NON_EXISTENT_CHECKSUM' +DIRECTORY_CHECKSUM = 'DIRECTORY_CHECKSUM' + +class LocalizationFileVersionConflictException(Exception): + pass + +class LocalizationFileDoesNotExistException(Exception): + pass + +class LocalizationFileIsNotDirectoryException(Exception): + pass + +class LocalizationContext(object): + """A localization context defines the scope of a localization file. + + For example the base localization context includes all the default files + installed with EDEX, while a particular user context has custom files for + that user. + + A localization context consists of a level and name. The level defines what + kind of entity this context is valid for, such as 'base', 'site', or 'user'. + The name identifies the specific entity, for example the name of a 'user' + level context is usually the username. The 'base' level does not have a name + because there cannot be only one 'base' context. + + Attributes: + level: the localization level + name: the context name + """ + def __init__(self, level="base", name=None, type="common_static"): + if level != "base": + assert name is not None + self.level = level + self.name = name + self.type = type + def isBase(self): + return self.level == "base" + def _getUrlComponent(self): + if self.isBase(): + return self.type + '/' + "base/" + else: + return self.type + '/' + self.level + '/' + self.name + '/' + def __str__(self): + if self.isBase(): + return self.type + ".base" + else: + return self.type + "." + self.level + "." + self.name + def __eq__(self, other): + return self.level == other.level and \ + self.name == other.name and \ + self.type == other.type + def __hash__(self): + return hash((self.level, self.name, self.type)) + +class _LocalizationOutput(StringIO): + """A file-like object for writing a localization file. + + The contents being written are stored in memory and written to a + localization server only when the writing is finished. + + This object should be used as a context manager, a save operation will be + executed if the context exits with no errors. If errors occur the partial + contents are abandoned and the server is unchanged. + + It is also possible to save the contents to the server with the save() + method. + """ + def __init__(self, manager, file): + StringIO.__init__(self) + self._manager = manager + self._file = file + def save(self): + """Send the currently written contents to the server.""" + request = self._manager._buildRequest(self._file.context, self._file.path, method="PUT") + + request.add_data(self.getvalue()) + request.add_header("If-Match", self._file.checksum) + try: + urllib2.urlopen(request) + except urllib2.HTTPError as e: + if e.code == 409: + raise LocalizationFileVersionConflictException, e.read() + else: + raise e + def __enter__(self): + return self + def __exit__(self, exc_type, exc_value, traceback): + if exc_type is None: + self.save() + def __str__(self): + return '<' + self.__class__.__name__ + " for " + str(self._file) + '>' + +class LocalizationFile(object): + """A specific file stored in localization. + + A localization file is uniquely defined by the context and path. There can + only be one valid file for that path and localization at a time. To access + the contents of the file use the open method. + + Attributes: + context: A LocalizationContext + path: A path to this file + checksum: A string representation of a checksum generated from the file contents. + timnestamp: A datetime.datetime object indicating when the file was last modified. + """ + def __init__(self, manager, context, path, checksum, timestamp): + """Initialize a LocalizationFile with the given manager and attributes. + + Args: + manager: A LocalizationFileManager to assist with server communication + context: A LocalizationContext + path: A path to this file + checksum: A string representation of a checksum generated from the file contents. + timnestamp: A datetime.datetime object indicating when the file was last modified. + """ + self._manager = manager + self.context = context + self.path = path + self.checksum = checksum + self.timestamp = timestamp + def open(self, mode='r'): + """Open the file. + + This should always be called as as part of a with statement. When + writing the content is not saved on the server until leaving the with + statement normally, if an error occurs the server is left unchanged. + + Example: + with locFile.open('w') as output: + output.write('some content') + + Args: + mode: 'r' for reading the file, 'w' for writing + + Returns: + A file like object that can be used for reads or writes. + """ + if mode == 'r': + request = self._manager._buildRequest(self.context, self.path) + response = urllib2.urlopen(request) + # Not the recommended way of reading directories. + if not(self.isDirectory()): + checksum = response.headers["Content-MD5"] + if self.checksum != checksum: + raise RuntimeError, "Localization checksum mismatch " + self.checksum + " " + checksum + return contextlib.closing(response) + elif mode == 'w': + return _LocalizationOutput(self._manager, self) + else: + raise ValueError, "mode string must be 'r' or 'w' not " + str(r) + def delete(self): + """Delete this file from the server""" + request = self._manager._buildRequest(self.context, self.path, method='DELETE') + request.add_header("If-Match", self.checksum) + try: + urllib2.urlopen(request) + except urllib2.HTTPError as e: + if e.code == 409: + raise LocalizationFileVersionConflictException, e.read() + else: + raise e + def exists(self): + """Check if this file actually exists. + + Returns: + boolean indicating existence of this file + """ + return self.checksum != NON_EXISTENT_CHECKSUM + def isDirectory(self): + """Check if this file is a directory. + + A file must exist to be considered a directory. + + Returns: + boolean indicating directorocity of this file + """ + return self.checksum == DIRECTORY_CHECKSUM + def getCheckSum(self): + return self.checksum + def getContext(self): + return self.context + def getPath(self): + return self.path + def getTimeStamp(self): + return self.timestamp + def __str__(self): + return str(self.context) + "/" + self.path + def __eq__(self, other): + return self.context == other.context and \ + self.path == other.path and \ + self.checksum == other.checksum \ + and self.timestamp == other.timestamp + def __hash__(self): + return hash((self.context, self.path, self.checksum, self.timestamp)) + +def _getHost(): + import subprocess + host = subprocess.check_output( + "source /awips2/fxa/bin/setup.env; echo $DEFAULT_HOST", + shell=True).strip() + if host: + return host + return 'localhost' + +def _getSiteFromServer(host): + try: + from ufpy import ThriftClient + from dynamicserialize.dstypes.com.raytheon.uf.common.site.requests import GetPrimarySiteRequest + client = ThriftClient.ThriftClient(host) + return client.sendRequest(GetPrimarySiteRequest()) + except: + # Servers that don't have GFE installed will not return a site + pass + +def _getSiteFromEnv(): + site = os.environ.get('FXA_LOCAL_SITE') + if site is None: + site = os.environ.get('SITE_IDENTIFIER'); + return site + +def _getSite(host): + site = _getSiteFromEnv() + if not(site): + site = _getSiteFromServer(host) + return site + +def _parseJsonList(manager, response, context, path): + fileList = [] + jsonResponse = loadjson(response) + for name, jsonData in jsonResponse.items(): + checksum = jsonData["checksum"] + timestampString = jsonData["timestamp"] + timestamp = dateutil.parser.parse(timestampString) + newpath = urljoin(path, name) + fileList.append(LocalizationFile(manager, context, newpath, checksum, timestamp)) + return fileList + +def _parseXmlList(manager, response, context, path): + fileList = [] + for xmlData in parseXml(response).getroot().findall('file'): + name = xmlData.get("name") + checksum = xmlData.get("checksum") + timestampString = xmlData.get("timestamp") + timestamp = dateutil.parser.parse(timestampString) + newpath = urljoin(path, name) + fileList.append(LocalizationFile(manager, context, newpath, checksum, timestamp)) + return fileList + +class LocalizationFileManager(object): + """Connects to a server and retrieves LocalizationFiles.""" + def __init__(self, host=None, port=9581, path="/services/localization/", contexts=None, site=None, type="common_static"): + """Initializes a LocalizationFileManager with connection parameters and context information + + All arguments are optional and will use defaults or attempt to figure out appropriate values form the environment. + + Args: + host: A hostname of the localization server, such as 'ec'. + port: A port to use to connect to the localization server, usually 9581. + path: A path to reach the localization file service on the server. + contexts: A list of contexts to check for files, the order of the contexts will be used + for the order of incremental results and the priority of absolute results. + site: A site identifier to use for site specific contexts. This is only used if the contexts arg is None. + type: A localization type for contexts. This is only used if the contexts arg is None. + + """ + if host is None: + host = _getHost() + if contexts is None: + if site is None : + site = _getSite(host) + contexts = [LocalizationContext("base", None, type)] + if site: + contexts.append(LocalizationContext("configured", site, type)) + contexts.append(LocalizationContext("site", site, type)) + contexts.append(LocalizationContext("user", getuser(), type)) + netloc = host + ':' + str(port) + self._baseUrl = urlunparse(('http', netloc, path, None, None, None)) + self._contexts = contexts + def _buildRequest(self, context, path, method='GET'): + url = urljoin(self._baseUrl, context._getUrlComponent()) + url = urljoin(url, path) + request = urllib2.Request(url) + username = getuser() + # Currently password is ignored in the server + # this is the defacto standard for not providing one to this service. + password = username + base64string = b64encode('%s:%s' % (username, password)) + request.add_header("Authorization", "Basic %s" % base64string) + if method != 'GET': + request.get_method = lambda: method + return request + def _normalizePath(self, path): + if path == '' or path == '/': + path = '.' + if path[0] == '/': + path = path[1:] + return path + def _list(self, path): + path = self._normalizePath(path) + if path[-1] != '/': + path += '/' + fileList = [] + exists = False + for context in self._contexts: + try: + request = self._buildRequest(context, path) + request.add_header("Accept", "application/json, application/xml") + response = urllib2.urlopen(request) + exists = True + if not(response.geturl().endswith("/")): + # For ordinary files the server sends a redirect to remove the slash. + raise LocalizationFileIsNotDirectoryException, "Not a directory: " + path + elif response.headers["Content-Type"] == "application/xml": + fileList += _parseXmlList(self, response, context, path) + else: + fileList += _parseJsonList(self, response, context, path) + except urllib2.HTTPError as e: + if e.code != 404: + raise e + if not(exists): + raise LocalizationFileDoesNotExistException, "No such file or directory: " + path + return fileList + def _get(self, context, path): + path = self._normalizePath(path) + try: + request = self._buildRequest(context, path, method='HEAD') + resp = urllib2.urlopen(request) + if (resp.geturl().endswith("/")): + checksum = DIRECTORY_CHECKSUM; + else: + if "Content-MD5" not in resp.headers: + raise RuntimeError, "Missing Content-MD5 header in response from " + resp.geturl() + checksum = resp.headers["Content-MD5"] + if "Last-Modified" not in resp.headers: + raise RuntimeError, "Missing Last-Modified header in response from " + resp.geturl() + timestamp = dateutil.parser.parse(resp.headers["Last-Modified"]) + return LocalizationFile(self, context, path, checksum, timestamp) + except urllib2.HTTPError as e: + if e.code != 404: + raise e + else: + return LocalizationFile(self, context, path, NON_EXISTENT_CHECKSUM, None) + def listAbsolute(self, path): + """List the files in a localization directory, only a single file is returned for each unique path. + + If a file exists in more than one context then the highest level(furthest from base) is used. + + Args: + path: A path to a directory that should be the root of the listing + + Returns: + A list of LocalizationFiles + """ + merged = dict() + for file in self._list(path): + merged[file.path] = file + return sorted(merged.values(), key=lambda file: file.path) + def listIncremental(self, path): + """List the files in a localization directory, this includes all files for all contexts. + + Args: + path: A path to a directory that should be the root of the listing + + Returns: + A list of tuples, each tuple will contain one or more files for the + same paths but different contexts. Each tuple will be ordered the + same as the contexts in this manager, generally with 'base' first + and 'user' last. + """ + merged = dict() + for file in self._list(path): + if file.path in merged: + merged[file.path] += (file,) + else: + merged[file.path] = (file, ) + return sorted(merged.values(), key=lambda t: t[0].path) + def getAbsolute(self, path): + """Get a single localization file from the highest level context where it exists. + + Args: + path: A path to a localization file + + Returns: + A Localization File with the specified path or None if the file does not exist in any context. + + """ + for context in reversed(self._contexts): + f = self._get(context, path) + if f.exists(): + return f + def getIncremental(self, path): + """Get all the localization files that exist in any context for the provided path. + + Args: + path: A path to a localization file + + Returns: + A tuple containing all the files that exist for this path in any context. The tuple + will be ordered the same as the contexts in this manager, generally with 'base' first + and 'user' last. + """ + result = () + for context in self._contexts: + f = self._get(context, path) + if f.exists(): + result += (f,) + return result + def getSpecific(self, level, path): + """Get a specific localization file at a given level, the file may not exist. + + The file is returned for whichever context is valid for the provided level in this manager. + + For writing new files this is the only way to get access to a file that + does not exist in order to create it. + + Args: + level: the name of a localization level, such as "base", "site", "user" + path: A path to a localization file + + Returns: + A Localization File with the specified path and a context for the specified level. + """ + for context in self._contexts: + if context.level == level: + return self._get(context, path) + raise ValueError, "No context defined for level " + level + def __str__(self): + contextsStr = '[' + ' '.join((str(c) for c in self._contexts)) + ']' + return '<' + self.__class__.__name__ + " for " + self._baseUrl + ' ' + contextsStr + '>' diff --git a/awips/localization/__init__.py b/awips/localization/__init__.py new file mode 100644 index 0000000..f684128 --- /dev/null +++ b/awips/localization/__init__.py @@ -0,0 +1,32 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# +# __init__.py for ufpy.localization package +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# --------- -------- --------- -------------------------- +# 08/10/17 5731 bsteffen Initial Creation. + +__all__ = [ + ] \ No newline at end of file diff --git a/awips/qpidingest.py b/awips/qpidingest.py index f5aa83f..0a9dc1b 100644 --- a/awips/qpidingest.py +++ b/awips/qpidingest.py @@ -59,9 +59,13 @@ # .... # 06/13/2013 DR 16242 D. Friedman Add Qpid authentication info # 03/06/2014 DR 17907 D. Friedman Workaround for issue QPID-5569 +# 02/16/2017 DR 6084 bsteffen Support ssl connections # #=============================================================================== +import os +import os.path + import qpid from qpid.util import connect from qpid.connection import Connection @@ -71,17 +75,31 @@ QPID_USERNAME = 'guest' QPID_PASSWORD = 'guest' class IngestViaQPID: - def __init__(self, host='localhost', port=5672): + def __init__(self, host='localhost', port=5672, ssl=None): ''' Connect to QPID and make bindings to route message to external.dropbox queue @param host: string hostname of computer running EDEX and QPID (default localhost) @param port: integer port used to connect to QPID (default 5672) + @param ssl: boolean to determine whether ssl is used, default value of None will use ssl only if a client certificate is found. ''' try: # - self.socket = connect(host, port) - self.connection = Connection (sock=self.socket, username=QPID_USERNAME, password=QPID_PASSWORD) + socket = connect(host, port) + if "QPID_SSL_CERT_DB" in os.environ: + certdb = os.environ["QPID_SSL_CERT_DB"] + else: + certdb = os.path.expanduser("~/.qpid/") + if "QPID_SSL_CERT_NAME" in os.environ: + certname = os.environ["QPID_SSL_CERT_NAME"] + else: + certname = QPID_USERNAME + certfile = os.path.join(certdb, certname + ".crt") + if ssl or (ssl is None and os.path.exists(certfile)): + keyfile = os.path.join(certdb, certname + ".key") + trustfile = os.path.join(certdb, "root.crt") + socket = qpid.util.ssl(socket, keyfile=keyfile, certfile=certfile, ca_certs=trustfile) + self.connection = Connection (sock=socket, username=QPID_USERNAME, password=QPID_PASSWORD) self.connection.start() self.session = self.connection.session(str(uuid4())) self.session.exchange_bind(exchange='amq.direct', queue='external.dropbox', binding_key='external.dropbox') diff --git a/awips/test/Record.py b/awips/test/Record.py index cc11f29..3913707 100644 --- a/awips/test/Record.py +++ b/awips/test/Record.py @@ -45,4 +45,4 @@ class Record(): self.exc_text="TEST" def getMessage(self): - return self.message + return self.message \ No newline at end of file diff --git a/awips/test/Test b/awips/test/Test index 6ff8cc1..7fffc86 100644 --- a/awips/test/Test +++ b/awips/test/Test @@ -45,4 +45,3 @@ import Record avh = AlertVizHandler.AlertVizHandler(host=os.getenv("BROKER_ADDR","localhost"), port=9581, category='LOCAL', source='ANNOUNCER', level=logging.NOTSET) record = Record.Record(10) avh.emit(record) - diff --git a/awips/test/dafTests/baseBufrMosTestCase.py b/awips/test/dafTests/baseBufrMosTestCase.py index 652d838..7b3607a 100644 --- a/awips/test/dafTests/baseBufrMosTestCase.py +++ b/awips/test/dafTests/baseBufrMosTestCase.py @@ -19,8 +19,11 @@ ## from awips.dataaccess import DataAccessLayer as DAL +from shapely.geometry import box import baseDafTestCase +import params +import unittest # # Base TestCase for BufrMos* tests. @@ -31,7 +34,8 @@ import baseDafTestCase # ------------ ---------- ----------- -------------------------- # 01/19/16 4795 mapeters Initial Creation. # 04/11/16 5548 tgurney Cleanup -# +# 12/07/16 5981 tgurney Parameterize +# 12/15/16 5981 tgurney Add envelope test # # @@ -39,6 +43,8 @@ import baseDafTestCase class BufrMosTestCase(baseDafTestCase.DafTestCase): """Base class for testing DAF support of bufrmos data""" + data_params = "temperature", "dewpoint" + def testGetAvailableParameters(self): req = DAL.newDataRequest(self.datatype) self.runParametersTest(req) @@ -49,11 +55,19 @@ class BufrMosTestCase(baseDafTestCase.DafTestCase): def testGetAvailableTimes(self): req = DAL.newDataRequest(self.datatype) - req.setLocationNames("KOMA") + req.setLocationNames(params.OBS_STATION) self.runTimesTest(req) def testGetGeometryData(self): req = DAL.newDataRequest(self.datatype) - req.setLocationNames("KOMA") - req.setParameters("temperature", "dewpoint") + req.setLocationNames(params.OBS_STATION) + req.setParameters(*self.data_params) self.runGeometryDataTest(req) + + def testGetGeometryDataWithEnvelope(self): + req = DAL.newDataRequest(self.datatype) + req.setParameters(*self.data_params) + req.setEnvelope(params.ENVELOPE) + data = self.runGeometryDataTest(req) + for item in data: + self.assertTrue(params.ENVELOPE.contains(item.getGeometry())) diff --git a/awips/test/dafTests/baseDafTestCase.py b/awips/test/dafTests/baseDafTestCase.py index 9275082..469d205 100644 --- a/awips/test/dafTests/baseDafTestCase.py +++ b/awips/test/dafTests/baseDafTestCase.py @@ -50,6 +50,8 @@ import unittest # 10/05/16 5926 dgilling Better checks in runGeometryDataTest. # 11/08/16 5985 tgurney Do not check data times on # time-agnostic data +# 03/13/17 5981 tgurney Do not check valid period on +# data time # # @@ -166,10 +168,13 @@ class DafTestCase(unittest.TestCase): self.assertIsNotNone(geomData) if times: self.assertNotEqual(len(geomData), 0) + if not geomData: + raise unittest.SkipTest("No data available") print("Number of geometry records: " + str(len(geomData))) print("Sample geometry data:") for record in geomData[:self.sampleDataLimit]: - if checkDataTimes and times: + if (checkDataTimes and times and + "PERIOD_USED" not in record.getDataTime().getUtilityFlags()): self.assertIn(record.getDataTime(), times[:self.numTimesToLimit]) print("geometry=" + str(record.getGeometry()), end="") for p in req.getParameters(): @@ -184,6 +189,8 @@ class DafTestCase(unittest.TestCase): """ geomData = DAL.getGeometryData(req, timeRange) self.assertIsNotNone(geomData) + if not geomData: + raise unittest.SkipTest("No data available") print("Number of geometry records: " + str(len(geomData))) print("Sample geometry data:") for record in geomData[:self.sampleDataLimit]: @@ -207,6 +214,8 @@ class DafTestCase(unittest.TestCase): times = DafTestCase.getTimesIfSupported(req) gridData = DAL.getGridData(req, times[:self.numTimesToLimit]) self.assertIsNotNone(gridData) + if not gridData: + raise unittest.SkipTest("No data available") print("Number of grid records: " + str(len(gridData))) if len(gridData) > 0: print("Sample grid data shape:\n" + str(gridData[0].getRawData().shape) + "\n") diff --git a/awips/test/dafTests/baseRadarTestCase.py b/awips/test/dafTests/baseRadarTestCase.py new file mode 100644 index 0000000..8d39c68 --- /dev/null +++ b/awips/test/dafTests/baseRadarTestCase.py @@ -0,0 +1,194 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +from __future__ import print_function +from shapely.geometry import box +from ufpy.dataaccess import DataAccessLayer as DAL +from ufpy.ThriftClient import ThriftRequestException + +import baseDafTestCase +import params +import unittest + +# +# Tests common to all radar factories +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 01/19/16 4795 mapeters Initial Creation. +# 04/11/16 5548 tgurney Cleanup +# 04/18/16 5548 tgurney More cleanup +# 04/26/16 5587 tgurney Move identifier values tests +# out of base class +# 06/01/16 5587 tgurney Update testGetIdentifierValues +# 06/08/16 5574 mapeters Add advanced query tests +# 06/13/16 5574 tgurney Fix checks for None +# 06/14/16 5548 tgurney Undo previous change (broke +# test) +# 06/30/16 5725 tgurney Add test for NOT IN +# 08/25/16 2671 tgurney Rename to baseRadarTestCase +# and move factory-specific +# tests +# 12/07/16 5981 tgurney Parameterize +# +# + + +class BaseRadarTestCase(baseDafTestCase.DafTestCase): + """Tests common to all radar factories""" + + # datatype is specified by subclass + datatype = None + + radarLoc = params.RADAR.lower() + + def testGetAvailableParameters(self): + req = DAL.newDataRequest(self.datatype) + self.runParametersTest(req) + + def testGetAvailableLocations(self): + req = DAL.newDataRequest(self.datatype) + self.runLocationsTest(req) + + def testGetAvailableLevels(self): + req = DAL.newDataRequest(self.datatype) + self.runLevelsTest(req) + + def testGetAvailableLevelsWithInvalidLevelIdentifierThrowsException(self): + req = DAL.newDataRequest(self.datatype) + req.addIdentifier('level.one.field', 'invalidLevelField') + with self.assertRaises(ThriftRequestException) as cm: + self.runLevelsTest(req) + self.assertIn('IncompatibleRequestException', str(cm.exception)) + + def testGetAvailableTimes(self): + req = DAL.newDataRequest(self.datatype) + req.setEnvelope(params.ENVELOPE) + self.runTimesTest(req) + + def testGetIdentifierValues(self): + req = DAL.newDataRequest(self.datatype) + optionalIds = set(DAL.getOptionalIdentifiers(req)) + requiredIds = set(DAL.getRequiredIdentifiers(req)) + self.runGetIdValuesTest(optionalIds | requiredIds) + + def testGetInvalidIdentifierValuesThrowsException(self): + self.runInvalidIdValuesTest() + + def testGetNonexistentIdentifierValuesThrowsException(self): + self.runNonexistentIdValuesTest() + + def runConstraintTest(self, key, operator, value): + raise NotImplementedError + + def testGetDataWithEqualsString(self): + gridData = self.runConstraintTest('icao', '=', self.radarLoc) + for record in gridData: + self.assertEqual(record.getAttribute('icao'), self.radarLoc) + + def testGetDataWithEqualsUnicode(self): + gridData = self.runConstraintTest('icao', '=', unicode(self.radarLoc)) + for record in gridData: + self.assertEqual(record.getAttribute('icao'), self.radarLoc) + + def testGetDataWithEqualsInt(self): + gridData = self.runConstraintTest('icao', '=', 1000) + for record in gridData: + self.assertEqual(record.getAttribute('icao'), 1000) + + def testGetDataWithEqualsLong(self): + gridData = self.runConstraintTest('icao', '=', 1000L) + for record in gridData: + self.assertEqual(record.getAttribute('icao'), 1000) + + def testGetDataWithEqualsFloat(self): + gridData = self.runConstraintTest('icao', '=', 1.0) + for record in gridData: + self.assertEqual(round(record.getAttribute('icao'), 1), 1.0) + + def testGetDataWithEqualsNone(self): + gridData = self.runConstraintTest('icao', '=', None) + for record in gridData: + self.assertIsNone(record.getAttribute('icao')) + + def testGetDataWithNotEquals(self): + gridData = self.runConstraintTest('icao', '!=', self.radarLoc) + for record in gridData: + self.assertNotEqual(record.getAttribute('icao'), self.radarLoc) + + def testGetDataWithNotEqualsNone(self): + gridData = self.runConstraintTest('icao', '!=', None) + for record in gridData: + self.assertIsNotNone(record.getAttribute('icao')) + + def testGetDataWithGreaterThan(self): + gridData = self.runConstraintTest('icao', '>', self.radarLoc) + for record in gridData: + self.assertGreater(record.getAttribute('icao'), self.radarLoc) + + def testGetDataWithLessThan(self): + gridData = self.runConstraintTest('icao', '<', self.radarLoc) + for record in gridData: + self.assertLess(record.getAttribute('icao'), self.radarLoc) + + def testGetDataWithGreaterThanEquals(self): + gridData = self.runConstraintTest('icao', '>=', self.radarLoc) + for record in gridData: + self.assertGreaterEqual(record.getAttribute('icao'), self.radarLoc) + + def testGetDataWithLessThanEquals(self): + gridData = self.runConstraintTest('icao', '<=', self.radarLoc) + for record in gridData: + self.assertLessEqual(record.getAttribute('icao'), self.radarLoc) + + def testGetDataWithInTuple(self): + gridData = self.runConstraintTest('icao', 'in', (self.radarLoc, 'tpbi')) + for record in gridData: + self.assertIn(record.getAttribute('icao'), (self.radarLoc, 'tpbi')) + + def testGetDataWithInList(self): + gridData = self.runConstraintTest('icao', 'in', [self.radarLoc, 'tpbi']) + for record in gridData: + self.assertIn(record.getAttribute('icao'), (self.radarLoc, 'tpbi')) + + def testGetDataWithInGenerator(self): + generator = (item for item in (self.radarLoc, 'tpbi')) + gridData = self.runConstraintTest('icao', 'in', generator) + for record in gridData: + self.assertIn(record.getAttribute('icao'), (self.radarLoc, 'tpbi')) + + def testGetDataWithNotInList(self): + gridData = self.runConstraintTest('icao', 'not in', ['zzzz', self.radarLoc]) + for record in gridData: + self.assertNotIn(record.getAttribute('icao'), ('zzzz', self.radarLoc)) + + def testGetDataWithInvalidConstraintTypeThrowsException(self): + with self.assertRaises(ValueError): + self.runConstraintTest('icao', 'junk', self.radarLoc) + + def testGetDataWithInvalidConstraintValueThrowsException(self): + with self.assertRaises(TypeError): + self.runConstraintTest('icao', '=', {}) + + def testGetDataWithEmptyInConstraintThrowsException(self): + with self.assertRaises(ValueError): + self.runConstraintTest('icao', 'in', []) diff --git a/awips/test/dafTests/params.py b/awips/test/dafTests/params.py new file mode 100644 index 0000000..bdc6da7 --- /dev/null +++ b/awips/test/dafTests/params.py @@ -0,0 +1,43 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + + +# +# Site-specific parameters for DAF tests +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 12/07/16 5981 tgurney Initial creation +# 12/15/16 5981 tgurney Add ENVELOPE +# +# + +from shapely.geometry import box + +AIRPORT = 'OMA' +OBS_STATION = 'KOMA' +SITE_ID = 'OAX' +STATION_ID = '72558' +RADAR = 'KOAX' +SAMPLE_AREA = (-97.0, 41.0, -96.0, 42.0) + +ENVELOPE = box(*SAMPLE_AREA) \ No newline at end of file diff --git a/awips/test/dafTests/testBufrMosHpc.py b/awips/test/dafTests/testBufrMosHpc.py index e1ab295..5d1f8b4 100644 --- a/awips/test/dafTests/testBufrMosHpc.py +++ b/awips/test/dafTests/testBufrMosHpc.py @@ -22,6 +22,7 @@ from __future__ import print_function from awips.dataaccess import DataAccessLayer as DAL import baseBufrMosTestCase +import params import unittest # @@ -34,6 +35,8 @@ import unittest # 01/19/16 4795 mapeters Initial Creation. # 04/11/16 5548 tgurney Cleanup # 04/18/16 5548 tgurney More cleanup +# 12/07/16 5981 tgurney Parameterize +# 12/20/16 5981 tgurney Inherit all tests # # @@ -42,11 +45,6 @@ class BufrMosHpcTestCase(baseBufrMosTestCase.BufrMosTestCase): """Test DAF support for bufrmosHPC data""" datatype = "bufrmosHPC" + data_params = "forecastHr", "maxTemp24Hour" - # Most tests inherited from superclass - - def testGetGeometryData(self): - req = DAL.newDataRequest(self.datatype) - req.setLocationNames("KOMA") - req.setParameters("forecastHr", "maxTemp24Hour") - self.runGeometryDataTest(req) + # All tests inherited from superclass \ No newline at end of file diff --git a/awips/test/dafTests/testBufrMosMrf.py b/awips/test/dafTests/testBufrMosMrf.py index 27d1b0f..4062514 100644 --- a/awips/test/dafTests/testBufrMosMrf.py +++ b/awips/test/dafTests/testBufrMosMrf.py @@ -22,6 +22,7 @@ from __future__ import print_function from awips.dataaccess import DataAccessLayer as DAL import baseBufrMosTestCase +import params import unittest # @@ -34,6 +35,8 @@ import unittest # 01/19/16 4795 mapeters Initial Creation. # 04/11/16 5548 tgurney Cleanup # 04/18/16 5548 tgurney More cleanup +# 12/07/16 5981 tgurney Parameterize +# 12/20/16 5981 tgurney Inherit all tests # # @@ -42,11 +45,6 @@ class BufrMosMrfTestCase(baseBufrMosTestCase.BufrMosTestCase): """Test DAF support for bufrmosMRF data""" datatype = "bufrmosMRF" + data_params = "forecastHr", "maxTempDay" - # Most tests inherited from superclass - - def testGetGeometryData(self): - req = DAL.newDataRequest(self.datatype) - req.setLocationNames("KOMA") - req.setParameters("forecastHr", "maxTempDay") - self.runGeometryDataTest(req) + # All tests inherited from superclass diff --git a/awips/test/dafTests/testBufrUa.py b/awips/test/dafTests/testBufrUa.py index 42bfc05..084816e 100644 --- a/awips/test/dafTests/testBufrUa.py +++ b/awips/test/dafTests/testBufrUa.py @@ -23,6 +23,7 @@ from awips.dataaccess import DataAccessLayer as DAL from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint import baseDafTestCase +import params import unittest # @@ -38,6 +39,8 @@ import unittest # 06/09/16 5587 bsteffen Add getIdentifierValues tests # 06/13/16 5574 tgurney Add advanced query tests # 06/30/16 5725 tgurney Add test for NOT IN +# 12/07/16 5981 tgurney Parameterize +# 12/15/16 5981 tgurney Add envelope test # # @@ -47,8 +50,7 @@ class BufrUaTestCase(baseDafTestCase.DafTestCase): datatype = "bufrua" - location = "72558" - """stationid corresponding to KOAX""" + location = params.STATION_ID def testGetAvailableParameters(self): req = DAL.newDataRequest(self.datatype) @@ -91,6 +93,14 @@ class BufrUaTestCase(baseDafTestCase.DafTestCase): print("getGeometryData() complete\n\n") + def testGetGeometryDataWithEnvelope(self): + req = DAL.newDataRequest(self.datatype) + req.setParameters("staName", "rptType") + req.setEnvelope(params.ENVELOPE) + data = self.runGeometryDataTest(req) + for item in data: + self.assertTrue(params.ENVELOPE.contains(item.getGeometry())) + def testGetIdentifierValues(self): req = DAL.newDataRequest(self.datatype) optionalIds = set(DAL.getOptionalIdentifiers(req)) @@ -133,7 +143,6 @@ class BufrUaTestCase(baseDafTestCase.DafTestCase): # No float test because no float identifiers are available - def testGetDataWithEqualsNone(self): geometryData = self._runConstraintTest('reportType', '=', None) for record in geometryData: diff --git a/awips/test/dafTests/testClimate.py b/awips/test/dafTests/testClimate.py index 5c1ce5b..c2db2d4 100644 --- a/awips/test/dafTests/testClimate.py +++ b/awips/test/dafTests/testClimate.py @@ -26,6 +26,7 @@ from awips.dataaccess import DataAccessLayer as DAL from awips.ThriftClient import ThriftRequestException import baseDafTestCase +import params import unittest # @@ -44,6 +45,9 @@ import unittest # 06/21/16 5548 tgurney Skip tests that cause errors # 06/30/16 5725 tgurney Add test for NOT IN # 10/06/16 5926 dgilling Add additional time and location tests. +# 12/07/16 5981 tgurney Parameterize +# 12/20/16 5981 tgurney Add envelope test +# 08/16/17 6388 tgurney Test for duplicate data # # @@ -52,6 +56,7 @@ class ClimateTestCase(baseDafTestCase.DafTestCase): """Test DAF support for climate data""" datatype = 'climate' + obsStation = params.OBS_STATION def testGetAvailableParameters(self): req = DAL.newDataRequest(self.datatype) @@ -104,7 +109,7 @@ class ClimateTestCase(baseDafTestCase.DafTestCase): """ req = DAL.newDataRequest(self.datatype) req.addIdentifier('table', 'public.cli_asos_monthly') - req.setLocationNames('KOMA', 'KABR', 'KDMO') + req.setLocationNames(self.obsStation, 'KABR', 'KDMO') req.setParameters('maxtemp_mon', 'min_sea_press') self.runTimesTest(req) @@ -115,7 +120,7 @@ class ClimateTestCase(baseDafTestCase.DafTestCase): """ req = DAL.newDataRequest(self.datatype) req.addIdentifier('table', 'public.cli_asos_daily') - req.setLocationNames('KOMA', 'KABR', 'KDMO') + req.setLocationNames(self.obsStation, 'KABR', 'KDMO') req.setParameters('maxtemp_cal', 'min_press') self.runTimesTest(req) @@ -126,7 +131,7 @@ class ClimateTestCase(baseDafTestCase.DafTestCase): """ req = DAL.newDataRequest(self.datatype) req.addIdentifier('table', 'public.cli_mon_season_yr') - req.setLocationNames('KOMA', 'KABR', 'KDMO') + req.setLocationNames(self.obsStation, 'KABR', 'KDMO') req.setParameters('max_temp', 'precip_total') self.runTimesTest(req) @@ -137,7 +142,7 @@ class ClimateTestCase(baseDafTestCase.DafTestCase): """ req = DAL.newDataRequest(self.datatype) req.addIdentifier('table', 'public.daily_climate') - req.setLocationNames('KOMA', 'KABR', 'KDMO') + req.setLocationNames(self.obsStation, 'KABR', 'KDMO') req.setParameters('max_temp', 'precip', 'avg_wind_speed') self.runTimesTest(req) @@ -155,6 +160,15 @@ class ClimateTestCase(baseDafTestCase.DafTestCase): req.setParameters('maxtemp_mon', 'min_sea_press') self.runGeometryDataTest(req) + def testGetGeometryDataWithEnvelopeThrowsException(self): + # Envelope is not used + req = DAL.newDataRequest(self.datatype) + req.addIdentifier('table', 'public.cli_asos_monthly') + req.setParameters('maxtemp_mon', 'min_sea_press') + req.setEnvelope(params.ENVELOPE) + with self.assertRaises(Exception): + data = self.runGeometryDataTest(req) + def testGetGeometryDataForYearAndDayOfYearTable(self): """ Test retrieval of data for a climo table that uses year and @@ -243,14 +257,14 @@ class ClimateTestCase(baseDafTestCase.DafTestCase): return self.runGeometryDataTest(req) def testGetDataWithEqualsString(self): - geometryData = self._runConstraintTest('station_code', '=', 'KOMA') + geometryData = self._runConstraintTest('station_code', '=', self.obsStation) for record in geometryData: - self.assertEqual(record.getString('station_code'), 'KOMA') + self.assertEqual(record.getString('station_code'), self.obsStation) def testGetDataWithEqualsUnicode(self): - geometryData = self._runConstraintTest('station_code', '=', u'KOMA') + geometryData = self._runConstraintTest('station_code', '=', unicode(self.obsStation)) for record in geometryData: - self.assertEqual(record.getString('station_code'), 'KOMA') + self.assertEqual(record.getString('station_code'), self.obsStation) def testGetDataWithEqualsInt(self): geometryData = self._runConstraintTest('avg_daily_max', '=', 70) @@ -272,9 +286,9 @@ class ClimateTestCase(baseDafTestCase.DafTestCase): self.assertEqual(len(geometryData), 0) def testGetDataWithNotEquals(self): - geometryData = self._runConstraintTest('station_code', '!=', 'KOMA') + geometryData = self._runConstraintTest('station_code', '!=', self.obsStation) for record in geometryData: - self.assertNotEqual(record.getString('station_code'), 'KOMA') + self.assertNotEqual(record.getString('station_code'), self.obsStation) def testGetDataWithNotEqualsNone(self): geometryData = self._runConstraintTest('station_code', '!=', None) @@ -302,19 +316,19 @@ class ClimateTestCase(baseDafTestCase.DafTestCase): self.assertLessEqual(record.getNumber('avg_daily_max'), 70) def testGetDataWithInTuple(self): - collection = ('KOMA', 'KABR') + collection = (self.obsStation, 'KABR') geometryData = self._runConstraintTest('station_code', 'in', collection) for record in geometryData: self.assertIn(record.getString('station_code'), collection) def testGetDataWithInList(self): - collection = ['KOMA', 'KABR'] + collection = [self.obsStation, 'KABR'] geometryData = self._runConstraintTest('station_code', 'in', collection) for record in geometryData: self.assertIn(record.getString('station_code'), collection) def testGetDataWithInGenerator(self): - collection = ('KOMA', 'KABR') + collection = (self.obsStation, 'KABR') generator = (item for item in collection) geometryData = self._runConstraintTest('station_code', 'in', generator) for record in geometryData: @@ -328,7 +342,7 @@ class ClimateTestCase(baseDafTestCase.DafTestCase): def testGetDataWithInvalidConstraintTypeThrowsException(self): with self.assertRaises(ValueError): - self._runConstraintTest('station_code', 'junk', 'KOMA') + self._runConstraintTest('station_code', 'junk', self.obsStation) def testGetDataWithInvalidConstraintValueThrowsException(self): with self.assertRaises(TypeError): @@ -418,3 +432,13 @@ class ClimateTestCase(baseDafTestCase.DafTestCase): tr = TimeRange(startTime, endTime) self.runGeometryDataTestWithTimeRange(req, tr) + def testNoDuplicateData(self): + req = DAL.newDataRequest(self.datatype) + req.addIdentifier('table', 'public.cli_asos_monthly') + req.setLocationNames('KOMA') + req.setParameters('maxtemp_day1') + rows = DAL.getGeometryData(req, DAL.getAvailableTimes(req)[0:5]) + for i in range(len(rows)): + for j in range(len(rows)): + if i != j: + self.assertNotEqual(rows[i].__dict__, rows[j].__dict__) diff --git a/awips/test/dafTests/testCombinedTimeQuery.py b/awips/test/dafTests/testCombinedTimeQuery.py index 953302f..b3527db 100644 --- a/awips/test/dafTests/testCombinedTimeQuery.py +++ b/awips/test/dafTests/testCombinedTimeQuery.py @@ -49,7 +49,7 @@ class CombinedTimeQueryTestCase(unittest.TestCase): def testSuccessfulQuery(self): req = DAL.newDataRequest('grid') - req.setLocationNames('RAP13') + req.setLocationNames('RUC130') req.setParameters('T','GH') req.setLevels('300MB', '500MB','700MB') times = CTQ.getAvailableTimes(req); @@ -60,7 +60,7 @@ class CombinedTimeQueryTestCase(unittest.TestCase): Test that when a parameter is only available on one of the levels that no times are returned. """ req = DAL.newDataRequest('grid') - req.setLocationNames('RAP13') + req.setLocationNames('RUC130') req.setParameters('T','GH', 'LgSP1hr') req.setLevels('300MB', '500MB','700MB','0.0SFC') times = CTQ.getAvailableTimes(req); diff --git a/awips/test/dafTests/testCommonObsSpatial.py b/awips/test/dafTests/testCommonObsSpatial.py index 8a3e18d..918761e 100644 --- a/awips/test/dafTests/testCommonObsSpatial.py +++ b/awips/test/dafTests/testCommonObsSpatial.py @@ -24,6 +24,7 @@ from awips.dataaccess import DataAccessLayer as DAL from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint import baseDafTestCase +import params import unittest # @@ -42,6 +43,8 @@ import unittest # 06/13/16 5574 tgurney Add advanced query tests # 06/21/16 5548 tgurney Skip tests that cause errors # 06/30/16 5725 tgurney Add test for NOT IN +# 12/07/16 5981 tgurney Parameterize +# 01/06/17 5981 tgurney Do not check data times # @@ -50,9 +53,6 @@ class CommonObsSpatialTestCase(baseDafTestCase.DafTestCase): datatype = "common_obs_spatial" - envelope = box(-97.0, 41.0, -96.0, 42.0) - """Default request area (box around KOAX)""" - def testGetAvailableParameters(self): req = DAL.newDataRequest(self.datatype) self.runParametersTest(req) @@ -65,19 +65,11 @@ class CommonObsSpatialTestCase(baseDafTestCase.DafTestCase): def testGetIdentifierValues(self): self.runGetIdValuesTest(['country']) - @unittest.skip('avoid EDEX error') - def testGetInvalidIdentifierValuesThrowsException(self): - self.runInvalidIdValuesTest() - - @unittest.skip('avoid EDEX error') - def testGetNonexistentIdentifierValuesThrowsException(self): - self.runNonexistentIdValuesTest() - def testGetGeometryData(self): req = DAL.newDataRequest(self.datatype) - req.setEnvelope(self.envelope) + req.setEnvelope(params.ENVELOPE) req.setParameters("name", "stationid") - self.runGeometryDataTest(req) + self.runGeometryDataTest(req, checkDataTimes=False) def testRequestingTimesThrowsTimeAgnosticDataException(self): req = DAL.newDataRequest(self.datatype) @@ -88,7 +80,7 @@ class CommonObsSpatialTestCase(baseDafTestCase.DafTestCase): constraint = RequestConstraint.new(operator, value) req.addIdentifier(key, constraint) req.setParameters('catalogtype', 'elevation', 'state') - return self.runGeometryDataTest(req) + return self.runGeometryDataTest(req, checkDataTimes=False) def testGetDataWithEqualsString(self): geometryData = self._runConstraintTest('state', '=', 'NE') diff --git a/awips/test/dafTests/testFfmp.py b/awips/test/dafTests/testFfmp.py index 5fa7d93..d953348 100644 --- a/awips/test/dafTests/testFfmp.py +++ b/awips/test/dafTests/testFfmp.py @@ -23,6 +23,7 @@ from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import R from awips.dataaccess import DataAccessLayer as DAL import baseDafTestCase +import params import unittest # @@ -46,6 +47,8 @@ import unittest # PRTM parameter since it isn't # configured for ec-oma # 11/08/16 5985 tgurney Do not check data times +# 12/07/16 5981 tgurney Parameterize +# 12/20/16 5981 tgurney Do not check data times # # @@ -54,10 +57,11 @@ class FfmpTestCase(baseDafTestCase.DafTestCase): """Test DAF support for ffmp data""" datatype = 'ffmp' + location = params.RADAR.lower() @staticmethod def addIdentifiers(req): - req.addIdentifier('wfo', 'OAX') + req.addIdentifier('wfo', params.SITE_ID) req.addIdentifier('siteKey', 'hpe') req.addIdentifier('dataKey', 'hpe') req.addIdentifier('huc', 'ALL') @@ -99,8 +103,8 @@ class FfmpTestCase(baseDafTestCase.DafTestCase): req = DAL.newDataRequest(self.datatype) if id == 'accumHrs': req.setParameters('ARI6H2YR') - req.addIdentifier('wfo', 'OAX') - req.addIdentifier('siteKey', 'koax') + req.addIdentifier('wfo', params.SITE_ID) + req.addIdentifier('siteKey', self.location) req.addIdentifier('huc', 'ALL') idValues = DAL.getIdentifierValues(req, id) self.assertTrue(hasattr(idValues, '__iter__')) @@ -116,20 +120,20 @@ class FfmpTestCase(baseDafTestCase.DafTestCase): req = DAL.newDataRequest(self.datatype) constraint = RequestConstraint.new(operator, value) req.addIdentifier(key, constraint) - req.addIdentifier('wfo', 'OAX') + req.addIdentifier('wfo', params.SITE_ID) req.addIdentifier('huc', 'ALL') req.setParameters('QPFSCAN') return self.runGeometryDataTest(req, checkDataTimes=False) def testGetDataWithEqualsString(self): - geometryData = self._runConstraintTest('siteKey', '=', 'koax') + geometryData = self._runConstraintTest('siteKey', '=', self.location) for record in geometryData: - self.assertEqual(record.getAttribute('siteKey'), 'koax') + self.assertEqual(record.getAttribute('siteKey'), self.location) def testGetDataWithEqualsUnicode(self): - geometryData = self._runConstraintTest('siteKey', '=', u'koax') + geometryData = self._runConstraintTest('siteKey', '=', unicode(self.location)) for record in geometryData: - self.assertEqual(record.getAttribute('siteKey'), 'koax') + self.assertEqual(record.getAttribute('siteKey'), self.location) # No numeric tests since no numeric identifiers are available that support # RequestConstraints. @@ -140,9 +144,9 @@ class FfmpTestCase(baseDafTestCase.DafTestCase): self.assertIsNone(record.getAttribute('siteKey')) def testGetDataWithNotEquals(self): - geometryData = self._runConstraintTest('siteKey', '!=', 'koax') + geometryData = self._runConstraintTest('siteKey', '!=', self.location) for record in geometryData: - self.assertNotEqual(record.getAttribute('siteKey'), 'koax') + self.assertNotEqual(record.getAttribute('siteKey'), self.location) def testGetDataWithNotEqualsNone(self): geometryData = self._runConstraintTest('siteKey', '!=', None) @@ -150,40 +154,40 @@ class FfmpTestCase(baseDafTestCase.DafTestCase): self.assertIsNotNone(record.getAttribute('siteKey')) def testGetDataWithGreaterThan(self): - geometryData = self._runConstraintTest('siteKey', '>', 'koax') + geometryData = self._runConstraintTest('siteKey', '>', self.location) for record in geometryData: - self.assertGreater(record.getAttribute('siteKey'), 'koax') + self.assertGreater(record.getAttribute('siteKey'), self.location) def testGetDataWithLessThan(self): - geometryData = self._runConstraintTest('siteKey', '<', 'koax') + geometryData = self._runConstraintTest('siteKey', '<', self.location) for record in geometryData: - self.assertLess(record.getAttribute('siteKey'), 'koax') + self.assertLess(record.getAttribute('siteKey'), self.location) def testGetDataWithGreaterThanEquals(self): - geometryData = self._runConstraintTest('siteKey', '>=', 'koax') + geometryData = self._runConstraintTest('siteKey', '>=', self.location) for record in geometryData: - self.assertGreaterEqual(record.getAttribute('siteKey'), 'koax') + self.assertGreaterEqual(record.getAttribute('siteKey'), self.location) def testGetDataWithLessThanEquals(self): - geometryData = self._runConstraintTest('siteKey', '<=', 'koax') + geometryData = self._runConstraintTest('siteKey', '<=', self.location) for record in geometryData: - self.assertLessEqual(record.getAttribute('siteKey'), 'koax') + self.assertLessEqual(record.getAttribute('siteKey'), self.location) def testGetDataWithInList(self): - collection = ['koax', 'kuex'] + collection = [self.location, 'kuex'] geometryData = self._runConstraintTest('siteKey', 'in', collection) for record in geometryData: self.assertIn(record.getAttribute('siteKey'), collection) def testGetDataWithNotInList(self): - collection = ['koax', 'kuex'] + collection = [self.location, 'kuex'] geometryData = self._runConstraintTest('siteKey', 'not in', collection) for record in geometryData: self.assertNotIn(record.getAttribute('siteKey'), collection) def testGetDataWithInvalidConstraintTypeThrowsException(self): with self.assertRaises(ValueError): - self._runConstraintTest('siteKey', 'junk', 'koax') + self._runConstraintTest('siteKey', 'junk', self.location) def testGetDataWithInvalidConstraintValueThrowsException(self): with self.assertRaises(TypeError): @@ -194,11 +198,11 @@ class FfmpTestCase(baseDafTestCase.DafTestCase): self._runConstraintTest('siteKey', 'in', []) def testGetDataWithSiteKeyAndDataKeyConstraints(self): - siteKeys = ['koax', 'hpe'] + siteKeys = [self.location, 'hpe'] dataKeys = ['kuex', 'kdmx'] req = DAL.newDataRequest(self.datatype) - req.addIdentifier('wfo', 'OAX') + req.addIdentifier('wfo', params.SITE_ID) req.addIdentifier('huc', 'ALL') siteKeysConstraint = RequestConstraint.new('in', siteKeys) @@ -217,8 +221,8 @@ class FfmpTestCase(baseDafTestCase.DafTestCase): def testGetGuidanceDataWithoutAccumHrsIdentifierSet(self): # Test that accumHrs identifier is not required for guidance data req = DAL.newDataRequest(self.datatype) - req.addIdentifier('wfo', 'OAX') - req.addIdentifier('siteKey', 'koax') + req.addIdentifier('wfo', params.SITE_ID) + req.addIdentifier('siteKey', self.location) req.addIdentifier('huc', 'ALL') req.setParameters('FFG0124hr') self.runGeometryDataTest(req, checkDataTimes=False) \ No newline at end of file diff --git a/awips/test/dafTests/testGfe.py b/awips/test/dafTests/testGfe.py index d09b81a..74c8a31 100644 --- a/awips/test/dafTests/testGfe.py +++ b/awips/test/dafTests/testGfe.py @@ -21,8 +21,10 @@ from __future__ import print_function from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint from awips.dataaccess import DataAccessLayer as DAL +from shapely.geometry import box, Point import baseDafTestCase +import params import unittest # @@ -41,6 +43,12 @@ import unittest # 06/17/16 5574 mapeters Add advanced query tests # 06/30/16 5725 tgurney Add test for NOT IN # 11/07/16 5991 bsteffen Improve vector tests +# 12/07/16 5981 tgurney Parameterize +# 12/15/16 6040 tgurney Add testGetGridDataWithDbType +# 12/20/16 5981 tgurney Add envelope test +# 10/19/17 6491 tgurney Add test for dbtype identifier +# 11/10/17 6491 tgurney Replace modelName with +# parmId.dbId.modelName # # @@ -56,26 +64,49 @@ class GfeTestCase(baseDafTestCase.DafTestCase): def testGetAvailableLocations(self): req = DAL.newDataRequest(self.datatype) - req.addIdentifier('modelName', 'Fcst') + req.addIdentifier('parmId.dbId.modelName', 'Fcst') self.runLocationsTest(req) def testGetAvailableTimes(self): req = DAL.newDataRequest(self.datatype) - req.addIdentifier('modelName', 'Fcst') - req.addIdentifier('siteId', 'OAX') + req.addIdentifier('parmId.dbId.modelName', 'Fcst') + req.addIdentifier('parmId.dbId.siteId', params.SITE_ID) self.runTimesTest(req) def testGetGridData(self): req = DAL.newDataRequest(self.datatype) - req.addIdentifier('modelName', 'Fcst') - req.addIdentifier('siteId', 'OAX') + req.addIdentifier('parmId.dbId.modelName', 'Fcst') + req.addIdentifier('parmId.dbId.siteId', params.SITE_ID) req.setParameters('T') - self.runGridDataTest(req) + gridDatas = self.runGridDataTest(req) + for gridData in gridDatas: + self.assertEqual(gridData.getAttribute('parmId.dbId.dbType'), '') + + def testGetGridDataWithEnvelope(self): + req = DAL.newDataRequest(self.datatype) + req.addIdentifier('parmId.dbId.modelName', 'Fcst') + req.addIdentifier('parmId.dbId.siteId', params.SITE_ID) + req.setParameters('T') + req.setEnvelope(params.ENVELOPE) + gridData = self.runGridDataTest(req) + if not gridData: + raise unittest.SkipTest('no data available') + lons, lats = gridData[0].getLatLonCoords() + lons = lons.reshape(-1) + lats = lats.reshape(-1) + + # Ensure all points are within one degree of the original box + # to allow slight margin of error for reprojection distortion. + testEnv = box(params.ENVELOPE.bounds[0] - 1, params.ENVELOPE.bounds[1] - 1, + params.ENVELOPE.bounds[2] + 1, params.ENVELOPE.bounds[3] + 1 ) + + for i in range(len(lons)): + self.assertTrue(testEnv.contains(Point(lons[i], lats[i]))) def testGetVectorGridData(self): req = DAL.newDataRequest(self.datatype) - req.addIdentifier('modelName', 'Fcst') - req.addIdentifier('siteId', 'OAX') + req.addIdentifier('parmId.dbId.modelName', 'Fcst') + req.addIdentifier('parmId.dbId.siteId', params.SITE_ID) req.setParameters('Wind') times = DAL.getAvailableTimes(req) if not(times): @@ -114,90 +145,76 @@ class GfeTestCase(baseDafTestCase.DafTestCase): req = DAL.newDataRequest(self.datatype) constraint = RequestConstraint.new(operator, value) req.addIdentifier(key, constraint) - req.setLocationNames('OAX') + req.setLocationNames(params.SITE_ID) req.setParameters('T') return self.runGridDataTest(req) - def testGetDataWithEqualsString(self): - geometryData = self._runConstraintTest('modelName', '=', 'Fcst') - for record in geometryData: - self.assertEqual(record.getAttribute('modelName'), 'Fcst') + def testGetDataWithModelNameEqualsString(self): + gridData = self._runConstraintTest('parmId.dbId.modelName', '=', 'Fcst') + for record in gridData: + self.assertEqual(record.getAttribute('parmId.dbId.modelName'), 'Fcst') + + def testGetDataWithDbTypeEqualsString(self): + gridData = self._runConstraintTest('parmId.dbId.dbType', '=', 'Prac') + for record in gridData: + self.assertEqual(record.getAttribute('parmId.dbId.dbType'), 'Prac') def testGetDataWithEqualsUnicode(self): - geometryData = self._runConstraintTest('modelName', '=', u'Fcst') - for record in geometryData: - self.assertEqual(record.getAttribute('modelName'), 'Fcst') + gridData = self._runConstraintTest('parmId.dbId.modelName', '=', u'Fcst') + for record in gridData: + self.assertEqual(record.getAttribute('parmId.dbId.modelName'), 'Fcst') # No numeric tests since no numeric identifiers are available. def testGetDataWithEqualsNone(self): - geometryData = self._runConstraintTest('modelName', '=', None) - for record in geometryData: - self.assertIsNone(record.getAttribute('modelName')) + gridData = self._runConstraintTest('parmId.dbId.modelName', '=', None) + for record in gridData: + self.assertIsNone(record.getAttribute('parmId.dbId.modelName')) def testGetDataWithNotEquals(self): - geometryData = self._runConstraintTest('modelName', '!=', 'Fcst') - for record in geometryData: - self.assertNotEqual(record.getAttribute('modelName'), 'Fcst') + gridData = self._runConstraintTest('parmId.dbId.modelName', '!=', 'Fcst') + for record in gridData: + self.assertNotEqual(record.getAttribute('parmId.dbId.modelName'), 'Fcst') def testGetDataWithNotEqualsNone(self): - geometryData = self._runConstraintTest('modelName', '!=', None) - for record in geometryData: - self.assertIsNotNone(record.getAttribute('modelName')) - - def testGetDataWithGreaterThan(self): - geometryData = self._runConstraintTest('modelName', '>', 'Fcst') - for record in geometryData: - self.assertGreater(record.getAttribute('modelName'), 'Fcst') - - def testGetDataWithLessThan(self): - geometryData = self._runConstraintTest('modelName', '<', 'Fcst') - for record in geometryData: - self.assertLess(record.getAttribute('modelName'), 'Fcst') - - def testGetDataWithGreaterThanEquals(self): - geometryData = self._runConstraintTest('modelName', '>=', 'Fcst') - for record in geometryData: - self.assertGreaterEqual(record.getAttribute('modelName'), 'Fcst') - - def testGetDataWithLessThanEquals(self): - geometryData = self._runConstraintTest('modelName', '<=', 'Fcst') - for record in geometryData: - self.assertLessEqual(record.getAttribute('modelName'), 'Fcst') + gridData = self._runConstraintTest('parmId.dbId.modelName', '!=', None) + for record in gridData: + self.assertIsNotNone(record.getAttribute('parmId.dbId.modelName')) def testGetDataWithInTuple(self): collection = ('Fcst', 'SAT') - geometryData = self._runConstraintTest('modelName', 'in', collection) - for record in geometryData: - self.assertIn(record.getAttribute('modelName'), collection) + gridData = self._runConstraintTest('parmId.dbId.modelName', 'in', collection) + for record in gridData: + self.assertIn(record.getAttribute('parmId.dbId.modelName'), collection) def testGetDataWithInList(self): collection = ['Fcst', 'SAT'] - geometryData = self._runConstraintTest('modelName', 'in', collection) - for record in geometryData: - self.assertIn(record.getAttribute('modelName'), collection) + gridData = self._runConstraintTest('parmId.dbId.modelName', 'in', collection) + for record in gridData: + self.assertIn(record.getAttribute('parmId.dbId.modelName'), collection) def testGetDataWithInGenerator(self): collection = ('Fcst', 'SAT') generator = (item for item in collection) - geometryData = self._runConstraintTest('modelName', 'in', generator) - for record in geometryData: - self.assertIn(record.getAttribute('modelName'), collection) + gridData = self._runConstraintTest('parmId.dbId.modelName', 'in', generator) + for record in gridData: + self.assertIn(record.getAttribute('parmId.dbId.modelName'), collection) def testGetDataWithNotInList(self): collection = ('Fcst', 'SAT') - geometryData = self._runConstraintTest('modelName', 'not in', collection) - for record in geometryData: - self.assertNotIn(record.getAttribute('modelName'), collection) + gridData = self._runConstraintTest('parmId.dbId.modelName', 'not in', collection) + for record in gridData: + self.assertNotIn(record.getAttribute('parmId.dbId.modelName'), collection) def testGetDataWithInvalidConstraintTypeThrowsException(self): with self.assertRaises(ValueError): - self._runConstraintTest('modelName', 'junk', 'Fcst') + self._runConstraintTest('parmId.dbId.modelName', 'junk', 'Fcst') def testGetDataWithInvalidConstraintValueThrowsException(self): with self.assertRaises(TypeError): - self._runConstraintTest('modelName', '=', {}) + self._runConstraintTest('parmId.dbId.modelName', '=', {}) def testGetDataWithEmptyInConstraintThrowsException(self): with self.assertRaises(ValueError): - self._runConstraintTest('modelName', 'in', []) \ No newline at end of file + self._runConstraintTest('parmId.dbId.modelName', 'in', []) + diff --git a/awips/test/dafTests/testGfeEditArea.py b/awips/test/dafTests/testGfeEditArea.py new file mode 100644 index 0000000..853a180 --- /dev/null +++ b/awips/test/dafTests/testGfeEditArea.py @@ -0,0 +1,220 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +from __future__ import print_function +from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint +from ufpy.dataaccess import DataAccessLayer as DAL +from ufpy.ThriftClient import ThriftRequestException + +import baseDafTestCase +import params + +# +# Test DAF support for GFE edit area data +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 06/08/17 6298 mapeters Initial Creation. +# 09/27/17 6463 tgurney Remove GID site identifier +# +# + + +class GfeEditAreaTestCase(baseDafTestCase.DafTestCase): + """Test DAF support for GFE edit area data""" + + datatype = 'gfeEditArea' + + siteIdKey = 'siteId' + + editAreaNames = ['ISC_NHA', 'SDZ066', 'StormSurgeWW_EditArea'] + + groupKey = 'group' + + groups = ['ISC', 'WFOs', 'FIPS_' + params.SITE_ID] + + def testGetAvailableParameters(self): + req = DAL.newDataRequest(self.datatype) + req.addIdentifier(self.siteIdKey, params.SITE_ID) + with self.assertRaises(ThriftRequestException): + self.runParametersTest(req) + + def testGetAvailableLocations(self): + req = DAL.newDataRequest(self.datatype) + req.addIdentifier(self.siteIdKey, params.SITE_ID) + self.runLocationsTest(req) + + def testGetAvailableTimes(self): + req = DAL.newDataRequest(self.datatype) + req.addIdentifier(self.siteIdKey, params.SITE_ID) + with self.assertRaises(ThriftRequestException): + self.runTimesTest(req) + + def testGetGeometryDataWithoutSiteIdThrowsException(self): + req = DAL.newDataRequest(self.datatype) + with self.assertRaises(ThriftRequestException): + self.runGeometryDataTest(req) + + def testGetGeometryData(self): + req = DAL.newDataRequest(self.datatype) + req.addIdentifier(self.siteIdKey, params.SITE_ID) + data = self.runGeometryDataTest(req) + for item in data: + self.assertEqual(params.SITE_ID, item.getAttribute(self.siteIdKey)) + + def testGetGeometryDataWithLocNames(self): + req = DAL.newDataRequest(self.datatype) + req.addIdentifier(self.siteIdKey, params.SITE_ID) + req.setLocationNames(*self.editAreaNames) + data = self.runGeometryDataTest(req) + for item in data: + self.assertEqual(params.SITE_ID, item.getAttribute(self.siteIdKey)) + self.assertIn(item.getLocationName(), self.editAreaNames) + + def testGetGeometryDataWithGroups(self): + req = DAL.newDataRequest(self.datatype) + req.addIdentifier(self.siteIdKey, params.SITE_ID) + req.addIdentifier(self.groupKey, RequestConstraint.new('in', self.groups)) + data = self.runGeometryDataTest(req) + for item in data: + self.assertEqual(params.SITE_ID, item.getAttribute(self.siteIdKey)) + self.assertIn(item.getAttribute(self.groupKey), self.groups) + + def testGetGeometryDataWithLocNamesAndGroupsThrowException(self): + req = DAL.newDataRequest(self.datatype) + req.addIdentifier(self.siteIdKey, params.SITE_ID) + req.setLocationNames(*self.editAreaNames) + req.addIdentifier(self.groupKey, RequestConstraint.new('in', self.groups)) + with self.assertRaises(ThriftRequestException): + self.runGeometryDataTest(req) + + def testGetGeometryDataWithEnvelope(self): + req = DAL.newDataRequest(self.datatype) + req.addIdentifier(self.siteIdKey, params.SITE_ID) + req.setEnvelope(params.ENVELOPE) + data = self.runGeometryDataTest(req) + for item in data: + self.assertEqual(params.SITE_ID, item.getAttribute(self.siteIdKey)) + self.assertTrue(params.ENVELOPE.intersects(item.getGeometry())) + + def testGetIdentifierValues(self): + req = DAL.newDataRequest(self.datatype) + optionalIds = set(DAL.getOptionalIdentifiers(req)) + requiredIds = set(DAL.getRequiredIdentifiers(req)) + self.runGetIdValuesTest(optionalIds | requiredIds) + + def testGetInvalidIdentifierValuesThrowsException(self): + self.runInvalidIdValuesTest() + + def testGetNonexistentIdentifierValuesThrowsException(self): + self.runNonexistentIdValuesTest() + + def _runConstraintTest(self, key, operator, value): + req = DAL.newDataRequest(self.datatype) + constraint = RequestConstraint.new(operator, value) + req.addIdentifier(key, constraint) + req.setLocationNames(*self.editAreaNames) + return self.runGeometryDataTest(req) + + def testGetDataWithEqualsString(self): + geomData = self._runConstraintTest(self.siteIdKey, '=', params.SITE_ID) + for record in geomData: + self.assertEqual(record.getAttribute(self.siteIdKey), params.SITE_ID) + + def testGetDataWithEqualsUnicode(self): + geomData = self._runConstraintTest(self.siteIdKey, '=', params.SITE_ID.decode('unicode-escape')) + for record in geomData: + self.assertEqual(record.getAttribute(self.siteIdKey), params.SITE_ID) + + # No numeric tests since no numeric identifiers are available. + + def testGetDataWithEqualsNone(self): + geomData = self._runConstraintTest(self.siteIdKey, '=', None) + for record in geomData: + self.assertIsNone(record.getAttribute(self.siteIdKey)) + + def testGetDataWithNotEquals(self): + geomData = self._runConstraintTest(self.siteIdKey, '!=', params.SITE_ID) + for record in geomData: + self.assertNotEqual(record.getAttribute(self.siteIdKey), params.SITE_ID) + + def testGetDataWithNotEqualsNone(self): + geomData = self._runConstraintTest(self.siteIdKey, '!=', None) + for record in geomData: + self.assertIsNotNone(record.getAttribute(self.siteIdKey)) + + def testGetDataWithGreaterThan(self): + geomData = self._runConstraintTest(self.siteIdKey, '>', params.SITE_ID) + for record in geomData: + self.assertGreater(record.getAttribute(self.siteIdKey), params.SITE_ID) + + def testGetDataWithLessThan(self): + geomData = self._runConstraintTest(self.siteIdKey, '<', params.SITE_ID) + for record in geomData: + self.assertLess(record.getAttribute(self.siteIdKey), params.SITE_ID) + + def testGetDataWithGreaterThanEquals(self): + geomData = self._runConstraintTest(self.siteIdKey, '>=', params.SITE_ID) + for record in geomData: + self.assertGreaterEqual(record.getAttribute(self.siteIdKey), params.SITE_ID) + + def testGetDataWithLessThanEquals(self): + geomData = self._runConstraintTest(self.siteIdKey, '<=', params.SITE_ID) + for record in geomData: + self.assertLessEqual(record.getAttribute(self.siteIdKey), params.SITE_ID) + + def testGetDataWithInTuple(self): + collection = (params.SITE_ID,) + geomData = self._runConstraintTest(self.siteIdKey, 'in', collection) + for record in geomData: + self.assertIn(record.getAttribute(self.siteIdKey), collection) + + def testGetDataWithInList(self): + collection = [params.SITE_ID,] + geomData = self._runConstraintTest(self.siteIdKey, 'in', collection) + for record in geomData: + self.assertIn(record.getAttribute(self.siteIdKey), collection) + + def testGetDataWithInGenerator(self): + collection = (params.SITE_ID,) + generator = (item for item in collection) + geomData = self._runConstraintTest(self.siteIdKey, 'in', generator) + for record in geomData: + self.assertIn(record.getAttribute(self.siteIdKey), collection) + + def testGetDataWithNotInList(self): + collection = [params.SITE_ID,] + geomData = self._runConstraintTest(self.siteIdKey, 'not in', collection) + for record in geomData: + self.assertNotIn(record.getAttribute(self.siteIdKey), collection) + + def testGetDataWithInvalidConstraintTypeThrowsException(self): + with self.assertRaises(ValueError): + self._runConstraintTest(self.siteIdKey, 'junk', params.SITE_ID) + + def testGetDataWithInvalidConstraintValueThrowsException(self): + with self.assertRaises(TypeError): + self._runConstraintTest(self.siteIdKey, '=', {}) + + def testGetDataWithEmptyInConstraintThrowsException(self): + with self.assertRaises(ValueError): + self._runConstraintTest(self.siteIdKey, 'in', []) diff --git a/awips/test/dafTests/testGrid.py b/awips/test/dafTests/testGrid.py index 3c16d8a..5962320 100644 --- a/awips/test/dafTests/testGrid.py +++ b/awips/test/dafTests/testGrid.py @@ -25,6 +25,7 @@ from awips.dataaccess import DataAccessLayer as DAL from awips.ThriftClient import ThriftRequestException import baseDafTestCase +import params import unittest # @@ -44,6 +45,9 @@ import unittest # 10/13/16 5942 bsteffen Test envelopes # 11/08/16 5985 tgurney Skip certain tests when no # data is available +# 12/07/16 5981 tgurney Parameterize +# 01/06/17 5981 tgurney Skip envelope test when no +# data is available # @@ -54,8 +58,6 @@ class GridTestCase(baseDafTestCase.DafTestCase): model = 'GFS160' - envelope = box(-97.0, 41.0, -96.0, 42.0) - def testGetAvailableParameters(self): req = DAL.newDataRequest(self.datatype) req.addIdentifier('info.datasetId', self.model) @@ -110,18 +112,18 @@ class GridTestCase(baseDafTestCase.DafTestCase): req.addIdentifier('info.datasetId', self.model) req.setLevels('2FHAG') req.setParameters('T') - req.setEnvelope(self.envelope) + req.setEnvelope(params.ENVELOPE) gridData = self.runGridDataTest(req) - if not gridData: - raise unittest.SkipTest('no data available') + if len(gridData) == 0: + raise unittest.SkipTest("No data available") lons, lats = gridData[0].getLatLonCoords() lons = lons.reshape(-1) lats = lats.reshape(-1) # Ensure all points are within one degree of the original box # to allow slight margin of error for reprojection distortion. - testEnv = box(self.envelope.bounds[0] - 1, self.envelope.bounds[1] - 1, - self.envelope.bounds[2] + 1, self.envelope.bounds[3] + 1 ) + testEnv = box(params.ENVELOPE.bounds[0] - 1, params.ENVELOPE.bounds[1] - 1, + params.ENVELOPE.bounds[2] + 1, params.ENVELOPE.bounds[3] + 1 ) for i in range(len(lons)): self.assertTrue(testEnv.contains(Point(lons[i], lats[i]))) @@ -283,4 +285,4 @@ class GridTestCase(baseDafTestCase.DafTestCase): with self.assertRaises(ThriftRequestException) as cm: self.runGridDataTest(req) self.assertIn('IncompatibleRequestException', str(cm.exception)) - self.assertIn('info.level.masterLevel.name', str(cm.exception)) \ No newline at end of file + self.assertIn('info.level.masterLevel.name', str(cm.exception)) diff --git a/awips/test/dafTests/testMaps.py b/awips/test/dafTests/testMaps.py index 0b9f716..4c716b5 100644 --- a/awips/test/dafTests/testMaps.py +++ b/awips/test/dafTests/testMaps.py @@ -40,6 +40,7 @@ import unittest # 06/13/16 5574 mapeters Add advanced query tests # 06/21/16 5548 tgurney Skip tests that cause errors # 06/30/16 5725 tgurney Add test for NOT IN +# 01/06/17 5981 tgurney Do not check data times # # @@ -71,7 +72,7 @@ class MapsTestCase(baseDafTestCase.DafTestCase): req.setLocationNames('OAX') req.addIdentifier('cwa', 'OAX') req.setParameters('countyname', 'state', 'fips') - self.runGeometryDataTest(req) + self.runGeometryDataTest(req, checkDataTimes=False) def testRequestingTimesThrowsTimeAgnosticDataException(self): req = DAL.newDataRequest(self.datatype) @@ -104,22 +105,6 @@ class MapsTestCase(baseDafTestCase.DafTestCase): with self.assertRaises(ThriftRequestException): idValues = DAL.getIdentifierValues(req, 'state') - @unittest.skip('avoid EDEX error') - def testGetColumnIdValuesWithNonexistentTableThrowsException(self): - req = DAL.newDataRequest(self.datatype) - req.addIdentifier('table', 'mapdata.nonexistentjunk') - req.addIdentifier('geomField', 'the_geom') - with self.assertRaises(ThriftRequestException): - idValues = DAL.getIdentifierValues(req, 'state') - - @unittest.skip('avoid EDEX error') - def testGetNonexistentColumnIdValuesThrowsException(self): - req = DAL.newDataRequest(self.datatype) - req.addIdentifier('table', 'mapdata.county') - req.addIdentifier('geomField', 'the_geom') - with self.assertRaises(ThriftRequestException): - idValues = DAL.getIdentifierValues(req, 'nonexistentjunk') - def testGetInvalidIdentifierValuesThrowsException(self): self.runInvalidIdValuesTest() @@ -134,7 +119,7 @@ class MapsTestCase(baseDafTestCase.DafTestCase): constraint = RequestConstraint.new(operator, value) req.addIdentifier(key, constraint) req.setParameters('state', 'reservoir', 'area_sq_mi') - return self.runGeometryDataTest(req) + return self.runGeometryDataTest(req, checkDataTimes=False) def testGetDataWithEqualsString(self): geometryData = self._runConstraintTest('state', '=', 'NE') diff --git a/awips/test/dafTests/testModelSounding.py b/awips/test/dafTests/testModelSounding.py index ac8fb79..1e2454e 100644 --- a/awips/test/dafTests/testModelSounding.py +++ b/awips/test/dafTests/testModelSounding.py @@ -23,6 +23,7 @@ from awips.dataaccess import DataAccessLayer as DAL from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint import baseDafTestCase +import params import unittest # @@ -40,6 +41,9 @@ import unittest # 06/30/16 5725 tgurney Add test for NOT IN # 11/10/16 5985 tgurney Mark expected failures prior # to 17.3.1 +# 12/07/16 5981 tgurney Parameterize +# 12/19/16 5981 tgurney Remove pre-17.3 expected fails +# 12/20/16 5981 tgurney Add envelope test # # @@ -51,31 +55,25 @@ class ModelSoundingTestCase(baseDafTestCase.DafTestCase): def testGetAvailableParameters(self): req = DAL.newDataRequest(self.datatype) - self.runParametersTest(req) def testGetAvailableLocations(self): req = DAL.newDataRequest(self.datatype) req.addIdentifier("reportType", "ETA") - self.runLocationsTest(req) def testGetAvailableTimes(self): req = DAL.newDataRequest(self.datatype) req.addIdentifier("reportType", "ETA") - req.setLocationNames("KOMA") - + req.setLocationNames(params.OBS_STATION) self.runTimesTest(req) - @unittest.expectedFailure def testGetGeometryData(self): req = DAL.newDataRequest(self.datatype) req.addIdentifier("reportType", "ETA") - req.setLocationNames("KOMA") + req.setLocationNames(params.OBS_STATION) req.setParameters("temperature", "pressure", "specHum", "sfcPress", "temp2", "q2") - print("Testing getGeometryData()") - geomData = DAL.getGeometryData(req) print("Number of geometry records: " + str(len(geomData))) print("Sample geometry data:") @@ -84,18 +82,32 @@ class ModelSoundingTestCase(baseDafTestCase.DafTestCase): # One dimensional parameters are reported on the 0.0UNKNOWN level. # 2D parameters are reported on MB levels from pressure. if record.getLevel() == "0.0UNKNOWN": - print(" sfcPress=" + record.getString("sfcPress") + record.getUnit("sfcPress"), end="") - print(" temp2=" + record.getString("temp2") + record.getUnit("temp2"), end="") - print(" q2=" + record.getString("q2") + record.getUnit("q2"), end="") - + print(" sfcPress=" + record.getString("sfcPress") + + record.getUnit("sfcPress"), end="") + print(" temp2=" + record.getString("temp2") + + record.getUnit("temp2"), end="") + print(" q2=" + record.getString("q2") + + record.getUnit("q2"), end="") else: - print(" pressure=" + record.getString("pressure") + record.getUnit("pressure"), end="") - print(" temperature=" + record.getString("temperature") + record.getUnit("temperature"), end="") - print(" specHum=" + record.getString("specHum") + record.getUnit("specHum"), end="") + print(" pressure=" + record.getString("pressure") + + record.getUnit("pressure"), end="") + print(" temperature=" + record.getString("temperature") + + record.getUnit("temperature"), end="") + print(" specHum=" + record.getString("specHum") + + record.getUnit("specHum"), end="") print(" geometry=" + str(record.getGeometry())) - print("getGeometryData() complete\n\n") + def testGetGeometryDataWithEnvelope(self): + req = DAL.newDataRequest(self.datatype) + req.addIdentifier("reportType", "ETA") + req.setEnvelope(params.ENVELOPE) + req.setParameters("temperature", "pressure", "specHum", "sfcPress", "temp2", "q2") + print("Testing getGeometryData()") + data = DAL.getGeometryData(req) + for item in data: + self.assertTrue(params.ENVELOPE.contains(item.getGeometry())) + def testGetIdentifierValues(self): req = DAL.newDataRequest(self.datatype) optionalIds = set(DAL.getOptionalIdentifiers(req)) @@ -111,7 +123,7 @@ class ModelSoundingTestCase(baseDafTestCase.DafTestCase): req = DAL.newDataRequest(self.datatype) constraint = RequestConstraint.new(operator, value) req.setParameters('dataURI') - req.setLocationNames('KOMA', 'KORD', 'KOFK', 'KLNK') + req.setLocationNames(params.OBS_STATION, 'KORD', 'KOFK', 'KLNK') req.addIdentifier(key, constraint) return self.runGeometryDataTest(req) @@ -123,13 +135,11 @@ class ModelSoundingTestCase(baseDafTestCase.DafTestCase): # # Can also eyeball the number of returned records. - @unittest.expectedFailure def testGetDataWithEqualsString(self): geometryData = self._runConstraintTest('reportType', '=', 'ETA') for record in geometryData: self.assertIn('/ETA/', record.getString('dataURI')) - @unittest.expectedFailure def testGetDataWithEqualsUnicode(self): geometryData = self._runConstraintTest('reportType', '=', u'ETA') for record in geometryData: @@ -137,37 +147,29 @@ class ModelSoundingTestCase(baseDafTestCase.DafTestCase): # No numeric tests since no numeric identifiers are available. - @unittest.expectedFailure def testGetDataWithEqualsNone(self): geometryData = self._runConstraintTest('reportType', '=', None) - @unittest.expectedFailure def testGetDataWithNotEquals(self): geometryData = self._runConstraintTest('reportType', '!=', 'ETA') for record in geometryData: self.assertNotIn('/ETA/', record.getString('dataURI')) - @unittest.expectedFailure def testGetDataWithNotEqualsNone(self): geometryData = self._runConstraintTest('reportType', '!=', None) - @unittest.expectedFailure def testGetDataWithGreaterThan(self): geometryData = self._runConstraintTest('reportType', '>', 'ETA') - @unittest.expectedFailure def testGetDataWithLessThan(self): geometryData = self._runConstraintTest('reportType', '<', 'ETA') - @unittest.expectedFailure def testGetDataWithGreaterThanEquals(self): geometryData = self._runConstraintTest('reportType', '>=', 'ETA') - @unittest.expectedFailure def testGetDataWithLessThanEquals(self): geometryData = self._runConstraintTest('reportType', '<=', 'ETA') - @unittest.expectedFailure def testGetDataWithInTuple(self): collection = ('ETA', 'GFS') geometryData = self._runConstraintTest('reportType', 'in', collection) @@ -175,7 +177,6 @@ class ModelSoundingTestCase(baseDafTestCase.DafTestCase): dataURI = record.getString('dataURI') self.assertTrue('/ETA/' in dataURI or '/GFS/' in dataURI) - @unittest.expectedFailure def testGetDataWithInList(self): collection = ['ETA', 'GFS'] geometryData = self._runConstraintTest('reportType', 'in', collection) @@ -183,7 +184,6 @@ class ModelSoundingTestCase(baseDafTestCase.DafTestCase): dataURI = record.getString('dataURI') self.assertTrue('/ETA/' in dataURI or '/GFS/' in dataURI) - @unittest.expectedFailure def testGetDataWithInGenerator(self): collection = ('ETA', 'GFS') generator = (item for item in collection) diff --git a/awips/test/dafTests/testObs.py b/awips/test/dafTests/testObs.py index bf97bda..a4e92b8 100644 --- a/awips/test/dafTests/testObs.py +++ b/awips/test/dafTests/testObs.py @@ -23,6 +23,7 @@ from awips.dataaccess import DataAccessLayer as DAL from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint import baseDafTestCase +import params import unittest # @@ -38,6 +39,8 @@ import unittest # 06/09/16 5587 bsteffen Add getIdentifierValues tests # 06/13/16 5574 tgurney Add advanced query tests # 06/30/16 5725 tgurney Add test for NOT IN +# 12/07/16 5981 tgurney Parameterize +# 12/20/16 5981 tgurney Add envelope test # # @@ -57,14 +60,22 @@ class ObsTestCase(baseDafTestCase.DafTestCase): def testGetAvailableTimes(self): req = DAL.newDataRequest(self.datatype) - req.setLocationNames("KOMA") + req.setLocationNames(params.OBS_STATION) self.runTimesTest(req) def testGetGeometryData(self): req = DAL.newDataRequest(self.datatype) - req.setLocationNames("KOMA") + req.setLocationNames(params.OBS_STATION) req.setParameters("temperature", "seaLevelPress", "dewpoint") - self.runGeometryDataTest(req) + data = self.runGeometryDataTest(req) + + def testGetGeometryDataWithEnvelope(self): + req = DAL.newDataRequest(self.datatype) + req.setEnvelope(params.ENVELOPE) + req.setParameters("temperature", "seaLevelPress", "dewpoint") + data = self.runGeometryDataTest(req) + for item in data: + self.assertTrue(params.ENVELOPE.contains(item.getGeometry())) def testGetIdentifierValues(self): req = DAL.newDataRequest(self.datatype) @@ -81,7 +92,7 @@ class ObsTestCase(baseDafTestCase.DafTestCase): req = DAL.newDataRequest(self.datatype) constraint = RequestConstraint.new(operator, value) req.setParameters("temperature", "reportType") - req.setLocationNames("KOMA") + req.setLocationNames(params.OBS_STATION) req.addIdentifier(key, constraint) return self.runGeometryDataTest(req) diff --git a/awips/test/dafTests/testPirep.py b/awips/test/dafTests/testPirep.py index 876a378..7f38638 100644 --- a/awips/test/dafTests/testPirep.py +++ b/awips/test/dafTests/testPirep.py @@ -22,6 +22,7 @@ from __future__ import print_function from awips.dataaccess import DataAccessLayer as DAL import baseDafTestCase +import params import unittest # @@ -34,6 +35,8 @@ import unittest # 01/19/16 4795 mapeters Initial Creation. # 04/11/16 5548 tgurney Cleanup # 04/18/16 5548 tgurney More cleanup +# 12/07/16 5981 tgurney Parameterize +# 12/20/16 5981 tgurney Add envelope test # # @@ -53,16 +56,14 @@ class PirepTestCase(baseDafTestCase.DafTestCase): def testGetAvailableTimes(self): req = DAL.newDataRequest(self.datatype) - req.setLocationNames('OMA') + req.setLocationNames(params.AIRPORT) self.runTimesTest(req) def testGetGeometryData(self): req = DAL.newDataRequest(self.datatype) - req.setLocationNames('OMA') + req.setLocationNames(params.AIRPORT) req.setParameters("temperature", "windSpeed", "hazardType", "turbType") - print("Testing getGeometryData()") - geomData = DAL.getGeometryData(req) self.assertIsNotNone(geomData) print("Number of geometry records: " + str(len(geomData))) @@ -78,6 +79,13 @@ class PirepTestCase(baseDafTestCase.DafTestCase): print(" hazardType=" + record.getString("hazardType"), end="") print(" turbType=" + record.getString("turbType"), end="") print(" geometry=", record.getGeometry()) - print("getGeometryData() complete\n") + def testGetGeometryDataWithEnvelope(self): + req = DAL.newDataRequest(self.datatype) + req.setParameters("temperature", "windSpeed", "hazardType", "turbType") + req.setEnvelope(params.ENVELOPE) + print("Testing getGeometryData()") + data = DAL.getGeometryData(req) + for item in data: + self.assertTrue(params.ENVELOPE.contains(item.getGeometry())) diff --git a/awips/test/dafTests/testRadarGraphics.py b/awips/test/dafTests/testRadarGraphics.py new file mode 100644 index 0000000..0cb903d --- /dev/null +++ b/awips/test/dafTests/testRadarGraphics.py @@ -0,0 +1,95 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +import unittest + +from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint +from ufpy.dataaccess import DataAccessLayer as DAL + +import baseRadarTestCase +import params + + +# +# Test DAF support for radar graphics data +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 08/25/16 2671 tgurney Initial creation. +# 08/31/16 2671 tgurney Add mesocyclone +# 09/08/16 2671 tgurney Add storm track +# 09/27/16 2671 tgurney Add hail index +# 09/30/16 2671 tgurney Add TVS +# 12/07/16 5981 tgurney Parameterize +# 12/19/16 5981 tgurney Do not check data times on +# returned data +# +# +class RadarGraphicsTestCase(baseRadarTestCase.BaseRadarTestCase): + """Test DAF support for radar data""" + + datatype = 'radar' + + def runConstraintTest(self, key, operator, value): + req = DAL.newDataRequest(self.datatype) + constraint = RequestConstraint.new(operator, value) + req.addIdentifier(key, constraint) + req.setParameters('166') + # TODO: Cannot check datatimes on the result because the times returned + # by getAvailableTimes have level = -1.0, while the time on the actual + # data has the correct level set (>= 0.0). + return self.runGeometryDataTest(req, checkDataTimes=False) + + def testGetGeometryDataMeltingLayer(self): + req = DAL.newDataRequest(self.datatype) + req.setEnvelope(params.ENVELOPE) + req.setLocationNames(self.radarLoc) + req.setParameters('166') + self.runGeometryDataTest(req, checkDataTimes=False) + + def testGetGeometryDataMesocyclone(self): + req = DAL.newDataRequest(self.datatype) + req.setEnvelope(params.ENVELOPE) + req.setLocationNames(self.radarLoc) + req.setParameters('141') + self.runGeometryDataTest(req, checkDataTimes=False) + + def testGetGeometryDataStormTrack(self): + req = DAL.newDataRequest(self.datatype) + req.setEnvelope(params.ENVELOPE) + req.setLocationNames(self.radarLoc) + req.setParameters('58') + self.runGeometryDataTest(req, checkDataTimes=False) + + def testGetGeometryDataHailIndex(self): + req = DAL.newDataRequest(self.datatype) + req.setEnvelope(params.ENVELOPE) + req.setLocationNames(self.radarLoc) + req.setParameters('59') + self.runGeometryDataTest(req, checkDataTimes=False) + + def testGetGeometryDataTVS(self): + req = DAL.newDataRequest(self.datatype) + req.setEnvelope(params.ENVELOPE) + req.setLocationNames(self.radarLoc) + req.setParameters('61') + self.runGeometryDataTest(req, checkDataTimes=False) diff --git a/awips/test/dafTests/testRadarGrid.py b/awips/test/dafTests/testRadarGrid.py new file mode 100644 index 0000000..ed82f84 --- /dev/null +++ b/awips/test/dafTests/testRadarGrid.py @@ -0,0 +1,61 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +from ufpy.dataaccess import DataAccessLayer as DAL +from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint + +import baseRadarTestCase +import params +import unittest + +# +# Test DAF support for radar grid data +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 08/25/16 2671 tgurney Initial creation +# +# + + +class RadarTestCase(baseRadarTestCase.BaseRadarTestCase): + """Test DAF support for radar data""" + + datatype = 'radar' + + parameterList = ['94'] + + def runConstraintTest(self, key, operator, value): + req = DAL.newDataRequest(self.datatype) + constraint = RequestConstraint.new(operator, value) + req.addIdentifier(key, constraint) + req.setParameters(*self.parameterList) + # Don't test shapes since they may differ. + return self.runGridDataTest(req, testSameShape=False) + + def testGetGridData(self): + req = DAL.newDataRequest(self.datatype) + req.setEnvelope(params.ENVELOPE) + req.setLocationNames(self.radarLoc) + req.setParameters(*self.parameterList) + # Don't test shapes since they may differ. + self.runGridDataTest(req, testSameShape=False) diff --git a/awips/test/dafTests/testRadarSpatial.py b/awips/test/dafTests/testRadarSpatial.py index 039b3d2..6addec1 100644 --- a/awips/test/dafTests/testRadarSpatial.py +++ b/awips/test/dafTests/testRadarSpatial.py @@ -24,6 +24,7 @@ from awips.dataaccess import DataAccessLayer as DAL from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint import baseDafTestCase +import params import unittest # @@ -41,6 +42,8 @@ import unittest # superclass # 06/13/16 5574 tgurney Add advanced query tests # 06/30/16 5725 tgurney Add test for NOT IN +# 12/07/16 5981 tgurney Parameterize +# 01/06/17 5981 tgurney Do not check data times # # @@ -50,14 +53,9 @@ class RadarSpatialTestCase(baseDafTestCase.DafTestCase): datatype = "radar_spatial" - envelope = box(-97.0, 41.0, -96.0, 42.0) - """ - Default request area (box around KOAX) - """ - def testGetAvailableLocations(self): req = DAL.newDataRequest(self.datatype) - req.setEnvelope(self.envelope) + req.setEnvelope(params.ENVELOPE) self.runLocationsTest(req) def testGetAvailableParameters(self): @@ -71,7 +69,7 @@ class RadarSpatialTestCase(baseDafTestCase.DafTestCase): req = DAL.newDataRequest(self.datatype) req.setLocationNames("TORD", "TMDW") req.setParameters("wfo_id", "name", "elevmeter") - self.runGeometryDataTest(req) + self.runGeometryDataTest(req, checkDataTimes=False) def testRequestingTimesThrowsTimeAgnosticDataException(self): req = DAL.newDataRequest(self.datatype) @@ -82,17 +80,17 @@ class RadarSpatialTestCase(baseDafTestCase.DafTestCase): constraint = RequestConstraint.new(operator, value) req.addIdentifier(key, constraint) req.setParameters('elevmeter', 'eqp_elv', 'wfo_id', 'immutablex') - return self.runGeometryDataTest(req) + return self.runGeometryDataTest(req, checkDataTimes=False) def testGetDataWithEqualsString(self): - geometryData = self._runConstraintTest('wfo_id', '=', 'OAX') + geometryData = self._runConstraintTest('wfo_id', '=', params.SITE_ID) for record in geometryData: - self.assertEqual(record.getString('wfo_id'), 'OAX') + self.assertEqual(record.getString('wfo_id'), params.SITE_ID) def testGetDataWithEqualsUnicode(self): - geometryData = self._runConstraintTest('wfo_id', '=', u'OAX') + geometryData = self._runConstraintTest('wfo_id', '=', unicode(params.SITE_ID)) for record in geometryData: - self.assertEqual(record.getString('wfo_id'), 'OAX') + self.assertEqual(record.getString('wfo_id'), params.SITE_ID) def testGetDataWithEqualsInt(self): geometryData = self._runConstraintTest('immutablex', '=', 57) @@ -115,9 +113,9 @@ class RadarSpatialTestCase(baseDafTestCase.DafTestCase): self.assertEqual(record.getType('wfo_id'), 'NULL') def testGetDataWithNotEquals(self): - geometryData = self._runConstraintTest('wfo_id', '!=', 'OAX') + geometryData = self._runConstraintTest('wfo_id', '!=', params.SITE_ID) for record in geometryData: - self.assertNotEquals(record.getString('wfo_id'), 'OAX') + self.assertNotEquals(record.getString('wfo_id'), params.SITE_ID) def testGetDataWithNotEqualsNone(self): geometryData = self._runConstraintTest('wfo_id', '!=', None) @@ -145,33 +143,33 @@ class RadarSpatialTestCase(baseDafTestCase.DafTestCase): self.assertLessEqual(record.getNumber('eqp_elv'), 138) def testGetDataWithInTuple(self): - collection = ('OAX', 'GID') + collection = (params.SITE_ID, 'GID') geometryData = self._runConstraintTest('wfo_id', 'in', collection) for record in geometryData: self.assertIn(record.getString('wfo_id'), collection) def testGetDataWithInList(self): - collection = ['OAX', 'GID'] + collection = [params.SITE_ID, 'GID'] geometryData = self._runConstraintTest('wfo_id', 'in', collection) for record in geometryData: self.assertIn(record.getString('wfo_id'), collection) def testGetDataWithInGenerator(self): - collection = ('OAX', 'GID') + collection = (params.SITE_ID, 'GID') generator = (item for item in collection) geometryData = self._runConstraintTest('wfo_id', 'in', generator) for record in geometryData: self.assertIn(record.getString('wfo_id'), collection) def testGetDataWithNotInList(self): - collection = ['OAX', 'GID'] + collection = [params.SITE_ID, 'GID'] geometryData = self._runConstraintTest('wfo_id', 'not in', collection) for record in geometryData: self.assertNotIn(record.getString('wfo_id'), collection) def testGetDataWithInvalidConstraintTypeThrowsException(self): with self.assertRaises(ValueError): - self._runConstraintTest('wfo_id', 'junk', 'OAX') + self._runConstraintTest('wfo_id', 'junk', params.SITE_ID) def testGetDataWithInvalidConstraintValueThrowsException(self): with self.assertRaises(TypeError): diff --git a/awips/test/dafTests/testTopo.py b/awips/test/dafTests/testTopo.py index 1ed4131..4daf49e 100644 --- a/awips/test/dafTests/testTopo.py +++ b/awips/test/dafTests/testTopo.py @@ -24,7 +24,6 @@ from awips.ThriftClient import ThriftRequestException import baseDafTestCase import shapely.geometry -import unittest # # Test DAF support for topo data @@ -39,7 +38,9 @@ import unittest # 05/26/16 5587 tgurney Add test for # getIdentifierValues() # 06/01/16 5587 tgurney Update testGetIdentifierValues -# +# 07/18/17 6253 randerso Removed referenced to GMTED +# 02/20/18 7220 mapeters Added tests for getting filtered +# group/dataset identifier values # @@ -61,7 +62,7 @@ class TopoTestCase(baseDafTestCase.DafTestCase): print("Sample grid data shape:\n" + str(gridData[0].getRawData().shape) + "\n") print("Sample grid data:\n" + str(gridData[0].getRawData()) + "\n") - for topoFile in ["gmted2010", "gtopo30"]: + for topoFile in ["gtopo30"]: print("\n" + topoFile) req.addIdentifier("topoFile", topoFile) gridData = DAL.getGridData(req) @@ -89,6 +90,18 @@ class TopoTestCase(baseDafTestCase.DafTestCase): requiredIds = set(DAL.getRequiredIdentifiers(req)) self.runGetIdValuesTest(optionalIds | requiredIds) + def testGetFilteredDatasetValues(self): + req = DAL.newDataRequest(self.datatype) + req.addIdentifier('group', '/') + datasetVals = DAL.getIdentifierValues(req, 'dataset') + self.assertSequenceEqual(datasetVals, ['full']) + + def testGetFilteredGroupValues(self): + req = DAL.newDataRequest(self.datatype) + req.addIdentifier('dataset', '1') + groupVals = DAL.getIdentifierValues(req, 'group') + self.assertSequenceEqual(groupVals, ['/interpolated']) + def testGetInvalidIdentifierValuesThrowsException(self): self.runInvalidIdValuesTest() diff --git a/awips/test/dafTests/testWarning.py b/awips/test/dafTests/testWarning.py index fd0ef4e..5a30dee 100644 --- a/awips/test/dafTests/testWarning.py +++ b/awips/test/dafTests/testWarning.py @@ -42,7 +42,9 @@ import unittest # 06/13/16 5574 tgurney Fix checks for None # 06/21/16 5548 tgurney Skip tests that cause errors # 06/30/16 5725 tgurney Add test for NOT IN -# +# 12/12/16 5981 tgurney Improve test performance +# 02/20/18 7220 mapeters Added test for getting filtered +# column identifier values # @@ -81,22 +83,19 @@ class WarningTestCase(baseDafTestCase.DafTestCase): self.runGeometryDataTest(req) def testFilterOnLocationName(self): - allRecordsCount = len(self._getAllRecords()) allLocationNames = self._getLocationNames() - if allRecordsCount == 0: + if len(allLocationNames) == 0: errmsg = "No {0} data exists on {1}. Try again with {0} data." raise unittest.SkipTest(errmsg.format(self.datatype, DAL.THRIFT_HOST)) - if len(allLocationNames) != 1: - testCount = 3 # number of different location names to test - for locationName in allLocationNames[:testCount]: - req = DAL.newDataRequest() - req.setDatatype(self.datatype) - req.setParameters('id') - req.setLocationNames(locationName) - geomData = DAL.getGeometryData(req) - self.assertLess(len(geomData), allRecordsCount) - for geom in geomData: - self.assertEqual(geom.getLocationName(), locationName) + testCount = 3 # number of different location names to test + for locationName in allLocationNames[:testCount]: + req = DAL.newDataRequest() + req.setDatatype(self.datatype) + req.setParameters('id') + req.setLocationNames(locationName) + geomData = DAL.getGeometryData(req) + for geom in geomData: + self.assertEqual(geom.getLocationName(), locationName) def testFilterOnNonexistentLocationReturnsEmpty(self): req = DAL.newDataRequest() @@ -117,6 +116,13 @@ class WarningTestCase(baseDafTestCase.DafTestCase): def testGetColumnIdentifierValues(self): self.runGetIdValuesTest(['act']) + def testGetFilteredColumnIdentifierValues(self): + req = DAL.newDataRequest(self.datatype) + req.addIdentifier('sig', 'W') + phensigs = DAL.getIdentifierValues(req, 'phensig') + for phensig in phensigs: + self.assertTrue(phensig.endswith('.W')) + @unittest.skip('avoid EDEX error') def testGetInvalidIdentifierValuesThrowsException(self): self.runInvalidIdValuesTest() diff --git a/awips/test/localization/__init__.py b/awips/test/localization/__init__.py new file mode 100644 index 0000000..a178a51 --- /dev/null +++ b/awips/test/localization/__init__.py @@ -0,0 +1,32 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + + +# +# __init__.py for ufpy.test.localization package +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# --------- -------- --------- -------------------------- +# 08/07/17 5731 bsteffen Initial Creation. + +__all__ = [] diff --git a/awips/test/localization/testLocalizationFileManager.py b/awips/test/localization/testLocalizationFileManager.py new file mode 100644 index 0000000..64b197b --- /dev/null +++ b/awips/test/localization/testLocalizationFileManager.py @@ -0,0 +1,172 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# +# Tests for the LocalizationFileManager +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# --------- -------- --------- -------------------------- +# 08/09/17 5731 bsteffen Initial Creation. + +import unittest + +from ufpy.localization.LocalizationFileManager import (LocalizationFileManager, + LocalizationFileVersionConflictException, + LocalizationContext, + LocalizationFileIsNotDirectoryException, + LocalizationFileDoesNotExistException) + +testFile = "purge/defaultPurgeRules.xml" +testContent = "05-05:05:05" +testDir = "purge/" +testNewFile = "purge/testPurgeRules.xml" + +class ContextTestCase(unittest.TestCase): + def test_eq(self): + c1 = LocalizationContext() + c2 = LocalizationContext() + self.assertEqual(c1,c2) + c3 = LocalizationContext("site", "test") + c4 = LocalizationContext("site", "test") + self.assertEqual(c3,c4) + self.assertNotEqual(c1,c3) + + def test_hash(self): + c1 = LocalizationContext() + c2 = LocalizationContext() + self.assertEqual(hash(c1),hash(c2)) + c3 = LocalizationContext("site", "test") + c4 = LocalizationContext("site", "test") + self.assertEqual(hash(c3),hash(c4)) + +class LFMTestCase(unittest.TestCase): + def setUp(self): + self.manager = LocalizationFileManager() + userFile = self.manager.getSpecific("user", testFile) + if userFile.exists(): + userFile.delete() + newFile = self.manager.getSpecific("user", testNewFile) + if newFile.exists(): + newFile.delete() + def test_gets(self): + startingIncremental = self.manager.getIncremental(testFile) + baseFile = self.manager.getSpecific("base", testFile) + self.assertEqual(baseFile, startingIncremental[0]) + self.assertTrue(baseFile.exists()) + self.assertFalse(baseFile.isDirectory()) + userFile = self.manager.getSpecific("user", testFile) + self.assertFalse(userFile.exists()) + with userFile.open("w") as stream: + stream.write(testContent) + userFile = self.manager.getSpecific("user", testFile) + self.assertTrue(userFile.exists()) + with userFile.open('r') as stream: + self.assertEqual(stream.read(), testContent) + absFile = self.manager.getAbsolute(testFile) + self.assertEqual(absFile, userFile) + endingIncremental = self.manager.getIncremental(testFile) + self.assertEqual(len(startingIncremental) + 1, len(endingIncremental)) + self.assertEqual(userFile, endingIncremental[-1]) + self.assertEqual(baseFile, endingIncremental[0]) + + + userFile.delete() + userFile = self.manager.getSpecific("user", testFile) + self.assertFalse(userFile.exists()) + + def test_concurrent_edit(self): + userFile1 = self.manager.getSpecific("user", testFile) + userFile2 = self.manager.getSpecific("user", testFile) + self.assertFalse(userFile1.exists()) + self.assertFalse(userFile2.exists()) + with self.assertRaises(LocalizationFileVersionConflictException): + with userFile1.open("w") as stream1: + stream1.write(testContent) + with userFile2.open("w") as stream2: + stream2.write(testContent) + + userFile = self.manager.getSpecific("user", testFile) + userFile.delete() + + def test_dir(self): + dir = self.manager.getAbsolute(testDir) + self.assertTrue(dir.isDirectory()) + with self.assertRaises(Exception): + dir.delete() + + def test_list(self): + abs1 = self.manager.listAbsolute(testDir) + inc1 = self.manager.listIncremental(testDir) + self.assertEqual(len(abs1), len(inc1)) + for i in range(len(abs1)): + self.assertEquals(abs1[i], inc1[i][-1]) + + userFile = self.manager.getSpecific("user", testNewFile) + self.assertNotIn(userFile, abs1) + + with userFile.open("w") as stream: + stream.write(testContent) + userFile = self.manager.getSpecific("user", testNewFile) + + + abs2 = self.manager.listAbsolute(testDir) + inc2 = self.manager.listIncremental(testDir) + self.assertEqual(len(abs2), len(inc2)) + for i in range(len(abs2)): + self.assertEquals(abs2[i], inc2[i][-1]) + + self.assertEquals(len(abs1) + 1, len(abs2)) + self.assertIn(userFile, abs2) + + userFile.delete() + + def test_list_file(self): + with self.assertRaises(LocalizationFileIsNotDirectoryException): + self.manager.listIncremental(testFile) + + def test_list_nonexistant(self): + with self.assertRaises(LocalizationFileDoesNotExistException): + self.manager.listIncremental('dontNameYourDirectoryThis') + + def test_root_variants(self): + list1 = self.manager.listAbsolute(".") + list2 = self.manager.listAbsolute("") + list3 = self.manager.listAbsolute("/") + self.assertEquals(list1,list2) + self.assertEquals(list2,list3) + + def test_slashiness(self): + raw = testDir + if raw[0] == '/': + raw = raw[1:] + if raw[-1] == '/': + raw = raw[:-1] + list1 = self.manager.listAbsolute(raw) + list2 = self.manager.listAbsolute(raw + "/") + list3 = self.manager.listAbsolute("/" + raw) + self.assertEquals(list1,list2) + self.assertEquals(list2,list3) + + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/awips/test/localization/testLocalizationRest.py b/awips/test/localization/testLocalizationRest.py new file mode 100644 index 0000000..98b264b --- /dev/null +++ b/awips/test/localization/testLocalizationRest.py @@ -0,0 +1,359 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +import unittest +import urllib2 + +from HTMLParser import HTMLParser +from xml.etree.ElementTree import parse as parseXml +from json import load as loadjson +from urlparse import urljoin +from base64 import b64encode + +# +# Test the localizaiton REST service. +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# --------- -------- --------- -------------------------- +# 08/07/17 5731 bsteffen Initial Creation. + +baseURL = "http://localhost:9581/services/localization/" +testSite = "OAX" +testDir = "menus" +testFile = "test.xml" +username = "test" +password = username + +base64string = b64encode('%s:%s' % (username, password)) +authString = "Basic %s" % base64string + +class ValidHTMLParser(HTMLParser): + """Simple HTML parser that performs very minimal validation. + + This ensures that all start and end tags match, and also that there are + some tags. It also accumulates the text of all links in the html file + in the link_texts attribute, which can be used for further validation. + """ + + def __init__(self, testcase): + HTMLParser.__init__(self) + self._testcase = testcase + self._tags = [] + self._any = False + self.link_texts = [] + + def handle_starttag(self, tag, attrs): + self._tags.append(tag) + self._any = True + + def handle_endtag(self, tag): + self._testcase.assertNotEquals([], self._tags, "Unstarted end tag " + tag) + self._testcase.assertEquals(tag, self._tags.pop()) + + def handle_data(self, data): + if self._tags[-1] == "a": + self.link_texts.append(data) + + def close(self): + HTMLParser.close(self) + self._testcase.assertTrue(self._any) + self._testcase.assertEquals([], self._tags) + + + +class AbstractListingTestCase(): + """Base test case for testing listings, retrieves data as html, xml, and json. + + Sub classes should implement assertValidHtml, assertValidXml, and + assertValidJson to ensure that the content returned matches what was + expected. + """ + + def assertRequestGetsHtml(self, request): + response = urllib2.urlopen(request) + self.assertEquals(response.headers["Content-Type"], "text/html") + body = response.read() + parser = ValidHTMLParser(self) + parser.feed(body) + parser.close() + self.assertValidHtml(parser) + + def assertValidHtml(self, parser): + """Intended to be overriden by subclasses to validate HTML content. + + The argument is a populated instance of ValidHTMLParser. + """ + pass + + def test_default(self): + request = urllib2.Request(self.url) + self.assertRequestGetsHtml(request) + + def test_last_slash(self): + if self.url.endswith("/"): + request = urllib2.Request(self.url[:-1]) + else: + request = urllib2.Request(self.url + "/") + self.assertRequestGetsHtml(request) + + def test_wild_mime(self): + request = urllib2.Request(self.url) + request.add_header("Accept", "*/*") + self.assertRequestGetsHtml(request) + request.add_header("Accept", "text/*") + self.assertRequestGetsHtml(request) + + def test_html(self): + request = urllib2.Request(self.url) + request.add_header("Accept", "text/html") + self.assertRequestGetsHtml(request) + + def test_json(self): + request = urllib2.Request(self.url) + request.add_header("Accept", "application/json") + response = urllib2.urlopen(request) + self.assertEquals(response.headers["Content-Type"], "application/json") + jsonData = loadjson(response) + self.assertValidJson(jsonData) + + + def assertValidJson(self, jsonData): + """Intended to be overriden by subclasses to validate JSON content. + + The argument is a python object as returned from json.load + """ + pass + + def test_xml(self): + request = urllib2.Request(self.url) + request.add_header("Accept", "application/xml") + response = urllib2.urlopen(request) + self.assertEquals(response.headers["Content-Type"], "application/xml") + xmlData = parseXml(response) + self.assertValidXml(xmlData) + + def assertValidXml(self, xmlData): + """Intended to be overriden by subclasses to validate XML content. + + The argument is an ElementTree + """ + pass + + def test_delete(self): + request = urllib2.Request(self.url) + request.get_method = lambda: "DELETE" + with self.assertRaises(urllib2.HTTPError) as cm: + response = urllib2.urlopen(request) + self.assertEqual(405, cm.exception.code) + + def test_put(self): + request = urllib2.Request(self.url) + request.get_method = lambda: "PUT" + request.add_data("Test Data") + with self.assertRaises(urllib2.HTTPError) as cm: + response = urllib2.urlopen(request) + self.assertEqual(405, cm.exception.code) + + def test_unacceptable(self): + request = urllib2.Request(self.url) + request.add_header("Accept", "application/fakemimetype") + with self.assertRaises(urllib2.HTTPError) as cm: + response = urllib2.urlopen(request) + self.assertEqual(406, cm.exception.code) + request.add_header("Accept", "fakemimetype/*") + with self.assertRaises(urllib2.HTTPError) as cm: + response = urllib2.urlopen(request) + self.assertEqual(406, cm.exception.code) + + def test_accept_quality_factor(self): + request = urllib2.Request(self.url) + request.add_header("Accept", "application/xml; q=0.8, application/json; q=0.2") + response = urllib2.urlopen(request) + self.assertEquals(response.headers["Content-Type"], "application/xml") + xmlData = parseXml(response) + self.assertValidXml(xmlData) + + request.add_header("Accept", "application/xml; q=0.2, application/json; q=0.8") + response = urllib2.urlopen(request) + self.assertEquals(response.headers["Content-Type"], "application/json") + jsonData = loadjson(response) + self.assertValidJson(jsonData) + + request.add_header("Accept", "application/xml, application/json; q=0.8") + response = urllib2.urlopen(request) + self.assertEquals(response.headers["Content-Type"], "application/xml") + xmlData = parseXml(response) + self.assertValidXml(xmlData) + + request.add_header("Accept", "application/fakemimetype, application/json; q=0.8") + response = urllib2.urlopen(request) + self.assertEquals(response.headers["Content-Type"], "application/json") + jsonData = loadjson(response) + self.assertValidJson(jsonData) + +class RootTestCase(AbstractListingTestCase, unittest.TestCase): + """Test that the root of the localization service returns listing of localization types.""" + def setUp(self): + self.url = baseURL + def assertValidHtml(self, parser): + self.assertIn("common_static/", parser.link_texts) + def assertValidJson(self, jsonData): + self.assertIn("common_static/", jsonData) + def assertValidXml(self, xmlData): + root = xmlData.getroot() + self.assertEquals(root.tag, "entries") + names = [e.text for e in root.findall("entry")] + self.assertIn("common_static/", names) + +class TypeTestCase(AbstractListingTestCase, unittest.TestCase): + """Test that common_static will list context levels.""" + def setUp(self): + self.url = urljoin(baseURL, "common_static/") + def assertValidHtml(self, parser): + self.assertIn("base/", parser.link_texts) + self.assertIn("site/", parser.link_texts) + def assertValidJson(self, jsonData): + self.assertIn("base/", jsonData) + self.assertIn("site/", jsonData) + def assertValidXml(self, xmlData): + root = xmlData.getroot() + self.assertEquals(root.tag, "entries") + names = [e.text for e in root.findall("entry")] + self.assertIn("base/", names) + self.assertIn("site/", names) + +class LevelTestCase(AbstractListingTestCase, unittest.TestCase): + """Test that common_static/site will list sites.""" + def setUp(self): + self.url = urljoin(baseURL, "common_static/site/") + def assertValidHtml(self, parser): + self.assertIn(testSite +"/", parser.link_texts) + def assertValidJson(self, jsonData): + self.assertIn(testSite +"/", jsonData) + def assertValidXml(self, xmlData): + root = xmlData.getroot() + self.assertEquals(root.tag, "entries") + names = [e.text for e in root.findall("entry")] + self.assertIn(testSite +"/", names) + +class AbstractFileListingTestCase(AbstractListingTestCase): + """Base test case for a file listing""" + + def assertValidHtml(self, parser): + self.assertIn(testDir +"/", parser.link_texts) + self.assertEquals(parser.link_texts, sorted(parser.link_texts)) + def assertValidJson(self, jsonData): + self.assertIn(testDir +"/", jsonData) + def assertValidXml(self, xmlData): + root = xmlData.getroot() + self.assertEquals(root.tag, "files") + names = [e.get("name") for e in root.findall("file")] + self.assertIn(testDir +"/", names) + self.assertEquals(names, sorted(names)) + +class BaseFileListingTestCase(AbstractFileListingTestCase, unittest.TestCase): + """Test that common_static/base lists files""" + def setUp(self): + self.url = urljoin(baseURL, "common_static/base/") + +class SiteFileListingTestCase(AbstractFileListingTestCase, unittest.TestCase): + """Test that common_static/site// lists files""" + def setUp(self): + self.url = urljoin(baseURL, "common_static/site/" + testSite + "/") + +class FileTestCase(unittest.TestCase): + """Test retrieval, modification and deletion of an individual.""" + def setUp(self): + self.url = urljoin(baseURL, "common_static/user/" + username + "/" + testFile) + # The file should not exist before the test, but if it does then delete it + # This is some of the same functionality we are testing so if setup fails + # then the test would probably fail anyway + try: + request = urllib2.Request(self.url) + response = urllib2.urlopen(request) + request = urllib2.Request(self.url) + request.get_method = lambda: "DELETE" + request.add_header("Authorization", authString) + request.add_header("If-Match", response.headers["Content-MD5"]) + response = urllib2.urlopen(request) + except urllib2.HTTPError as e: + if e.code != 404: + raise e + def test_file_operations(self): + """Run through a typical set of file interactions and verify everything works correctly.""" + request = urllib2.Request(self.url) + request.get_method = lambda: "PUT" + request.add_data("Test Data") + with self.assertRaises(urllib2.HTTPError) as cm: + response = urllib2.urlopen(request) + self.assertEqual(401, cm.exception.code) + + request.add_header("Authorization", authString) + with self.assertRaises(urllib2.HTTPError) as cm: + response = urllib2.urlopen(request) + self.assertEqual(409, cm.exception.code) + + request.add_header("If-Match", "NON_EXISTENT_CHECKSUM") + response = urllib2.urlopen(request) + + + request = urllib2.Request(self.url) + response = urllib2.urlopen(request) + self.assertEquals(response.read(), "Test Data") + + request = urllib2.Request(self.url + "/") + response = urllib2.urlopen(request) + self.assertEquals(response.read(), "Test Data") + + request = urllib2.Request(self.url) + request.get_method = lambda: "PUT" + request.add_data("Test Data2") + request.add_header("If-Match", response.headers["Content-MD5"]) + request.add_header("Authorization", authString) + response = urllib2.urlopen(request) + checksum = response.headers["Content-MD5"] + + request = urllib2.Request(self.url) + response = urllib2.urlopen(request) + self.assertEquals(response.read(), "Test Data2") + + request = urllib2.Request(self.url) + request.get_method = lambda: "DELETE" + with self.assertRaises(urllib2.HTTPError) as cm: + response = urllib2.urlopen(request) + self.assertEqual(401, cm.exception.code) + + request.add_header("Authorization", authString) + with self.assertRaises(urllib2.HTTPError) as cm: + response = urllib2.urlopen(request) + self.assertEqual(409, cm.exception.code) + + request.add_header("If-Match", checksum) + response = urllib2.urlopen(request) + + request = urllib2.Request(self.url) + with self.assertRaises(urllib2.HTTPError) as cm: + response = urllib2.urlopen(request) + self.assertEqual(404, cm.exception.code) + +if __name__ == '__main__': + unittest.main() diff --git a/awips/test/testQpidTimeToLive.py b/awips/test/testQpidTimeToLive.py index 7176324..6172735 100644 --- a/awips/test/testQpidTimeToLive.py +++ b/awips/test/testQpidTimeToLive.py @@ -78,6 +78,7 @@ class ListenThread(threading.Thread): self.qs.close() + def main(): print "Starting up at", time.strftime('%H:%M:%S') @@ -95,5 +96,9 @@ def main(): finally: thread.stop() + if __name__ == '__main__': main() + + + diff --git a/dynamicserialize/DynamicSerializationManager.py b/dynamicserialize/DynamicSerializationManager.py index 2615149..60d8512 100644 --- a/dynamicserialize/DynamicSerializationManager.py +++ b/dynamicserialize/DynamicSerializationManager.py @@ -66,4 +66,4 @@ class DynamicSerializationManager: return self.transport.getvalue() def _serialize(self, ctx, obj): - ctx.serializeMessage(obj) + ctx.serializeMessage(obj) \ No newline at end of file diff --git a/dynamicserialize/ThriftSerializationContext.py b/dynamicserialize/ThriftSerializationContext.py index 0ed39d2..75aa08e 100644 --- a/dynamicserialize/ThriftSerializationContext.py +++ b/dynamicserialize/ThriftSerializationContext.py @@ -29,7 +29,7 @@ # match what they should be in the destination language. # # -# SOFTWARE HISTORY +# SOFTWARE HISTORY # # Date Ticket# Engineer Description # ------------ ---------- ----------- -------------------------- @@ -37,20 +37,27 @@ # 06/12/13 #2099 dgilling Implement readObject() and # writeObject(). # Apr 24, 2015 4425 nabowle Add Double support +# Oct 17, 2016 5919 njensen Optimized for speed # # from thrift.Thrift import TType -import inspect, sys, types +import inspect +import sys +import types +import time import dynamicserialize from dynamicserialize import dstypes, adapters import SelfDescribingBinaryProtocol import numpy +DS_LEN = len('dynamicserialize.dstypes.') + dsObjTypes = {} + def buildObjMap(module): - if module.__dict__.has_key('__all__'): + if '__all__' in module.__dict__: for i in module.__all__: name = module.__name__ + '.' + i __import__(name) @@ -59,7 +66,7 @@ def buildObjMap(module): clzName = module.__name__[module.__name__.rfind('.') + 1:] clz = module.__dict__[clzName] tname = module.__name__ - tname = tname.replace('dynamicserialize.dstypes.', '') + tname = tname[DS_LEN:] dsObjTypes[tname] = clz buildObjMap(dstypes) @@ -72,7 +79,7 @@ pythonToThriftMap = { types.DictionaryType: TType.MAP, type(set([])): TType.SET, types.FloatType: SelfDescribingBinaryProtocol.FLOAT, - #types.FloatType: TType.DOUBLE, + # types.FloatType: TType.DOUBLE, types.BooleanType: TType.BOOL, types.InstanceType: TType.STRUCT, types.NoneType: TType.VOID, @@ -87,7 +94,9 @@ pythonToThriftMap = { numpy.int64: TType.I64 } -primitiveSupport = (TType.BYTE, TType.I16, TType.I32, TType.I64, SelfDescribingBinaryProtocol.FLOAT, TType.DOUBLE) +primitiveSupport = (TType.BYTE, TType.I16, TType.I32, TType.I64, + SelfDescribingBinaryProtocol.FLOAT, TType.DOUBLE) + class ThriftSerializationContext(object): @@ -95,52 +104,51 @@ class ThriftSerializationContext(object): self.serializationManager = serializationManager self.protocol = selfDescribingBinaryProtocol self.typeDeserializationMethod = { - TType.STRING: self.protocol.readString, - TType.I16: self.protocol.readI16, - TType.I32: self.protocol.readI32, - TType.LIST: self._deserializeArray, - TType.MAP: self._deserializeMap, - TType.SET: self._deserializeSet, - SelfDescribingBinaryProtocol.FLOAT: self.protocol.readFloat, - TType.BYTE: self.protocol.readByte, - TType.I64: self.protocol.readI64, - TType.DOUBLE: self.protocol.readDouble, - TType.BOOL: self.protocol.readBool, - TType.STRUCT: self.deserializeMessage, - TType.VOID: lambda: None - } + TType.STRING: self.protocol.readString, + TType.I16: self.protocol.readI16, + TType.I32: self.protocol.readI32, + TType.LIST: self._deserializeArray, + TType.MAP: self._deserializeMap, + TType.SET: self._deserializeSet, + SelfDescribingBinaryProtocol.FLOAT: self.protocol.readFloat, + TType.BYTE: self.protocol.readByte, + TType.I64: self.protocol.readI64, + TType.DOUBLE: self.protocol.readDouble, + TType.BOOL: self.protocol.readBool, + TType.STRUCT: self.deserializeMessage, + TType.VOID: lambda: None + } self.typeSerializationMethod = { - TType.STRING: self.protocol.writeString, - TType.I16: self.protocol.writeI16, - TType.I32: self.protocol.writeI32, - TType.LIST: self._serializeArray, - TType.MAP: self._serializeMap, - TType.SET: self._serializeSet, - SelfDescribingBinaryProtocol.FLOAT: self.protocol.writeFloat, - TType.BYTE: self.protocol.writeByte, - TType.I64: self.protocol.writeI64, - TType.DOUBLE: self.protocol.writeDouble, - TType.BOOL: self.protocol.writeBool, - TType.STRUCT: self.serializeMessage, - TType.VOID: lambda x: None - } + TType.STRING: self.protocol.writeString, + TType.I16: self.protocol.writeI16, + TType.I32: self.protocol.writeI32, + TType.LIST: self._serializeArray, + TType.MAP: self._serializeMap, + TType.SET: self._serializeSet, + SelfDescribingBinaryProtocol.FLOAT: self.protocol.writeFloat, + TType.BYTE: self.protocol.writeByte, + TType.I64: self.protocol.writeI64, + TType.DOUBLE: self.protocol.writeDouble, + TType.BOOL: self.protocol.writeBool, + TType.STRUCT: self.serializeMessage, + TType.VOID: lambda x: None + } self.listDeserializationMethod = { - TType.BYTE: self.protocol.readI8List, - TType.I16: self.protocol.readI16List, - TType.I32: self.protocol.readI32List, - TType.I64: self.protocol.readI64List, - SelfDescribingBinaryProtocol.FLOAT: self.protocol.readF32List, - TType.DOUBLE: self.protocol.readF64List - } + TType.BYTE: self.protocol.readI8List, + TType.I16: self.protocol.readI16List, + TType.I32: self.protocol.readI32List, + TType.I64: self.protocol.readI64List, + SelfDescribingBinaryProtocol.FLOAT: self.protocol.readF32List, + TType.DOUBLE: self.protocol.readF64List + } self.listSerializationMethod = { - TType.BYTE: self.protocol.writeI8List, - TType.I16: self.protocol.writeI16List, - TType.I32: self.protocol.writeI32List, - TType.I64: self.protocol.writeI64List, - SelfDescribingBinaryProtocol.FLOAT: self.protocol.writeF32List, - TType.DOUBLE: self.protocol.writeF64List - } - + TType.BYTE: self.protocol.writeI8List, + TType.I16: self.protocol.writeI16List, + TType.I32: self.protocol.writeI32List, + TType.I64: self.protocol.writeI64List, + SelfDescribingBinaryProtocol.FLOAT: self.protocol.writeF32List, + TType.DOUBLE: self.protocol.writeF64List + } def readMessageStart(self): msg = self.protocol.readMessageBegin() @@ -151,17 +159,19 @@ class ThriftSerializationContext(object): def deserializeMessage(self): name = self.protocol.readStructBegin() - name = name.replace('_', '.') if name.isdigit(): obj = self._deserializeType(int(name)) return obj - elif adapters.classAdapterRegistry.has_key(name): + name = name.replace('_', '.') + if name in adapters.classAdapterRegistry: return adapters.classAdapterRegistry[name].deserialize(self) - elif name.find('$') > -1: - # it's an inner class, we're going to hope it's an enum, treat it special + elif '$' in name: + # it's an inner class, we're going to hope it's an enum, treat it + # special fieldName, fieldType, fieldId = self.protocol.readFieldBegin() if fieldName != '__enumValue__': - raise dynamiceserialize.SerializationException("Expected to find enum payload. Found: " + fieldName) + raise dynamiceserialize.SerializationException( + "Expected to find enum payload. Found: " + fieldName) obj = self.protocol.readString() self.protocol.readFieldEnd() return obj @@ -176,37 +186,30 @@ class ThriftSerializationContext(object): return obj def _deserializeType(self, b): - if self.typeDeserializationMethod.has_key(b): + try: return self.typeDeserializationMethod[b]() - else: - raise dynamicserialize.SerializationException("Unsupported type value " + str(b)) - + except KeyError: + raise dynamicserialize.SerializationException( + "Unsupported type value " + str(b)) def _deserializeField(self, structname, obj): fieldName, fieldType, fieldId = self.protocol.readFieldBegin() if fieldType == TType.STOP: return False elif fieldType != TType.VOID: -# if adapters.fieldAdapterRegistry.has_key(structname) and adapters.fieldAdapterRegistry[structname].has_key(fieldName): -# result = adapters.fieldAdapterRegistry[structname][fieldName].deserialize(self) -# else: result = self._deserializeType(fieldType) lookingFor = "set" + fieldName[0].upper() + fieldName[1:] try: setMethod = getattr(obj, lookingFor) - - if callable(setMethod): - setMethod(result) - else: - raise dynamicserialize.SerializationException("Couldn't find setter method " + lookingFor) + setMethod(result) except: - raise dynamicserialize.SerializationException("Couldn't find setter method " + lookingFor) + raise dynamicserialize.SerializationException( + "Couldn't find setter method " + lookingFor) self.protocol.readFieldEnd() return True - def _deserializeArray(self): listType, size = self.protocol.readListBegin() result = [] @@ -241,19 +244,20 @@ class ThriftSerializationContext(object): def _lookupType(self, obj): pyt = type(obj) - if pythonToThriftMap.has_key(pyt): + if pyt in pythonToThriftMap: return pythonToThriftMap[pyt] - elif pyt.__module__.startswith('dynamicserialize.dstypes'): + elif pyt.__module__[:DS_LEN - 1] == ('dynamicserialize.dstypes'): return pythonToThriftMap[types.InstanceType] else: - raise dynamicserialize.SerializationException("Don't know how to serialize object of type: " + str(pyt)) + raise dynamicserialize.SerializationException( + "Don't know how to serialize object of type: " + str(pyt)) def serializeMessage(self, obj): tt = self._lookupType(obj) if tt == TType.STRUCT: - fqn = obj.__module__.replace('dynamicserialize.dstypes.', '') - if adapters.classAdapterRegistry.has_key(fqn): + fqn = obj.__module__[DS_LEN:] + if fqn in adapters.classAdapterRegistry: # get proper class name when writing class name to serialization stream # in case we have a special inner-class case m = sys.modules[adapters.classAdapterRegistry[fqn].__name__] @@ -273,7 +277,7 @@ class ThriftSerializationContext(object): val = m[1]() ft = self._lookupType(val) if ft == TType.STRUCT: - fc = val.__module__.replace('dynamicserialize.dstypes.', '') + fc = val.__module__[DS_LEN:] self._serializeField(fieldname, ft, fid, val) else: self._serializeField(fieldname, ft, fid, val) @@ -293,17 +297,18 @@ class ThriftSerializationContext(object): self.protocol.writeFieldEnd() def _serializeType(self, fieldValue, fieldType): - if self.typeSerializationMethod.has_key(fieldType): + if fieldType in self.typeSerializationMethod: return self.typeSerializationMethod[fieldType](fieldValue) else: - raise dynamicserialize.SerializationException("Unsupported type value " + str(fieldType)) + raise dynamicserialize.SerializationException( + "Unsupported type value " + str(fieldType)) def _serializeArray(self, obj): size = len(obj) if size: if type(obj) is numpy.ndarray: - t = pythonToThriftMap[obj.dtype.type] - size = obj.size + t = pythonToThriftMap[obj.dtype.type] + size = obj.size else: t = self._lookupType(obj[0]) else: @@ -331,7 +336,6 @@ class ThriftSerializationContext(object): self.listSerializationMethod[t](obj) self.protocol.writeListEnd() - def _serializeMap(self, obj): size = len(obj) self.protocol.writeMapBegin(TType.VOID, TType.VOID, size) diff --git a/dynamicserialize/adapters/__init__.py b/dynamicserialize/adapters/__init__.py index d7e643f..f72aaa8 100644 --- a/dynamicserialize/adapters/__init__.py +++ b/dynamicserialize/adapters/__init__.py @@ -1,19 +1,19 @@ ## # This software was developed and / or modified by Raytheon Company, # pursuant to Contract DG133W-05-CQ-1067 with the US Government. -# +# # U.S. EXPORT CONTROLLED TECHNICAL DATA # This software product contains export-restricted data whose # export/transfer/disclosure is restricted by U.S. law. Dissemination # to non-U.S. persons whether in the United States or abroad requires # an export license or other authorization. -# +# # Contractor Name: Raytheon Company # Contractor Address: 6825 Pine Street, Suite 340 # Mail Stop B8 # Omaha, NE 68106 # 402.291.0100 -# +# # See the AWIPS II Master Rights File ("Master Rights File.pdf") for # further licensing information. ## @@ -21,56 +21,80 @@ # # __init__.py for Dynamic Serialize adapters. -# -# +# +# Plugins can contribute to dynamicserialize.adapters by either including their +# classes directly in pythonPackages/dynamicserialize/adapters/ within their +# plugin. The plugin's adapter will automatically be added to __all__ at runtime +# and registered. +# Plugins should not include a custom __init__.py in +# pythonPackages/dynamicserialize/adapters/ because it will overwrite this file. +# If custom package initialization is needed, a subpackage should be created +# with an __init__.py that includes the following: +# +# __all__ = ['CustomAdapter1', 'CustomAdapter2'] +# from dynamicserialize.adapters import registerAdapters +# registerAdapters(__name__, __all__) +# +# # SOFTWARE HISTORY -# +# # Date Ticket# Engineer Description # ------------ ---------- ----------- -------------------------- -# 08/31/10 njensen Initial Creation. -# 03/20/13 #1774 randerso Added TimeConstraintsAdapter -# 04/22/13 #1949 rjpeter Added LockTableAdapter -# 02/06/14 #2672 bsteffen Added JTSEnvelopeAdapter -# 06/22/2015 #4573 randerso Added JobProgressAdapter -# 09/21/2015 #4486 rjpeter Added FormattedDateAdapter -# 06/23/2016 #5696 rjpeter Added CommutativeTimestampAdapter +# 08/31/10 njensen Initial Creation. +# 03/20/13 #1774 randerso Added TimeConstraintsAdapter +# 04/22/13 #1949 rjpeter Added LockTableAdapter +# 02/06/14 #2672 bsteffen Added JTSEnvelopeAdapter +# 06/22/2015 #4573 randerso Added JobProgressAdapter +# 09/21/2015 #4486 rjpeter Added FormattedDateAdapter +# 06/23/2016 #5696 rjpeter Added CommutativeTimestampAdapter +# 10/17/2016 #5919 njensen Added GeomDataRespAdapter +# 01/09/2017 #5997 nabowle Allow contribution from plugins. # __all__ = [ - 'PointAdapter', - 'StackTraceElementAdapter', - 'WsIdAdapter', - 'CalendarAdapter', - 'GregorianCalendarAdapter', - 'ActiveTableModeAdapter', - 'DateAdapter', - 'FormattedDateAdapter', - 'LocalizationLevelSerializationAdapter', - 'LocalizationTypeSerializationAdapter', - 'GeometryTypeAdapter', - 'CoordAdapter', - 'TimeRangeTypeAdapter', - 'ParmIDAdapter', - 'DatabaseIDAdapter', - 'TimestampAdapter', - 'CommutativeTimestampAdapter', - 'EnumSetAdapter', - 'FloatBufferAdapter', - 'ByteBufferAdapter', - 'TimeConstraintsAdapter', - 'LockTableAdapter', - 'JTSEnvelopeAdapter', - 'JobProgressAdapter', - ] - + 'PointAdapter', + 'StackTraceElementAdapter', + 'CalendarAdapter', + 'GregorianCalendarAdapter', + 'DateAdapter', + 'GeometryTypeAdapter', + 'CoordAdapter', + 'TimestampAdapter', + 'EnumSetAdapter', + 'FloatBufferAdapter', + 'ByteBufferAdapter', + 'JTSEnvelopeAdapter' +] + classAdapterRegistry = {} - + def getAdapterRegistry(): - import sys - for x in __all__: - exec 'import ' + x - m = sys.modules['dynamicserialize.adapters.' + x] + import pkgutil + + discoveredPackages = [] + # allow other plugins to contribute to adapters by dropping their adapter or + # package into the dynamicserialize.adapters package + for _, modname, ispkg in pkgutil.iter_modules(__path__): + if ispkg: + discoveredPackages.append(modname) + else: + if modname not in __all__: + __all__.append(modname) + + registerAdapters(__name__, __all__) + + for pkg in discoveredPackages: + __import__(__name__ + '.' + pkg) + + +def registerAdapters(package, modules): + import sys + if not package.endswith('.'): + package += '.' + for x in modules: + exec 'import ' + package + x + m = sys.modules[package + x] d = m.__dict__ if d.has_key('ClassAdapter'): if isinstance(m.ClassAdapter, list): @@ -80,9 +104,8 @@ def getAdapterRegistry(): clzName = m.ClassAdapter classAdapterRegistry[clzName] = m else: - raise LookupError('Adapter class ' + x + ' has no ClassAdapter field ' + \ + raise LookupError('Adapter class ' + x + ' has no ClassAdapter field ' + 'and cannot be registered.') - + getAdapterRegistry() - diff --git a/dynamicserialize/dstypes/com/raytheon/uf/common/__init__.py b/dynamicserialize/dstypes/com/raytheon/uf/common/__init__.py index dfd5b4a..40e4cff 100644 --- a/dynamicserialize/dstypes/com/raytheon/uf/common/__init__.py +++ b/dynamicserialize/dstypes/com/raytheon/uf/common/__init__.py @@ -21,22 +21,7 @@ # File auto-generated by PythonFileGenerator __all__ = [ - 'activetable', - 'alertviz', - 'auth', - 'dataaccess', - 'dataplugin', - 'dataquery', - 'datastorage', - 'localization', - 'management', - 'message', - 'plugin', - 'pointdata', - 'pypies', - 'serialization', - 'site', - 'time' + 'dataplugin' ] diff --git a/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/__init__.py b/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/__init__.py index 006c19d..acb7c9b 100644 --- a/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/__init__.py +++ b/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/__init__.py @@ -21,13 +21,7 @@ # File auto-generated by PythonFileGenerator __all__ = [ - 'events', - 'gfe', - 'grid', - 'level', - 'message', - 'radar', - 'text' + 'events' ] diff --git a/dynamicserialize/dstypes/gov/noaa/nws/ncep/common/dataplugin/pgen/request/RetrieveActivityMapRequest.py b/dynamicserialize/dstypes/gov/noaa/nws/ncep/common/dataplugin/pgen/request/RetrieveActivityMapRequest.py index 1139684..f5279ab 100644 --- a/dynamicserialize/dstypes/gov/noaa/nws/ncep/common/dataplugin/pgen/request/RetrieveActivityMapRequest.py +++ b/dynamicserialize/dstypes/gov/noaa/nws/ncep/common/dataplugin/pgen/request/RetrieveActivityMapRequest.py @@ -2,7 +2,7 @@ # File auto-generated against equivalent DynamicSerialize Java class # # SOFTWARE HISTORY - +# # Date Ticket# Engineer Description # ------------ ---------- ----------- -------------------------- # May 05, 2016 root Generated diff --git a/dynamicserialize/dstypes/java/sql/Timestamp.py b/dynamicserialize/dstypes/java/sql/Timestamp.py index 2426b8f..06d5ac1 100644 --- a/dynamicserialize/dstypes/java/sql/Timestamp.py +++ b/dynamicserialize/dstypes/java/sql/Timestamp.py @@ -40,4 +40,4 @@ class Timestamp(Date): super(Timestamp, self).__init__(time) def __repr__(self): - return strftime("%Y-%m-%d %H:%M:%S.", gmtime(self.time/1000.0)) + '{:03d}'.format(self.time%1000) + return strftime("%Y-%m-%d %H:%M:%S.", gmtime(self.time/1000.0)) + '{:03d}'.format(self.time%1000) \ No newline at end of file