xan/python-awips
1
0
Fork 0
mirror of https://github.com/Unidata/python-awips.git synced 2025-02-23 14:57:56 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

First step on bringing over v20 python code

Browse source 
- brought over all the pythonPackages from the following repos/packages:
  - awips2/pythonPackages
  - awips2/edexOsgi/com.raytheon.uf.common.alertviz/pythonPackages
  - awips2/edexOsgi/com.raytheon.uf.common.mpe/pythonPackages
  - awips2/edexOsgi/com.raytheon.uf.common.dataplugin.text/pythonPackages
  - awips2/edexOsgi/com.raytheon.uf.common.dataplugin.grid/pythonPackages
  - awips2/edexOsgi/com.raytheon.uf.common.activetable/pythonPackages
  - awips2/edexOsgi/com.raytheon.uf.common.management/pythonPackages
  - awips2/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/pythonPackages
  - awips2/edexOsgi/com.raytheon.uf.common.dataplugin.radar/pythonPackages
  - awips2/edexOsgi/com.raytheon.uf.common.site/pythonPackages
  - awips2-core/common/com.raytheon.uf.common.auth/pythonPackages
  - awips2-core/common/com.raytheon.uf.common.message/pythonPackages
  - awips2-core/common/com.raytheon.uf.common.localization/pythonPackages
  - awips2-core/common/com.raytheon.uf.common.datastorage/pythonPackages
  - awips2-core/common/com.raytheon.uf.common.pointdata/pythonPackages
  - awips2-core/common/com.raythoen.uf.common.pypies/pythonPackages
  - awips2-core/common/com.raytheon.uf.common.dataaccess/pythonPackages
  - awips2-core/common/com.raytheon.uf.common.dataplugin.level/pythonPackages
  - awips2-core/common/com.raytheon.uf.common.serialization/pythonPackages
  - awips2-core/common/com.raytheon.uf.common.time/pythonPackages
  - awips2-core/common/com.raytheon.uf.common.dataplugin/pythonPackages
  - awips2-core/common/com.raytheon.uf.common.dataquery/pythonPackages
- updated the _init_.py file in dynamicserialize/dstypes/com/raytheon/uf/common to have all the proper elements listed
- started to replace "ufpy" instances with "awips"
  - awips/test/testQpidTimeToLive.py
  - awips/test/dafTests/testWarning.py
  - awips/test/dafTests/testCombinedTimeQuery.py

- will continue the "ufpy" replacement in a separate commit for easier clarity
This commit is contained in:
Shay Carter 2023-08-23 11:45:18 -06:00
parent 77a7355896
commit fd295d2865
443 changed files with 16089 additions and 2334 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

87
awips/AlertVizHandler.py Normal file
View file

@ -0,0 +1,87 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Pure python logging mechanism for logging to AlertViz from
# pure python (ie not JEP). DO NOT USE IN PYTHON CALLED
# FROM JAVA.
#
# Sends local-delivery messages only, but needs to know the EDEX host name in
# order to forward the message. If the DEFAULT_HOST environment variable is
# not set correctly then this will not work.
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 08/18/10 njensen Initial Creation.
# Apr 16, 2020 8144 tgurney Reference AlertViz stomp port
# specified in NotificationMessage
# Aug 25, 2020 8220 tgurney Change local-delivery strategy
#
#
#
import logging
import os
from . import NotificationMessage
import socket
class AlertVizHandler(logging.Handler):
def __init__(self, host=None, port=None,
category='LOCAL', source='ANNOUNCER', level=logging.NOTSET, filters=None):
logging.Handler.__init__(self, level)
if host is None:
host = os.getenv('DEFAULT_HOST', 'localhost')
if port is None:
port = os.getenv('DEFAULT_PORT', 9581)
self._category = category
self._host = host
self._port = port
self._source = source
self._filters = filters
filters['WORKSTATION'] = socket.getfqdn()
def emit(self, record):
"Implements logging.Handler's interface. Record argument is a logging.LogRecord."
priority = None
if record.levelno >= 50:
priority = 'CRITICAL'
elif record.levelno >= 40:
priority = 'SIGNIFICANT'
elif record.levelno >= 30:
priority = 'PROBLEM'
elif record.levelno >= 20:
priority = 'EVENTA'
elif record.levelno >= 10:
priority = 'EVENTB'
else:
priority = 'VERBOSE'
msg = self.format(record)
notify = NotificationMessage.NotificationMessage(self._host, self._port, msg, priority, self._category, self._source,
filters=self._filters)
notify.send()

56
awips/ConfigFileUtil.py Normal file
View file

@ -0,0 +1,56 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# A set of utility functions for dealing with configuration files.
#
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 09/27/10 dgilling Initial Creation.
#
#
#
def parseKeyValueFile(fileName):
propDict= dict()
try:
propFile= open(fileName, "rU")
for propLine in propFile:
propDef= propLine.strip()
if len(propDef) == 0:
continue
if propDef[0] in ( '#' ):
continue
punctuation= [ propDef.find(c) for c in ':= ' ] + [ len(propDef) ]
found= min( [ pos for pos in punctuation if pos != -1 ] )
name= propDef[:found].rstrip()
value= propDef[found:].lstrip(":= ").rstrip()
propDict[name]= value
propFile.close()
except:
pass
return propDict

47
awips/DateTimeConverter.py
View file

@ -1,10 +1,30 @@
# #
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
# #
#
# Functions for converting between the various "Java" dynamic serialize types
# used by EDEX to the native python time datetime.
#
#
#
#
# SOFTWARE HISTORY
#
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 06/24/15 #4480 dgilling Initial Creation.
@ -17,22 +37,23 @@ from dynamicserialize.dstypes.java.util import Date
from dynamicserialize.dstypes.java.sql import Timestamp
from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange
MAX_TIME = pow(2, 31) - 1
MICROS_IN_SECOND = 1000000
def convertToDateTime(timeArg):
"""
Converts the given object to a python datetime object. Supports native
"""
Converts the given object to a python datetime object. Supports native
python representations like datetime and struct_time, but also
the dynamicserialize types like Date and Timestamp. Raises TypeError
if no conversion can be performed.
Args:
timeArg: a python object representing a date and time. Supported
types include datetime, struct_time, float, int, long and the
dynamicserialize types Date and Timestamp.
Returns:
A datetime that represents the same date/time as the passed in object.
"""
@ -56,7 +77,6 @@ def convertToDateTime(timeArg):
objType = str(type(timeArg))
raise TypeError("Cannot convert object of type " + objType + " to datetime.")
def _convertSecsAndMicros(seconds, micros):
if seconds < MAX_TIME:
rval = datetime.datetime.utcfromtimestamp(seconds)
@ -65,20 +85,19 @@ def _convertSecsAndMicros(seconds, micros):
rval = datetime.datetime.utcfromtimestamp(MAX_TIME) + extraTime
return rval.replace(microsecond=micros)
def constructTimeRange(*args):
"""
"""
Builds a python dynamicserialize TimeRange object from the given
arguments.
Args:
args*: must be a TimeRange or a pair of objects that can be
args*: must be a TimeRange or a pair of objects that can be
converted to a datetime via convertToDateTime().
Returns:
A TimeRange.
"""
if len(args) == 1 and isinstance(args[0], TimeRange):
return args[0]
if len(args) != 2:

136
awips/NotificationMessage.py Executable file
View file

@ -0,0 +1,136 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
import socket
import sys
from . import ThriftClient
from dynamicserialize.dstypes.com.raytheon.uf.common.alertviz import AlertVizRequest
#
# Provides a capability of constructing notification messages and sending
# them to a STOMP data source.
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 09/30/08 chammack Initial Creation.
# 11/03/10 5849 cjeanbap Moved to ufpy package from
# com.raytheon.uf.tools.cli
# 01/07/11 5645 cjeanbap Added audio file to Status Message.
# 05/27/11 3050 cjeanbap Added if-statement to check Priority
# value
# 07/27/15 4654 skorolev Added filters
# 11/11/15 5120 rferrel Cannot serialize empty filters.
# 03/05/18 6899 dgilling Update to latest version of stomp.py API.
# 09/14/18 7464 dgilling Only serialize audioFile if provided.
# Apr 16, 2020 8144 tgurney Change AlertViz stomp port to
# calculated value based on uid
# May 15, 2020 8144 tgurney Remove local-delivery logic
# (no longer works as of 19.3.4)
class NotificationMessage:
priorityMap = {
0: 'CRITICAL',
1: 'SIGNIFICANT',
2: 'PROBLEM',
3: 'EVENTA',
4: 'EVENTB',
5: 'VERBOSE'}
def __init__(self, host='localhost', port=9581, message='', priority='PROBLEM', category="LOCAL", source="ANNOUNCER", audioFile=None, filters=None):
self.host = host
self.port = port
self.message = message
self.audioFile = audioFile
self.source = source
self.category = category
self.filters = filters
priorityInt = None
try:
priorityInt = int(priority)
except:
pass
if priorityInt is None:
#UFStatus.java contains mapping of Priority to Logging level mapping
if priority == 'CRITICAL' or priority == 'FATAL':
priorityInt = int(0)
elif priority == 'SIGNIFICANT' or priority == 'ERROR':
priorityInt = int(1)
elif priority == 'PROBLEM' or priority == 'WARN':
priorityInt = int(2)
elif priority == 'EVENTA' or priority == 'INFO':
priorityInt = int(3)
elif priority == 'EVENTB':
priorityInt = int(4)
elif priority == 'VERBOSE' or priority == 'DEBUG':
priorityInt = int(5)
if (priorityInt < 0 or priorityInt > 5):
print("Error occurred, supplied an invalid Priority value:", str(priorityInt))
print("Priority values are 0, 1, 2, 3, 4 and 5.")
sys.exit(1)
if priorityInt is not None:
self.priority = self.priorityMap[priorityInt]
else:
self.priority = priority
def send(self):
alertVizRequest = createRequest(self.message, self.priority, self.source, self.category, self.audioFile, self.filters)
thriftClient = ThriftClient.ThriftClient(self.host, self.port, "/services")
serverResponse = None
try:
serverResponse = thriftClient.sendRequest(alertVizRequest)
except Exception as ex:
print("Caught exception submitting AlertVizRequest:", str(ex))
if (serverResponse != "None"):
print("Error occurred submitting Notification Message to AlertViz receiver:", serverResponse)
sys.exit(1)
else:
print("Response:", str(serverResponse))
def createRequest(message, priority, source, category, audioFile, filters):
obj = AlertVizRequest()
obj.setMachine(socket.gethostname())
obj.setPriority(priority)
obj.setCategory(category)
obj.setSourceKey(source)
obj.setMessage(message)
if (audioFile is not None):
obj.setAudioFile(audioFile)
else:
obj.setAudioFile('\0')
obj.setFilters(filters)
return obj
if __name__ == '__main__':
main()

193
awips/QpidSubscriber.py
View file

@ -1,103 +1,170 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Provides a Python-based interface for subscribing to qpid queues and topics.
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------- -------- ------------ --------------------------------------------
# Nov 17, 2010 njensen Initial Creation.
# Aug 15, 2013 2169 bkowal Optionally gzip decompress any data that is read.
# Aug 04, 2016 2416 tgurney Add queueStarted property
# Feb 16, 2017 6084 bsteffen Support ssl connections
# Sep 07, 2017 6175 tgurney Remove "decompressing" log message
# Jul 23, 2019 7724 mrichardson Upgrade Qpid to Qpid Proton
# Nov 04, 2019 7724 tgurney Fix topic creation
# Jun 24, 2020 8187 randerso Added qpid connection_id
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 11/17/10 njensen Initial Creation.
# 08/15/13 2169 bkowal Optionally gzip decompress any data that is read.
# 08/04/16 2416 tgurney Add queueStarted property
# 02/16/17 6084 bsteffen Support ssl connections
# 09/07/17 6175 tgurney Remove "decompressing" log message
#
#
from __future__ import print_function
import logging
import os
import os.path
import qpid
import pwd
import socket
import zlib
from ssl import SSLContext, PROTOCOL_TLS
from Queue import Empty
from qpid.exceptions import Closed
from proton import SSLDomain
from proton.handlers import MessagingHandler
from proton.reactor import Container
logging.basicConfig(level=logging.INFO, datefmt='%H:%M:%S',
format='[%(process)s] %(asctime)s %(levelname)s: %(message)s')
log = logging.getLogger('QpidSubscriber')
class QpidSubscriber:
SSL_PASSWORD = 'password'
QPID_USERNAME = 'guest'
QPID_PASSWORD = 'guest'
def __init__(self, host='127.0.0.1', port=5672, decompress=False, ssl=None):
class QpidSubscriber(MessagingHandler):
def __init__(self, host='127.0.0.1', port=5672, decompress=False, ssl=None, program="QpidSubscriber"):
super(QpidSubscriber, self).__init__(auto_accept=True)
#__init__ should only handle setting up properties;
# any connection and subscription actions should be handled
# by the reactor functions
self.queues = {}
self.scheme = 'amqp'
self.rest_scheme = 'https'
self.ssl_context = None
self.host = host
self.port = port
self.decompress = decompress
socket = qpid.util.connect(host, port)
self.__queueStarted = False
self.__subscribed = False
pwuid = pwd.getpwuid(os.getuid())
if "QPID_SSL_CERT_DB" in os.environ:
certdb = os.environ["QPID_SSL_CERT_DB"]
certdbloc = os.environ["QPID_SSL_CERT_DB"]
else:
certdb = os.path.expanduser("~/.qpid/")
certdbloc = pwuid.pw_dir + "/.qpid/"
if "QPID_SSL_CERT_NAME" in os.environ:
certname = os.environ["QPID_SSL_CERT_NAME"]
else:
certname = "guest"
certfile = os.path.join(certdb, certname + ".crt")
if ssl or (ssl is None and os.path.exists(certfile)):
keyfile = os.path.join(certdb, certname + ".key")
trustfile = os.path.join(certdb, "root.crt")
socket = qpid.util.ssl(socket, keyfile=keyfile, certfile=certfile, ca_certs=trustfile)
self.__connection = qpid.connection.Connection(sock=socket, username='guest', password='guest')
self.__connection.start()
self.__session = self.__connection.session(str(qpid.datatypes.uuid4()))
self.subscribed = True
self.__queueStarted = False
certname = QPID_USERNAME
certfile = os.path.join(certdbloc, certname + ".crt")
certkey = os.path.join(certdbloc, certname + ".key")
if ssl or (ssl is None and os.path.isfile(certfile) and os.path.isfile(certkey)):
self.scheme = "amqps"
self.rest_scheme = 'https'
self.ssl_context = SSLContext(PROTOCOL_TLS)
self.ssl_context.load_cert_chain(certfile, certkey)
self.cert_file = certfile
self.cert_key = certkey
self.url = '{}://{}:{}@{}:{}'.format(self.scheme, QPID_USERNAME, QPID_PASSWORD, self.host, self.port)
self.clientID = ":".join([
socket.gethostname(),
pwuid.pw_name,
program,
str(os.getpid()),
])
def topicSubscribe(self, topicName, callback):
self.topicName = topicName
self.callback = callback
Container(self).run()
def on_start(self, event):
'''
# if the queue is edex.alerts, set decompress to true always for now to
# maintain compatibility with existing python scripts.
if topicName == 'edex.alerts':
'''
if self.topicName == 'edex.alerts':
self.decompress = True
print("Establishing connection to broker on", self.host)
queueName = topicName + self.__session.name
self.__session.queue_declare(queue=queueName, exclusive=True, auto_delete=True,
arguments={'qpid.max_count': 100, 'qpid.policy_type': 'ring'})
self.__session.exchange_bind(exchange='amq.topic', queue=queueName, binding_key=topicName)
self.__innerSubscribe(queueName, callback)
self.container = event.container
queueName = 'amq.topic/' + self.topicName
def __innerSubscribe(self, serverQueueName, callback):
local_queue_name = 'local_queue_' + serverQueueName
queue = self.__session.incoming(local_queue_name)
self.__session.message_subscribe(serverQueueName, destination=local_queue_name)
queue.start()
print("Connection complete to broker on", self.host)
self.ssl_domain = None
if self.scheme == "amqps" and self.cert_file and self.cert_key:
self.ssl_domain = SSLDomain(mode=SSLDomain.MODE_CLIENT)
self.ssl_domain.set_credentials(self.cert_file, self.cert_key, SSL_PASSWORD)
event.container.container_id = self.clientID
self.conn = event.container.connect(self.url, ssl_domain=self.ssl_domain)
self.receiver = event.container.create_receiver(self.conn, queueName)
self.__queueStarted = True
self.__subscribed = True
while self.subscribed:
def on_message(self, event):
message = event.message
content = message.body
self.process_message(content)
if not self.__subscribed:
self.close()
def process_message(self, content):
if (self.decompress):
try:
message = queue.get(timeout=10)
content = message.body
self.__session.message_accept(qpid.datatypes.RangedSet(message.id))
if self.decompress:
try:
# http://stackoverflow.com/questions/2423866/python-decompressing-gzip-chunk-by-chunk
d = zlib.decompressobj(16+zlib.MAX_WBITS)
content = d.decompress(content)
except ValueError:
# decompression failed, return the original content
pass
callback(content)
except Empty:
# http://stackoverflow.com/questions/2423866/python-decompressing-gzip-chunk-by-chunk
d = zlib.decompressobj(16 + zlib.MAX_WBITS)
content = d.decompress(content)
except Exception:
# decompression failed, return the original content
pass
except Closed:
self.close()
self.callback(content)
def close(self):
self.__queueStarted = False
self.subscribed = False
self.unsubscribe()
try:
self.__session.close(timeout=10)
except ValueError:
self.receiver.close()
self.conn.close()
except:
# already closed
pass
@property
def queueStarted(self):
return self.__queueStarted
@property
def subscribed(self):
return self.__subscribed
def unsubscribe(self):
self.__subscribed = False

100
awips/ThriftClient.py
View file

@ -1,80 +1,102 @@
#
# Provides a Python-based interface for executing Thrift requests.
#
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 09/20/10 dgilling Initial Creation.
#
#
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
import http.client
try:
import http.client as httpcl
except ImportError:
import httplib as httpcl
from dynamicserialize import DynamicSerializationManager
class ThriftClient:
#
# Provides a Python-based interface for executing Thrift requests.
#
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 09/20/10 dgilling Initial Creation.
#
#
#
class ThriftClient:
# How to call this constructor:
# 1. Pass in all arguments separately (e.g.,
# 1. Pass in all arguments separately (e.g.,
# ThriftClient.ThriftClient("localhost", 9581, "/services"))
# will return a Thrift client pointed at http://localhost:9581/services.
# 2. Pass in all arguments through the host string (e.g.,
# 2. Pass in all arguments through the host string (e.g.,
# ThriftClient.ThriftClient("localhost:9581/services"))
# will return a Thrift client pointed at http://localhost:9581/services.
# 3. Pass in host/port arguments through the host string (e.g.,
# 3. Pass in host/port arguments through the host string (e.g.,
# ThriftClient.ThriftClient("localhost:9581", "/services"))
# will return a Thrift client pointed at http://localhost:9581/services.
def __init__(self, host, port=9581, uri="/services"):
hostParts = host.split("/", 1)
if len(hostParts) > 1:
if (len(hostParts) > 1):
hostString = hostParts[0]
self.__uri = "/" + hostParts[1]
self.__httpConn = httpcl.HTTPConnection(hostString)
self.__httpConn = http.client.HTTPConnection(hostString)
else:
if port is None:
self.__httpConn = httpcl.HTTPConnection(host)
if (port is None):
self.__httpConn = http.client.HTTPConnection(host)
else:
self.__httpConn = httpcl.HTTPConnection(host, port)
self.__httpConn = http.client.HTTPConnection(host, port)
self.__uri = uri
self.__dsm = DynamicSerializationManager.DynamicSerializationManager()
def sendRequest(self, request, uri="/thrift"):
message = self.__dsm.serializeObject(request)
self.__httpConn.connect()
self.__httpConn.request("POST", self.__uri + uri, message)
response = self.__httpConn.getresponse()
if response.status != 200:
if (response.status != 200):
raise ThriftRequestException("Unable to post request to server")
rval = self.__dsm.deserializeBytes(response.read())
self.__httpConn.close()
# let's verify we have an instance of ServerErrorResponse
# IF we do, through an exception up to the caller along
# with the original Java stack trace
# ELSE: we have a valid response and pass it back
try:
forceError = rval.getException()
raise ThriftRequestException(forceError)
forceError = rval.getException()
raise ThriftRequestException(forceError)
except AttributeError:
pass
return rval
class ThriftRequestException(Exception):
def __init__(self, value):
self.parameter = value
def __str__(self):
return repr(self.parameter)

56
awips/TimeUtil.py
View file

@ -1,3 +1,22 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
# ----------------------------------------------------------------------------
# This software is in the public domain, furnished "as is", without technical
# support, and with no warranty, express or implied, as to its usefulness for
@ -9,9 +28,9 @@
# Author: hansen/romberg
# ----------------------------------------------------------------------------
import string
import time
# Given the timeStr, return the offset (in seconds)
# from the current time.
# Also return the launchStr i.e. Programs launched from this
@ -20,7 +39,7 @@ import time
# negative for time in the past.
#
# May still want it to be normalized to the most recent midnight.
#
#
# NOTES about synchronizing:
# --With synchronizing on, the "current time" for all processes started
# within a given hour will be the same.
@ -33,7 +52,7 @@ import time
# For example, if someone starts the GFE at 12:59 and someone
# else starts it at 1:01, they will have different offsets and
# current times.
# --With synchronizing off, when the process starts, the current time
# --With synchronizing off, when the process starts, the current time
# matches the drtTime in the command line. However, with synchronizing
# on, the current time will be offset by the fraction of the hour at
# which the process was started. Examples:
@ -44,15 +63,14 @@ import time
# Synchronizing on:
# GFE Spatial Editor at StartUp: 20040616_0030
#
def determineDrtOffset(timeStr):
launchStr = timeStr
# Check for time difference
if timeStr.find(",") >= 0:
if timeStr.find(",") >=0:
times = timeStr.split(",")
t1 = makeTime(times[0])
t2 = makeTime(times[1])
#print "time offset", t1-t2, (t1-t2)/3600
return t1-t2, launchStr
# Check for synchronized mode
synch = 0
@ -60,28 +78,30 @@ def determineDrtOffset(timeStr):
timeStr = timeStr[1:]
synch = 1
drt_t = makeTime(timeStr)
#print "input", year, month, day, hour, minute
gm = time.gmtime()
cur_t = time.mktime(gm)
# Synchronize to most recent hour
# i.e. "truncate" cur_t to most recent hour.
#print "gmtime", gm
if synch:
cur_t = time.mktime((gm[0], gm[1], gm[2], gm[3], 0, 0, 0, 0, 0))
curStr = '%4s%2s%2s_%2s00\n' % (repr(gm[0]), repr(gm[1]),
repr(gm[2]), repr(gm[3]))
curStr = curStr.replace(' ', '0')
curStr = time.strftime('%Y%m%d_%H00\n', gm)
launchStr = timeStr + "," + curStr
offset = drt_t - cur_t
#print "drt, cur", drt_t, cur_t
offset = drt_t - cur_t
#print "offset", offset, offset/3600, launchStr
return int(offset), launchStr
def makeTime(timeStr):
year = string.atoi(timeStr[0:4])
month = string.atoi(timeStr[4:6])
day = string.atoi(timeStr[6:8])
hour = string.atoi(timeStr[9:11])
minute = string.atoi(timeStr[11:13])
year = int(timeStr[0:4])
month = int(timeStr[4:6])
day = int(timeStr[6:8])
hour = int(timeStr[9:11])
minute = int(timeStr[11:13])
# Do not use daylight savings because gmtime is not in daylight
# savings time.
return time.mktime((year, month, day, hour, minute, 0, 0, 0, 0))

35
awips/UsageArgumentParser.py
View file

@ -1,3 +1,22 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# SOFTWARE HISTORY
#
@ -5,6 +24,7 @@
# ------------- -------- --------- ---------------------------------------------
# Feb 13, 2017 6092 randerso Added StoreTimeAction
#
##
import argparse
import sys
@ -15,7 +35,6 @@ from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects i
TIME_FORMAT = "%Y%m%d_%H%M"
class UsageArgumentParser(argparse.ArgumentParser):
"""
A subclass of ArgumentParser that overrides error() to print the
@ -26,8 +45,7 @@ class UsageArgumentParser(argparse.ArgumentParser):
self.print_help()
sys.exit(2)
# Custom actions for ArgumentParser objects
## Custom actions for ArgumentParser objects ##
class StoreDatabaseIDAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
did = DatabaseID(values)
@ -36,19 +54,18 @@ class StoreDatabaseIDAction(argparse.Action):
else:
parser.error("DatabaseID [" + values + "] not a valid identifier")
class AppendParmNameAndLevelAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
tx = ParmID.parmNameAndLevel(values)
comp = tx[0] + '_' + tx[1]
if (hasattr(namespace, self.dest)) and (getattr(namespace, self.dest) is not None):
if (hasattr(namespace, self.dest)) and \
(getattr(namespace, self.dest) is not None):
currentValues = getattr(namespace, self.dest)
currentValues.append(comp)
setattr(namespace, self.dest, currentValues)
else:
setattr(namespace, self.dest, [comp])
class StoreTimeAction(argparse.Action):
"""
argparse.Action subclass to validate GFE formatted time strings
@ -57,6 +74,8 @@ class StoreTimeAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
try:
timeStruct = time.strptime(values, TIME_FORMAT)
setattr(namespace, self.dest, timeStruct)
except ValueError:
except:
parser.error(str(values) + " is not a valid time string of the format YYYYMMDD_hhmm")
setattr(namespace, self.dest, timeStruct)

22
awips/UsageOptionParser.py
View file

@ -1,7 +1,26 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
import sys
from optparse import OptionParser
class UsageOptionParser(OptionParser):
"""
A subclass of OptionParser that prints that overrides error() to print the
@ -16,3 +35,4 @@ class UsageOptionParser(OptionParser):
sys.stderr.write(msg)
sys.stderr.write("\n")
sys.exit(2)

36
awips/__init__.py
View file

@ -1,3 +1,37 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
__all__ = [
#
# __init__.py for ufpy package
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 09/21/10 dgilling Initial Creation.
#
#
#
__all__ = [
]

48
awips/dataaccess/CombinedTimeQuery.py
View file

@ -1,23 +1,41 @@
# #
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
# #
#
# Method for performing a DAF time query where all parameter/level/location
# combinations must be available at the same time.
#
#
#
#
#
# SOFTWARE HISTORY
#
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 06/22/16 #5591 bsteffen Initial Creation.
#
from awips.dataaccess import DataAccessLayer
from ufpy.dataaccess import DataAccessLayer
def getAvailableTimes(request, refTimeOnly=False):
return __getAvailableTimesForEachParameter(request, refTimeOnly)
def __getAvailableTimesForEachParameter(request, refTimeOnly=False):
parameters = request.getParameters()
if parameters:
@ -36,7 +54,6 @@ def __getAvailableTimesForEachParameter(request, refTimeOnly=False):
else:
return __getAvailableTimesForEachLevel(request, refTimeOnly)
def __getAvailableTimesForEachLevel(request, refTimeOnly=False):
levels = request.getLevels()
if levels:
@ -55,7 +72,6 @@ def __getAvailableTimesForEachLevel(request, refTimeOnly=False):
else:
return __getAvailableTimesForEachLocation(request, refTimeOnly)
def __getAvailableTimesForEachLocation(request, refTimeOnly=False):
locations = request.getLocationNames()
if locations:
@ -73,12 +89,12 @@ def __getAvailableTimesForEachLocation(request, refTimeOnly=False):
return times
else:
return DataAccessLayer.getAvailableTimes(request, refTimeOnly)
def __cloneRequest(request):
return DataAccessLayer.newDataRequest(datatype=request.getDatatype(),
parameters=request.getParameters(),
levels=request.getLevels(),
locationNames=request.getLocationNames(),
envelope=request.getEnvelope(),
**request.getIdentifiers())
return DataAccessLayer.newDataRequest(datatype = request.getDatatype(),
parameters = request.getParameters(),
levels = request.getLevels(),
locationNames = request.getLocationNames(),
envelope = request.getEnvelope(),
**request.getIdentifiers())

227
awips/dataaccess/DataAccessLayer.py
View file

@ -1,33 +1,62 @@
# #
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# Published interface for awips.dataaccess package
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
# #
#
# Published interface for ufpy.dataaccess package
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ------- ---------- -------------------------
# 12/10/12 njensen Initial Creation.
# Feb 14, 2013 1614 bsteffen refactor data access framework to use single request.
# 04/10/13 1871 mnash move getLatLonCoords to JGridData and add default args
# 05/29/13 2023 dgilling Hook up ThriftClientRouter.
# 03/03/14 2673 bsteffen Add ability to query only ref times.
# 07/22/14 3185 njensen Added optional/default args to newDataRequest
# 07/30/14 3185 njensen Renamed valid identifiers to optional
# Apr 26, 2015 4259 njensen Updated for new JEP API
# Apr 13, 2016 5379 tgurney Add getIdentifierValues(), getRequiredIdentifiers(),
# and getOptionalIdentifiers()
# Oct 07, 2016 ---- mjames@ucar Added getForecastRun
# Oct 18, 2016 5916 bsteffen Add setLazyLoadGridLatLon
# Oct 11, 2018 ---- mjames@ucar Added getMetarObs() getSynopticObs()
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 12/10/12 njensen Initial Creation.
# Feb 14, 2013 1614 bsteffen refactor data access framework
# to use single request.
# 04/10/13 1871 mnash move getLatLonCoords to JGridData and add default args
# 05/29/13 2023 dgilling Hook up ThriftClientRouter.
# 03/03/14 2673 bsteffen Add ability to query only ref times.
# 07/22/14 3185 njensen Added optional/default args to newDataRequest
# 07/30/14 3185 njensen Renamed valid identifiers to optional
# Apr 26, 2015 4259 njensen Updated for new JEP API
# Apr 13, 2016 5379 tgurney Add getIdentifierValues()
# Jun 01, 2016 5587 tgurney Add new signatures for
# getRequiredIdentifiers() and
# getOptionalIdentifiers()
# Oct 18, 2016 5916 bsteffen Add setLazyLoadGridLatLon
#
#
import sys
import subprocess
import warnings
THRIFT_HOST = "edex"
THRIFT_HOST = subprocess.check_output(
"source /awips2/fxa/bin/setup.env; echo $DEFAULT_HOST",
shell=True).decode().strip()
USING_NATIVE_THRIFT = False
if 'jep' in sys.modules:
# intentionally do not catch if this fails to import, we want it to
# be obvious that something is configured wrong when running from within
@ -35,152 +64,11 @@ if 'jep' in sys.modules:
import JepRouter
router = JepRouter
else:
from awips.dataaccess import ThriftClientRouter
from ufpy.dataaccess import ThriftClientRouter
router = ThriftClientRouter.ThriftClientRouter(THRIFT_HOST)
USING_NATIVE_THRIFT = True
def getRadarProductIDs(availableParms):
"""
Get only the numeric idetifiers for NEXRAD3 products.
Args:
availableParms: Full list of radar parameters
Returns:
List of filtered parameters
"""
productIDs = []
for p in list(availableParms):
try:
if isinstance(int(p), int):
productIDs.append(str(p))
except ValueError:
pass
return productIDs
def getRadarProductNames(availableParms):
"""
Get only the named idetifiers for NEXRAD3 products.
Args:
availableParms: Full list of radar parameters
Returns:
List of filtered parameters
"""
productNames = []
for p in list(availableParms):
if len(p) > 3:
productNames.append(p)
return productNames
def getMetarObs(response):
"""
Processes a DataAccessLayer "obs" response into a dictionary,
with special consideration for multi-value parameters
"presWeather", "skyCover", and "skyLayerBase".
Args:
response: DAL getGeometry() list
Returns:
A dictionary of METAR obs
"""
from datetime import datetime
single_val_params = ["timeObs", "stationName", "longitude", "latitude",
"temperature", "dewpoint", "windDir",
"windSpeed", "seaLevelPress"]
multi_val_params = ["presWeather", "skyCover", "skyLayerBase"]
params = single_val_params + multi_val_params
station_names, pres_weather, sky_cov, sky_layer_base = [], [], [], []
obs = dict({params: [] for params in params})
for ob in response:
avail_params = ob.getParameters()
if "presWeather" in avail_params:
pres_weather.append(ob.getString("presWeather"))
elif "skyCover" in avail_params and "skyLayerBase" in avail_params:
sky_cov.append(ob.getString("skyCover"))
sky_layer_base.append(ob.getNumber("skyLayerBase"))
else:
# If we already have a record for this stationName, skip
if ob.getString('stationName') not in station_names:
station_names.append(ob.getString('stationName'))
for param in single_val_params:
if param in avail_params:
if param == 'timeObs':
obs[param].append(datetime.fromtimestamp(ob.getNumber(param) / 1000.0))
else:
try:
obs[param].append(ob.getNumber(param))
except TypeError:
obs[param].append(ob.getString(param))
else:
obs[param].append(None)
obs['presWeather'].append(pres_weather)
obs['skyCover'].append(sky_cov)
obs['skyLayerBase'].append(sky_layer_base)
pres_weather = []
sky_cov = []
sky_layer_base = []
return obs
def getSynopticObs(response):
"""
Processes a DataAccessLayer "sfcobs" response into a dictionary
of available parameters.
Args:
response: DAL getGeometry() list
Returns:
A dictionary of synop obs
"""
from datetime import datetime
station_names = []
params = response[0].getParameters()
sfcobs = dict({params: [] for params in params})
for sfcob in response:
# If we already have a record for this stationId, skip
if sfcob.getString('stationId') not in station_names:
station_names.append(sfcob.getString('stationId'))
for param in params:
if param == 'timeObs':
sfcobs[param].append(datetime.fromtimestamp(sfcob.getNumber(param) / 1000.0))
else:
try:
sfcobs[param].append(sfcob.getNumber(param))
except TypeError:
sfcobs[param].append(sfcob.getString(param))
return sfcobs
def getForecastRun(cycle, times):
"""
Get the latest forecast run (list of objects) from all
all cycles and times returned from DataAccessLayer "grid"
response.
Args:
cycle: Forecast cycle reference time
times: All available times/cycles
Returns:
DataTime array for a single forecast run
"""
fcstRun = []
for t in times:
if str(t)[:19] == str(cycle):
fcstRun.append(t)
return fcstRun
def getAvailableTimes(request, refTimeOnly=False):
"""
@ -189,7 +77,7 @@ def getAvailableTimes(request, refTimeOnly=False):
Args:
request: the IDataRequest to get data for
refTimeOnly: optional, use True if only unique refTimes should be
returned (without a forecastHr)
returned (without a forecastHr)
Returns:
a list of DataTimes
@ -206,7 +94,7 @@ def getGridData(request, times=[]):
Args:
request: the IDataRequest to get data for
times: a list of DataTimes, a TimeRange, or None if the data is time
agnostic
agnostic
Returns:
a list of IGridData
@ -223,10 +111,10 @@ def getGeometryData(request, times=[]):
Args:
request: the IDataRequest to get data for
times: a list of DataTimes, a TimeRange, or None if the data is time
agnostic
agnostic
Returns:
a list of IGeometryData
a list of IGeometryData
"""
return router.getGeometryData(request, times)
@ -319,9 +207,8 @@ def getIdentifierValues(request, identifierKey):
"""
return router.getIdentifierValues(request, identifierKey)
def newDataRequest(datatype=None, **kwargs):
"""
""""
Creates a new instance of IDataRequest suitable for the runtime environment.
All args are optional and exist solely for convenience.
@ -331,14 +218,13 @@ def newDataRequest(datatype=None, **kwargs):
levels: a list of levels to set on the request
locationNames: a list of locationNames to set on the request
envelope: an envelope to limit the request
kwargs: any leftover kwargs will be set as identifiers
**kwargs: any leftover kwargs will be set as identifiers
Returns:
a new IDataRequest
"""
return router.newDataRequest(datatype, **kwargs)
def getSupportedDatatypes():
"""
Gets the datatypes that are supported by the framework
@ -356,7 +242,7 @@ def changeEDEXHost(newHostName):
method will throw a TypeError.
Args:
newHostName: the EDEX host to connect to
newHostHame: the EDEX host to connect to
"""
if USING_NATIVE_THRIFT:
global THRIFT_HOST
@ -366,7 +252,6 @@ def changeEDEXHost(newHostName):
else:
raise TypeError("Cannot call changeEDEXHost when using JepRouter.")
def setLazyLoadGridLatLon(lazyLoadGridLatLon):
"""
Provide a hint to the Data Access Framework indicating whether to load the
@ -379,7 +264,7 @@ def setLazyLoadGridLatLon(lazyLoadGridLatLon):
set to False if it is guaranteed that all lat/lon information is needed and
it would be better to get any performance overhead for generating the
lat/lon data out of the way during the initial request.
Args:
lazyLoadGridLatLon: Boolean value indicating whether to lazy load.
@ -388,4 +273,4 @@ def setLazyLoadGridLatLon(lazyLoadGridLatLon):
router.setLazyLoadGridLatLon(lazyLoadGridLatLon)
except AttributeError:
# The router is not required to support this capability.
pass
pass

73
awips/dataaccess/DataNotificationLayer.py
View file

@ -1,13 +1,34 @@
# #
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# Published interface for retrieving data updates via awips.dataaccess package
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# SOFTWARE HISTORY
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
# #
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# May 26, 2016 2416 rjpeter Initial Creation.
# Aug 1, 2016 2416 tgurney Finish implementation
# Published interface for retrieving data updates via ufpy.dataaccess package
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------- -------- ------------ --------------------------------------------
# May 26, 2016 2416 rjpeter Initial Creation.
# Aug 01, 2016 2416 tgurney Finish implementation
# Nov 05, 2019 7884 tgurney Python 3 fixes
# Jun 24, 2020 8187 randerso Added program for qpid connection_id
#
#
@ -17,7 +38,7 @@ retrieval of new data as it is coming into the system.
There are two ways to access this feature:
1. The DataQueue module (awips.dataaccess.DataQueue) offers a collection that
1. The DataQueue module (ufpy.dataaccess.DataQueue) offers a collection that
automatically fills up with new data as it receives notifications. See that
module for more information.
@ -29,8 +50,8 @@ each time new data is received.
Example code follows. This example prints temperature as observed from KOMA
each time a METAR is received from there.
from awips.dataaccess import DataAccessLayer as DAL
from awips.dataaccess import DataNotificationLayer as DNL
from ufpy.dataaccess import DataAccessLayer as DAL
from ufpy.dataaccess import DataNotificationLayer as DNL
def process_obs(list_of_data):
for item in list_of_data:
@ -39,25 +60,25 @@ each time a METAR is received from there.
request = DAL.newDataRequest('obs')
request.setParameters('temperature')
request.setLocationNames('KOMA')
notifier = DNL.getGeometryDataUpdates(request)
notifier.subscribe(process_obs)
# process_obs will called with a list of data each time new data comes in
"""
import re
import sys
from awips.dataaccess.PyGeometryNotification import PyGeometryNotification
from awips.dataaccess.PyGridNotification import PyGridNotification
import subprocess
from ufpy.dataaccess.PyGeometryNotification import PyGeometryNotification
from ufpy.dataaccess.PyGridNotification import PyGridNotification
THRIFT_HOST = "edex"
THRIFT_HOST = subprocess.check_output(
"source /awips2/fxa/bin/setup.env; echo $DEFAULT_HOST",
shell=True).decode().strip()
USING_NATIVE_THRIFT = False
JMS_HOST_PATTERN = re.compile('tcp://([^:]+):([0-9]+)')
if 'jep' in sys.modules:
# intentionally do not catch if this fails to import, we want it to
# be obvious that something is configured wrong when running from within
@ -65,7 +86,7 @@ if 'jep' in sys.modules:
import JepRouter
router = JepRouter
else:
from awips.dataaccess import ThriftClientRouter
from ufpy.dataaccess import ThriftClientRouter
router = ThriftClientRouter.ThriftClientRouter(THRIFT_HOST)
USING_NATIVE_THRIFT = True
@ -73,9 +94,9 @@ else:
def _getJmsConnectionInfo(notifFilterResponse):
serverString = notifFilterResponse.getJmsConnectionInfo()
try:
host, port = JMS_HOST_PATTERN.match(serverString).groups()
except AttributeError:
raise RuntimeError('Got bad JMS connection info from server: ' + serverString)
host, port = serverString.split(':')
except Exception as e:
raise ValueError('Got bad JMS connection info from server: "' + serverString + '"') from e
return {'host': host, 'port': port}
@ -91,10 +112,9 @@ def getGridDataUpdates(request):
calling its subscribe() method
"""
response = router.getNotificationFilter(request)
notificationFilter = response.getNotificationFilter()
filter = response.getNotificationFilter()
jmsInfo = _getJmsConnectionInfo(response)
notifier = PyGridNotification(request, notificationFilter,
requestHost=THRIFT_HOST, **jmsInfo)
notifier = PyGridNotification(request, filter, requestHost=THRIFT_HOST, program="daf-gridDataUpdates", **jmsInfo)
return notifier
@ -110,10 +130,9 @@ def getGeometryDataUpdates(request):
calling its subscribe() method
"""
response = router.getNotificationFilter(request)
notificationFilter = response.getNotificationFilter()
filter = response.getNotificationFilter()
jmsInfo = _getJmsConnectionInfo(response)
notifier = PyGeometryNotification(request, notificationFilter,
requestHost=THRIFT_HOST, **jmsInfo)
notifier = PyGeometryNotification(request, filter, requestHost=THRIFT_HOST, program="daf-geometryDataUpdates", **jmsInfo)
return notifier
@ -124,7 +143,7 @@ def changeEDEXHost(newHostName):
method will throw a TypeError.
Args:
newHostName: the EDEX host to connect to
newHostHame: the EDEX host to connect to
"""
if USING_NATIVE_THRIFT:
global THRIFT_HOST

43
awips/dataaccess/DataQueue.py
View file

@ -1,3 +1,23 @@
# #
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
# #
#
# Convenience class for using the DAF's notifications feature. This is a
# collection that, once connected to EDEX by calling start(), fills with
@ -13,25 +33,24 @@
# 07/29/16 2416 tgurney Initial creation
#
from awips.dataaccess import DataNotificationLayer as DNL
from ufpy.dataaccess import DataNotificationLayer as DNL
import time
from threading import Thread
import sys
if sys.version_info.major == 2:
from Queue import Queue, Empty
else: # Python 3 module renamed to 'queue'
from queue import Queue, Empty
from queue import Queue, Empty
# Used to indicate a DataQueue that will produce geometry data.
"""Used to indicate a DataQueue that will produce geometry data."""
GEOMETRY = object()
# Used to indicate a DataQueue that will produce grid data.
"""Used to indicate a DataQueue that will produce grid data."""
GRID = object()
# Default maximum queue size.
"""Default maximum queue size."""
_DEFAULT_MAXSIZE = 100
@ -128,12 +147,12 @@ class DataQueue(object):
return self._queue.get(block, timeout)
except Empty:
return None
def get_all(self):
"""
Get all data waiting for processing, in a single list. Always returns
immediately. Returns an empty list if no data has arrived yet.
Returns:
List of IData
"""
@ -187,4 +206,4 @@ class DataQueue(object):
return self
def __exit__(self, *unused):
self.close()
self.close()

49
awips/dataaccess/PyData.py
View file

@ -1,23 +1,40 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Implements IData for use by native Python clients to the Data Access
# Framework.
#
#
#
#
# SOFTWARE HISTORY
#
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 06/03/13 dgilling Initial Creation.
# 10/05/18 mjames@ucar Encode/decode attribute names.
#
#
#
from awips.dataaccess import IData
import six
from ufpy.dataaccess import IData
class PyData(IData):
def __init__(self, dataRecord):
self.__time = dataRecord.getTime()
self.__level = dataRecord.getLevel()
@ -26,19 +43,15 @@ class PyData(IData):
def getAttribute(self, key):
return self.__attributes[key]
def getAttributes(self):
return self.__attributes.keys()
return list(self.__attributes.keys())
def getDataTime(self):
return self.__time
def getLevel(self):
if six.PY2:
return self.__level
if not isinstance(self.__level, str):
return self.__level.decode('utf-8')
return self.__level
def getLocationName(self):
return self.__locationName

93
awips/dataaccess/PyGeometryData.py
View file

@ -1,81 +1,76 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Implements IGeometryData for use by native Python clients to the Data Access
# Framework.
#
#
#
#
# SOFTWARE HISTORY
#
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 06/03/13 dgilling Initial Creation.
# 01/06/14 2537 bsteffen Share geometry WKT.
# 03/19/14 2882 dgilling Raise an exception when getNumber()
# is called for data that is not a
# is called for data that is not a
# numeric Type.
# 06/09/16 5574 mapeters Handle 'SHORT' type in getNumber().
# 10/05/18 mjames@ucar Encode/decode string, number val, and type
#
#
#
from awips.dataaccess import IGeometryData
from awips.dataaccess import PyData
import six
from ufpy.dataaccess import IGeometryData
from ufpy.dataaccess import PyData
class PyGeometryData(IGeometryData, PyData.PyData):
def __init__(self, geoDataRecord, geometry):
PyData.PyData.__init__(self, geoDataRecord)
self.__geometry = geometry
self.__dataMap = {}
tempDataMap = geoDataRecord.getDataMap()
for key, value in list(tempDataMap.items()):
for key, value in tempDataMap.items():
self.__dataMap[key] = (value[0], value[1], value[2])
def getGeometry(self):
return self.__geometry
def getParameters(self):
if six.PY2:
return list(self.__dataMap.keys())
else:
return [x.decode('utf-8') for x in list(self.__dataMap.keys())]
def getParameters(self):
return list(self.__dataMap.keys())
def getString(self, param):
if six.PY2:
return self.__dataMap[param][0]
value = self.__dataMap[param.encode('utf-8')][0]
if isinstance(value, bytes):
return str(value.decode('utf-8'))
value = self.__dataMap[param][0]
return str(value)
def getNumber(self, param):
t = self.getType(param)
if six.PY2:
value = self.__dataMap[param][0]
else:
value = self.__dataMap[param.encode('utf-8')][0]
if t == 'INT' or t == 'SHORT' or t == 'LONG':
def getNumber(self, param):
value = self.__dataMap[param][0]
t = self.getType(param)
if t in ('INT', 'SHORT', 'LONG'):
return int(value)
elif t == 'FLOAT':
return float(value)
elif t == 'DOUBLE':
elif t in ('DOUBLE', 'FLOAT'):
return float(value)
else:
raise TypeError("Data for parameter " + param + " is not a numeric type.")
def getUnit(self, param):
if six.PY2:
return self.__dataMap[param][2]
unit = self.__dataMap[param.encode('utf-8')][2]
if unit is not None:
return unit.decode('utf-8')
return unit
return self.__dataMap[param][2]
def getType(self, param):
if six.PY2:
return self.__dataMap[param][1]
datatype = self.__dataMap[param.encode('utf-8')][1]
if datatype is not None:
return datatype.decode('utf-8')
return datatype
return self.__dataMap[param][1]

38
awips/dataaccess/PyGeometryNotification.py
View file

@ -1,19 +1,39 @@
# #
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
# #
#
# Notification object that produces geometry data
#
#
#
#
# SOFTWARE HISTORY
#
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 07/22/16 2416 tgurney Initial creation
# 09/07/17 6175 tgurney Override messageReceived
# 07/22/2016 2416 tgurney Initial creation
# 09/07/2017 6175 tgurney Override messageReceived
# 11/05/2019 7884 tgurney Add missing import
#
import traceback
import dynamicserialize
from awips.dataaccess.PyNotification import PyNotification
import traceback
from ufpy.dataaccess.PyNotification import PyNotification
class PyGeometryNotification(PyNotification):
@ -28,7 +48,7 @@ class PyGeometryNotification(PyNotification):
try:
data = self.getData(self.request, list(dataTimes))
self.callback(data)
except ValueError:
except Exception as e:
traceback.print_exc()
def getData(self, request, dataTimes):

49
awips/dataaccess/PyGridData.py
View file

@ -1,25 +1,45 @@
# #
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
# #
#
# Implements IGridData for use by native Python clients to the Data Access
# Framework.
#
#
#
#
# SOFTWARE HISTORY
#
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 06/03/13 #2023 dgilling Initial Creation.
# 10/13/16 #5916 bsteffen Correct grid shape, allow lat/lon
# 11/10/16 #5900 bsteffen Correct grid shape
# to be requested by a delegate
#
#
#
import numpy
import warnings
import six
from awips.dataaccess import IGridData
from awips.dataaccess import PyData
from ufpy.dataaccess import IGridData
from ufpy.dataaccess import PyData
NO_UNIT_CONVERT_WARNING = """
The ability to unit convert grid data is not currently available in this version of the Data Access Framework.
@ -27,8 +47,8 @@ The ability to unit convert grid data is not currently available in this version
class PyGridData(IGridData, PyData.PyData):
def __init__(self, gridDataRecord, nx, ny, latLonGrid=None, latLonDelegate=None):
def __init__(self, gridDataRecord, nx, ny, latLonGrid = None, latLonDelegate = None):
PyData.PyData.__init__(self, gridDataRecord)
nx = nx
ny = ny
@ -38,16 +58,13 @@ class PyGridData(IGridData, PyData.PyData):
self.__latLonGrid = latLonGrid
self.__latLonDelegate = latLonDelegate
def getParameter(self):
return self.__parameter
def getUnit(self):
if six.PY2:
return self.__unit
if self.__unit is not None and not isinstance(self.__unit, str):
return self.__unit.decode('utf-8')
return self.__unit
def getRawData(self, unit=None):
# TODO: Find a proper python library that deals will with numpy and
# javax.measure style unit strings and hook it in to this method to
@ -55,7 +72,7 @@ class PyGridData(IGridData, PyData.PyData):
if unit is not None:
warnings.warn(NO_UNIT_CONVERT_WARNING, stacklevel=2)
return self.__gridData
def getLatLonCoords(self):
if self.__latLonGrid is not None:
return self.__latLonGrid

30
awips/dataaccess/PyGridNotification.py
View file

@ -1,3 +1,23 @@
# #
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
# #
#
# Notification object that produces grid data
#
@ -6,14 +26,14 @@
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 06/03/16 2416 rjpeter Initial Creation.
# 09/06/17 6175 tgurney Override messageReceived
# 06/03/2016 2416 rjpeter Initial Creation.
# 09/06/2017 6175 tgurney Override messageReceived
# 11/05/2019 7884 tgurney Add missing import
#
import dynamicserialize
import traceback
from awips.dataaccess.PyNotification import PyNotification
from ufpy.dataaccess.PyNotification import PyNotification
class PyGridNotification(PyNotification):
@ -33,7 +53,7 @@ class PyGridNotification(PyNotification):
newReq.setParameters(self.request.getParameters())
data = self.getData(newReq, [])
self.callback(data)
except ValueError:
except Exception as e:
traceback.print_exc()
def getData(self, request, dataTimes):

65
awips/dataaccess/PyNotification.py
View file

@ -1,39 +1,61 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Implements IData for use by native Python clients to the Data Access
# Framework.
#
# SOFTWARE HISTORY
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# Jun 22, 2016 2416 rjpeter Initial creation
# Jul 22, 2016 2416 tgurney Finish implementation
# Sep 07, 2017 6175 tgurney Override messageReceived in subclasses
# Date Ticket# Engineer Description
# ------------- -------- ------------ --------------------------------------------
# Jun 22, 2016 2416 rjpeter Initial creation
# Jul 22, 2016 2416 tgurney Finish implementation
# Sep 07, 2017 6175 tgurney Override messageReceived in subclasses
# Nov 05, 2019 7884 tgurney Fix in subscribed()
# Jun 24, 2020 8187 randerso Added program for qpid connection_id
#
from six import with_metaclass
import abc
from awips.dataaccess import DataAccessLayer
from awips.dataaccess import INotificationSubscriber
from awips.QpidSubscriber import QpidSubscriber
from ufpy.dataaccess import DataAccessLayer
from ufpy.dataaccess import INotificationSubscriber
from ufpy.QpidSubscriber import QpidSubscriber
from dynamicserialize.dstypes.com.raytheon.uf.common.time import DataTime
class PyNotification(with_metaclass(abc.ABCMeta, INotificationSubscriber)):
class PyNotification(INotificationSubscriber, metaclass=abc.ABCMeta):
"""
Receives notifications for new data and retrieves the data that meets
specified filtering criteria.
"""
def __init__(self, request, notificationFilter, host='localhost',
port=5672, requestHost='localhost'):
def __init__(self, request, filter, host='localhost', port=5672, requestHost='localhost', program="PyNotification"):
self.DAL = DataAccessLayer
self.DAL.changeEDEXHost(requestHost)
self.request = request
self.notificationFilter = notificationFilter
self.__topicSubscriber = QpidSubscriber(host, port, decompress=True)
self.notificationFilter = filter
self.host = host
self.port = port
self.program=program
self.__topicName = "edex.alerts"
self.callback = None
@ -47,12 +69,12 @@ class PyNotification(with_metaclass(abc.ABCMeta, INotificationSubscriber)):
"""
assert hasattr(callback, '__call__'), 'callback arg must be callable'
self.callback = callback
self.__topicSubscriber.topicSubscribe(self.__topicName, self.messageReceived)
self.qs = QpidSubscriber(host=self.host, port=self.port, decompress=True, program=self.program)
self.qs.topicSubscribe(self.__topicName, self.messageReceived)
# Blocks here
def close(self):
if self.__topicSubscriber.subscribed:
self.__topicSubscriber.close()
self.qs.close()
def getDataTime(self, dataURI):
dataTimeStr = dataURI.split('/')[2]
@ -82,4 +104,7 @@ class PyNotification(with_metaclass(abc.ABCMeta, INotificationSubscriber)):
@property
def subscribed(self):
"""True if currently subscribed to notifications."""
return self.__topicSubscriber.queueStarted
try:
return self.qs.queueStarted
except AttributeError:
return False

283
awips/dataaccess/SoundingsSupport.py Normal file
View file

@ -0,0 +1,283 @@
# #
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
# #
#
# Classes for retrieving soundings based on gridded data from the Data Access
# Framework
#
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 06/24/15 #4480 dgilling Initial Creation.
#
from collections import defaultdict
from shapely.geometry import Point
from ufpy import DateTimeConverter
from ufpy.dataaccess import DataAccessLayer
from dynamicserialize.dstypes.com.raytheon.uf.common.time import DataTime
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.level import Level
def getSounding(modelName, weatherElements, levels, samplePoint, refTime=None, timeRange=None):
""""
Performs a series of Data Access Framework requests to retrieve a sounding object
based on the specified request parameters.
Args:
modelName: the grid model datasetid to use as the basis of the sounding.
weatherElements: a list of parameters to return in the sounding.
levels: a list of levels to sample the given weather elements at
samplePoint: a lat/lon pair to perform the sampling of data at.
refTime: (optional) the grid model reference time to use for the sounding.
If not specified, the latest ref time in the system will be used.
timeRange: (optional) a TimeRange to specify which forecast hours to use.
If not specified, will default to all forecast hours.
Returns:
A _SoundingCube instance, which acts a 3-tiered dictionary, keyed
by DataTime, then by level and finally by weather element. If no
data is available for the given request parameters, None is returned.
"""
(locationNames, parameters, levels, envelope, refTime, timeRange) = \
__sanitizeInputs(modelName, weatherElements, levels, samplePoint, refTime, timeRange)
requestArgs = { 'datatype' : 'grid',
'locationNames' : locationNames,
'parameters' : parameters,
'levels' : levels,
'envelope' : envelope,
}
req = DataAccessLayer.newDataRequest(**requestArgs)
forecastHours = __determineForecastHours(req, refTime, timeRange)
if not forecastHours:
return None
response = DataAccessLayer.getGeometryData(req, forecastHours)
soundingObject = _SoundingCube(response)
return soundingObject
def setEDEXHost(host):
"""
Changes the EDEX host the Data Access Framework is communicating with.
Args:
host: the EDEX host to connect to
"""
if host:
DataAccessLayer.changeEDEXHost(str(host))
def __sanitizeInputs(modelName, weatherElements, levels, samplePoint, refTime, timeRange):
locationNames = [str(modelName)]
parameters = __buildStringList(weatherElements)
levels = __buildStringList(levels)
envelope = Point(samplePoint)
if refTime is not None:
refTime = DataTime(refTime=DateTimeConverter.convertToDateTime(refTime))
if timeRange is not None:
timeRange = DateTimeConverter.constructTimeRange(*timeRange)
return (locationNames, parameters, levels, envelope, refTime, timeRange)
def __determineForecastHours(request, refTime, timeRange):
dataTimes = DataAccessLayer.getAvailableTimes(request, False)
timesGen = [(DataTime(refTime=dataTime.getRefTime()), dataTime) for dataTime in dataTimes]
dataTimesMap = defaultdict(list)
for baseTime, dataTime in timesGen:
dataTimesMap[baseTime].append(dataTime)
if refTime is None:
refTime = max(dataTimesMap.keys())
forecastHours = dataTimesMap[refTime]
if timeRange is None:
return forecastHours
else:
return [forecastHour for forecastHour in forecastHours if timeRange.contains(forecastHour.getValidPeriod())]
def __buildStringList(param):
if __notStringIter(param):
return [str(item) for item in param]
else:
return [str(param)]
def __notStringIter(iterable):
if not isinstance(iterable, str):
try:
iter(iterable)
return True
except TypeError:
return False
class _SoundingCube(object):
"""
The top-level sounding object returned when calling SoundingsSupport.getSounding.
This object acts as a 3-tiered dict which is keyed by time then level
then parameter name. Calling times() will return all valid keys into this
object.
"""
def __init__(self, geometryDataObjects):
self._dataDict = {}
self._sortedTimes = []
if geometryDataObjects:
for geometryData in geometryDataObjects:
dataTime = geometryData.getDataTime()
level = geometryData.getLevel()
for parameter in geometryData.getParameters():
self.__addItem(parameter, dataTime, level, geometryData.getNumber(parameter))
def __addItem(self, parameter, dataTime, level, value):
timeLayer = self._dataDict.get(dataTime, _SoundingTimeLayer(dataTime))
self._dataDict[dataTime] = timeLayer
timeLayer._addItem(parameter, level, value)
if dataTime not in self._sortedTimes:
self._sortedTimes.append(dataTime)
self._sortedTimes.sort()
def __getitem__(self, key):
return self._dataDict[key]
def __len__(self):
return len(self._dataDict)
def times(self):
"""
Returns the valid times for this sounding.
Returns:
A list containing the valid DataTimes for this sounding in order.
"""
return self._sortedTimes
class _SoundingTimeLayer(object):
"""
The second-level sounding object returned when calling SoundingsSupport.getSounding.
This object acts as a 2-tiered dict which is keyed by level then parameter
name. Calling levels() will return all valid keys into this
object. Calling time() will return the DataTime for this particular layer.
"""
def __init__(self, dataTime):
self._dataTime = dataTime
self._dataDict = {}
def _addItem(self, parameter, level, value):
asString = str(level)
levelLayer = self._dataDict.get(asString, _SoundingTimeAndLevelLayer(self._dataTime, asString))
levelLayer._addItem(parameter, value)
self._dataDict[asString] = levelLayer
def __getitem__(self, key):
asString = str(key)
if asString in self._dataDict:
return self._dataDict[asString]
else:
raise KeyError("Level " + str(key) + " is not a valid level for this sounding.")
def __len__(self):
return len(self._dataDict)
def time(self):
"""
Returns the DataTime for this sounding cube layer.
Returns:
The DataTime for this sounding layer.
"""
return self._dataTime
def levels(self):
"""
Returns the valid levels for this sounding.
Returns:
A list containing the valid levels for this sounding in order of
closest to surface to highest from surface.
"""
sortedLevels = [Level(level) for level in list(self._dataDict.keys())]
sortedLevels.sort()
return [str(level) for level in sortedLevels]
class _SoundingTimeAndLevelLayer(object):
"""
The bottom-level sounding object returned when calling SoundingsSupport.getSounding.
This object acts as a dict which is keyed by parameter name. Calling
parameters() will return all valid keys into this object. Calling time()
will return the DataTime for this particular layer. Calling level() will
return the level for this layer.
"""
def __init__(self, time, level):
self._time = time
self._level = level
self._parameters = {}
def _addItem(self, parameter, value):
self._parameters[parameter] = value
def __getitem__(self, key):
return self._parameters[key]
def __len__(self):
return len(self._parameters)
def level(self):
"""
Returns the level for this sounding cube layer.
Returns:
The level for this sounding layer.
"""
return self._level
def parameters(self):
"""
Returns the valid parameters for this sounding.
Returns:
A list containing the valid parameter names.
"""
return list(self._parameters.keys())
def time(self):
"""
Returns the DataTime for this sounding cube layer.
Returns:
The DataTime for this sounding layer.
"""
return self._time

64
awips/dataaccess/ThriftClientRouter.py
View file

@ -1,7 +1,28 @@
# #
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
# #
#
# Routes requests to the Data Access Framework through Python Thrift.
#
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
@ -22,8 +43,8 @@
# 10/26/16 5919 njensen Speed up geometry creation in getGeometryData()
#
import numpy
import six
import shapely.wkb
from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.impl import DefaultDataRequest
@ -40,9 +61,9 @@ from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import G
from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetSupportedDatatypesRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetNotificationFilterRequest
from awips import ThriftClient
from awips.dataaccess import PyGeometryData
from awips.dataaccess import PyGridData
from ufpy import ThriftClient
from ufpy.dataaccess import PyGeometryData
from ufpy.dataaccess import PyGridData
class LazyGridLatLon(object):
@ -121,13 +142,7 @@ class ThriftClientRouter(object):
retVal = []
for gridDataRecord in response.getGridData():
locationName = gridDataRecord.getLocationName()
if locationName is not None:
if six.PY2:
locData = locSpecificData[locationName]
else:
locData = locSpecificData[locationName.encode('utf-8')]
else:
locData = locSpecificData[locationName]
locData = locSpecificData[locationName]
if self._lazyLoadGridLatLon:
retVal.append(PyGridData.PyGridData(gridDataRecord, locData[
0], locData[1], latLonDelegate=locData[2]))
@ -165,20 +180,12 @@ class ThriftClientRouter(object):
locNamesRequest = GetAvailableLocationNamesRequest()
locNamesRequest.setRequestParameters(request)
response = self._client.sendRequest(locNamesRequest)
if six.PY2:
return response
if response is not None:
return [x.decode('utf-8') for x in response]
return response
def getAvailableParameters(self, request):
paramReq = GetAvailableParametersRequest()
paramReq.setRequestParameters(request)
response = self._client.sendRequest(paramReq)
if six.PY2:
return response
if response is not None:
return [x.decode('utf-8') for x in response]
return response
def getAvailableLevels(self, request):
@ -194,10 +201,6 @@ class ThriftClientRouter(object):
idReq = GetRequiredIdentifiersRequest()
idReq.setRequest(request)
response = self._client.sendRequest(idReq)
if six.PY2:
return response
if response is not None:
return [x.decode('utf-8') for x in response]
return response
def getOptionalIdentifiers(self, request):
@ -207,10 +210,6 @@ class ThriftClientRouter(object):
idReq = GetOptionalIdentifiersRequest()
idReq.setRequest(request)
response = self._client.sendRequest(idReq)
if six.PY2:
return response
if response is not None:
return [x.decode('utf-8') for x in response]
return response
def getIdentifierValues(self, request, identifierKey):
@ -218,14 +217,9 @@ class ThriftClientRouter(object):
idValReq.setIdentifierKey(identifierKey)
idValReq.setRequestParameters(request)
response = self._client.sendRequest(idValReq)
if six.PY2:
return response
if response is not None:
return [x.decode('utf-8') for x in response]
return response
def newDataRequest(self, datatype, parameters=[], levels=[], locationNames=[],
envelope=None, **kwargs):
def newDataRequest(self, datatype, parameters=[], levels=[], locationNames=[], envelope=None, **kwargs):
req = DefaultDataRequest()
if datatype:
req.setDatatype(datatype)
@ -244,10 +238,6 @@ class ThriftClientRouter(object):
def getSupportedDatatypes(self):
response = self._client.sendRequest(GetSupportedDatatypesRequest())
if six.PY2:
return response
if response is not None:
return [x.decode('utf-8') for x in response]
return response
def getNotificationFilter(self, request):

47
awips/dataaccess/__init__.py
View file

@ -1,5 +1,26 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# __init__.py for awips.dataaccess package
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# __init__.py for ufpy.dataaccess package
#
#
# SOFTWARE HISTORY
@ -18,20 +39,12 @@
#
__all__ = [
'IData',
'IDataRequest',
'IGeometryData',
'IGridData',
'IGeometryData',
'INotificationFilter',
'INotificationSubscriber'
]
]
import abc
from six import with_metaclass
class IDataRequest(with_metaclass(abc.ABCMeta, object)):
class IDataRequest(object, metaclass=abc.ABCMeta):
"""
An IDataRequest to be submitted to the DataAccessLayer to retrieve data.
"""
@ -151,7 +164,8 @@ class IDataRequest(with_metaclass(abc.ABCMeta, object)):
return
class IData(with_metaclass(abc.ABCMeta, object)):
class IData(object, metaclass=abc.ABCMeta):
"""
An IData representing data returned from the DataAccessLayer.
"""
@ -211,6 +225,7 @@ class IData(with_metaclass(abc.ABCMeta, object)):
return
class IGridData(IData):
"""
An IData representing grid data that is returned by the DataAccessLayer.
@ -258,6 +273,7 @@ class IGridData(IData):
return
class IGeometryData(IData):
"""
An IData representing geometry data that is returned by the DataAccessLayer.
@ -336,7 +352,7 @@ class IGeometryData(IData):
return
class INotificationSubscriber(with_metaclass(abc.ABCMeta, object)):
class INotificationSubscriber(object, metaclass=abc.ABCMeta):
"""
An INotificationSubscriber representing a notification filter returned from
the DataNotificationLayer.
@ -359,8 +375,7 @@ class INotificationSubscriber(with_metaclass(abc.ABCMeta, object)):
"""Closes the notification subscriber"""
pass
class INotificationFilter(with_metaclass(abc.ABCMeta, object)):
class INotificationFilter(object, metaclass=abc.ABCMeta):
"""
Represents data required to filter a set of URIs and
return a corresponding list of IDataRequest to retrieve data for.

107
awips/gfe/IFPClient.py
View file

@ -1,16 +1,25 @@
#
# Provides a Python-based interface for executing GFE requests.
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 07/26/12 dgilling Initial Creation.
#
#
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from ufpy import ThriftClient
from awips import ThriftClient
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import DatabaseID
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import ParmID
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import CommitGridsRequest
@ -23,6 +32,21 @@ from dynamicserialize.dstypes.com.raytheon.uf.common.site.requests import GetAct
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.server.message import ServerResponse
#
# Provides a Python-based interface for executing GFE requests.
#
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 07/26/12 dgilling Initial Creation.
#
#
#
class IFPClient(object):
def __init__(self, host, port, user, site=None, progName=None):
self.__thrift = ThriftClient.ThriftClient(host, port)
@ -33,15 +57,14 @@ class IFPClient(object):
if len(sr.getPayload()) > 0:
site = sr.getPayload()[0]
self.__siteId = site
def commitGrid(self, request):
if isinstance(request, CommitGridRequest):
if type(request) is CommitGridRequest:
return self.__commitGrid([request])
elif self.__isHomogenousIterable(request, CommitGridRequest):
return self.__commitGrid([cgr for cgr in request])
raise TypeError("Invalid type: " + str(type(request)) +
" for commitGrid(). Only accepts CommitGridRequest or lists of CommitGridRequest.")
raise TypeError("Invalid type: " + str(type(request)) + " specified to commitGrid(). Only accepts CommitGridRequest or lists of CommitGridRequest.")
def __commitGrid(self, requests):
ssr = ServerResponse()
request = CommitGridsRequest()
@ -49,26 +72,25 @@ class IFPClient(object):
sr = self.__makeRequest(request)
ssr.setMessages(sr.getMessages())
return ssr
def getParmList(self, pid):
argType = type(pid)
def getParmList(self, id):
argType = type(id)
if argType is DatabaseID:
return self.__getParmList([pid])
elif self.__isHomogenousIterable(pid, DatabaseID):
return self.__getParmList([dbid for dbid in pid])
raise TypeError("Invalid type: " + str(argType) +
" for getParmList(). Only accepts DatabaseID or lists of DatabaseID.")
return self.__getParmList([id])
elif self.__isHomogenousIterable(id, DatabaseID):
return self.__getParmList([dbid for dbid in id])
raise TypeError("Invalid type: " + str(argType) + " specified to getParmList(). Only accepts DatabaseID or lists of DatabaseID.")
def __getParmList(self, ids):
ssr = ServerResponse()
request = GetParmListRequest()
request.setDbIds(ids)
sr = self.__makeRequest(request)
ssr.setMessages(sr.getMessages())
parmlist = sr.getPayload() if sr.getPayload() is not None else []
ssr.setPayload(parmlist)
list = sr.getPayload() if sr.getPayload() is not None else []
ssr.setPayload(list)
return ssr
def __isHomogenousIterable(self, iterable, classType):
try:
iterator = iter(iterable)
@ -78,23 +100,22 @@ class IFPClient(object):
except TypeError:
return False
return True
def getGridInventory(self, parmID):
if isinstance(parmID, ParmID):
if type(parmID) is ParmID:
sr = self.__getGridInventory([parmID])
inventoryList = []
list = []
try:
inventoryList = sr.getPayload()[parmID]
list = sr.getPayload()[parmID]
except KeyError:
# no-op, we've already default the TimeRange list to empty
pass
sr.setPayload(inventoryList)
sr.setPayload(list)
return sr
elif self.__isHomogenousIterable(parmID, ParmID):
return self.__getGridInventory([pid for pid in parmID])
raise TypeError("Invalid type: " + str(type(parmID)) +
" specified to getGridInventory(). Accepts ParmID or lists of ParmID.")
return self.__getGridInventory([id for id in parmID])
raise TypeError("Invalid type: " + str(type(parmID)) + " specified to getGridInventory(). Only accepts ParmID or lists of ParmID.")
def __getGridInventory(self, parmIDs):
ssr = ServerResponse()
request = GetGridInventoryRequest()
@ -104,7 +125,7 @@ class IFPClient(object):
trs = sr.getPayload() if sr.getPayload() is not None else {}
ssr.setPayload(trs)
return ssr
def getSelectTR(self, name):
request = GetSelectTimeRangeRequest()
request.setName(name)
@ -113,7 +134,7 @@ class IFPClient(object):
ssr.setMessages(sr.getMessages())
ssr.setPayload(sr.getPayload())
return ssr
def getSiteID(self):
ssr = ServerResponse()
request = GetActiveSitesRequest()
@ -122,7 +143,7 @@ class IFPClient(object):
ids = sr.getPayload() if sr.getPayload() is not None else []
sr.setPayload(ids)
return sr
def __makeRequest(self, request):
try:
request.setSiteID(self.__siteId)
@ -132,7 +153,7 @@ class IFPClient(object):
request.setWorkstationID(self.__wsId)
except AttributeError:
pass
sr = ServerResponse()
response = None
try:
@ -148,5 +169,5 @@ class IFPClient(object):
except AttributeError:
# not a server response, nothing else to do
pass
return sr

36
awips/gfe/__init__.py
View file

@ -1,3 +1,37 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
__all__ = [
#
# __init__.py for ufpy.gfe package
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 07/26/12 dgilling Initial Creation.
#
#
#
__all__ = [
]

64
awips/ignite_password.py Normal file
View file

@ -0,0 +1,64 @@
##############################################################################
# Encryption/decryption for ignite passwords
#
# TODO RODO #8677: The ignite password encryption/decryption code in this and
# associated files is based off similar JMS password code that exists in a
# later version, so the similar code should be consolidated later on.
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------- -------- -------------- -------------------------------------------
# Oct 12, 2021 8667 mapeters Initial version
# Mar 03, 2022 8762 mapeters Handle cache VM jars being moved under
# /awips2/ignite
#
##############################################################################
import os
import subprocess
class IgniteConfigurationException(Exception):
"""Exception subclass for ignite password errors."""
pass
def updateIgnitePasswords(keystore_password, truststore_password, password_props_path):
igniteJar = __findPluginJar("com.raytheon.uf.common.datastore.ignite")
cryptoJar = __findPluginJar("com.raytheon.uf.common.security")
statusJar = __findPluginJar("com.raytheon.uf.common.status")
apacheJar = __findFossJar("org.apache.commons.codec")
classPath = ":".join([igniteJar, cryptoJar, statusJar, apacheJar])
passwords_dict = {'a2.ignite.keystore.password': keystore_password, 'a2.ignite.truststore.password': truststore_password}
for password_key, password in passwords_dict.items():
# need full java path since this is run as root, which doesn't have appropriate path vars set
process = subprocess.run(["/awips2/java/bin/java", "-cp", classPath, "com.raytheon.uf.common.datastore.ignite.IgnitePasswordUtils", "--update", password, password_key, password_props_path],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
process.check_returncode()
except subprocess.CalledProcessError:
raise IgniteConfigurationException(f"Failed to update {password_key}: {process.stderr.decode()}")
def __findPluginJar(pluginName):
# for cache VMs
ignitePluginJar = f"/awips2/ignite/lib/plugins/{pluginName}.jar"
if os.path.isfile(ignitePluginJar):
return ignitePluginJar
# for edex VMs
edexPluginJar = f"/awips2/edex/lib/plugins/{pluginName}.jar"
if os.path.isfile(edexPluginJar):
return edexPluginJar
raise RuntimeError(f"Could not locate plugin {pluginName}.jar")
def __findFossJar(libraryName):
# for cache VMs
igniteFossDir = f"/awips2/ignite/lib/dependencies/{libraryName}"
if os.path.isdir(igniteFossDir):
return f"{igniteFossDir}/*"
# for edex VMs
edexFossDir = f"/awips2/edex/lib/dependencies/{libraryName}"
if os.path.isdir(edexFossDir):
return f"{edexFossDir}/*"
raise RuntimeError(f"Could not locate plugin {libraryName}")

477
awips/localization/LocalizationFileManager.py Normal file
View file

@ -0,0 +1,477 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Library for accessing localization files from python.
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# --------- -------- --------- --------------------------
# 08/09/17 5731 bsteffen Initial Creation.
# 04/23/19 7756 mapeters Make user context name determination work with IdM
# 04/22/20 7883 tgurney Python 3 string/bytes fixes
# 01/10/22 8735 mapeters Set Content-Type for PUTs
import urllib.request, urllib.error, urllib.parse
from json import load as loadjson
from xml.etree.ElementTree import parse as parseXml
from base64 import b64encode
from io import BytesIO
import dateutil.parser
import contextlib
import os
from urllib.parse import urlunparse, urljoin
from . import LocalizationUtil
NON_EXISTENT_CHECKSUM = 'NON_EXISTENT_CHECKSUM'
DIRECTORY_CHECKSUM = 'DIRECTORY_CHECKSUM'
class LocalizationFileVersionConflictException(Exception):
pass
class LocalizationFileDoesNotExistException(Exception):
pass
class LocalizationFileIsNotDirectoryException(Exception):
pass
class LocalizationContext(object):
"""A localization context defines the scope of a localization file.
For example the base localization context includes all the default files
installed with EDEX, while a particular user context has custom files for
that user.
A localization context consists of a level and name. The level defines what
kind of entity this context is valid for, such as 'base', 'site', or 'user'.
The name identifies the specific entity, for example the name of a 'user'
level context is usually the username. The 'base' level does not have a name
because there cannot be only one 'base' context.
Attributes:
level: the localization level
name: the context name
"""
def __init__(self, level="base", name=None, localizationType="common_static"):
if level != "base":
assert name is not None
self.level = level
self.name = name
self.type = localizationType
def isBase(self):
return self.level == "base"
def _getUrlComponent(self):
if self.isBase():
return self.type + '/' + "base/"
else:
return self.type + '/' + self.level + '/' + self.name + '/'
def __str__(self):
if self.isBase():
return self.type + ".base"
else:
return self.type + "." + self.level + "." + self.name
def __eq__(self, other):
return self.level == other.level and \
self.name == other.name and \
self.type == other.type
def __hash__(self):
return hash((self.level, self.name, self.type))
class _LocalizationOutput(BytesIO):
"""A file-like object for writing a localization file.
The contents being written are stored in memory and written to a
localization server only when the writing is finished.
This object should be used as a context manager, a save operation will be
executed if the context exits with no errors. If errors occur the partial
contents are abandoned and the server is unchanged.
It is also possible to save the contents to the server with the save()
method.
"""
def __init__(self, manager, lFile):
super().__init__()
self._manager = manager
self._file = lFile
def save(self):
"""Send the currently written contents to the server."""
request = self._manager._buildRequest(self._file.context, self._file.path, method="PUT")
request.data = self.getvalue()
request.add_header("If-Match", self._file.checksum)
# An empty file is created if Content-Type isn't specified (defaults to
# "application/x-www-form-urlencoded"). We aren't encoding the
# request.data bytes in any special way for either text files (e.g. .py
# or .xml) or binary files (e.g. .png), so this works for both.
request.add_header("Content-Type", "application/octet-stream")
try:
urllib.request.urlopen(request)
except urllib.error.HTTPError as e:
if e.code == 409:
raise LocalizationFileVersionConflictException(e.read())
else:
raise e
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
if exc_type is None:
self.save()
def __str__(self):
return '<' + self.__class__.__name__ + " for " + str(self._file) + '>'
class LocalizationFile(object):
"""A specific file stored in localization.
A localization file is uniquely defined by the context and path. There can
only be one valid file for that path and localization at a time. To access
the contents of the file use the open method.
Attributes:
context: A LocalizationContext
path: A path to this file
checksum: A string representation of a checksum generated from the file contents.
timnestamp: A datetime.datetime object indicating when the file was last modified.
"""
def __init__(self, manager, context, path, checksum, timestamp):
"""Initialize a LocalizationFile with the given manager and attributes.
Args:
manager: A LocalizationFileManager to assist with server communication
context: A LocalizationContext
path: A path to this file
checksum: A string representation of a checksum generated from the file contents.
timnestamp: A datetime.datetime object indicating when the file was last modified.
"""
self._manager = manager
self.context = context
self.path = path
self.checksum = checksum
self.timestamp = timestamp
def open(self, mode='r'):
"""Open the file.
This should always be called as as part of a with statement. When
writing the content is not saved on the server until leaving the with
statement normally, if an error occurs the server is left unchanged.
Example:
with locFile.open('w') as output:
output.write('some content')
Args:
mode: 'r' for reading the file, 'w' for writing
Returns:
A file like object that can be used for reads or writes.
"""
if mode == 'r':
request = self._manager._buildRequest(self.context, self.path)
response = urllib.request.urlopen(request)
# Not the recommended way of reading directories.
if not(self.isDirectory()):
checksum = response.headers["Content-MD5"]
if self.checksum != checksum:
raise RuntimeError("Localization checksum mismatch " + self.checksum + " " + checksum)
return contextlib.closing(response)
elif mode == 'w':
return _LocalizationOutput(self._manager, self)
else:
raise ValueError("mode string must be 'r' or 'w' not " + str(r))
def delete(self):
"""Delete this file from the server"""
request = self._manager._buildRequest(self.context, self.path, method='DELETE')
request.add_header("If-Match", self.checksum)
try:
urllib.request.urlopen(request)
except urllib.error.HTTPError as e:
if e.code == 409:
raise LocalizationFileVersionConflictException(e.read())
else:
raise e
def exists(self):
"""Check if this file actually exists.
Returns:
boolean indicating existence of this file
"""
return self.checksum != NON_EXISTENT_CHECKSUM
def isDirectory(self):
"""Check if this file is a directory.
A file must exist to be considered a directory.
Returns:
boolean indicating directorocity of this file
"""
return self.checksum == DIRECTORY_CHECKSUM
def getCheckSum(self):
return self.checksum
def getContext(self):
return self.context
def getPath(self):
return self.path
def getTimeStamp(self):
return self.timestamp
def __str__(self):
return str(self.context) + "/" + self.path
def __eq__(self, other):
return self.context == other.context and \
self.path == other.path and \
self.checksum == other.checksum \
and self.timestamp == other.timestamp
def __hash__(self):
return hash((self.context, self.path, self.checksum, self.timestamp))
def _getHost():
import subprocess
host = subprocess.check_output(
"source /awips2/fxa/bin/setup.env; echo $DEFAULT_HOST",
shell=True).strip().decode()
if host:
return host
return 'localhost'
def _getSiteFromServer(host):
try:
from ufpy import ThriftClient
from dynamicserialize.dstypes.com.raytheon.uf.common.site.requests import GetPrimarySiteRequest
client = ThriftClient.ThriftClient(host)
return client.sendRequest(GetPrimarySiteRequest())
except:
# Servers that don't have GFE installed will not return a site
pass
def _getSiteFromEnv():
site = os.environ.get('FXA_LOCAL_SITE')
if site is None:
site = os.environ.get('SITE_IDENTIFIER')
return site
def _getSite(host):
site = _getSiteFromEnv()
if not(site):
site = _getSiteFromServer(host)
return site
def _parseJsonList(manager, response, context, path):
fileList = []
jsonResponse = loadjson(response)
for name, jsonData in jsonResponse.items():
checksum = jsonData["checksum"]
timestampString = jsonData["timestamp"]
timestamp = dateutil.parser.parse(timestampString)
newpath = urljoin(path, name)
fileList.append(LocalizationFile(manager, context, newpath, checksum, timestamp))
return fileList
def _parseXmlList(manager, response, context, path):
fileList = []
for xmlData in parseXml(response).getroot().findall('file'):
name = xmlData.get("name")
checksum = xmlData.get("checksum")
timestampString = xmlData.get("timestamp")
timestamp = dateutil.parser.parse(timestampString)
newpath = urljoin(path, name)
fileList.append(LocalizationFile(manager, context, newpath, checksum, timestamp))
return fileList
class LocalizationFileManager(object):
"""Connects to a server and retrieves LocalizationFiles."""
def __init__(self, host=None, port=9581, path="/services/localization/", contexts=None, site=None, localizationType="common_static"):
"""Initializes a LocalizationFileManager with connection parameters and context information
All arguments are optional and will use defaults or attempt to figure out appropriate values form the environment.
Args:
host: A hostname of the localization server, such as 'ev'.
port: A port to use to connect to the localization server, usually 9581.
path: A path to reach the localization file service on the server.
contexts: A list of contexts to check for files, the order of the contexts will be used
for the order of incremental results and the priority of absolute results.
site: A site identifier to use for site specific contexts. This is only used if the contexts arg is None.
localizationType: A localization type for contexts. This is only used if the contexts arg is None.
"""
if host is None:
host = _getHost()
if contexts is None:
if site is None:
site = _getSite(host)
contexts = [LocalizationContext("base", None, localizationType)]
if site:
contexts.append(LocalizationContext("configured", site, localizationType))
contexts.append(LocalizationContext("site", site, localizationType))
contexts.append(LocalizationContext("user", LocalizationUtil.getUser(), localizationType))
netloc = host + ':' + str(port)
self._baseUrl = urlunparse(('http', netloc, path, None, None, None))
self._contexts = contexts
def _buildRequest(self, context, path, method='GET'):
url = urljoin(self._baseUrl, context._getUrlComponent())
url = urljoin(url, path)
request = urllib.request.Request(url, method=method)
username = LocalizationUtil.getUser()
# Currently password is ignored in the server
# this is the defacto standard for not providing one to this service.
password = username
base64string = b64encode(b'%s:%s' % (username.encode(), password.encode()))
request.add_header("Authorization", "Basic %s" % base64string.decode())
return request
def _normalizePath(self, path):
if path == '' or path == '/':
path = '.'
if path[0] == '/':
path = path[1:]
return path
def _list(self, path):
path = self._normalizePath(path)
if path[-1] != '/':
path += '/'
fileList = []
exists = False
for context in self._contexts:
try:
request = self._buildRequest(context, path)
request.add_header("Accept", "application/json, application/xml")
response = urllib.request.urlopen(request)
exists = True
if not(response.geturl().endswith("/")):
# For ordinary files the server sends a redirect to remove the slash.
raise LocalizationFileIsNotDirectoryException("Not a directory: " + path)
elif response.headers["Content-Type"] == "application/xml":
fileList += _parseXmlList(self, response, context, path)
else:
fileList += _parseJsonList(self, response, context, path)
except urllib.error.HTTPError as e:
if e.code != 404:
raise e
if not(exists):
raise LocalizationFileDoesNotExistException("No such file or directory: " + path)
return fileList
def _get(self, context, path):
path = self._normalizePath(path)
try:
request = self._buildRequest(context, path, method='HEAD')
resp = urllib.request.urlopen(request)
if (resp.geturl().endswith("/")):
checksum = DIRECTORY_CHECKSUM
else:
if "Content-MD5" not in resp.headers:
raise RuntimeError("Missing Content-MD5 header in response from " + resp.geturl())
checksum = resp.headers["Content-MD5"]
if "Last-Modified" not in resp.headers:
raise RuntimeError("Missing Last-Modified header in response from " + resp.geturl())
timestamp = dateutil.parser.parse(resp.headers["Last-Modified"])
return LocalizationFile(self, context, path, checksum, timestamp)
except urllib.error.HTTPError as e:
if e.code != 404:
raise e
else:
return LocalizationFile(self, context, path, NON_EXISTENT_CHECKSUM, None)
def listAbsolute(self, path):
"""List the files in a localization directory, only a single file is returned for each unique path.
If a file exists in more than one context then the highest level(furthest from base) is used.
Args:
path: A path to a directory that should be the root of the listing
Returns:
A list of LocalizationFiles
"""
merged = dict()
for lFile in self._list(path):
merged[lFile.path] = lFile
return sorted(merged.values(), key=lambda lFile: lFile.path)
def listIncremental(self, path):
"""List the files in a localization directory, this includes all files for all contexts.
Args:
path: A path to a directory that should be the root of the listing
Returns:
A list of tuples, each tuple will contain one or more files for the
same paths but different contexts. Each tuple will be ordered the
same as the contexts in this manager, generally with 'base' first
and 'user' last.
"""
merged = dict()
for lFile in self._list(path):
if lFile.path in merged:
merged[lFile.path] += (lFile,)
else:
merged[lFile.path] = (lFile,)
return sorted(merged.values(), key=lambda t: t[0].path)
def getAbsolute(self, path):
"""Get a single localization file from the highest level context where it exists.
Args:
path: A path to a localization file
Returns:
A Localization File with the specified path or None if the file does not exist in any context.
"""
for context in reversed(self._contexts):
f = self._get(context, path)
if f.exists():
return f
def getIncremental(self, path):
"""Get all the localization files that exist in any context for the provided path.
Args:
path: A path to a localization file
Returns:
A tuple containing all the files that exist for this path in any context. The tuple
will be ordered the same as the contexts in this manager, generally with 'base' first
and 'user' last.
"""
result = ()
for context in self._contexts:
f = self._get(context, path)
if f.exists():
result += (f,)
return result
def getSpecific(self, level, path):
"""Get a specific localization file at a given level, the file may not exist.
The file is returned for whichever context is valid for the provided level in this manager.
For writing new files this is the only way to get access to a file that
does not exist in order to create it.
Args:
level: the name of a localization level, such as "base", "site", "user"
path: A path to a localization file
Returns:
A Localization File with the specified path and a context for the specified level.
"""
for context in self._contexts:
if context.level == level:
return self._get(context, path)
raise ValueError("No context defined for level " + level)
def __str__(self):
contextsStr = '[' + ' '.join((str(c) for c in self._contexts)) + ']'
return '<' + self.__class__.__name__ + " for " + self._baseUrl + ' ' + contextsStr + '>'

45
awips/localization/LocalizationUtil.py Normal file
View file

@ -0,0 +1,45 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Utilities for localization.
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# --------- -------- --------- --------------------------
# 04/23/19 7756 mapeters Initial creation
import getpass
def getUser():
'''
Get the user context name.
'''
try:
# Match Java's way of determining the user if we have Jep access
from java.lang import System
user = System.getProperty('user.name')
except:
# Otherwise use built-in getpass module. With IdM, this can return
# user.name@REALM, so strip the @REALM portion if it exists.
user = getpass.getuser()
user = user.split('@')[0]
return user

32
awips/localization/__init__.py Normal file
View file

@ -0,0 +1,32 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# __init__.py for ufpy.localization package
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# --------- -------- --------- --------------------------
# 08/10/17 5731 bsteffen Initial Creation.
__all__ = [
]

139
awips/qpidingest.py
View file

@ -1,19 +1,10 @@
# ===============================================================================
#===============================================================================
# qpidingest.py
#
# @author: Aaron Anderson
# @organization: NOAA/WDTB OU/CIMMS
# @version: 1.0 02/19/2010
# @requires: QPID Python Client available from http://qpid.apache.org/download.html
# The Python Client is located under Single Component Package/Client
#
# From the README.txt Installation Instructions
# = INSTALLATION =
# Extract the release archive into a directory of your choice and set
# your PYTHONPATH accordingly:
#
# tar -xzf qpid-python-<version>.tar.gz -C <install-prefix>
# export PYTHONPATH=<install-prefix>/qpid-<version>/python
# @requires: awips2-python and awips2-qpid-proton-python RPMs
#
# ***EDEX and QPID must be running for this module to work***
#
@ -38,7 +29,7 @@
# EXAMPLE:
# Simple example program:
#
# ------------------------------------------------------------------------------
#------------------------------------------------------------------------------
# import qpidingest
# #Tell EDEX to ingest a metar file from data_store. The filepath is
# #/data_store/20100218/metar/00/standard/20100218_005920_SAUS46KSEW.metar
@ -50,82 +41,102 @@
#
# conn.sendmessage('/data_store/20100218/metar/18/standard/20100218_185755_SAUS46KLOX.metar','SAUS46 KLOX')
# conn.close()
# -------------------------------------------------------------------------------
#-------------------------------------------------------------------------------
#
# SOFTWARE HISTORY
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# ....
# 06/13/2013 DR 16242 D. Friedman Add Qpid authentication info
# 03/06/2014 DR 17907 D. Friedman Workaround for issue QPID-5569
# 02/16/2017 DR 6084 bsteffen Support ssl connections
# Date Ticket# Engineer Description
# ------------- -------- ------------ ------------------------------------------
# Jun 13, 2013 16242 D. Friedman Add Qpid authentication info
# Mar 06, 2014 17907 D. Friedman Workaround for issue QPID-5569
# Feb 16, 2017 6084 bsteffen Support ssl connections
# Jun 14, 2019 7870 mrichardson MHS env workaround
# Jul 23, 2019 7724 mrichardson Upgrade Qpid to Qpid Proton
# Nov 06, 2019 7724 tgurney Remove the unnecessary
# QpidQueueManager
# Dec 12, 2019 7995 dgilling Revert interface changes from #7724.
# Jul 07, 2020 8187 randerso Added qpid connection_id
#
# ===============================================================================
#===============================================================================
import logging
import os
import pwd
import os.path
import socket
import qpid
from qpid.util import connect
from qpid.connection import Connection
from qpid.datatypes import Message, uuid4
import proton
import proton.utils
import proton.reactor
QPID_USERNAME = 'guest'
QPID_PASSWORD = 'guest'
log = logging.getLogger("qpidingest")
class QpidIngestException(Exception):
"""Exception subclass for broker communication exceptions."""
pass
class IngestViaQPID:
def __init__(self, host='localhost', port=5672, ssl=None):
"""
def __init__(self, host="localhost", port=5672, program="qpidingest"):
'''
Connect to QPID and make bindings to route message to external.dropbox queue
@param host: string hostname of computer running EDEX and QPID (default localhost)
@param port: integer port used to connect to QPID (default 5672)
@param ssl: boolean to determine whether ssl is used, default value of None will use
ssl only if a client certificate is found.
"""
'''
pwuid = pwd.getpwuid(os.getuid())
certdb = os.getenv("QPID_SSL_CERT_DB", os.path.join(pwuid.pw_dir, ".qpid"))
certname = os.getenv("QPID_SSL_CERT_NAME", "guest")
cert_password = os.getenv("QPID_SSL_CERT_PASSWORD", "password")
certfile = os.path.join(certdb, f"{certname}.crt")
keyfile = os.path.join(certdb, f"{certname}.key")
url = f"amqps://{host}:{port}"
ADDRESS = "external.dropbox"
ssl_domain = proton.SSLDomain(mode=proton.SSLDomain.MODE_CLIENT)
ssl_domain.set_credentials(certfile, keyfile, cert_password)
clientID = ":".join([
socket.gethostname(),
pwuid.pw_name,
program,
str(os.getpid()),
])
try:
#
socket = connect(host, port)
if "QPID_SSL_CERT_DB" in os.environ:
certdb = os.environ["QPID_SSL_CERT_DB"]
else:
certdb = os.path.expanduser("~/.qpid/")
if "QPID_SSL_CERT_NAME" in os.environ:
certname = os.environ["QPID_SSL_CERT_NAME"]
else:
certname = QPID_USERNAME
certfile = os.path.join(certdb, certname + ".crt")
if ssl or (ssl is None and os.path.exists(certfile)):
keyfile = os.path.join(certdb, certname + ".key")
trustfile = os.path.join(certdb, "root.crt")
socket = qpid.util.ssl(socket, keyfile=keyfile, certfile=certfile, ca_certs=trustfile)
self.connection = Connection(sock=socket, username=QPID_USERNAME, password=QPID_PASSWORD)
self.connection.start()
self.session = self.connection.session(str(uuid4()))
self.session.exchange_bind(exchange='amq.direct', queue='external.dropbox', binding_key='external.dropbox')
print('Connected to Qpid')
except ValueError:
print('Unable to connect to Qpid')
container = proton.reactor.Container()
container.container_id = clientID
self._conn = proton.utils.BlockingConnection(url, ssl_domain=ssl_domain)
self._sender = self._conn.create_sender(ADDRESS)
log.debug("Connected to broker [%s], endpoint [%s].", url, ADDRESS)
except proton.ProtonException as e:
log.exception("Failed to connect to broker [%s].", url)
raise QpidIngestException("Failed to connect to broker [{}].".format(url)) from e
def sendmessage(self, filepath, header):
"""
'''
This function sends a message to the external.dropbox queue providing the path
to the file to be ingested and a header to determine the plugin to be used to
decode the file.
@param filepath: string full path to file to be ingested
@param header: string header used to determine plugin decoder to use
"""
props = self.session.delivery_properties(routing_key='external.dropbox')
head = self.session.message_properties(application_headers={'header': header},
user_id=QPID_USERNAME)
self.session.message_transfer(destination='amq.direct', message=Message(props, head, filepath))
'''
try:
self._sender.send(proton.Message(body=filepath, subject=header))
except proton.ProtonException as e:
log.exception("Failed to send file [%s] to broker.", filepath)
raise QpidIngestException("Failed to send file [{}] to broker.".format(filepath)) from e
def close(self):
"""
'''
After all messages are sent call this function to close connection and make sure
there are no threads left open
"""
self.session.close(timeout=10)
print('Connection to Qpid closed')
'''
try:
self._sender.close()
self._conn.close()
log.debug("Disconnected from broker.")
except proton.ProtonException as e:
log.warning("Failed to disconnect from broker.", exc_info=True)
raise QpidIngestException("Failed to disconnect from broker.") from e

50
awips/test/Record.py
View file

@ -1,12 +1,48 @@
import sys
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Pure python logging mechanism for logging to AlertViz from
# pure python (ie not JEP). DO NOT USE IN PYTHON CALLED
# FROM JAVA.
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 11/03/10 5849 cjeanbap Initial Creation.
#
#
#
import os
import sys
class Record():
def __init__(self, level=0, msg='Test Message'):
self.levelno = level
self.message = msg
self.exc_info = sys.exc_info()
self.exc_text = "TEST"
self.levelno=level
self.message=msg
self.exc_info=sys.exc_info()
self.exc_text="TEST"
def getMessage(self):
return self.message
return self.message

48
awips/test/Test Normal file
View file

@ -0,0 +1,48 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Pure python logging mechanism for logging to AlertViz from
# pure python (ie not JEP). DO NOT USE IN PYTHON CALLED
# FROM JAVA.
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 11/03/10 5849 cjeanbap Initial Creation.
#
#
#
## to execute type python Test
import os
import logging
from awips import AlertVizHandler
import Record
avh = AlertVizHandler.AlertVizHandler(host=os.getenv("BROKER_ADDR","localhost"), port=9581, category='LOCAL', source='ANNOUNCER', level=logging.NOTSET)
record = Record.Record(10)
avh.emit(record)

34
awips/test/__init__.py
View file

@ -1,2 +1,34 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# __init__.py for ufpy package
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 11/03/10 5489 cjeanbap Initial Creation.
#
#
#
__all__ = []

34
awips/test/dafTests/__init__.py
View file

@ -1,2 +1,36 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# __init__.py for ufpy.test.dafTests package
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 02/09/2016 4795 mapeters Initial creation.
# 04/12/2016 5548 tgurney Cleanup
#
#
#
__all__ = []

34
awips/test/dafTests/baseBufrMosTestCase.py
View file

@ -1,3 +1,28 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from ufpy.dataaccess import DataAccessLayer as DAL
from . import baseDafTestCase
from . import params
#
# Base TestCase for BufrMos* tests.
#
@ -12,17 +37,12 @@
#
#
from awips.dataaccess import DataAccessLayer as DAL
from awips.test.dafTests import baseDafTestCase
from awips.test.dafTests import params
class BufrMosTestCase(baseDafTestCase.DafTestCase):
"""Base class for testing DAF support of bufrmos data"""
data_params = "temperature", "dewpoint"
def testGetAvailableParameters(self):
req = DAL.newDataRequest(self.datatype)
self.runParametersTest(req)

80
awips/test/dafTests/baseDafTestCase.py
View file

@ -1,3 +1,32 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from ufpy.dataaccess import DataAccessLayer as DAL
from ufpy.ThriftClient import ThriftRequestException
import os
import numpy
import unittest
#
# Base TestCase for DAF tests. This class provides helper methods and
# tests common to all DAF test cases.
@ -24,16 +53,11 @@
# time-agnostic data
# 03/13/17 5981 tgurney Do not check valid period on
# data time
# 04/14/22 8845 njensen Add checks for NaNs in geometry
# data tests
#
#
from __future__ import print_function
from awips.dataaccess import DataAccessLayer as DAL
from awips.ThriftClient import ThriftRequestException
import os
import unittest
class DafTestCase(unittest.TestCase):
@ -56,7 +80,7 @@ class DafTestCase(unittest.TestCase):
def setUpClass(cls):
host = os.environ.get('DAF_TEST_HOST')
if host is None:
host = 'edex-cloud.unidata.ucar.edu'
host = 'localhost'
DAL.changeEDEXHost(host)
@staticmethod
@ -68,13 +92,13 @@ class DafTestCase(unittest.TestCase):
try:
times = DAL.getAvailableTimes(req)
except ThriftRequestException as e:
if 'TimeAgnosticDataException' not in str(e):
if not 'TimeAgnosticDataException' in str(e):
raise
return times
def testDatatypeIsSupported(self):
allSupported = DAL.getSupportedDatatypes()
self.assertIn(self.datatype, allSupported)
allSupported = (item.lower() for item in DAL.getSupportedDatatypes())
self.assertIn(self.datatype.lower(), allSupported)
def testGetRequiredIdentifiers(self):
req = DAL.newDataRequest(self.datatype)
@ -89,21 +113,21 @@ class DafTestCase(unittest.TestCase):
print("Optional identifiers:", optional)
def runGetIdValuesTest(self, identifiers):
for identifier in identifiers:
if identifier.lower() == 'datauri':
for id in identifiers:
if id.lower() == 'datauri':
continue
req = DAL.newDataRequest(self.datatype)
idValues = DAL.getIdentifierValues(req, identifier)
idValues = DAL.getIdentifierValues(req, id)
self.assertTrue(hasattr(idValues, '__iter__'))
def runInvalidIdValuesTest(self):
badString = 'id from ' + self.datatype + '; select 1;'
with self.assertRaises(ThriftRequestException):
with self.assertRaises(ThriftRequestException) as cm:
req = DAL.newDataRequest(self.datatype)
DAL.getIdentifierValues(req, badString)
def runNonexistentIdValuesTest(self):
with self.assertRaises(ThriftRequestException):
with self.assertRaises(ThriftRequestException) as cm:
req = DAL.newDataRequest(self.datatype)
DAL.getIdentifierValues(req, 'idthatdoesnotexist')
@ -145,9 +169,22 @@ class DafTestCase(unittest.TestCase):
times = DafTestCase.getTimesIfSupported(req)
geomData = DAL.getGeometryData(req, times[:self.numTimesToLimit])
self.assertIsNotNone(geomData)
if times:
self.assertNotEqual(len(geomData), 0)
if not geomData:
raise unittest.SkipTest("No data available")
print("Number of geometry records: " + str(len(geomData)))
print("Sample geometry data:")
for record in geomData[:self.sampleDataLimit]:
if (checkDataTimes and times and
"PERIOD_USED" not in record.getDataTime().getUtilityFlags()):
self.assertIn(record.getDataTime(), times[:self.numTimesToLimit])
print("geometry=" + str(record.getGeometry()), end="")
for p in req.getParameters():
print(" " + p + "=" + record.getString(p), end="")
if record.getType(p) in ['FLOAT', 'DOUBLE']:
self.assertFalse(numpy.isnan(record.getNumber(p)))
print()
return geomData
def runGeometryDataTestWithTimeRange(self, req, timeRange):
@ -160,6 +197,16 @@ class DafTestCase(unittest.TestCase):
if not geomData:
raise unittest.SkipTest("No data available")
print("Number of geometry records: " + str(len(geomData)))
print("Sample geometry data:")
for record in geomData[:self.sampleDataLimit]:
self.assertGreaterEqual(record.getDataTime().getRefTime().getTime(), timeRange.getStartInMillis())
self.assertLessEqual(record.getDataTime().getRefTime().getTime(), timeRange.getEndInMillis())
print("geometry=" + str(record.getGeometry()), end="")
for p in req.getParameters():
print(" " + p + "=" + record.getString(p), end="")
if record.getType(p) in ['FLOAT', 'DOUBLE']:
self.assertFalse(numpy.isnan(record.getNumber(p)))
print()
return geomData
def runGridDataTest(self, req, testSameShape=True):
@ -168,7 +215,6 @@ class DafTestCase(unittest.TestCase):
request.
Args:
req: the grid request
testSameShape: whether or not to verify that all the retrieved data
have the same shape (most data don't change shape)
"""

41
awips/test/dafTests/baseRadarTestCase.py
View file

@ -1,3 +1,30 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from ufpy.dataaccess import DataAccessLayer as DAL
from ufpy.ThriftClient import ThriftRequestException
from . import baseDafTestCase
from . import params
#
# Tests common to all radar factories
#
@ -23,13 +50,6 @@
#
#
from __future__ import print_function
from awips.dataaccess import DataAccessLayer as DAL
from awips.ThriftClient import ThriftRequestException
from awips.test.dafTests import baseDafTestCase
from awips.test.dafTests import params
class BaseRadarTestCase(baseDafTestCase.DafTestCase):
"""Tests common to all radar factories"""
@ -50,7 +70,7 @@ class BaseRadarTestCase(baseDafTestCase.DafTestCase):
def testGetAvailableLevels(self):
req = DAL.newDataRequest(self.datatype)
self.runLevelsTest(req)
def testGetAvailableLevelsWithInvalidLevelIdentifierThrowsException(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('level.one.field', 'invalidLevelField')
@ -88,11 +108,6 @@ class BaseRadarTestCase(baseDafTestCase.DafTestCase):
for record in gridData:
self.assertEqual(record.getAttribute('icao'), 1000)
def testGetDataWithEqualsLong(self):
gridData = self.runConstraintTest('icao', '=', 1000)
for record in gridData:
self.assertEqual(record.getAttribute('icao'), 1000)
def testGetDataWithEqualsFloat(self):
gridData = self.runConstraintTest('icao', '=', 1.0)
for record in gridData:

28
awips/test/dafTests/params.py
View file

@ -1,3 +1,24 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Site-specific parameters for DAF tests
#
@ -7,17 +28,18 @@
# ------------ ---------- ----------- --------------------------
# 12/07/16 5981 tgurney Initial creation
# 12/15/16 5981 tgurney Add ENVELOPE
# 04/14/22 8845 njensen Add POINT
#
#
from shapely.geometry import box
from shapely.geometry import box, Point
AIRPORT = 'OMA'
OBS_STATION = 'KOMA'
SITE_ID = 'OAX'
STATION_ID = '72558'
RADAR = 'KOAX'
POINT = Point(-96.25, 41.16)
SAMPLE_AREA = (-97.0, 41.0, -96.0, 42.0)
ENVELOPE = box(*SAMPLE_AREA)
ENVELOPE = box(*SAMPLE_AREA)

29
awips/test/dafTests/testAcars.py
View file

@ -1,3 +1,28 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from ufpy.dataaccess import DataAccessLayer as DAL
from . import baseDafTestCase
#
# Test DAF support for ACARS data
#
@ -11,10 +36,6 @@
#
#
from __future__ import print_function
from awips.dataaccess import DataAccessLayer as DAL
from awips.test.dafTests import baseDafTestCase
class AcarsTestCase(baseDafTestCase.DafTestCase):
"""Test DAF support for ACARS data"""

32
awips/test/dafTests/testAirep.py
View file

@ -1,3 +1,29 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from ufpy.dataaccess import DataAccessLayer as DAL
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from . import baseDafTestCase
#
# Test DAF support for airep data
#
@ -14,12 +40,6 @@
#
#
from __future__ import print_function
from awips.dataaccess import DataAccessLayer as DAL
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from awips.test.dafTests import baseDafTestCase
class AirepTestCase(baseDafTestCase.DafTestCase):
"""Test DAF support for airep data"""

91
awips/test/dafTests/testBinLightning.py
View file

@ -1,3 +1,31 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from ufpy.dataaccess import DataAccessLayer as DAL
from ufpy.ThriftClient import ThriftRequestException
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from . import baseDafTestCase
#
# Test DAF support for binlightning data
#
@ -18,20 +46,14 @@
# 06/13/16 5574 tgurney Typo
# 06/30/16 5725 tgurney Add test for NOT IN
# 11/08/16 5985 tgurney Do not check data times
#
#
#
from __future__ import print_function
from awips.dataaccess import DataAccessLayer as DAL
from awips.ThriftClient import ThriftRequestException
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from awips.test.dafTests import baseDafTestCase
class BinLightningTestCase(baseDafTestCase.DafTestCase):
"""Test DAF support for binlightning data"""
datatype = "binlightning"
source = "GLMfl"
def testGetAvailableParameters(self):
req = DAL.newDataRequest(self.datatype)
@ -39,18 +61,18 @@ class BinLightningTestCase(baseDafTestCase.DafTestCase):
def testGetAvailableTimes(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('source', self.source)
req.addIdentifier("source", "NLDN")
self.runTimesTest(req)
def testGetGeometryDataSingleSourceSingleParameter(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('source', self.source)
req.addIdentifier("source", "NLDN")
req.setParameters('intensity')
self.runGeometryDataTest(req, checkDataTimes=False)
def testGetGeometryDataInvalidParamRaisesIncompatibleRequestException(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('source', self.source)
req.addIdentifier("source", "NLDN")
req.setParameters('blahblahblah')
with self.assertRaises(ThriftRequestException) as cm:
self.runGeometryDataTest(req)
@ -58,7 +80,7 @@ class BinLightningTestCase(baseDafTestCase.DafTestCase):
def testGetGeometryDataSingleSourceAllParameters(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('source', self.source)
req.addIdentifier("source", "NLDN")
req.setParameters(*DAL.getAvailableParameters(req))
self.runGeometryDataTest(req, checkDataTimes=False)
@ -82,20 +104,15 @@ class BinLightningTestCase(baseDafTestCase.DafTestCase):
return self.runGeometryDataTest(req, checkDataTimes=False)
def testGetDataWithEqualsString(self):
geomData = self._runConstraintTest('source', '=', self.source)
geomData = self._runConstraintTest('source', '=', 'NLDN')
for record in geomData:
self.assertEqual(record.getAttribute('source'), self.source)
self.assertEqual(record.getAttribute('source'), 'NLDN')
def testGetDataWithEqualsInt(self):
geomData = self._runConstraintTest('source', '=', 1000)
for record in geomData:
self.assertEqual(record.getAttribute('source'), 1000)
def testGetDataWithEqualsLong(self):
geomData = self._runConstraintTest('source', '=', 1000)
for record in geomData:
self.assertEqual(record.getAttribute('source'), 1000)
def testGetDataWithEqualsFloat(self):
geomData = self._runConstraintTest('source', '=', 1.0)
for record in geomData:
@ -107,9 +124,9 @@ class BinLightningTestCase(baseDafTestCase.DafTestCase):
self.assertIsNone(record.getAttribute('source'))
def testGetDataWithNotEquals(self):
geomData = self._runConstraintTest('source', '!=', self.source)
geomData = self._runConstraintTest('source', '!=', 'NLDN')
for record in geomData:
self.assertNotEqual(record.getAttribute('source'), self.source)
self.assertNotEqual(record.getAttribute('source'), 'NLDN')
def testGetDataWithNotEqualsNone(self):
geomData = self._runConstraintTest('source', '!=', None)
@ -117,49 +134,49 @@ class BinLightningTestCase(baseDafTestCase.DafTestCase):
self.assertIsNotNone(record.getAttribute('source'))
def testGetDataWithGreaterThan(self):
geomData = self._runConstraintTest('source', '>', self.source)
geomData = self._runConstraintTest('source', '>', 'NLDN')
for record in geomData:
self.assertGreater(record.getAttribute('source'), self.source)
self.assertGreater(record.getAttribute('source'), 'NLDN')
def testGetDataWithLessThan(self):
geomData = self._runConstraintTest('source', '<', self.source)
geomData = self._runConstraintTest('source', '<', 'NLDN')
for record in geomData:
self.assertLess(record.getAttribute('source'), self.source)
self.assertLess(record.getAttribute('source'), 'NLDN')
def testGetDataWithGreaterThanEquals(self):
geomData = self._runConstraintTest('source', '>=', self.source)
geomData = self._runConstraintTest('source', '>=', 'NLDN')
for record in geomData:
self.assertGreaterEqual(record.getAttribute('source'), self.source)
self.assertGreaterEqual(record.getAttribute('source'), 'NLDN')
def testGetDataWithLessThanEquals(self):
geomData = self._runConstraintTest('source', '<=', self.source)
geomData = self._runConstraintTest('source', '<=', 'NLDN')
for record in geomData:
self.assertLessEqual(record.getAttribute('source'), self.source)
self.assertLessEqual(record.getAttribute('source'), 'NLDN')
def testGetDataWithInTuple(self):
geomData = self._runConstraintTest('source', 'in', (self.source, 'GLMev'))
geomData = self._runConstraintTest('source', 'in', ('NLDN', 'ENTLN'))
for record in geomData:
self.assertIn(record.getAttribute('source'), (self.source, 'GLMev'))
self.assertIn(record.getAttribute('source'), ('NLDN', 'ENTLN'))
def testGetDataWithInList(self):
geomData = self._runConstraintTest('source', 'in', [self.source, 'GLMev'])
geomData = self._runConstraintTest('source', 'in', ['NLDN', 'ENTLN'])
for record in geomData:
self.assertIn(record.getAttribute('source'), (self.source, 'GLMev'))
self.assertIn(record.getAttribute('source'), ('NLDN', 'ENTLN'))
def testGetDataWithInGenerator(self):
generator = (item for item in (self.source, 'GLMev'))
generator = (item for item in ('NLDN', 'ENTLN'))
geomData = self._runConstraintTest('source', 'in', generator)
for record in geomData:
self.assertIn(record.getAttribute('source'), (self.source, 'GLMev'))
self.assertIn(record.getAttribute('source'), ('NLDN', 'ENTLN'))
def testGetDataWithNotInList(self):
geomData = self._runConstraintTest('source', 'not in', [self.source, 'blah'])
geomData = self._runConstraintTest('source', 'not in', ['NLDN', 'blah'])
for record in geomData:
self.assertNotIn(record.getAttribute('source'), (self.source, 'blah'))
self.assertNotIn(record.getAttribute('source'), ('NLDN', 'blah'))
def testGetDataWithInvalidConstraintTypeThrowsException(self):
with self.assertRaises(ValueError):
self._runConstraintTest('source', 'junk', self.source)
self._runConstraintTest('source', 'junk', 'NLDN')
def testGetDataWithInvalidConstraintValueThrowsException(self):
with self.assertRaises(TypeError):

44
awips/test/dafTests/testBufrMosAvn.py Normal file
View file

@ -0,0 +1,44 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from . import baseBufrMosTestCase
#
# Test DAF support for bufrmosAVN data
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 01/19/16 4795 mapeters Initial Creation.
# 04/11/16 5548 tgurney Cleanup
# 04/18/16 5548 tgurney More cleanup
#
#
class BufrMosAvnTestCase(baseBufrMosTestCase.BufrMosTestCase):
"""Test DAF support for bufrmosAVN data"""
datatype = "bufrmosAVN"
# All tests inherited from superclass

44
awips/test/dafTests/testBufrMosEta.py Normal file
View file

@ -0,0 +1,44 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from . import baseBufrMosTestCase
#
# Test DAF support for bufrmosETA data
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 01/19/16 4795 mapeters Initial Creation.
# 04/11/16 5548 tgurney Cleanup
# 04/18/16 5548 tgurney More cleanup
#
#
class BufrMosEtaTestCase(baseBufrMosTestCase.BufrMosTestCase):
"""Test DAF support for bufrmosETA data"""
datatype = "bufrmosETA"
# All tests inherited from superclass

26
awips/test/dafTests/testBufrMosGfs.py
View file

@ -1,3 +1,27 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from . import baseBufrMosTestCase
#
# Test DAF support for bufrmosGFS data
#
@ -11,8 +35,6 @@
#
#
from awips.test.dafTests import baseBufrMosTestCase
class BufrMosGfsTestCase(baseBufrMosTestCase.BufrMosTestCase):
"""Test DAF support for bufrmosGFS data"""

46
awips/test/dafTests/testBufrMosHpc.py Normal file
View file

@ -0,0 +1,46 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from . import baseBufrMosTestCase
#
# Test DAF support for bufrmosHPC data
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 01/19/16 4795 mapeters Initial Creation.
# 04/11/16 5548 tgurney Cleanup
# 04/18/16 5548 tgurney More cleanup
# 12/07/16 5981 tgurney Parameterize
# 12/20/16 5981 tgurney Inherit all tests
#
#
class BufrMosHpcTestCase(baseBufrMosTestCase.BufrMosTestCase):
"""Test DAF support for bufrmosHPC data"""
datatype = "bufrmosHPC"
data_params = "forecastHr", "maxTemp24Hour"
# All tests inherited from superclass

44
awips/test/dafTests/testBufrMosLamp.py Normal file
View file

@ -0,0 +1,44 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from . import baseBufrMosTestCase
#
# Test DAF support for bufrmosLAMP data
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 01/19/16 4795 mapeters Initial Creation.
# 04/11/16 5548 tgurney Cleanup
# 04/18/16 5548 tgurney More cleanup
#
#
class BufrMosLampTestCase(baseBufrMosTestCase.BufrMosTestCase):
"""Test DAF support for bufrmosLAMP data"""
datatype = "bufrmosLAMP"
# All tests inherited from superclass

45
awips/test/dafTests/testBufrMosMrf.py Normal file
View file

@ -0,0 +1,45 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from . import baseBufrMosTestCase
#
# Test DAF support for bufrmosMRF data
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 01/19/16 4795 mapeters Initial Creation.
# 04/11/16 5548 tgurney Cleanup
# 04/18/16 5548 tgurney More cleanup
# 12/07/16 5981 tgurney Parameterize
# 12/20/16 5981 tgurney Inherit all tests
#
#
class BufrMosMrfTestCase(baseBufrMosTestCase.BufrMosTestCase):
"""Test DAF support for bufrmosMRF data"""
datatype = "bufrmosMRF"
data_params = "forecastHr", "maxTempDay"
# All tests inherited from superclass

34
awips/test/dafTests/testBufrUa.py
View file

@ -1,9 +1,30 @@
from __future__ import print_function
from awips.dataaccess import DataAccessLayer as DAL
# #
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
# #
from ufpy.dataaccess import DataAccessLayer as DAL
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from awips.test.dafTests import baseDafTestCase
from awips.test.dafTests import params
from . import baseDafTestCase
from . import params
import unittest
#
# Test DAF support for bufrua data
@ -110,11 +131,6 @@ class BufrUaTestCase(baseDafTestCase.DafTestCase):
for record in geometryData:
self.assertEqual(record.getString('rptType'), '2022')
def testGetDataWithEqualsLong(self):
geometryData = self._runConstraintTest('reportType', '=', 2022)
for record in geometryData:
self.assertEqual(record.getString('rptType'), '2022')
# No float test because no float identifiers are available
def testGetDataWithEqualsNone(self):

419
awips/test/dafTests/testClimate.py Normal file
View file

@ -0,0 +1,419 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
import datetime
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange
from ufpy.dataaccess import DataAccessLayer as DAL
from ufpy.ThriftClient import ThriftRequestException
from . import baseDafTestCase
from . import params
#
# Test DAF support for climate data
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 01/19/16 4795 mapeters Initial Creation.
# 04/11/16 5548 tgurney Cleanup
# 04/18/16 5548 tgurney More cleanup
# 04/26/16 5587 tgurney Add identifier values tests
# 06/09/16 5574 mapeters Add advanced query tests, Short parameter test
# 06/13/16 5574 tgurney Fix checks for None
# 06/21/16 5548 tgurney Skip tests that cause errors
# 06/30/16 5725 tgurney Add test for NOT IN
# 10/06/16 5926 dgilling Add additional time and location tests.
# 12/07/16 5981 tgurney Parameterize
# 12/20/16 5981 tgurney Add envelope test
# 08/16/17 6388 tgurney Test for duplicate data
#
#
class ClimateTestCase(baseDafTestCase.DafTestCase):
"""Test DAF support for climate data"""
datatype = 'climate'
obsStation = params.OBS_STATION
def testGetAvailableParameters(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_monthly')
self.runParametersTest(req)
def testGetAvailableLocations(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_monthly')
self.runLocationsTest(req)
def testGetAvailableLocationsForRptTable(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.rpt')
self.runLocationsTest(req)
def testGetAvailableLocationsForStationId(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.day_climate_norm')
self.runLocationsTest(req)
def testGetAvailableLocationsForInformId(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_mon_season_yr')
self.runLocationsTest(req)
def testGetAvailableLocationsWithConstraints(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_monthly')
req.addIdentifier('maxtemp_mon', RequestConstraint.new('>', 95))
self.runLocationsTest(req)
def testGetAvailableLocationsWithInvalidTable(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.boolean_values')
with self.assertRaises(ThriftRequestException) as cm:
DAL.getAvailableLocationNames(req)
self.assertIn('IncompatibleRequestException', str(cm.exception))
def testGetAvailableTimes(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_monthly')
req.setParameters('maxtemp_mon', 'min_sea_press')
self.runTimesTest(req)
def testGetAvailableTimesWithLocationNamesForYearMonth(self):
"""
Test retrieval of times for a climo table that uses year and
month columns to build DataTimes.
"""
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_monthly')
req.setLocationNames(self.obsStation, 'KABR', 'KDMO')
req.setParameters('maxtemp_mon', 'min_sea_press')
self.runTimesTest(req)
def testGetAvailableTimesWithLocationNamesForYearDayOfYear(self):
"""
Test retrieval of times for a climo table that uses year and
day_of_year columns to build DataTimes.
"""
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_daily')
req.setLocationNames(self.obsStation, 'KABR', 'KDMO')
req.setParameters('maxtemp_cal', 'min_press')
self.runTimesTest(req)
def testGetAvailableTimesWithLocationNamesForPeriod(self):
"""
Test retrieval of times for a climo table that uses
period_start and period_end columns to build DataTimes.
"""
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_mon_season_yr')
req.setLocationNames(self.obsStation, 'KABR', 'KDMO')
req.setParameters('max_temp', 'precip_total')
self.runTimesTest(req)
def testGetAvailableTimesWithLocationNamesForDate(self):
"""
Test retrieval of times for a climo table that uses a date
column to build DataTimes.
"""
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.daily_climate')
req.setLocationNames(self.obsStation, 'KABR', 'KDMO')
req.setParameters('max_temp', 'precip', 'avg_wind_speed')
self.runTimesTest(req)
def testGetAvailableTimesWithConstraint(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_monthly')
req.addIdentifier('maxtemp_mon', RequestConstraint.new('<', 75))
req.setParameters('maxtemp_mon', 'min_sea_press')
self.runTimesTest(req)
def testGetGeometryData(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_monthly')
req.setLocationNames('KFNB')
req.setParameters('maxtemp_mon', 'min_sea_press')
self.runGeometryDataTest(req)
def testGetGeometryDataWithEnvelopeThrowsException(self):
# Envelope is not used
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_monthly')
req.setParameters('maxtemp_mon', 'min_sea_press')
req.setEnvelope(params.ENVELOPE)
with self.assertRaises(Exception):
self.runGeometryDataTest(req)
def testGetGeometryDataForYearAndDayOfYearTable(self):
"""
Test retrieval of data for a climo table that uses year and
day_of_year columns to build DataTimes.
"""
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_daily')
req.setLocationNames('KFNB')
req.setParameters('maxtemp_cal', 'min_press')
self.runGeometryDataTest(req)
def testGetGeometryDataForPeriodTable(self):
"""
Test retrieval of data for a climo table that uses a period_start and
period_end columns to build DataTimes.
"""
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_mon_season_yr')
req.setLocationNames('KFNB')
req.setParameters('max_temp', 'precip_total')
self.runGeometryDataTest(req)
def testGetGeometryDataForDateTable(self):
"""
Test retrieval of data for a climo table that uses a date column to
build DataTimes.
"""
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.daily_climate')
req.setLocationNames('KFNB')
req.setParameters('max_temp', 'precip', 'avg_wind_speed')
self.runGeometryDataTest(req)
def testGetGeometryDataWithShortParameter(self):
"""
Test that a parameter that is stored in Java as a Short is correctly
retrieved as a number.
"""
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'cli_asos_monthly')
req.setParameters('month')
geometryData = self.runGeometryDataTest(req)
for record in geometryData:
self.assertIsNotNone(record.getNumber('month'))
def testGetTableIdentifierValues(self):
self.runGetIdValuesTest(['table'])
def testGetColumnIdValuesWithTable(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_monthly')
idValues = DAL.getIdentifierValues(req, 'year')
self.assertTrue(hasattr(idValues, '__iter__'))
def testGetColumnIdValuesWithoutTableThrowsException(self):
req = DAL.newDataRequest(self.datatype)
with self.assertRaises(ThriftRequestException):
DAL.getIdentifierValues(req, 'year')
def testGetInvalidIdentifierValuesThrowsException(self):
self.runInvalidIdValuesTest()
def testGetNonexistentIdentifierValuesThrowsException(self):
self.runNonexistentIdValuesTest()
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'cli_asos_monthly')
constraint = RequestConstraint.new(operator, value)
req.addIdentifier(key, constraint)
req.setParameters('station_code', 'avg_daily_max')
return self.runGeometryDataTest(req)
def testGetDataWithEqualsString(self):
geometryData = self._runConstraintTest('station_code', '=', self.obsStation)
for record in geometryData:
self.assertEqual(record.getString('station_code'), self.obsStation)
def testGetDataWithEqualsInt(self):
geometryData = self._runConstraintTest('avg_daily_max', '=', 70)
for record in geometryData:
self.assertEqual(record.getNumber('avg_daily_max'), 70)
def testGetDataWithEqualsFloat(self):
geometryData = self._runConstraintTest('avg_daily_max', '=', 69.2)
for record in geometryData:
self.assertEqual(round(record.getNumber('avg_daily_max'), 1), 69.2)
def testGetDataWithEqualsNone(self):
geometryData = self._runConstraintTest('station_code', '=', None)
self.assertEqual(len(geometryData), 0)
def testGetDataWithNotEquals(self):
geometryData = self._runConstraintTest('station_code', '!=', self.obsStation)
for record in geometryData:
self.assertNotEqual(record.getString('station_code'), self.obsStation)
def testGetDataWithNotEqualsNone(self):
geometryData = self._runConstraintTest('station_code', '!=', None)
for record in geometryData:
self.assertNotEqual(record.getType('station_code'), 'NULL')
def testGetDataWithGreaterThan(self):
geometryData = self._runConstraintTest('avg_daily_max', '>', 70)
for record in geometryData:
self.assertGreater(record.getNumber('avg_daily_max'), 70)
def testGetDataWithLessThan(self):
geometryData = self._runConstraintTest('avg_daily_max', '<', 70)
for record in geometryData:
self.assertLess(record.getNumber('avg_daily_max'), 70)
def testGetDataWithGreaterThanEquals(self):
geometryData = self._runConstraintTest('avg_daily_max', '>=', 70)
for record in geometryData:
self.assertGreaterEqual(record.getNumber('avg_daily_max'), 70)
def testGetDataWithLessThanEquals(self):
geometryData = self._runConstraintTest('avg_daily_max', '<=', 70)
for record in geometryData:
self.assertLessEqual(record.getNumber('avg_daily_max'), 70)
def testGetDataWithInTuple(self):
collection = (self.obsStation, 'KABR')
geometryData = self._runConstraintTest('station_code', 'in', collection)
for record in geometryData:
self.assertIn(record.getString('station_code'), collection)
def testGetDataWithInList(self):
collection = [self.obsStation, 'KABR']
geometryData = self._runConstraintTest('station_code', 'in', collection)
for record in geometryData:
self.assertIn(record.getString('station_code'), collection)
def testGetDataWithInGenerator(self):
collection = (self.obsStation, 'KABR')
generator = (item for item in collection)
geometryData = self._runConstraintTest('station_code', 'in', generator)
for record in geometryData:
self.assertIn(record.getString('station_code'), collection)
def testGetDataWithNotInList(self):
collection = ['KORD', 'KABR']
geometryData = self._runConstraintTest('station_code', 'not in', collection)
for record in geometryData:
self.assertNotIn(record.getString('station_code'), collection)
def testGetDataWithInvalidConstraintTypeThrowsException(self):
with self.assertRaises(ValueError):
self._runConstraintTest('station_code', 'junk', self.obsStation)
def testGetDataWithInvalidConstraintValueThrowsException(self):
with self.assertRaises(TypeError):
self._runConstraintTest('station_code', '=', {})
def testGetDataWithEmptyInConstraintThrowsException(self):
with self.assertRaises(ValueError):
self._runConstraintTest('station_code', 'in', [])
def testGetDataWithTimeRangeWithYearAndMonth1(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_monthly')
req.setLocationNames('KFNB')
req.setParameters('maxtemp_mon', 'min_sea_press')
startTime = datetime.datetime(2009, 1, 1)
endTime = datetime.datetime(2009, 12, 31)
tr = TimeRange(startTime, endTime)
self.runGeometryDataTestWithTimeRange(req, tr)
def testGetDataWithTimeRangeWithYearAndMonth2(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_monthly')
req.setLocationNames('KFNB')
req.setParameters('maxtemp_mon', 'min_sea_press')
startTime = datetime.datetime(2008, 1, 1)
endTime = datetime.datetime(2009, 3, 31)
tr = TimeRange(startTime, endTime)
self.runGeometryDataTestWithTimeRange(req, tr)
def testGetDataWithTimeRangeWithYearAndMonth3(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_monthly')
req.setLocationNames('KFNB')
req.setParameters('maxtemp_mon', 'min_sea_press')
startTime = datetime.datetime(2007, 7, 1)
endTime = datetime.datetime(2009, 3, 31)
tr = TimeRange(startTime, endTime)
self.runGeometryDataTestWithTimeRange(req, tr)
def testGetDataWithTimeRangeWithYearAndDayOfYear1(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_daily')
req.setLocationNames('KFNB')
req.setParameters('maxtemp_cal', 'min_press')
startTime = datetime.datetime(2009, 1, 1)
endTime = datetime.datetime(2009, 7, 31)
tr = TimeRange(startTime, endTime)
self.runGeometryDataTestWithTimeRange(req, tr)
def testGetDataWithTimeRangeWithYearAndDayOfYear2(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_daily')
req.setLocationNames('KFNB')
req.setParameters('maxtemp_cal', 'min_press')
startTime = datetime.datetime(2008, 7, 1)
endTime = datetime.datetime(2009, 3, 31)
tr = TimeRange(startTime, endTime)
self.runGeometryDataTestWithTimeRange(req, tr)
def testGetDataWithTimeRangeWithYearAndDayOfYear3(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_daily')
req.setLocationNames('KFNB')
req.setParameters('maxtemp_cal', 'min_press')
startTime = datetime.datetime(2007, 7, 1)
endTime = datetime.datetime(2009, 3, 31)
tr = TimeRange(startTime, endTime)
self.runGeometryDataTestWithTimeRange(req, tr)
def testGetDataWithTimeRangeWithPeriodTable(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_mon_season_yr')
req.setLocationNames('KFNB')
req.setParameters('max_temp', 'precip_total')
startTime = datetime.datetime(2007, 7, 1)
endTime = datetime.datetime(2009, 3, 31)
tr = TimeRange(startTime, endTime)
self.runGeometryDataTestWithTimeRange(req, tr)
def testGetDataWithTimeRangeWithForDateTable(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.daily_climate')
req.setLocationNames('KFNB')
req.setParameters('max_temp', 'precip', 'avg_wind_speed')
startTime = datetime.datetime(2007, 7, 1)
endTime = datetime.datetime(2009, 3, 31)
tr = TimeRange(startTime, endTime)
self.runGeometryDataTestWithTimeRange(req, tr)
def testNoDuplicateData(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_monthly')
req.setLocationNames('KOMA')
req.setParameters('maxtemp_day1')
rows = DAL.getGeometryData(req, DAL.getAvailableTimes(req)[0:5])
for i in range(len(rows)):
for j in range(len(rows)):
if i != j:
self.assertNotEqual(rows[i].__dict__, rows[j].__dict__)

56
awips/test/dafTests/testCombinedTimeQuery.py
View file

@ -1,3 +1,30 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from awips.dataaccess import DataAccessLayer as DAL
from awips.dataaccess import CombinedTimeQuery as CTQ
import unittest
import os
#
# Test the CombinedTimedQuery module
#
@ -11,39 +38,30 @@
#
#
from awips.dataaccess import DataAccessLayer as DAL
from awips.dataaccess import CombinedTimeQuery as CTQ
import unittest
import os
class CombinedTimeQueryTestCase(unittest.TestCase):
modelName = "RAP13"
@classmethod
def setUp(cls):
host = os.environ.get('DAF_TEST_HOST')
if host is None:
host = 'edex-cloud.unidata.ucar.edu'
host = 'localhost'
DAL.changeEDEXHost(host)
def testSuccessfulQuery(self):
req = DAL.newDataRequest('grid')
req.setLocationNames(self.modelName)
req.setParameters('T', 'GH')
req.setLevels('300MB', '500MB', '700MB')
times = CTQ.getAvailableTimes(req)
req.setLocationNames('RUC130')
req.setParameters('T','GH')
req.setLevels('300MB', '500MB','700MB')
times = CTQ.getAvailableTimes(req);
self.assertNotEqual(len(times), 0)
def testNonIntersectingQuery(self):
"""
Test that when a parameter is only available on one of the levels that no times are returned.
"""
req = DAL.newDataRequest('grid')
req.setLocationNames(self.modelName)
req.setParameters('T', 'GH', 'LgSP1hr')
req.setLevels('300MB', '500MB', '700MB', '0.0SFC')
times = CTQ.getAvailableTimes(req)
req.setLocationNames('RUC130')
req.setParameters('T','GH', 'LgSP1hr')
req.setLevels('300MB', '500MB','700MB','0.0SFC')
times = CTQ.getAvailableTimes(req);
self.assertEqual(len(times), 0)

38
awips/test/dafTests/testCommonObsSpatial.py
View file

@ -1,3 +1,30 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from ufpy.dataaccess import DataAccessLayer as DAL
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from . import baseDafTestCase
from . import params
#
# Test DAF support for common_obs_spatial data
#
@ -18,12 +45,6 @@
# 01/06/17 5981 tgurney Do not check data times
#
from __future__ import print_function
from awips.dataaccess import DataAccessLayer as DAL
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from awips.test.dafTests import baseDafTestCase
from awips.test.dafTests import params
class CommonObsSpatialTestCase(baseDafTestCase.DafTestCase):
"""Test DAF support for common_obs_spatial data"""
@ -69,11 +90,6 @@ class CommonObsSpatialTestCase(baseDafTestCase.DafTestCase):
for record in geometryData:
self.assertEqual(record.getNumber('catalogtype'), 32)
def testGetDataWithEqualsLong(self):
geometryData = self._runConstraintTest('elevation', '=', 0)
for record in geometryData:
self.assertEqual(record.getNumber('elevation'), 0)
# No float test since there are no float identifiers available. Attempting
# to filter a non-float identifier on a float value raises an exception.

44
awips/test/dafTests/testDataTime.py
View file

@ -1,3 +1,27 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from dynamicserialize.dstypes.com.raytheon.uf.common.time import DataTime
import unittest
#
# Unit tests for Python implementation of RequestConstraint
#
@ -9,10 +33,6 @@
#
#
from dynamicserialize.dstypes.com.raytheon.uf.common.time import DataTime
import unittest
class DataTimeTestCase(unittest.TestCase):
@ -37,14 +57,14 @@ class DataTimeTestCase(unittest.TestCase):
self.assertEqual(expected, str(DataTime(s)))
s = s.replace(' ', '_')
self.assertEqual(expected, str(DataTime(s)))
def testFromStrWithFcstTimeHr(self):
s = '2016-08-02 01:23:45 (17)'
expected = s
self.assertEqual(expected, str(DataTime(s)))
s = s.replace(' ', '_')
self.assertEqual(expected, str(DataTime(s)))
def testFromStrWithFcstTimeHrZeroMillis(self):
s = '2016-08-02 01:23:45.0 (17)'
expected = '2016-08-02 01:23:45 (17)'
@ -58,7 +78,7 @@ class DataTimeTestCase(unittest.TestCase):
self.assertEqual(expected, str(DataTime(s)))
s = s.replace(' ', '_')
self.assertEqual(expected, str(DataTime(s)))
def testFromStrWithFcstTimeHrMin(self):
s = '2016-08-02 01:23:45 (17:34)'
expected = s
@ -72,28 +92,28 @@ class DataTimeTestCase(unittest.TestCase):
self.assertEqual(expected, str(DataTime(s)))
s = s.replace(' ', '_')
self.assertEqual(expected, str(DataTime(s)))
def testFromStrWithPeriod(self):
s = '2016-08-02 01:23:45[2016-08-02 02:34:45--2016-08-02 03:45:56]'
expected = s
self.assertEqual(expected, str(DataTime(s)))
s = s.replace(' ', '_')
self.assertEqual(expected, str(DataTime(s)))
def testFromStrWithPeriodZeroMillis(self):
s = '2016-08-02 01:23:45.0[2016-08-02 02:34:45.0--2016-08-02 03:45:56.0]'
expected = '2016-08-02 01:23:45[2016-08-02 02:34:45--2016-08-02 03:45:56]'
self.assertEqual(expected, str(DataTime(s)))
s = s.replace(' ', '_')
self.assertEqual(expected, str(DataTime(s)))
def testFromStrWithEverything(self):
s = '2016-08-02 01:23:45.0_(17:34)[2016-08-02 02:34:45.0--2016-08-02 03:45:56.0]'
expected = '2016-08-02 01:23:45 (17:34)[2016-08-02 02:34:45--2016-08-02 03:45:56]'
self.assertEqual(expected, str(DataTime(s)))
s = s.replace(' ', '_')
self.assertEqual(expected, str(DataTime(s)))
def testDataTimeReconstructItselfFromString(self):
times = [
'2016-08-02 01:23:45',
@ -111,4 +131,4 @@ class DataTimeTestCase(unittest.TestCase):
'2016-08-02 01:23:45.456_(17:34)[2016-08-02_02:34:45.0--2016-08-02_03:45:56.0]'
]
for time in times:
self.assertEqual(DataTime(time), DataTime(str(DataTime(time))), time)
self.assertEqual(DataTime(time), DataTime(str(DataTime(time))), time)

222
awips/test/dafTests/testFfmp.py Normal file
View file

@ -0,0 +1,222 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from ufpy.dataaccess import DataAccessLayer as DAL
from . import baseDafTestCase
from . import params
#
# Test DAF support for ffmp data
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 01/19/16 4795 mapeters Initial Creation.
# 04/11/16 5548 tgurney Cleanup
# 04/18/16 5548 tgurney More cleanup
# 04/18/16 5587 tgurney Add test for sane handling of
# zero records returned
# 06/20/16 5587 tgurney Add identifier values tests
# 07/01/16 5728 mapeters Add advanced query tests,
# include huc and accumHrs in
# id values tests, test that
# accumHrs id is never required
# 08/03/16 5728 mapeters Fixed minor bugs, replaced
# PRTM parameter since it isn't
# configured for ev-oma
# 11/08/16 5985 tgurney Do not check data times
# 12/07/16 5981 tgurney Parameterize
# 12/20/16 5981 tgurney Do not check data times
#
#
class FfmpTestCase(baseDafTestCase.DafTestCase):
"""Test DAF support for ffmp data"""
datatype = 'ffmp'
location = params.RADAR.lower()
@staticmethod
def addIdentifiers(req):
req.addIdentifier('wfo', params.SITE_ID)
req.addIdentifier('siteKey', 'hpe')
req.addIdentifier('dataKey', 'hpe')
req.addIdentifier('huc', 'ALL')
def testGetAvailableParameters(self):
req = DAL.newDataRequest(self.datatype)
self.runParametersTest(req)
def testGetAvailableLocations(self):
req = DAL.newDataRequest(self.datatype)
self.addIdentifiers(req)
self.runLocationsTest(req)
def testGetAvailableTimes(self):
req = DAL.newDataRequest(self.datatype)
self.addIdentifiers(req)
req.setParameters('DHRMOSAIC')
self.runTimesTest(req)
def testGetGeometryData(self):
req = DAL.newDataRequest(self.datatype)
self.addIdentifiers(req)
req.setParameters('DHRMOSAIC')
self.runGeometryDataTest(req, checkDataTimes=False)
def testGetGeometryDataEmptyResult(self):
req = DAL.newDataRequest(self.datatype)
self.addIdentifiers(req)
req.setParameters('blah blah blah') # force 0 records returned
result = self.runGeometryDataTest(req, checkDataTimes=False)
self.assertEqual(len(result), 0)
def testGetIdentifierValues(self):
req = DAL.newDataRequest(self.datatype)
optionalIds = set(DAL.getOptionalIdentifiers(req))
requiredIds = set(DAL.getRequiredIdentifiers(req))
ids = requiredIds | optionalIds
for id in ids:
req = DAL.newDataRequest(self.datatype)
if id == 'accumHrs':
req.setParameters('ARI6H2YR')
req.addIdentifier('wfo', params.SITE_ID)
req.addIdentifier('siteKey', self.location)
req.addIdentifier('huc', 'ALL')
idValues = DAL.getIdentifierValues(req, id)
self.assertTrue(hasattr(idValues, '__iter__'))
print(id + " values: " + str(idValues))
def testGetInvalidIdentifierValuesThrowsException(self):
self.runInvalidIdValuesTest()
def testGetNonexistentIdentifierValuesThrowsException(self):
self.runNonexistentIdValuesTest()
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.addIdentifier(key, constraint)
req.addIdentifier('wfo', params.SITE_ID)
req.addIdentifier('huc', 'ALL')
req.setParameters('QPFSCAN')
return self.runGeometryDataTest(req, checkDataTimes=False)
def testGetDataWithEqualsString(self):
geometryData = self._runConstraintTest('siteKey', '=', self.location)
for record in geometryData:
self.assertEqual(record.getAttribute('siteKey'), self.location)
# No numeric tests since no numeric identifiers are available that support
# RequestConstraints.
def testGetDataWithEqualsNone(self):
geometryData = self._runConstraintTest('siteKey', '=', None)
for record in geometryData:
self.assertIsNone(record.getAttribute('siteKey'))
def testGetDataWithNotEquals(self):
geometryData = self._runConstraintTest('siteKey', '!=', self.location)
for record in geometryData:
self.assertNotEqual(record.getAttribute('siteKey'), self.location)
def testGetDataWithNotEqualsNone(self):
geometryData = self._runConstraintTest('siteKey', '!=', None)
for record in geometryData:
self.assertIsNotNone(record.getAttribute('siteKey'))
def testGetDataWithGreaterThan(self):
geometryData = self._runConstraintTest('siteKey', '>', self.location)
for record in geometryData:
self.assertGreater(record.getAttribute('siteKey'), self.location)
def testGetDataWithLessThan(self):
geometryData = self._runConstraintTest('siteKey', '<', self.location)
for record in geometryData:
self.assertLess(record.getAttribute('siteKey'), self.location)
def testGetDataWithGreaterThanEquals(self):
geometryData = self._runConstraintTest('siteKey', '>=', self.location)
for record in geometryData:
self.assertGreaterEqual(record.getAttribute('siteKey'), self.location)
def testGetDataWithLessThanEquals(self):
geometryData = self._runConstraintTest('siteKey', '<=', self.location)
for record in geometryData:
self.assertLessEqual(record.getAttribute('siteKey'), self.location)
def testGetDataWithInList(self):
collection = [self.location, 'kuex']
geometryData = self._runConstraintTest('siteKey', 'in', collection)
for record in geometryData:
self.assertIn(record.getAttribute('siteKey'), collection)
def testGetDataWithNotInList(self):
collection = [self.location, 'kuex']
geometryData = self._runConstraintTest('siteKey', 'not in', collection)
for record in geometryData:
self.assertNotIn(record.getAttribute('siteKey'), collection)
def testGetDataWithInvalidConstraintTypeThrowsException(self):
with self.assertRaises(ValueError):
self._runConstraintTest('siteKey', 'junk', self.location)
def testGetDataWithInvalidConstraintValueThrowsException(self):
with self.assertRaises(TypeError):
self._runConstraintTest('siteKey', '=', {})
def testGetDataWithEmptyInConstraintThrowsException(self):
with self.assertRaises(ValueError):
self._runConstraintTest('siteKey', 'in', [])
def testGetDataWithSiteKeyAndDataKeyConstraints(self):
siteKeys = [self.location, 'hpe']
dataKeys = ['kuex', 'kdmx']
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('wfo', params.SITE_ID)
req.addIdentifier('huc', 'ALL')
siteKeysConstraint = RequestConstraint.new('in', siteKeys)
req.addIdentifier('siteKey', siteKeysConstraint)
dataKeysConstraint = RequestConstraint.new('in', dataKeys)
req.addIdentifier('dataKey', dataKeysConstraint)
req.setParameters('QPFSCAN')
geometryData = self.runGeometryDataTest(req, checkDataTimes=False)
for record in geometryData:
self.assertIn(record.getAttribute('siteKey'), siteKeys)
# dataKey attr. is comma-separated list of dataKeys that had data
for dataKey in record.getAttribute('dataKey').split(','):
self.assertIn(dataKey, dataKeys)
def testGetGuidanceDataWithoutAccumHrsIdentifierSet(self):
# Test that accumHrs identifier is not required for guidance data
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('wfo', params.SITE_ID)
req.addIdentifier('siteKey', self.location)
req.addIdentifier('huc', 'ALL')
req.setParameters('FFG0124hr')
self.runGeometryDataTest(req, checkDataTimes=False)

49
awips/test/dafTests/testGfe.py
View file

@ -1,3 +1,32 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from ufpy.dataaccess import DataAccessLayer as DAL
from shapely.geometry import box, Point
from . import baseDafTestCase
from . import params
import unittest
#
# Test DAF support for GFE data
#
@ -23,15 +52,6 @@
#
#
from __future__ import print_function
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from awips.dataaccess import DataAccessLayer as DAL
from shapely.geometry import box, Point
from awips.test.dafTests import baseDafTestCase
from awips.test.dafTests import params
import unittest
class GfeTestCase(baseDafTestCase.DafTestCase):
"""Test DAF support for GFE data"""
@ -78,7 +98,7 @@ class GfeTestCase(baseDafTestCase.DafTestCase):
# Ensure all points are within one degree of the original box
# to allow slight margin of error for reprojection distortion.
testEnv = box(params.ENVELOPE.bounds[0] - 1, params.ENVELOPE.bounds[1] - 1,
params.ENVELOPE.bounds[2] + 1, params.ENVELOPE.bounds[3] + 1)
params.ENVELOPE.bounds[2] + 1, params.ENVELOPE.bounds[3] + 1 )
for i in range(len(lons)):
self.assertTrue(testEnv.contains(Point(lons[i], lats[i])))
@ -89,17 +109,17 @@ class GfeTestCase(baseDafTestCase.DafTestCase):
req.addIdentifier('parmId.dbId.siteId', params.SITE_ID)
req.setParameters('Wind')
times = DAL.getAvailableTimes(req)
if not times:
if not(times):
raise unittest.SkipTest('No Wind Data available for testing')
gridData = DAL.getGridData(req, [times[0]])
rawWind = None
rawDir = None
for grid in gridData:
if grid.getParameter() == 'Wind':
self.assertEqual(grid.getUnit(), 'kts')
self.assertEqual(grid.getUnit(),'kts')
rawWind = grid.getRawData()
elif grid.getParameter() == 'WindDirection':
self.assertEqual(grid.getUnit(), 'deg')
self.assertEqual(grid.getUnit(),'deg')
rawDir = grid.getRawData()
self.assertIsNotNone(rawWind, 'Wind Magnitude grid is not present')
self.assertIsNotNone(rawDir, 'Wind Direction grid is not present')
@ -107,7 +127,7 @@ class GfeTestCase(baseDafTestCase.DafTestCase):
self.assertTrue((rawWind >= 0).all(), 'Wind Speed should not contain negative values')
self.assertTrue((rawDir >= 0).all(), 'Wind Direction should not contain negative values')
self.assertTrue((rawDir <= 360).all(), 'Wind Direction should be less than or equal to 360')
self.assertFalse((rawDir == rawWind).all(), 'Wind Direction should be different from Wind Speed')
self.assertFalse((rawDir == rawWind).all(), 'Wind Direction should be different from Wind Speed')
def testGetIdentifierValues(self):
req = DAL.newDataRequest(self.datatype)
@ -192,3 +212,4 @@ class GfeTestCase(baseDafTestCase.DafTestCase):
def testGetDataWithEmptyInConstraintThrowsException(self):
with self.assertRaises(ValueError):
self._runConstraintTest('parmId.dbId.modelName', 'in', [])

215
awips/test/dafTests/testGfeEditArea.py Normal file
View file

@ -0,0 +1,215 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from ufpy.dataaccess import DataAccessLayer as DAL
from ufpy.ThriftClient import ThriftRequestException
from . import baseDafTestCase
from . import params
#
# Test DAF support for GFE edit area data
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 06/08/17 6298 mapeters Initial Creation.
# 09/27/17 6463 tgurney Remove GID site identifier
#
#
class GfeEditAreaTestCase(baseDafTestCase.DafTestCase):
"""Test DAF support for GFE edit area data"""
datatype = 'gfeEditArea'
siteIdKey = 'siteId'
editAreaNames = ['ISC_NHA', 'SDZ066', 'StormSurgeWW_EditArea']
groupKey = 'group'
groups = ['ISC', 'WFOs', 'FIPS_' + params.SITE_ID]
def testGetAvailableParameters(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier(self.siteIdKey, params.SITE_ID)
with self.assertRaises(ThriftRequestException):
self.runParametersTest(req)
def testGetAvailableLocations(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier(self.siteIdKey, params.SITE_ID)
self.runLocationsTest(req)
def testGetAvailableTimes(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier(self.siteIdKey, params.SITE_ID)
with self.assertRaises(ThriftRequestException):
self.runTimesTest(req)
def testGetGeometryDataWithoutSiteIdThrowsException(self):
req = DAL.newDataRequest(self.datatype)
with self.assertRaises(ThriftRequestException):
self.runGeometryDataTest(req)
def testGetGeometryData(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier(self.siteIdKey, params.SITE_ID)
data = self.runGeometryDataTest(req)
for item in data:
self.assertEqual(params.SITE_ID, item.getAttribute(self.siteIdKey))
def testGetGeometryDataWithLocNames(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier(self.siteIdKey, params.SITE_ID)
req.setLocationNames(*self.editAreaNames)
data = self.runGeometryDataTest(req)
for item in data:
self.assertEqual(params.SITE_ID, item.getAttribute(self.siteIdKey))
self.assertIn(item.getLocationName(), self.editAreaNames)
def testGetGeometryDataWithGroups(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier(self.siteIdKey, params.SITE_ID)
req.addIdentifier(self.groupKey, RequestConstraint.new('in', self.groups))
data = self.runGeometryDataTest(req)
for item in data:
self.assertEqual(params.SITE_ID, item.getAttribute(self.siteIdKey))
self.assertIn(item.getAttribute(self.groupKey), self.groups)
def testGetGeometryDataWithLocNamesAndGroupsThrowException(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier(self.siteIdKey, params.SITE_ID)
req.setLocationNames(*self.editAreaNames)
req.addIdentifier(self.groupKey, RequestConstraint.new('in', self.groups))
with self.assertRaises(ThriftRequestException):
self.runGeometryDataTest(req)
def testGetGeometryDataWithEnvelope(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier(self.siteIdKey, params.SITE_ID)
req.setEnvelope(params.ENVELOPE)
data = self.runGeometryDataTest(req)
for item in data:
self.assertEqual(params.SITE_ID, item.getAttribute(self.siteIdKey))
self.assertTrue(params.ENVELOPE.intersects(item.getGeometry()))
def testGetIdentifierValues(self):
req = DAL.newDataRequest(self.datatype)
optionalIds = set(DAL.getOptionalIdentifiers(req))
requiredIds = set(DAL.getRequiredIdentifiers(req))
self.runGetIdValuesTest(optionalIds | requiredIds)
def testGetInvalidIdentifierValuesThrowsException(self):
self.runInvalidIdValuesTest()
def testGetNonexistentIdentifierValuesThrowsException(self):
self.runNonexistentIdValuesTest()
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.addIdentifier(key, constraint)
req.setLocationNames(*self.editAreaNames)
return self.runGeometryDataTest(req)
def testGetDataWithEqualsString(self):
geomData = self._runConstraintTest(self.siteIdKey, '=', params.SITE_ID)
for record in geomData:
self.assertEqual(record.getAttribute(self.siteIdKey), params.SITE_ID)
# No numeric tests since no numeric identifiers are available.
def testGetDataWithEqualsNone(self):
geomData = self._runConstraintTest(self.siteIdKey, '=', None)
for record in geomData:
self.assertIsNone(record.getAttribute(self.siteIdKey))
def testGetDataWithNotEquals(self):
geomData = self._runConstraintTest(self.siteIdKey, '!=', params.SITE_ID)
for record in geomData:
self.assertNotEqual(record.getAttribute(self.siteIdKey), params.SITE_ID)
def testGetDataWithNotEqualsNone(self):
geomData = self._runConstraintTest(self.siteIdKey, '!=', None)
for record in geomData:
self.assertIsNotNone(record.getAttribute(self.siteIdKey))
def testGetDataWithGreaterThan(self):
geomData = self._runConstraintTest(self.siteIdKey, '>', params.SITE_ID)
for record in geomData:
self.assertGreater(record.getAttribute(self.siteIdKey), params.SITE_ID)
def testGetDataWithLessThan(self):
geomData = self._runConstraintTest(self.siteIdKey, '<', params.SITE_ID)
for record in geomData:
self.assertLess(record.getAttribute(self.siteIdKey), params.SITE_ID)
def testGetDataWithGreaterThanEquals(self):
geomData = self._runConstraintTest(self.siteIdKey, '>=', params.SITE_ID)
for record in geomData:
self.assertGreaterEqual(record.getAttribute(self.siteIdKey), params.SITE_ID)
def testGetDataWithLessThanEquals(self):
geomData = self._runConstraintTest(self.siteIdKey, '<=', params.SITE_ID)
for record in geomData:
self.assertLessEqual(record.getAttribute(self.siteIdKey), params.SITE_ID)
def testGetDataWithInTuple(self):
collection = (params.SITE_ID,)
geomData = self._runConstraintTest(self.siteIdKey, 'in', collection)
for record in geomData:
self.assertIn(record.getAttribute(self.siteIdKey), collection)
def testGetDataWithInList(self):
collection = [params.SITE_ID,]
geomData = self._runConstraintTest(self.siteIdKey, 'in', collection)
for record in geomData:
self.assertIn(record.getAttribute(self.siteIdKey), collection)
def testGetDataWithInGenerator(self):
collection = (params.SITE_ID,)
generator = (item for item in collection)
geomData = self._runConstraintTest(self.siteIdKey, 'in', generator)
for record in geomData:
self.assertIn(record.getAttribute(self.siteIdKey), collection)
def testGetDataWithNotInList(self):
collection = [params.SITE_ID,]
geomData = self._runConstraintTest(self.siteIdKey, 'not in', collection)
for record in geomData:
self.assertNotIn(record.getAttribute(self.siteIdKey), collection)
def testGetDataWithInvalidConstraintTypeThrowsException(self):
with self.assertRaises(ValueError):
self._runConstraintTest(self.siteIdKey, 'junk', params.SITE_ID)
def testGetDataWithInvalidConstraintValueThrowsException(self):
with self.assertRaises(TypeError):
self._runConstraintTest(self.siteIdKey, '=', {})
def testGetDataWithEmptyInConstraintThrowsException(self):
with self.assertRaises(ValueError):
self._runConstraintTest(self.siteIdKey, 'in', [])

62
awips/test/dafTests/testGrid.py
View file

@ -1,3 +1,33 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from shapely.geometry import box, Point
from ufpy.dataaccess import DataAccessLayer as DAL
from ufpy.ThriftClient import ThriftRequestException
from . import baseDafTestCase
from . import params
import unittest
#
# Test DAF support for grid data
#
@ -18,18 +48,9 @@
# 12/07/16 5981 tgurney Parameterize
# 01/06/17 5981 tgurney Skip envelope test when no
# data is available
# 04/14/22 8845 njensen Added testGetDataAtPoint
#
from __future__ import print_function
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from awips.dataaccess import DataAccessLayer as DAL
from awips.ThriftClient import ThriftRequestException
from shapely.geometry import box, Point
from awips.test.dafTests import baseDafTestCase
from awips.test.dafTests import params
import unittest
class GridTestCase(baseDafTestCase.DafTestCase):
"""Test DAF support for grid data"""
@ -86,6 +107,7 @@ class GridTestCase(baseDafTestCase.DafTestCase):
def testGetNonexistentIdentifierValuesThrowsException(self):
self.runNonexistentIdValuesTest()
def testGetDataWithEnvelope(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('info.datasetId', self.model)
@ -98,15 +120,16 @@ class GridTestCase(baseDafTestCase.DafTestCase):
lons, lats = gridData[0].getLatLonCoords()
lons = lons.reshape(-1)
lats = lats.reshape(-1)
# Ensure all points are within one degree of the original box
# to allow slight margin of error for reprojection distortion.
testEnv = box(params.ENVELOPE.bounds[0] - 1, params.ENVELOPE.bounds[1] - 1,
params.ENVELOPE.bounds[2] + 1, params.ENVELOPE.bounds[3] + 1)
params.ENVELOPE.bounds[2] + 1, params.ENVELOPE.bounds[3] + 1 )
for i in range(len(lons)):
self.assertTrue(testEnv.contains(Point(lons[i], lats[i])))
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
@ -127,11 +150,6 @@ class GridTestCase(baseDafTestCase.DafTestCase):
for record in gridData:
self.assertEqual(record.getAttribute('info.level.levelonevalue'), 2000)
def testGetDataWithEqualsLong(self):
gridData = self._runConstraintTest('info.level.levelonevalue', '=', 2000)
for record in gridData:
self.assertEqual(record.getAttribute('info.level.levelonevalue'), 2000)
def testGetDataWithEqualsFloat(self):
gridData = self._runConstraintTest('info.level.levelonevalue', '=', 2000.0)
for record in gridData:
@ -259,3 +277,11 @@ class GridTestCase(baseDafTestCase.DafTestCase):
self.runGridDataTest(req)
self.assertIn('IncompatibleRequestException', str(cm.exception))
self.assertIn('info.level.masterLevel.name', str(cm.exception))
def testGetDataAtPoint(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('info.datasetId', self.model)
req.setLevels('2FHAG')
req.setParameters('T')
req.setEnvelope(params.POINT)
self.runGeometryDataTest(req)

243
awips/test/dafTests/testHydro.py Normal file
View file

@ -0,0 +1,243 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
import datetime
from ufpy.dataaccess import DataAccessLayer as DAL
from ufpy.ThriftClient import ThriftRequestException
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange
from . import baseDafTestCase
#
# Test DAF support for hydro data
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 01/19/16 4795 mapeters Initial Creation.
# 04/11/16 5548 tgurney Cleanup
# 04/18/16 5548 tgurney More cleanup
# 04/21/16 5596 tgurney Add tests to verify #5596
# 04/26/16 5587 tgurney Add identifier values tests
# 06/09/16 5574 tgurney Add advanced query tests
# 06/13/16 5574 tgurney Fix checks for None
# 06/21/16 5548 tgurney Skip tests that cause errors
# 06/30/16 5725 tgurney Add test for NOT IN
# 10/06/16 5926 dgilling Add additional location tests.
#
#
class HydroTestCase(baseDafTestCase.DafTestCase):
"""Test DAF support for hydro data"""
datatype = 'hydro'
def testGetAvailableParameters(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'height')
self.runParametersTest(req)
def testGetAvailableParametersFullyQualifiedTable(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.height')
self.runParametersTest(req)
def testGetAvailableParamsNoTableThrowsInvalidIdentifiersException(self):
req = DAL.newDataRequest(self.datatype)
with self.assertRaises(ThriftRequestException) as cm:
self.runParametersTest(req)
self.assertIn('InvalidIdentifiersException', str(cm.exception))
def testGetAvailableLocations(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'height')
self.runLocationsTest(req)
def testGetAvailableLocationsWithConstraint(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'height')
req.addIdentifier('value', RequestConstraint.new('>', 5.0))
self.runLocationsTest(req)
def testGetAvailableLocationsWithInvalidTable(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'city')
with self.assertRaises(ThriftRequestException) as cm:
DAL.getAvailableLocationNames(req)
self.assertIn('IncompatibleRequestException', str(cm.exception))
def testGetAvailableTimes(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'height')
req.setParameters('lid', 'quality_code')
self.runTimesTest(req)
def testGetGeometryDataWithoutLocationSpecified(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'height')
req.setParameters('lid', 'quality_code')
self.runGeometryDataTest(req)
def testGetGeometryDataWithLocationSpecified(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'fcstheight')
locs = DAL.getAvailableLocationNames(req)
if locs:
req.setLocationNames(locs[0])
req.setParameters('probability', 'value')
data = self.runGeometryDataTest(req)
self.assertNotEqual(len(data), 0)
def testGetTableIdentifierValues(self):
self.runGetIdValuesTest(['table'])
def testGetColumnIdValuesWithTable(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'height')
idValues = DAL.getIdentifierValues(req, 'lid')
self.assertTrue(hasattr(idValues, '__iter__'))
def testGetColumnIdValuesWithoutTableThrowsException(self):
req = DAL.newDataRequest(self.datatype)
with self.assertRaises(ThriftRequestException):
DAL.getIdentifierValues(req, 'lid')
def testGetInvalidIdentifierValuesThrowsException(self):
self.runInvalidIdValuesTest()
def testGetNonexistentIdentifierValuesThrowsException(self):
self.runNonexistentIdValuesTest()
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.addIdentifier(key, constraint)
req.addIdentifier('table', 'height')
req.addIdentifier('ts', 'RG')
req.setParameters('value', 'lid', 'quality_code')
return self.runGeometryDataTest(req)
def testGetDataWithEqualsString(self):
geometryData = self._runConstraintTest('value', '=', '3')
for record in geometryData:
self.assertEqual(record.getNumber('value'), 3)
def testGetDataWithEqualsInt(self):
geometryData = self._runConstraintTest('value', '=', 3)
for record in geometryData:
self.assertEqual(record.getNumber('value'), 3)
def testGetDataWithEqualsFloat(self):
geometryData = self._runConstraintTest('value', '=', 3.0)
for record in geometryData:
self.assertEqual(round(record.getNumber('value'), 1), 3.0)
def testGetDataWithEqualsNone(self):
geometryData = self._runConstraintTest('value', '=', None)
self.assertEqual(len(geometryData), 0)
def testGetDataWithNotEquals(self):
geometryData = self._runConstraintTest('value', '!=', 3)
for record in geometryData:
self.assertNotEqual(record.getNumber('value'), '3')
def testGetDataWithNotEqualsNone(self):
geometryData = self._runConstraintTest('value', '!=', None)
self.assertNotEqual(len(geometryData), 0)
for record in geometryData:
self.assertNotEqual(record.getType('value'), 'NULL')
def testGetDataWithGreaterThan(self):
geometryData = self._runConstraintTest('value', '>', 3)
for record in geometryData:
self.assertGreater(record.getNumber('value'), 3)
def testGetDataWithLessThan(self):
geometryData = self._runConstraintTest('value', '<', 3)
for record in geometryData:
self.assertLess(record.getNumber('value'), 3)
def testGetDataWithGreaterThanEquals(self):
geometryData = self._runConstraintTest('value', '>=', 3)
for record in geometryData:
self.assertGreaterEqual(record.getNumber('value'), 3)
def testGetDataWithLessThanEquals(self):
geometryData = self._runConstraintTest('value', '<=', 3)
for record in geometryData:
self.assertLessEqual(record.getNumber('value'), 3)
def testGetDataWithInTuple(self):
collection = (3, 4)
geometryData = self._runConstraintTest('value', 'in', collection)
for record in geometryData:
self.assertIn(record.getNumber('value'), collection)
def testGetDataWithInList(self):
collection = [3, 4]
geometryData = self._runConstraintTest('value', 'in', collection)
for record in geometryData:
self.assertIn(record.getNumber('value'), collection)
def testGetDataWithInGenerator(self):
collection = (3, 4)
generator = (item for item in collection)
geometryData = self._runConstraintTest('value', 'in', generator)
for record in geometryData:
self.assertIn(record.getNumber('value'), collection)
def testGetDataWithNotInList(self):
collection = [3, 4]
geometryData = self._runConstraintTest('value', 'not in', collection)
for record in geometryData:
self.assertNotIn(record.getNumber('value'), collection)
def testGetDataWithTimeRange(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'height')
req.addIdentifier('ts', 'RG')
req.setParameters('value', 'lid', 'quality_code')
times = DAL.getAvailableTimes(req)
limitTimes = times[-self.numTimesToLimit:]
startTime = datetime.datetime.utcfromtimestamp(limitTimes[0].getRefTime().getTime()/1000)
endTime = datetime.datetime.utcnow()
tr = TimeRange(startTime, endTime)
self.runGeometryDataTestWithTimeRange(req, tr)
def testGetDataWithInvalidConstraintTypeThrowsException(self):
with self.assertRaises(ValueError):
self._runConstraintTest('value', 'junk', 3)
def testGetDataWithInvalidConstraintValueThrowsException(self):
with self.assertRaises(TypeError):
self._runConstraintTest('value', '=', {})
def testGetDataWithEmptyInConstraintThrowsException(self):
with self.assertRaises(ValueError):
self._runConstraintTest('value', 'in', [])
def testGetDataWithNestedInConstraintThrowsException(self):
collection = ('3', '4', ())
with self.assertRaises(TypeError):
self._runConstraintTest('value', 'in', collection)

84
awips/test/dafTests/testLdadMesonet.py Normal file
View file

@ -0,0 +1,84 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from shapely.geometry import Polygon
from ufpy.dataaccess import DataAccessLayer as DAL
from . import baseDafTestCase
#
# Test DAF support for ldadmesonet data
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 01/19/16 4795 mapeters Initial Creation.
# 04/11/16 5548 tgurney Cleanup
# 04/18/16 5548 tgurney More cleanup
# 01/20/17 6095 tgurney Add null identifiers test
#
#
class LdadMesonetTestCase(baseDafTestCase.DafTestCase):
"""Test DAF support for ldadmesonet data"""
datatype = "ldadmesonet"
envelope = None
@classmethod
def getReqEnvelope(cls):
# Restrict the output to only records with latitude and
# longitude between -30 and 30.
if not cls.envelope:
vertices = [(-30, -30), (-30, 30), (30, 30), (30, -30)]
polygon = Polygon(vertices)
cls.envelope = polygon.envelope
return cls.envelope
def testGetAvailableParameters(self):
req = DAL.newDataRequest(self.datatype)
self.runParametersTest(req)
def testGetAvailableLocations(self):
req = DAL.newDataRequest(self.datatype)
req.setEnvelope(self.getReqEnvelope())
self.runLocationsTest(req)
def testGetAvailableTimes(self):
req = DAL.newDataRequest(self.datatype)
req.setEnvelope(self.getReqEnvelope())
self.runTimesTest(req)
def testGetGeometryData(self):
req = DAL.newDataRequest(self.datatype)
req.setParameters("highLevelCloud", "pressure")
req.setEnvelope(self.getReqEnvelope())
self.runGeometryDataTest(req)
def testGetGeometryDataNullIdentifiers(self):
req = DAL.newDataRequest(self.datatype)
req.setParameters("highLevelCloud", "pressure")
req.setEnvelope(self.getReqEnvelope())
req.identifiers = None
self.runGeometryDataTest(req)

39
awips/test/dafTests/testMaps.py
View file

@ -1,3 +1,30 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from ufpy.dataaccess import DataAccessLayer as DAL
from ufpy.ThriftClient import ThriftRequestException
from . import baseDafTestCase
#
# Test DAF support for maps data
#
@ -16,13 +43,6 @@
#
#
from __future__ import print_function
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from awips.dataaccess import DataAccessLayer as DAL
from awips.ThriftClient import ThriftRequestException
from awips.test.dafTests import baseDafTestCase
class MapsTestCase(baseDafTestCase.DafTestCase):
"""Test DAF support for maps data"""
@ -110,11 +130,6 @@ class MapsTestCase(baseDafTestCase.DafTestCase):
for record in geometryData:
self.assertEqual(record.getNumber('reservoir'), 1)
def testGetDataWithEqualsLong(self):
geometryData = self._runConstraintTest('reservoir', '=', 1)
for record in geometryData:
self.assertEqual(record.getNumber('reservoir'), 1)
def testGetDataWithEqualsFloat(self):
geometryData = self._runConstraintTest('area_sq_mi', '=', 5.00)
for record in geometryData:

45
awips/test/dafTests/testModelSounding.py
View file

@ -1,3 +1,31 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from ufpy.dataaccess import DataAccessLayer as DAL
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from . import baseDafTestCase
from . import params
import unittest
#
# Test DAF support for modelsounding data
#
@ -19,39 +47,30 @@
#
#
from __future__ import print_function
from awips.dataaccess import DataAccessLayer as DAL
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from awips.test.dafTests import baseDafTestCase
from awips.test.dafTests import params
class ModelSoundingTestCase(baseDafTestCase.DafTestCase):
"""Test DAF support for modelsounding data"""
datatype = "modelsounding"
reporttype = "ETA"
def testGetAvailableParameters(self):
req = DAL.newDataRequest(self.datatype)
self.runParametersTest(req)
def testGetAvailableLocations(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier("reportType", self.reporttype)
req.addIdentifier("reportType", "ETA")
self.runLocationsTest(req)
def testGetAvailableTimes(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier("reportType", self.reporttype)
req.addIdentifier("reportType", "ETA")
req.setLocationNames(params.OBS_STATION)
self.runTimesTest(req)
def testGetGeometryData(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier("reportType", self.reporttype)
req.addIdentifier("reportType", "ETA")
req.setLocationNames(params.OBS_STATION)
req.setParameters("temperature", "pressure", "specHum", "sfcPress", "temp2", "q2")
print("Testing getGeometryData()")
@ -81,7 +100,7 @@ class ModelSoundingTestCase(baseDafTestCase.DafTestCase):
def testGetGeometryDataWithEnvelope(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier("reportType", self.reporttype)
req.addIdentifier("reportType", "ETA")
req.setEnvelope(params.ENVELOPE)
req.setParameters("temperature", "pressure", "specHum", "sfcPress", "temp2", "q2")
print("Testing getGeometryData()")

34
awips/test/dafTests/testObs.py
View file

@ -1,3 +1,30 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from ufpy.dataaccess import DataAccessLayer as DAL
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from . import baseDafTestCase
from . import params
#
# Test DAF support for obs data
#
@ -16,13 +43,6 @@
#
#
from __future__ import print_function
from awips.dataaccess import DataAccessLayer as DAL
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from awips.test.dafTests import baseDafTestCase
from awips.test.dafTests import params
class ObsTestCase(baseDafTestCase.DafTestCase):
"""Test DAF support for obs data"""

91
awips/test/dafTests/testPirep.py Normal file
View file

@ -0,0 +1,91 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from ufpy.dataaccess import DataAccessLayer as DAL
from . import baseDafTestCase
from . import params
import unittest
#
# Test DAF support for pirep data
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 01/19/16 4795 mapeters Initial Creation.
# 04/11/16 5548 tgurney Cleanup
# 04/18/16 5548 tgurney More cleanup
# 12/07/16 5981 tgurney Parameterize
# 12/20/16 5981 tgurney Add envelope test
#
#
class PirepTestCase(baseDafTestCase.DafTestCase):
"""Test DAF support for pirep data"""
datatype = "pirep"
def testGetAvailableParameters(self):
req = DAL.newDataRequest(self.datatype)
self.runParametersTest(req)
def testGetAvailableLocations(self):
req = DAL.newDataRequest(self.datatype)
self.runLocationsTest(req)
def testGetAvailableTimes(self):
req = DAL.newDataRequest(self.datatype)
req.setLocationNames(params.AIRPORT)
self.runTimesTest(req)
def testGetGeometryData(self):
req = DAL.newDataRequest(self.datatype)
req.setLocationNames(params.AIRPORT)
req.setParameters("temperature", "windSpeed", "hazardType", "turbType")
print("Testing getGeometryData()")
geomData = DAL.getGeometryData(req)
self.assertIsNotNone(geomData)
print("Number of geometry records: " + str(len(geomData)))
print("Sample geometry data:")
for record in geomData[:self.sampleDataLimit]:
print("level=", record.getLevel(), end="")
# One dimensional parameters are reported on the 0.0UNKNOWN level.
# 2D parameters are reported on MB levels from pressure.
if record.getLevel() == "0.0UNKNOWN":
print(" temperature=" + record.getString("temperature") + record.getUnit("temperature"), end="")
print(" windSpeed=" + record.getString("windSpeed") + record.getUnit("windSpeed"), end="")
else:
print(" hazardType=" + record.getString("hazardType"), end="")
print(" turbType=" + record.getString("turbType"), end="")
print(" geometry=", record.getGeometry())
print("getGeometryData() complete\n")
def testGetGeometryDataWithEnvelope(self):
req = DAL.newDataRequest(self.datatype)
req.setParameters("temperature", "windSpeed", "hazardType", "turbType")
req.setEnvelope(params.ENVELOPE)
print("Testing getGeometryData()")
data = DAL.getGeometryData(req)
for item in data:
self.assertTrue(params.ENVELOPE.contains(item.getGeometry()))

44
awips/test/dafTests/testPracticeWarning.py Normal file
View file

@ -0,0 +1,44 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from . import testWarning
#
# Test DAF support for practicewarning data
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 01/19/16 4795 mapeters Initial Creation.
# 04/11/16 5548 tgurney Cleanup
# 04/18/16 5548 tgurney More cleanup
# 06/10/16 5548 tgurney Inherit all tests from
# warning
#
class PracticeWarningTestCase(testWarning.WarningTestCase):
"""Test DAF support for practicewarning data"""
datatype = "practicewarning"
# All tests taken from testWarning

35
awips/test/dafTests/testRadarGraphics.py
View file

@ -1,3 +1,30 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from ufpy.dataaccess import DataAccessLayer as DAL
from . import baseRadarTestCase
from . import params
#
# Test DAF support for radar graphics data
#
@ -15,14 +42,6 @@
# returned data
#
#
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from awips.dataaccess import DataAccessLayer as DAL
from awips.test.dafTests import baseRadarTestCase
from awips.test.dafTests import params
class RadarGraphicsTestCase(baseRadarTestCase.BaseRadarTestCase):
"""Test DAF support for radar data"""

31
awips/test/dafTests/testRadarGrid.py
View file

@ -1,3 +1,28 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from ufpy.dataaccess import DataAccessLayer as DAL
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from . import baseRadarTestCase
from . import params
#
# Test DAF support for radar grid data
#
@ -9,12 +34,6 @@
#
#
from awips.dataaccess import DataAccessLayer as DAL
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from awips.test.dafTests import baseRadarTestCase
from awips.test.dafTests import params
class RadarTestCase(baseRadarTestCase.BaseRadarTestCase):
"""Test DAF support for radar data"""

41
awips/test/dafTests/testRadarSpatial.py
View file

@ -1,3 +1,30 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from ufpy.dataaccess import DataAccessLayer as DAL
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from . import baseDafTestCase
from . import params
#
# Test DAF support for radar_spatial data
#
@ -18,13 +45,6 @@
#
#
from __future__ import print_function
from awips.dataaccess import DataAccessLayer as DAL
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from awips.test.dafTests import baseDafTestCase
from awips.test.dafTests import params
class RadarSpatialTestCase(baseDafTestCase.DafTestCase):
"""Test DAF support for radar_spatial data"""
@ -70,11 +90,6 @@ class RadarSpatialTestCase(baseDafTestCase.DafTestCase):
for record in geometryData:
self.assertEqual(record.getNumber('immutablex'), 57)
def testGetDataWithEqualsLong(self):
geometryData = self._runConstraintTest('immutablex', '=', 57)
for record in geometryData:
self.assertEqual(record.getNumber('immutablex'), 57)
def testGetDataWithEqualsFloat(self):
geometryData = self._runConstraintTest('immutablex', '=', 57.0)
for record in geometryData:
@ -88,7 +103,7 @@ class RadarSpatialTestCase(baseDafTestCase.DafTestCase):
def testGetDataWithNotEquals(self):
geometryData = self._runConstraintTest('wfo_id', '!=', params.SITE_ID)
for record in geometryData:
self.assertNotEquals(record.getString('wfo_id'), params.SITE_ID)
self.assertNotEqual(record.getString('wfo_id'), params.SITE_ID)
def testGetDataWithNotEqualsNone(self):
geometryData = self._runConstraintTest('wfo_id', '!=', None)

103
awips/test/dafTests/testRequestConstraint.py
View file

@ -1,3 +1,27 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
import unittest
#
# Unit tests for Python implementation of RequestConstraint
#
@ -9,10 +33,6 @@
#
#
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
import unittest
class RequestConstraintTestCase(unittest.TestCase):
@ -27,9 +47,7 @@ class RequestConstraintTestCase(unittest.TestCase):
self.assertTrue(new('=', 3).evaluate(3))
self.assertTrue(new('=', 3).evaluate('3'))
self.assertTrue(new('=', '3').evaluate(3))
self.assertTrue(new('=', 12345).evaluate(12345))
self.assertTrue(new('=', 'a').evaluate('a'))
self.assertTrue(new('=', 'a').evaluate(u'a'))
self.assertTrue(new('=', 1.0001).evaluate(2.0 - 0.999999))
self.assertTrue(new('=', 1.00001).evaluate(1))
self.assertFalse(new('=', 'a').evaluate(['a']))
@ -48,9 +66,7 @@ class RequestConstraintTestCase(unittest.TestCase):
self.assertFalse(new('!=', 3).evaluate('3'))
self.assertFalse(new('!=', '3').evaluate(3))
self.assertFalse(new('!=', 3).evaluate(3))
self.assertFalse(new('!=', 12345).evaluate(12345))
self.assertFalse(new('!=', 'a').evaluate('a'))
self.assertFalse(new('!=', 'a').evaluate(u'a'))
self.assertFalse(new('!=', 1.0001).evaluate(2.0 - 0.9999))
def testEvaluateGreaterThan(self):
@ -59,7 +75,6 @@ class RequestConstraintTestCase(unittest.TestCase):
self.assertTrue(new('>', 'a').evaluate('b'))
self.assertTrue(new('>', 3).evaluate(4))
self.assertFalse(new('>', 20).evaluate(3))
self.assertFalse(new('>', 12345).evaluate(12345))
self.assertFalse(new('>', 'a').evaluate('a'))
self.assertFalse(new('>', 'z').evaluate('a'))
self.assertFalse(new('>', 4).evaluate(3))
@ -67,7 +82,6 @@ class RequestConstraintTestCase(unittest.TestCase):
def testEvaluateGreaterThanEquals(self):
new = RequestConstraint.new
self.assertTrue(new('>=', 3).evaluate(3))
self.assertTrue(new('>=', 12345).evaluate(12345))
self.assertTrue(new('>=', 'a').evaluate('a'))
self.assertTrue(new('>=', 1.0001).evaluate(1.0002))
self.assertTrue(new('>=', 'a').evaluate('b'))
@ -81,7 +95,6 @@ class RequestConstraintTestCase(unittest.TestCase):
self.assertTrue(new('<', 'z').evaluate('a'))
self.assertTrue(new('<', 30).evaluate(4))
self.assertFalse(new('<', 3).evaluate(3))
self.assertFalse(new('<', 12345).evaluate(12345))
self.assertFalse(new('<', 'a').evaluate('a'))
self.assertFalse(new('<', 1.0001).evaluate(1.0002))
self.assertFalse(new('<', 'a').evaluate('b'))
@ -92,7 +105,6 @@ class RequestConstraintTestCase(unittest.TestCase):
self.assertTrue(new('<=', 'z').evaluate('a'))
self.assertTrue(new('<=', 20).evaluate(3))
self.assertTrue(new('<=', 3).evaluate(3))
self.assertTrue(new('<=', 12345).evaluate(12345))
self.assertTrue(new('<=', 'a').evaluate('a'))
self.assertFalse(new('<=', 1.0001).evaluate(1.0002))
self.assertFalse(new('<=', 'a').evaluate('b'))
@ -134,6 +146,73 @@ class RequestConstraintTestCase(unittest.TestCase):
self.assertFalse(new('not in', 'a').evaluate('a'))
self.assertFalse(new('not in', [1.0001, 2, 3]).evaluate(2.0 - 0.9999))
def testEvaluateLike(self):
# cannot make "like" with RequestConstraint.new()
new = self._newRequestConstraint
self.assertTrue(new('LIKE', 'a').evaluate('a'))
self.assertTrue(new('LIKE', 'a%').evaluate('a'))
self.assertTrue(new('LIKE', 'a%').evaluate('abcd'))
self.assertTrue(new('LIKE', '%a').evaluate('a'))
self.assertTrue(new('LIKE', '%a').evaluate('bcda'))
self.assertTrue(new('LIKE', '%').evaluate(''))
self.assertTrue(new('LIKE', '%').evaluate('anything'))
self.assertTrue(new('LIKE', 'a%d').evaluate('ad'))
self.assertTrue(new('LIKE', 'a%d').evaluate('abcd'))
self.assertTrue(new('LIKE', 'aa.()!{[]^%$').evaluate('aa.()!{[]^zzz$'))
self.assertTrue(new('LIKE', 'a__d%').evaluate('abcdefg'))
self.assertFalse(new('LIKE', 'a%').evaluate('b'))
self.assertFalse(new('LIKE', 'a%').evaluate('ba'))
self.assertFalse(new('LIKE', '%a').evaluate('b'))
self.assertFalse(new('LIKE', '%a').evaluate('ab'))
self.assertFalse(new('LIKE', 'a%').evaluate('A'))
self.assertFalse(new('LIKE', 'A%').evaluate('a'))
self.assertFalse(new('LIKE', 'a%d').evaluate('da'))
self.assertFalse(new('LIKE', 'a__d%').evaluate('abccdefg'))
self.assertFalse(new('LIKE', '....').evaluate('aaaa'))
self.assertFalse(new('LIKE', '.*').evaluate('anything'))
def testEvaluateILike(self):
# cannot make "ilike" with RequestConstraint.new()
new = self._newRequestConstraint
self.assertTrue(new('ILIKE', 'a').evaluate('a'))
self.assertTrue(new('ILIKE', 'a%').evaluate('a'))
self.assertTrue(new('ILIKE', 'a%').evaluate('abcd'))
self.assertTrue(new('ILIKE', '%a').evaluate('a'))
self.assertTrue(new('ILIKE', '%a').evaluate('bcda'))
self.assertTrue(new('ILIKE', '%').evaluate(''))
self.assertTrue(new('ILIKE', '%').evaluate('anything'))
self.assertTrue(new('ILIKE', 'a%d').evaluate('ad'))
self.assertTrue(new('ILIKE', 'a%d').evaluate('abcd'))
self.assertTrue(new('ILIKE', 'a').evaluate('A'))
self.assertTrue(new('ILIKE', 'a%').evaluate('A'))
self.assertTrue(new('ILIKE', 'a%').evaluate('ABCD'))
self.assertTrue(new('ILIKE', '%a').evaluate('A'))
self.assertTrue(new('ILIKE', '%a').evaluate('BCDA'))
self.assertTrue(new('ILIKE', '%').evaluate(''))
self.assertTrue(new('ILIKE', '%').evaluate('anything'))
self.assertTrue(new('ILIKE', 'a%d').evaluate('AD'))
self.assertTrue(new('ILIKE', 'a%d').evaluate('ABCD'))
self.assertTrue(new('ILIKE', 'A').evaluate('a'))
self.assertTrue(new('ILIKE', 'A%').evaluate('a'))
self.assertTrue(new('ILIKE', 'A%').evaluate('abcd'))
self.assertTrue(new('ILIKE', '%A').evaluate('a'))
self.assertTrue(new('ILIKE', '%A').evaluate('bcda'))
self.assertTrue(new('ILIKE', '%').evaluate(''))
self.assertTrue(new('ILIKE', '%').evaluate('anything'))
self.assertTrue(new('ILIKE', 'A%D').evaluate('ad'))
self.assertTrue(new('ILIKE', 'A%D').evaluate('abcd'))
self.assertTrue(new('ILIKE', 'aa.()!{[]^%$').evaluate('AA.()!{[]^zzz$'))
self.assertTrue(new('ILIKE', 'a__d%').evaluate('abcdefg'))
self.assertTrue(new('ILIKE', 'a__d%').evaluate('ABCDEFG'))
self.assertFalse(new('ILIKE', 'a%').evaluate('b'))
self.assertFalse(new('ILIKE', 'a%').evaluate('ba'))
self.assertFalse(new('ILIKE', '%a').evaluate('b'))
self.assertFalse(new('ILIKE', '%a').evaluate('ab'))
self.assertFalse(new('ILIKE', 'a%d').evaluate('da'))
self.assertFalse(new('ILIKE', 'a__d%').evaluate('abccdefg'))
self.assertFalse(new('ILIKE', '....').evaluate('aaaa'))
self.assertFalse(new('ILIKE', '.*').evaluate('anything'))
def testEvaluateBetween(self):
# cannot make "between" with RequestConstraint.new()
new = self._newRequestConstraint

38
awips/test/dafTests/testSatellite.py
View file

@ -1,3 +1,30 @@
#!/awips2/python/bin/python3
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from ufpy.dataaccess import DataAccessLayer as DAL
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from . import baseDafTestCase
#
# Test DAF support for satellite data
#
@ -17,12 +44,6 @@
#
#
from __future__ import print_function
from awips.dataaccess import DataAccessLayer as DAL
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from awips.test.dafTests import baseDafTestCase
class SatelliteTestCase(baseDafTestCase.DafTestCase):
"""Test DAF support for satellite data"""
@ -78,11 +99,6 @@ class SatelliteTestCase(baseDafTestCase.DafTestCase):
for record in gridData:
self.assertEqual(record.getAttribute('creatingEntity'), 1000)
def testGetDataWithEqualsLong(self):
gridData = self._runConstraintTest('creatingEntity', '=', 1000)
for record in gridData:
self.assertEqual(record.getAttribute('creatingEntity'), 1000)
def testGetDataWithEqualsFloat(self):
gridData = self._runConstraintTest('creatingEntity', '=', 1.0)
for record in gridData:

37
awips/test/dafTests/testSfcObs.py
View file

@ -1,3 +1,29 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from ufpy.dataaccess import DataAccessLayer as DAL
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from . import baseDafTestCase
#
# Test DAF support for sfcobs data
#
@ -15,12 +41,6 @@
#
#
from __future__ import print_function
from awips.dataaccess import DataAccessLayer as DAL
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from awips.test.dafTests import baseDafTestCase
class SfcObsTestCase(baseDafTestCase.DafTestCase):
"""Test DAF support for sfcobs data"""
@ -81,11 +101,6 @@ class SfcObsTestCase(baseDafTestCase.DafTestCase):
for record in geometryData:
self.assertEqual(record.getString('reportType'), '1004')
def testGetDataWithEqualsLong(self):
geometryData = self._runConstraintTest('reportType', '=', 1004)
for record in geometryData:
self.assertEqual(record.getString('reportType'), '1004')
# No float test because no float identifiers are available
def testGetDataWithEqualsNone(self):

49
awips/test/dafTests/testTopo.py
View file

@ -1,3 +1,30 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from ufpy.dataaccess import DataAccessLayer as DAL
from ufpy.ThriftClient import ThriftRequestException
from . import baseDafTestCase
import shapely.geometry
#
# Test DAF support for topo data
#
@ -12,15 +39,10 @@
# getIdentifierValues()
# 06/01/16 5587 tgurney Update testGetIdentifierValues
# 07/18/17 6253 randerso Removed referenced to GMTED
# 02/20/18 7220 mapeters Added tests for getting filtered
# group/dataset identifier values
#
from __future__ import print_function
from awips.dataaccess import DataAccessLayer as DAL
from awips.ThriftClient import ThriftRequestException
import shapely.geometry
from awips.test.dafTests import baseDafTestCase
class TopoTestCase(baseDafTestCase.DafTestCase):
"""Test DAF support for topo data"""
@ -49,6 +71,7 @@ class TopoTestCase(baseDafTestCase.DafTestCase):
print("Sample grid data shape:\n" + str(gridData[0].getRawData().shape) + "\n")
print("Sample grid data:\n" + str(gridData[0].getRawData()) + "\n")
def testRequestingTooMuchDataThrowsResponseTooLargeException(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier("group", "/")
@ -67,6 +90,18 @@ class TopoTestCase(baseDafTestCase.DafTestCase):
requiredIds = set(DAL.getRequiredIdentifiers(req))
self.runGetIdValuesTest(optionalIds | requiredIds)
def testGetFilteredDatasetValues(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('group', '/')
datasetVals = DAL.getIdentifierValues(req, 'dataset')
self.assertSequenceEqual(datasetVals, ['full'])
def testGetFilteredGroupValues(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('dataset', '1')
groupVals = DAL.getIdentifierValues(req, 'group')
self.assertSequenceEqual(groupVals, ['/interpolated'])
def testGetInvalidIdentifierValuesThrowsException(self):
self.runInvalidIdValuesTest()

55
awips/test/dafTests/testWarning.py
View file

@ -1,3 +1,30 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from awips.dataaccess import DataAccessLayer as DAL
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from . import baseDafTestCase
import unittest
#
# Test DAF support for warning data
#
@ -16,15 +43,9 @@
# 06/21/16 5548 tgurney Skip tests that cause errors
# 06/30/16 5725 tgurney Add test for NOT IN
# 12/12/16 5981 tgurney Improve test performance
# 02/20/18 7220 mapeters Added test for getting filtered
# column identifier values
#
#
from __future__ import print_function
from awips.dataaccess import DataAccessLayer as DAL
from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
from awips.test.dafTests import baseDafTestCase
import unittest
class WarningTestCase(baseDafTestCase.DafTestCase):
@ -95,13 +116,12 @@ class WarningTestCase(baseDafTestCase.DafTestCase):
def testGetColumnIdentifierValues(self):
self.runGetIdValuesTest(['act'])
@unittest.skip('avoid EDEX error')
def testGetInvalidIdentifierValuesThrowsException(self):
self.runInvalidIdValuesTest()
@unittest.skip('avoid EDEX error')
def testGetNonexistentIdentifierValuesThrowsException(self):
self.runNonexistentIdValuesTest()
def testGetFilteredColumnIdentifierValues(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('sig', 'W')
phensigs = DAL.getIdentifierValues(req, 'phensig')
for phensig in phensigs:
self.assertTrue(phensig.endswith('.W'))
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
@ -120,11 +140,6 @@ class WarningTestCase(baseDafTestCase.DafTestCase):
for record in geometryData:
self.assertEqual(record.getString('etn'), '1000')
def testGetDataWithEqualsLong(self):
geometryData = self._runConstraintTest('etn', '=', 1000)
for record in geometryData:
self.assertEqual(record.getString('etn'), '1000')
def testGetDataWithEqualsFloat(self):
geometryData = self._runConstraintTest('etn', '=', 1.0)
for record in geometryData:

32
awips/test/localization/__init__.py Normal file
View file

@ -0,0 +1,32 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# __init__.py for ufpy.test.localization package
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# --------- -------- --------- --------------------------
# 08/07/17 5731 bsteffen Initial Creation.
__all__ = []

BIN
awips/test/localization/smallTestImage.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 379 B

91
awips/test/localization/testLF.py Normal file
View file

@ -0,0 +1,91 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 2120 South 72nd Street, Suite 900
# Omaha, NE 68124
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Tests for the lf utility script
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# --------- -------- --------- --------------------------
# 01/11/22 8735 mapeters Initial creation
import os
import subprocess
import unittest
class TxtFileTestCase(unittest.TestCase):
def setUp(self):
"""Test file needs to not exist at the start of the test. Delete it,
suppressing any errors (an error would be printed if it doesn't exist).
"""
self.locDir = "lfTestDir"
self.locFile = os.path.join(self.locDir, "TestFile.txt")
subprocess.run(["lf", "rm", "-o", self.locFile], stderr=subprocess.DEVNULL)
def test_file_operations(self):
"""Run through a typical set of interactions with a .txt file and
verify everything works correctly.
"""
subprocess.run(["lf", "write", self.locFile], input=b"Test Data", check=True)
proc = subprocess.run(["lf", "ls", "-l", self.locDir], stdout=subprocess.PIPE, check=True)
self.assertIn(self.locFile, proc.stdout.decode())
proc = subprocess.run(["lf", "read", self.locFile], stdout=subprocess.PIPE, check=True)
self.assertEqual(proc.stdout.decode(), "Test Data")
subprocess.run(["lf", "rm", "-o", self.locFile], check=True)
proc = subprocess.run(["lf", "ls", "-l", self.locDir], stdout=subprocess.PIPE, check=True)
self.assertNotIn(self.locFile, proc.stdout.decode())
class PngFileTestCase(unittest.TestCase):
def setUp(self):
"""Test file needs to not exist at the start of the test. Delete it,
suppressing any errors (an error would be printed if it doesn't exist).
"""
self.locDir = "lfTestDir"
self.locFile = os.path.join(self.locDir, "TestImage.png")
subprocess.run(["lf", "rm", "-o", self.locFile], stderr=subprocess.DEVNULL)
def test_file_operations(self):
"""Run through a typical set of interactions with a .png file and
verify everything works correctly.
"""
cwdPath = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
localPngPath = os.path.join(cwdPath, "smallTestImage.png")
with open(localPngPath, "rb") as f:
subprocess.run(["lf", "write", self.locFile], stdin=f, check=True)
proc = subprocess.run(["lf", "ls", self.locDir], stdout=subprocess.PIPE, check=True)
self.assertIn(self.locFile, proc.stdout.decode().splitlines())
proc = subprocess.run(["lf", "read", self.locFile], stdout=subprocess.PIPE, check=True)
with open(localPngPath, "rb") as f:
localPngBytes = f.read()
self.assertEqual(proc.stdout, localPngBytes)
subprocess.run(["lf", "rm", "-o", self.locFile], check=True)
proc = subprocess.run(["lf", "ls", self.locDir], stdout=subprocess.PIPE, check=True)
self.assertNotIn(self.locFile, proc.stdout.decode().splitlines())
if __name__ == '__main__':
unittest.main()

173
awips/test/localization/testLocalizationFileManager.py Normal file
View file

@ -0,0 +1,173 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Tests for the LocalizationFileManager
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# --------- -------- --------- --------------------------
# 08/09/17 5731 bsteffen Initial Creation.
# 01/06/22 8735 mapeters Python 3 fix
import unittest
from ufpy.localization.LocalizationFileManager import (LocalizationFileManager,
LocalizationFileVersionConflictException,
LocalizationContext,
LocalizationFileIsNotDirectoryException,
LocalizationFileDoesNotExistException)
testFile = "purge/defaultPurgeRules.xml"
testContent = b"<purgeRuleSet><defaultRule><period>05-05:05:05</period></defaultRule></purgeRuleSet>"
testDir = "purge/"
testNewFile = "purge/testPurgeRules.xml"
class ContextTestCase(unittest.TestCase):
def test_eq(self):
c1 = LocalizationContext()
c2 = LocalizationContext()
self.assertEqual(c1, c2)
c3 = LocalizationContext("site", "test")
c4 = LocalizationContext("site", "test")
self.assertEqual(c3, c4)
self.assertNotEqual(c1, c3)
def test_hash(self):
c1 = LocalizationContext()
c2 = LocalizationContext()
self.assertEqual(hash(c1), hash(c2))
c3 = LocalizationContext("site", "test")
c4 = LocalizationContext("site", "test")
self.assertEqual(hash(c3), hash(c4))
class LFMTestCase(unittest.TestCase):
def setUp(self):
self.manager = LocalizationFileManager()
userFile = self.manager.getSpecific("user", testFile)
if userFile.exists():
userFile.delete()
newFile = self.manager.getSpecific("user", testNewFile)
if newFile.exists():
newFile.delete()
def test_gets(self):
startingIncremental = self.manager.getIncremental(testFile)
baseFile = self.manager.getSpecific("base", testFile)
self.assertEqual(baseFile, startingIncremental[0])
self.assertTrue(baseFile.exists())
self.assertFalse(baseFile.isDirectory())
userFile = self.manager.getSpecific("user", testFile)
self.assertFalse(userFile.exists())
with userFile.open("w") as stream:
stream.write(testContent)
userFile = self.manager.getSpecific("user", testFile)
self.assertTrue(userFile.exists())
with userFile.open('r') as stream:
self.assertEqual(stream.read(), testContent)
absFile = self.manager.getAbsolute(testFile)
self.assertEqual(absFile, userFile)
endingIncremental = self.manager.getIncremental(testFile)
self.assertEqual(len(startingIncremental) + 1, len(endingIncremental))
self.assertEqual(userFile, endingIncremental[-1])
self.assertEqual(baseFile, endingIncremental[0])
userFile.delete()
userFile = self.manager.getSpecific("user", testFile)
self.assertFalse(userFile.exists())
def test_concurrent_edit(self):
userFile1 = self.manager.getSpecific("user", testFile)
userFile2 = self.manager.getSpecific("user", testFile)
self.assertFalse(userFile1.exists())
self.assertFalse(userFile2.exists())
with self.assertRaises(LocalizationFileVersionConflictException):
with userFile1.open("w") as stream1:
stream1.write(testContent)
with userFile2.open("w") as stream2:
stream2.write(testContent)
userFile = self.manager.getSpecific("user", testFile)
userFile.delete()
def test_dir(self):
locDir = self.manager.getAbsolute(testDir)
self.assertTrue(locDir.isDirectory())
with self.assertRaises(Exception):
locDir.delete()
def test_list(self):
abs1 = self.manager.listAbsolute(testDir)
inc1 = self.manager.listIncremental(testDir)
self.assertEqual(len(abs1), len(inc1))
for i in range(len(abs1)):
self.assertEqual(abs1[i], inc1[i][-1])
userFile = self.manager.getSpecific("user", testNewFile)
self.assertNotIn(userFile, abs1)
with userFile.open("w") as stream:
stream.write(testContent)
userFile = self.manager.getSpecific("user", testNewFile)
abs2 = self.manager.listAbsolute(testDir)
inc2 = self.manager.listIncremental(testDir)
self.assertEqual(len(abs2), len(inc2))
for i in range(len(abs2)):
self.assertEqual(abs2[i], inc2[i][-1])
self.assertEqual(len(abs1) + 1, len(abs2))
self.assertIn(userFile, abs2)
userFile.delete()
def test_list_file(self):
with self.assertRaises(LocalizationFileIsNotDirectoryException):
self.manager.listIncremental(testFile)
def test_list_nonexistant(self):
with self.assertRaises(LocalizationFileDoesNotExistException):
self.manager.listIncremental('dontNameYourDirectoryThis')
def test_root_variants(self):
list1 = self.manager.listAbsolute(".")
list2 = self.manager.listAbsolute("")
list3 = self.manager.listAbsolute("/")
self.assertEqual(list1, list2)
self.assertEqual(list2, list3)
def test_slashiness(self):
raw = testDir
if raw[0] == '/':
raw = raw[1:]
if raw[-1] == '/':
raw = raw[:-1]
list1 = self.manager.listAbsolute(raw)
list2 = self.manager.listAbsolute(raw + "/")
list3 = self.manager.listAbsolute("/" + raw)
self.assertEqual(list1, list2)
self.assertEqual(list2, list3)
if __name__ == '__main__':
unittest.main()

365
awips/test/localization/testLocalizationRest.py Normal file
View file

@ -0,0 +1,365 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
import unittest
import urllib.request, urllib.error, urllib.parse
from html.parser import HTMLParser
from xml.etree.ElementTree import parse as parseXml
from json import load as loadjson
from urllib.parse import urljoin
from base64 import b64encode
#
# Test the localization REST service.
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# --------- -------- --------- --------------------------
# 08/07/17 5731 bsteffen Initial Creation.
# 01/06/22 8735 mapeters Python 3 fixes, test configured level instead of
# site since it more reliably has menus/ dir
baseURL = "http://localhost:9581/services/localization/"
testSite = "OAX"
testDir = "menus"
testFile = "test.xml"
username = "test"
password = username
base64string = b64encode(b'%s:%s' % (username.encode(), password.encode()))
authString = "Basic %s" % base64string.decode()
class ValidHTMLParser(HTMLParser):
"""Simple HTML parser that performs very minimal validation.
This ensures that all start and end tags match, and also that there are
some tags. It also accumulates the text of all links in the html file
in the link_texts attribute, which can be used for further validation.
"""
def __init__(self, testcase):
HTMLParser.__init__(self)
self._testcase = testcase
self._tags = []
self._any = False
self.link_texts = []
def handle_starttag(self, tag, attrs): # NOSONAR
"""Suppressing Pylint:W0613 "Unused argument" on above line since this
is a method override.
"""
self._tags.append(tag)
self._any = True
def handle_endtag(self, tag):
self._testcase.assertNotEqual([], self._tags, "Unstarted end tag " + tag)
self._testcase.assertEqual(tag, self._tags.pop())
def handle_data(self, data):
if self._tags[-1] == "a":
self.link_texts.append(data)
def close(self):
HTMLParser.close(self)
self._testcase.assertTrue(self._any)
self._testcase.assertEqual([], self._tags)
class AbstractListingTestCase():
"""Base test case for testing listings, retrieves data as html, xml, and json.
Sub classes should implement assertValidHtml, assertValidXml, and
assertValidJson to ensure that the content returned matches what was
expected.
"""
def assertRequestGetsHtml(self, request):
response = urllib.request.urlopen(request)
self.assertEqual(response.headers["Content-Type"], "text/html")
body = response.read().decode()
parser = ValidHTMLParser(self)
parser.feed(body)
parser.close()
self.assertValidHtml(parser)
def assertValidHtml(self, parser):
"""Intended to be overriden by subclasses to validate HTML content.
The argument is a populated instance of ValidHTMLParser.
"""
pass
def test_default(self):
request = urllib.request.Request(self.url)
self.assertRequestGetsHtml(request)
def test_last_slash(self):
if self.url.endswith("/"):
request = urllib.request.Request(self.url[:-1])
else:
request = urllib.request.Request(self.url + "/")
self.assertRequestGetsHtml(request)
def test_wild_mime(self):
request = urllib.request.Request(self.url)
request.add_header("Accept", "*/*")
self.assertRequestGetsHtml(request)
request.add_header("Accept", "text/*")
self.assertRequestGetsHtml(request)
def test_html(self):
request = urllib.request.Request(self.url)
request.add_header("Accept", "text/html")
self.assertRequestGetsHtml(request)
def test_json(self):
request = urllib.request.Request(self.url)
request.add_header("Accept", "application/json")
response = urllib.request.urlopen(request)
self.assertEqual(response.headers["Content-Type"], "application/json")
jsonData = loadjson(response)
self.assertValidJson(jsonData)
def assertValidJson(self, jsonData):
"""Intended to be overriden by subclasses to validate JSON content.
The argument is a python object as returned from json.load
"""
pass
def test_xml(self):
request = urllib.request.Request(self.url)
request.add_header("Accept", "application/xml")
response = urllib.request.urlopen(request)
self.assertEqual(response.headers["Content-Type"], "application/xml")
xmlData = parseXml(response)
self.assertValidXml(xmlData)
def assertValidXml(self, xmlData):
"""Intended to be overriden by subclasses to validate XML content.
The argument is an ElementTree
"""
pass
def test_delete(self):
request = urllib.request.Request(self.url, method="DELETE")
with self.assertRaises(urllib.error.HTTPError) as cm:
urllib.request.urlopen(request)
self.assertEqual(405, cm.exception.code)
def test_put(self):
request = urllib.request.Request(self.url, method="PUT")
request.data = b"Test Data"
with self.assertRaises(urllib.error.HTTPError) as cm:
urllib.request.urlopen(request)
self.assertEqual(405, cm.exception.code)
def test_unacceptable(self):
request = urllib.request.Request(self.url)
request.add_header("Accept", "application/fakemimetype")
with self.assertRaises(urllib.error.HTTPError) as cm:
urllib.request.urlopen(request)
self.assertEqual(406, cm.exception.code)
request.add_header("Accept", "fakemimetype/*")
with self.assertRaises(urllib.error.HTTPError) as cm:
urllib.request.urlopen(request)
self.assertEqual(406, cm.exception.code)
def test_accept_quality_factor(self):
request = urllib.request.Request(self.url)
request.add_header("Accept", "application/xml; q=0.8, application/json; q=0.2")
response = urllib.request.urlopen(request)
self.assertEqual(response.headers["Content-Type"], "application/xml")
xmlData = parseXml(response)
self.assertValidXml(xmlData)
request.add_header("Accept", "application/xml; q=0.2, application/json; q=0.8")
response = urllib.request.urlopen(request)
self.assertEqual(response.headers["Content-Type"], "application/json")
jsonData = loadjson(response)
self.assertValidJson(jsonData)
request.add_header("Accept", "application/xml, application/json; q=0.8")
response = urllib.request.urlopen(request)
self.assertEqual(response.headers["Content-Type"], "application/xml")
xmlData = parseXml(response)
self.assertValidXml(xmlData)
request.add_header("Accept", "application/fakemimetype, application/json; q=0.8")
response = urllib.request.urlopen(request)
self.assertEqual(response.headers["Content-Type"], "application/json")
jsonData = loadjson(response)
self.assertValidJson(jsonData)
class RootTestCase(AbstractListingTestCase, unittest.TestCase):
"""Test that the root of the localization service returns listing of localization types."""
def setUp(self):
self.url = baseURL
def assertValidHtml(self, parser):
self.assertIn("common_static/", parser.link_texts)
def assertValidJson(self, jsonData):
self.assertIn("common_static/", jsonData)
def assertValidXml(self, xmlData):
root = xmlData.getroot()
self.assertEqual(root.tag, "entries")
names = [e.text for e in root.findall("entry")]
self.assertIn("common_static/", names)
class TypeTestCase(AbstractListingTestCase, unittest.TestCase):
"""Test that common_static will list context levels."""
def setUp(self):
self.url = urljoin(baseURL, "common_static/")
def assertValidHtml(self, parser):
self.assertIn("base/", parser.link_texts)
self.assertIn("site/", parser.link_texts)
def assertValidJson(self, jsonData):
self.assertIn("base/", jsonData)
self.assertIn("site/", jsonData)
def assertValidXml(self, xmlData):
root = xmlData.getroot()
self.assertEqual(root.tag, "entries")
names = [e.text for e in root.findall("entry")]
self.assertIn("base/", names)
self.assertIn("site/", names)
class LevelTestCase(AbstractListingTestCase, unittest.TestCase):
"""Test that common_static/site will list sites."""
def setUp(self):
self.url = urljoin(baseURL, "common_static/site/")
def assertValidHtml(self, parser):
self.assertIn(testSite + "/", parser.link_texts)
def assertValidJson(self, jsonData):
self.assertIn(testSite + "/", jsonData)
def assertValidXml(self, xmlData):
root = xmlData.getroot()
self.assertEqual(root.tag, "entries")
names = [e.text for e in root.findall("entry")]
self.assertIn(testSite + "/", names)
class AbstractFileListingTestCase(AbstractListingTestCase):
"""Base test case for a file listing"""
def assertValidHtml(self, parser):
self.assertIn(testDir + "/", parser.link_texts)
self.assertEqual(parser.link_texts, sorted(parser.link_texts))
def assertValidJson(self, jsonData):
self.assertIn(testDir + "/", jsonData)
def assertValidXml(self, xmlData):
root = xmlData.getroot()
self.assertEqual(root.tag, "files")
names = [e.get("name") for e in root.findall("file")]
self.assertIn(testDir + "/", names)
self.assertEqual(names, sorted(names))
class BaseFileListingTestCase(AbstractFileListingTestCase, unittest.TestCase):
"""Test that common_static/base lists files"""
def setUp(self):
self.url = urljoin(baseURL, "common_static/base/")
class ConfiguredFileListingTestCase(AbstractFileListingTestCase, unittest.TestCase):
"""Test that common_static/site/<testSite>/ lists files"""
def setUp(self):
self.url = urljoin(baseURL, "common_static/configured/" + testSite + "/")
class FileTestCase(unittest.TestCase):
"""Test retrieval, modification and deletion of an individual."""
def setUp(self):
self.url = urljoin(baseURL, "common_static/user/" + username + "/" + testFile)
# The file should not exist before the test, but if it does then delete it
# This is some of the same functionality we are testing so if setup fails
# then the test would probably fail anyway
try:
request = urllib.request.Request(self.url)
response = urllib.request.urlopen(request)
request = urllib.request.Request(self.url, method="DELETE")
request.add_header("Authorization", authString)
request.add_header("If-Match", response.headers["Content-MD5"])
response = urllib.request.urlopen(request)
except urllib.error.HTTPError as e:
if e.code != 404:
raise e
def test_file_operations(self):
"""Run through a typical set of file interactions and verify everything works correctly."""
request = urllib.request.Request(self.url, method="PUT")
request.data = b"Test Data"
with self.assertRaises(urllib.error.HTTPError) as cm:
response = urllib.request.urlopen(request)
self.assertEqual(401, cm.exception.code)
request = urllib.request.Request(self.url, method="PUT")
request.data = b"Test Data"
request.add_header("Authorization", authString)
with self.assertRaises(urllib.error.HTTPError) as cm:
response = urllib.request.urlopen(request)
self.assertEqual(409, cm.exception.code)
request = urllib.request.Request(self.url, method="PUT")
request.data = b"Test Data"
request.add_header("Authorization", authString)
request.add_header("If-Match", "NON_EXISTENT_CHECKSUM")
request.add_header("Content-Type", "text/plain")
response = urllib.request.urlopen(request)
request = urllib.request.Request(self.url)
response = urllib.request.urlopen(request)
self.assertEqual(response.read().decode(), "Test Data")
request = urllib.request.Request(self.url + "/")
response = urllib.request.urlopen(request)
self.assertEqual(response.read().decode(), "Test Data")
request = urllib.request.Request(self.url, method="PUT")
request.data = b"Test Data2"
request.add_header("If-Match", response.headers["Content-MD5"])
request.add_header("Authorization", authString)
request.add_header("Content-Type", "text/plain")
response = urllib.request.urlopen(request)
checksum = response.headers["Content-MD5"]
request = urllib.request.Request(self.url)
response = urllib.request.urlopen(request)
self.assertEqual(response.read().decode(), "Test Data2")
request = urllib.request.Request(self.url, method="DELETE")
with self.assertRaises(urllib.error.HTTPError) as cm:
response = urllib.request.urlopen(request)
self.assertEqual(401, cm.exception.code)
request.add_header("Authorization", authString)
with self.assertRaises(urllib.error.HTTPError) as cm:
response = urllib.request.urlopen(request)
self.assertEqual(409, cm.exception.code)
request.add_header("If-Match", checksum)
response = urllib.request.urlopen(request)
request = urllib.request.Request(self.url)
with self.assertRaises(urllib.error.HTTPError) as cm:
response = urllib.request.urlopen(request)
self.assertEqual(404, cm.exception.code)
if __name__ == '__main__':
unittest.main()

98
awips/test/testQpidTimeToLive.py Normal file
View file

@ -0,0 +1,98 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------- -------- ------------ --------------------------------------------
# Mar 09, 2011 njensen Initial Creation.
# Aug 15, 2013 2169 bkowal Decompress data read from the queue
# Jun 24, 2020 8187 randerso Added program for qpid connection_id
#
#
import time
import threading
TIME_TO_SLEEP = 300
class ListenThread(threading.Thread):
def __init__(self, hostname, portNumber, topicName):
self.hostname = hostname
self.portNumber = portNumber
self.topicName = topicName
self.nMessagesReceived = 0
self.waitSecond = 0
self.stopped = False
threading.Thread.__init__(self)
def run(self):
from awips import QpidSubscriber
self.qs = QpidSubscriber.QpidSubscriber(host=self.hostname, port=self.portNumber, decompress=True, program="testQpidTimeToLive")
self.qs.topicSubscribe(self.topicName, self.receivedMessage)
def receivedMessage(self, msg):
print("Received message")
self.nMessagesReceived += 1
if self.waitSecond == 0:
fmsg = open('/tmp/rawMessage', 'w')
fmsg.write(msg)
fmsg.close()
while self.waitSecond < TIME_TO_SLEEP and not self.stopped:
if self.waitSecond % 60 == 0:
print(time.strftime('%H:%M:%S'), "Sleeping and stuck in not so infinite while loop")
self.waitSecond += 1
time.sleep(1)
print(time.strftime('%H:%M:%S'), "Received", self.nMessagesReceived, "messages")
def stop(self):
print("Stopping")
self.stopped = True
self.qs.close()
def main():
print("Starting up at", time.strftime('%H:%M:%S'))
topic = 'edex.alerts'
host = 'localhost'
port = 5672
thread = ListenThread(host, port, topic)
try:
thread.start()
while True:
time.sleep(3)
except KeyboardInterrupt:
pass
finally:
thread.stop()
if __name__ == '__main__':
main()

54
dynamicserialize/DynamicSerializationManager.py
View file

@ -1,45 +1,69 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# A port of the Java DynamicSerializeManager. Should be used to read/write
# DynamicSerialize binary data.
#
#
#
#
# SOFTWARE HISTORY
#
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 06/09/10 njensen Initial Creation.
#
#
#
from thrift.transport import TTransport
from . import SelfDescribingBinaryProtocol, ThriftSerializationContext
class DynamicSerializationManager:
def __init__(self):
self.transport = None
self.transport = None
def _deserialize(self, ctx):
return ctx.deserializeMessage()
def deserializeBytes(self, sbytes):
ctx = self._buildSerializationContext(sbytes)
def deserializeBytes(self, bytes):
ctx = self._buildSerializationContext(bytes)
ctx.readMessageStart()
obj = self._deserialize(ctx)
ctx.readMessageEnd()
return obj
def _buildSerializationContext(self, sbytes=None):
self.transport = TTransport.TMemoryBuffer(sbytes)
def _buildSerializationContext(self, bytes=None):
self.transport = TTransport.TMemoryBuffer(bytes)
protocol = SelfDescribingBinaryProtocol.SelfDescribingBinaryProtocol(self.transport)
return ThriftSerializationContext.ThriftSerializationContext(self, protocol)
def serializeObject(self, obj):
ctx = self._buildSerializationContext()
ctx.writeMessageStart("dynamicSerialize")
self._serialize(ctx, obj)
ctx.writeMessageEnd()
return self.transport.getvalue()
return self.transport.getvalue()
def _serialize(self, ctx, obj):
ctx.serializeMessage(obj)
ctx.serializeMessage(obj)

191
dynamicserialize/SelfDescribingBinaryProtocol.py
View file

@ -1,114 +1,151 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Partially compatible AWIPS-II Thrift Binary Protocol
#
#
# <B>Missing functionality:</B>
# <UL>
# <LI> Custom Serializers
# <LI> Inheritance
# </UL>
#
# SOFTWARE HISTORY
#
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 11/11/09 chammack Initial Creation.
# 06/09/10 njensen Added float, list methods
# Apr 24, 2015 4425 nabowle Add F64List support.
#
# 11/11/09 chammack Initial Creation.
# 06/09/10 njensen Added float, list methods
# Apr 24, 2015 4425 nabowle Add F64List support.
# Jun 25, 2019 7821 tgurney Replace numpy.getbuffer
# with memoryview
#
#
#
from thrift.protocol.TProtocol import *
from thrift.protocol.TBinaryProtocol import *
import struct
import numpy
from thrift.protocol.TProtocol import *
from thrift.protocol.TBinaryProtocol import *
######################################
# TODO Remove after switch to Python 3
# This is necessary because some APIs in Python 2 expect a buffer and not a
# memoryview.
import sys
if sys.version_info.major < 3:
memoryview = buffer
######################################
FLOAT = 64
intList = numpy.dtype(numpy.int32).newbyteorder('>')
floatList = numpy.dtype(numpy.float32).newbyteorder('>')
longList = numpy.dtype(numpy.int64).newbyteorder('>')
longList = numpy.dtype(numpy.int64).newbyteorder('>')
shortList = numpy.dtype(numpy.int16).newbyteorder('>')
byteList = numpy.dtype(numpy.int8).newbyteorder('>')
doubleList = numpy.dtype(numpy.float64).newbyteorder('>')
class SelfDescribingBinaryProtocol(TBinaryProtocol):
def readFieldBegin(self):
type = self.readByte()
if type == TType.STOP:
return (None, type, 0)
name = self.readString()
id = self.readI16()
return (name, type, id)
def readFieldBegin(self):
ftype = self.readByte()
if ftype == TType.STOP:
return None, ftype, 0
name = self.readString()
fid = self.readI16()
return name, ftype, fid
def readStructBegin(self):
return self.readString()
def readStructBegin(self):
return self.readString()
def writeStructBegin(self, name):
self.writeString(name)
def writeStructBegin(self, name):
self.writeString(name)
def writeFieldBegin(self, name, type, id):
self.writeByte(type)
self.writeString(name)
self.writeI16(id)
def writeFieldBegin(self, name, ftype, fid):
self.writeByte(ftype)
self.writeString(name)
self.writeI16(fid)
def readFloat(self):
d = self.readI32()
dAsBytes = struct.pack('i', d)
f = struct.unpack('f', dAsBytes)
return f[0]
def readFloat(self):
d = self.readI32()
dAsBytes = struct.pack('i', d)
f = struct.unpack('f', dAsBytes)
return f[0]
def writeFloat(self, f):
dAsBytes = struct.pack('f', f)
i = struct.unpack('i', dAsBytes)
self.writeI32(i[0])
def writeFloat(self, f):
dAsBytes = struct.pack('f', f)
i = struct.unpack('i', dAsBytes)
self.writeI32(i[0])
def readI32List(self, sz):
buff = self.trans.readAll(4*sz)
val = numpy.frombuffer(buff, dtype=intList, count=sz)
return val
def readI32List(self, sz):
buff = self.trans.readAll(4*sz)
val = numpy.frombuffer(buff, dtype=intList, count=sz)
return val
def readF32List(self, sz):
buff = self.trans.readAll(4*sz)
val = numpy.frombuffer(buff, dtype=floatList, count=sz)
return val
def readF32List(self, sz):
buff = self.trans.readAll(4*sz)
val = numpy.frombuffer(buff, dtype=floatList, count=sz)
return val
def readF64List(self, sz):
buff = self.trans.readAll(8*sz)
val = numpy.frombuffer(buff, dtype=doubleList, count=sz)
return val
def readF64List(self, sz):
buff = self.trans.readAll(8*sz)
val = numpy.frombuffer(buff, dtype=doubleList, count=sz)
return val
def readI64List(self, sz):
buff = self.trans.readAll(8*sz)
val = numpy.frombuffer(buff, dtype=longList, count=sz)
return val
def readI64List(self, sz):
buff = self.trans.readAll(8*sz)
val = numpy.frombuffer(buff, dtype=longList, count=sz)
return val
def readI16List(self, sz):
buff = self.trans.readAll(2*sz)
val = numpy.frombuffer(buff, dtype=shortList, count=sz)
return val
def readI16List(self, sz):
buff = self.trans.readAll(2*sz)
val = numpy.frombuffer(buff, dtype=shortList, count=sz)
return val
def readI8List(self, sz):
buff = self.trans.readAll(sz)
val = numpy.frombuffer(buff, dtype=byteList, count=sz)
return val
def readI8List(self, sz):
buff = self.trans.readAll(sz)
val = numpy.frombuffer(buff, dtype=byteList, count=sz)
return val
def writeI32List(self, buff):
b = numpy.asarray(buff, intList)
self.trans.write(memoryview(b))
def writeI32List(self, buff):
b = numpy.asarray(buff, intList)
self.trans.write(numpy.getbuffer(b))
def writeF32List(self, buff):
b = numpy.asarray(buff, floatList)
self.trans.write(memoryview(b))
def writeF32List(self, buff):
b = numpy.asarray(buff, floatList)
self.trans.write(numpy.getbuffer(b))
def writeF64List(self, buff):
b = numpy.asarray(buff, doubleList)
self.trans.write(memoryview(b))
def writeF64List(self, buff):
b = numpy.asarray(buff, doubleList)
self.trans.write(numpy.getbuffer(b))
def writeI64List(self, buff):
b = numpy.asarray(buff, longList)
self.trans.write(memoryview(b))
def writeI64List(self, buff):
b = numpy.asarray(buff, longList)
self.trans.write(numpy.getbuffer(b))
def writeI16List(self, buff):
b = numpy.asarray(buff, shortList)
self.trans.write(memoryview(b))
def writeI16List(self, buff):
b = numpy.asarray(buff, shortList)
self.trans.write(numpy.getbuffer(b))
def writeI8List(self, buff):
b = numpy.asarray(buff, byteList)
self.trans.write(numpy.getbuffer(b))
def writeI8List(self, buff):
b = numpy.asarray(buff, byteList)
self.trans.write(memoryview(b))

184
dynamicserialize/ThriftSerializationContext.py
View file

@ -1,3 +1,24 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# A port of the Java ThriftSerializationContext, used for reading/writing
# DynamicSerialize objects to/from thrift.
@ -17,19 +38,22 @@
# writeObject().
# Apr 24, 2015 4425 nabowle Add Double support
# Oct 17, 2016 5919 njensen Optimized for speed
# Sep 06, 2018 mjames@ucar Python3 compliance
# Oct 11, 2018 7306 dgilling Fix handling of ndarrays in
# _serializeArray.
# Apr 30, 2019 7814 dgilling Serialize python unicode strings.
# Jun 26, 2019 7888 tgurney Python 3 fixes
# Jul 23, 2019 7888 tgurney Fix handling of bytes as strings
# Sep 13, 2019 7888 tgurney Fix serialization of bytes
#
#
from thrift.Thrift import TType
import inspect
import sys
import types
import six
import numpy
from thrift.Thrift import TType
import dynamicserialize
from dynamicserialize import dstypes, adapters
from dynamicserialize import SelfDescribingBinaryProtocol
from . import SelfDescribingBinaryProtocol
import numpy
DS_LEN = len('dynamicserialize.dstypes.')
@ -49,61 +73,39 @@ def buildObjMap(module):
tname = tname[DS_LEN:]
dsObjTypes[tname] = clz
buildObjMap(dstypes)
if six.PY2:
pythonToThriftMap = {
types.StringType: TType.STRING,
types.IntType: TType.I32,
types.LongType: TType.I64,
types.ListType: TType.LIST,
unicode: TType.STRING,
types.DictionaryType: TType.MAP,
type(set([])): TType.SET,
types.FloatType: SelfDescribingBinaryProtocol.FLOAT,
# types.FloatType: TType.DOUBLE,
types.BooleanType: TType.BOOL,
types.InstanceType: TType.STRUCT,
types.NoneType: TType.VOID,
numpy.float32: SelfDescribingBinaryProtocol.FLOAT,
numpy.int32: TType.I32,
numpy.ndarray: TType.LIST,
numpy.object_: TType.STRING, # making an assumption here
numpy.string_: TType.STRING,
numpy.float64: TType.DOUBLE,
numpy.int16: TType.I16,
numpy.int8: TType.BYTE,
numpy.int64: TType.I64
}
else:
pythonToThriftMap = {
bytes: TType.STRING,
int: TType.I32,
int: TType.I64,
list: TType.LIST,
dict: TType.MAP,
type(set([])): TType.SET,
float: SelfDescribingBinaryProtocol.FLOAT,
# types.FloatType: TType.DOUBLE,
bool: TType.BOOL,
object: TType.STRUCT,
str: TType.STRING,
type(None): TType.VOID,
numpy.float32: SelfDescribingBinaryProtocol.FLOAT,
numpy.int32: TType.I32,
numpy.ndarray: TType.LIST,
numpy.object_: TType.STRING, # making an assumption here
numpy.string_: TType.STRING,
numpy.float64: TType.DOUBLE,
numpy.int16: TType.I16,
numpy.int8: TType.BYTE,
numpy.int64: TType.I64
}
pythonToThriftMap = {
bytes: TType.LIST,
str: TType.STRING,
int: TType.I64,
list: TType.LIST,
dict: TType.MAP,
set: TType.SET,
float: SelfDescribingBinaryProtocol.FLOAT,
# types.FloatType: TType.DOUBLE,
bool: TType.BOOL,
object: TType.STRUCT,
type(None): TType.VOID,
numpy.float32: SelfDescribingBinaryProtocol.FLOAT,
numpy.int32: TType.I32,
numpy.ndarray: TType.LIST,
numpy.object_: TType.STRING, # making an assumption here
numpy.string_: TType.STRING,
# numpy.bytes_ is the same as numpy.string_
numpy.unicode_: TType.STRING,
# numpy.str_ is the same as numpy.unicode_
numpy.float64: TType.DOUBLE,
numpy.int16: TType.I16,
numpy.int8: TType.BYTE,
numpy.int64: TType.I64
}
primitiveSupport = (TType.BYTE, TType.I16, TType.I32, TType.I64,
SelfDescribingBinaryProtocol.FLOAT, TType.DOUBLE)
BYTE_STRING_TYPES = {numpy.string_, numpy.bytes_, bytes}
class ThriftSerializationContext(object):
@ -126,7 +128,7 @@ class ThriftSerializationContext(object):
TType.VOID: lambda: None
}
self.typeSerializationMethod = {
TType.STRING: self.protocol.writeString,
TType.STRING: self.writeString,
TType.I16: self.protocol.writeI16,
TType.I32: self.protocol.writeI32,
TType.LIST: self._serializeArray,
@ -166,19 +168,18 @@ class ThriftSerializationContext(object):
def deserializeMessage(self):
name = self.protocol.readStructBegin()
name = name.decode('cp437')
name = name.replace('_', '.')
if name.isdigit():
obj = self._deserializeType(int(name))
return obj
name = name.replace('_', '.')
if name in adapters.classAdapterRegistry:
return adapters.classAdapterRegistry[name].deserialize(self)
elif '$' in name:
# it's an inner class, we're going to hope it's an enum, treat it
# special
fieldName, fieldType, fieldId = self.protocol.readFieldBegin()
if fieldName.decode('utf8') != '__enumValue__':
raise dynamicserialize.SerializationException(
if fieldName != '__enumValue__':
raise dynamiceserialize.SerializationException(
"Expected to find enum payload. Found: " + fieldName)
obj = self.protocol.readString()
self.protocol.readFieldEnd()
@ -187,7 +188,7 @@ class ThriftSerializationContext(object):
clz = dsObjTypes[name]
obj = clz()
while self._deserializeField(obj):
while self._deserializeField(name, obj):
pass
self.protocol.readStructEnd()
@ -200,19 +201,18 @@ class ThriftSerializationContext(object):
raise dynamicserialize.SerializationException(
"Unsupported type value " + str(b))
def _deserializeField(self, obj):
def _deserializeField(self, structname, obj):
fieldName, fieldType, fieldId = self.protocol.readFieldBegin()
if fieldType == TType.STOP:
return False
elif fieldType != TType.VOID:
result = self._deserializeType(fieldType)
fn_str = bytes.decode(fieldName)
lookingFor = "set" + fn_str[0].upper() + fn_str[1:]
lookingFor = "set" + fieldName[0].upper() + fieldName[1:]
try:
setMethod = getattr(obj, lookingFor)
setMethod(result)
except ValueError:
setMethod(result)
except:
raise dynamicserialize.SerializationException(
"Couldn't find setter method " + lookingFor)
@ -225,7 +225,7 @@ class ThriftSerializationContext(object):
if size:
if listType not in primitiveSupport:
m = self.typeDeserializationMethod[listType]
result = [m() for __ in range(size)]
result = [m() for n in range(size)]
else:
result = self.listDeserializationMethod[listType](size)
self.protocol.readListEnd()
@ -234,7 +234,7 @@ class ThriftSerializationContext(object):
def _deserializeMap(self):
keyType, valueType, size = self.protocol.readMapBegin()
result = {}
for __ in range(size):
for n in range(size):
# can't go off the type, due to java generics limitations dynamic serialize is
# serializing keys and values as void
key = self.typeDeserializationMethod[TType.STRUCT]()
@ -246,7 +246,7 @@ class ThriftSerializationContext(object):
def _deserializeSet(self):
setType, setSize = self.protocol.readSetBegin()
result = set([])
for __ in range(setSize):
for n in range(setSize):
result.add(self.typeDeserializationMethod[TType.STRUCT]())
self.protocol.readSetEnd()
return result
@ -256,11 +256,10 @@ class ThriftSerializationContext(object):
if pyt in pythonToThriftMap:
return pythonToThriftMap[pyt]
elif pyt.__module__[:DS_LEN - 1] == ('dynamicserialize.dstypes'):
if six.PY2:
return pythonToThriftMap[types.InstanceType]
return pythonToThriftMap[object]
raise dynamicserialize.SerializationException(
"Don't know how to serialize object of type: " + str(pyt))
else:
raise dynamicserialize.SerializationException(
"Don't know how to serialize object of type: " + str(pyt))
def serializeMessage(self, obj):
tt = self._lookupType(obj)
@ -314,29 +313,34 @@ class ThriftSerializationContext(object):
def _serializeArray(self, obj):
size = len(obj)
if size:
if isinstance(obj, numpy.ndarray):
t = pythonToThriftMap[obj.dtype.type]
size = obj.size
else:
t = self._lookupType(obj[0])
objType = type(obj)
if objType is numpy.ndarray:
t = pythonToThriftMap[obj.dtype.type]
size = obj.size
elif objType is bytes:
t = TType.BYTE
obj = list(obj)
elif size:
t = self._lookupType(obj[0])
else:
t = TType.STRUCT
self.protocol.writeListBegin(t, size)
if t == TType.STRING:
if isinstance(obj, numpy.ndarray):
# For TType.STRING we always assume that if the objType is bytes we
# want to decode it into a str
if objType is numpy.ndarray:
if len(obj.shape) == 1:
for x in obj:
s = str(x).strip()
s = self._decodeString(x).strip()
self.typeSerializationMethod[t](s)
else:
for x in obj:
for y in x:
s = str(y).strip()
s = self._decodeString(y).strip()
self.typeSerializationMethod[t](s)
else:
for x in obj:
s = str(x)
s = self._decodeString(x)
self.typeSerializationMethod[t](s)
elif t not in primitiveSupport:
for x in obj:
@ -348,9 +352,9 @@ class ThriftSerializationContext(object):
def _serializeMap(self, obj):
size = len(obj)
self.protocol.writeMapBegin(TType.VOID, TType.VOID, size)
for k in list(obj.keys()):
self.typeSerializationMethod[TType.STRUCT](k)
self.typeSerializationMethod[TType.STRUCT](obj[k])
for (key, value) in obj.items():
self.typeSerializationMethod[TType.STRUCT](key)
self.typeSerializationMethod[TType.STRUCT](value)
self.protocol.writeMapEnd()
def _serializeSet(self, obj):
@ -360,6 +364,13 @@ class ThriftSerializationContext(object):
self.typeSerializationMethod[TType.STRUCT](x)
self.protocol.writeSetEnd()
def _decodeString(self, s):
"""If s is a byte string, return s.decode(). Otherwise return str(s)"""
if type(s) in BYTE_STRING_TYPES:
return s.decode()
else:
return str(s)
def writeMessageStart(self, name):
self.protocol.writeMessageBegin(name, TType.VOID, 0)
@ -412,6 +423,7 @@ class ThriftSerializationContext(object):
return self.protocol.readString()
def writeString(self, s):
s = self._decodeString(s)
self.protocol.writeString(s)
def readBinary(self):

47
dynamicserialize/__init__.py
View file

@ -1,35 +1,58 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# TODO
#
#
# SOFTWARE HISTORY
#
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 08/20/10 njensen Initial Creation.
#
#
#
#
__all__ = ['SerializationException']
__all__ = [
]
from . import dstypes, adapters
from . import DynamicSerializationManager
class SerializationException(Exception):
def __init__(self, message=None):
self.message = message
def __str__(self):
if self.message:
return self.message
return self.message
else:
return ""
def serialize(obj):
dsm = DynamicSerializationManager.DynamicSerializationManager()
return dsm.serializeObject(obj)
def deserialize(objbytes):
def deserialize(bytes):
dsm = DynamicSerializationManager.DynamicSerializationManager()
return dsm.deserializeBytes(objbytes)
return dsm.deserializeBytes(bytes)

51
dynamicserialize/adapters/ActiveTableModeAdapter.py Normal file
View file

@ -0,0 +1,51 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Adapter for com.raytheon.uf.common.activetable.ActiveTableMode
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 09/29/10 wldougher Initial Creation.
#
#
#
from thrift.Thrift import TType
from dynamicserialize.dstypes.com.raytheon.uf.common.activetable import ActiveTableMode
ClassAdapter = 'com.raytheon.uf.common.activetable.ActiveTableMode'
def serialize(context, mode):
context.protocol.writeFieldBegin('__enumValue__', TType.STRING, 0)
context.writeString(mode.value)
def deserialize(context):
result = ActiveTableMode()
# Read the TType.STRING, "__enumValue__", and id.
# We're not interested in any of those, so just discard them.
context.protocol.readFieldBegin()
# now get the actual enum value
result.value = context.readString()
return result

37
dynamicserialize/adapters/ByteBufferAdapter.py
View file

@ -1,21 +1,46 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Adapter for java.nio.ByteBuffer
#
#
#
#
# SOFTWARE HISTORY
#
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 08/03/11 dgilling Initial Creation.
#
#
#
ClassAdapter = ['java.nio.ByteBuffer', 'java.nio.HeapByteBuffer']
ClassAdapter = ['java.nio.ByteBuffer', 'java.nio.HeapByteBuffer']
def serialize(context, bufferset):
def serialize(context, set):
raise NotImplementedError("Serialization of ByteBuffers is not supported.")
def deserialize(context):
byteBuf = context.readBinary()
return byteBuf

33
dynamicserialize/adapters/CalendarAdapter.py
View file

@ -1,24 +1,45 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Adapter for java.util.Calendar
#
#
#
#
# SOFTWARE HISTORY
#
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 09/29/10 wldougher Initial Creation.
#
#
#
from dynamicserialize.dstypes.java.util import Calendar
ClassAdapter = 'java.util.Calendar'
def serialize(context, calendar):
calTiM = calendar.getTimeInMillis()
context.writeI64(calTiM)
def deserialize(context):
result = Calendar()
result.setTimeInMillis(context.readI64())

35
dynamicserialize/adapters/CommutativeTimestampAdapter.py
View file

@ -1,25 +1,46 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Adapter for CommutativeTimestamp
#
#
#
#
# SOFTWARE HISTORY
#
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 9/21/2015 4486 rjpeter Initial creation.
# Jun 23, 2016 5696 rjpeter Handle CommutativeTimestamp.
#
#
from dynamicserialize.dstypes.com.raytheon.uf.common.time import CommutativeTimestamp
ClassAdapter = 'com.raytheon.uf.common.time.CommutativeTimestamp'
ClassAdapter = 'com.raytheon.uf.common.time.CommutativeTimestamp'
def serialize(context, date):
context.writeI64(date.getTime())
def deserialize(context):
result = CommutativeTimestamp()
result.setTime(context.readI64())
return result
return result

38
dynamicserialize/adapters/CoordAdapter.py
View file

@ -1,24 +1,45 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Adapter for com.vividsolutions.jts.geom.Coordinate
#
#
# Adapter for org.locationtech.jts.geom.Coordinate
#
#
# SOFTWARE HISTORY
#
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 01/20/11 dgilling Initial Creation.
#
#
#
from dynamicserialize.dstypes.com.vividsolutions.jts.geom import Coordinate
ClassAdapter = 'com.vividsolutions.jts.geom.Coordinate'
from dynamicserialize.dstypes.org.locationtech.jts.geom import Coordinate
ClassAdapter = 'org.locationtech.jts.geom.Coordinate'
def serialize(context, coordinate):
context.writeDouble(coordinate.getX())
context.writeDouble(coordinate.getY())
def deserialize(context):
x = context.readDouble()
y = context.readDouble()
@ -26,3 +47,4 @@ def deserialize(context):
coord.setX(x)
coord.setY(y)
return coord

37
dynamicserialize/adapters/DatabaseIDAdapter.py
View file

@ -1,23 +1,44 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Adapter for com.raytheon.uf.common.dataplugin.gfe.db.objects.DatabaseID
#
#
#
#
# SOFTWARE HISTORY
#
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 03/29/11 dgilling Initial Creation.
#
#
#
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import DatabaseID
ClassAdapter = 'com.raytheon.uf.common.dataplugin.gfe.db.objects.DatabaseID'
def serialize(context, dbId):
context.writeString(str(dbId))
def deserialize(context):
result = DatabaseID(context.readString())
return result
result = DatabaseID(context.readString())
return result

35
dynamicserialize/adapters/DateAdapter.py
View file

@ -1,24 +1,45 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Adapter for java.util.Date
#
#
#
#
# SOFTWARE HISTORY
#
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 12/06/10 dgilling Initial Creation.
#
#
#
from dynamicserialize.dstypes.java.util import Date
ClassAdapter = 'java.util.Date'
def serialize(context, date):
context.writeI64(date.getTime())
def deserialize(context):
result = Date()
result.setTime(context.readI64())
return result
return result

47
dynamicserialize/adapters/EnumSetAdapter.py
View file

@ -1,32 +1,57 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Adapter for java.util.EnumSet
#
#
#
#
# SOFTWARE HISTORY
#
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 07/28/11 dgilling Initial Creation.
# 12/02/13 2537 bsteffen Serialize empty enum sets.
#
#
#
from dynamicserialize.dstypes.java.util import EnumSet
ClassAdapter = ['java.util.EnumSet', 'java.util.RegularEnumSet']
ClassAdapter = ['java.util.EnumSet', 'java.util.RegularEnumSet']
def serialize(context, bufferset):
setSize = len(bufferset)
def serialize(context, set):
setSize = len(set)
context.writeI32(setSize)
context.writeString(bufferset.getEnumClass())
for val in bufferset:
context.writeString(set.getEnumClass())
for val in set:
context.writeString(val)
def deserialize(context):
setSize = context.readI32()
enumClassName = context.readString()
valList = []
for __ in range(setSize):
for i in range(setSize):
valList.append(context.readString())
return EnumSet(enumClassName, valList)
return EnumSet(enumClassName, valList)

37
dynamicserialize/adapters/FloatBufferAdapter.py
View file

@ -1,21 +1,46 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Adapter for java.nio.FloatBuffer
#
#
#
#
# SOFTWARE HISTORY
#
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 08/01/11 dgilling Initial Creation.
#
#
#
ClassAdapter = ['java.nio.FloatBuffer', 'java.nio.HeapFloatBuffer']
ClassAdapter = ['java.nio.FloatBuffer', 'java.nio.HeapFloatBuffer']
def serialize(context, bufferset):
def serialize(context, set):
raise NotImplementedError("Serialization of FloatBuffers is not supported.")
def deserialize(context):
floatBuf = context.readFloatArray()
return floatBuf

36
dynamicserialize/adapters/FormattedDateAdapter.py
View file

@ -1,24 +1,46 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Adapter for FormattedDate
#
#
#
#
# SOFTWARE HISTORY
#
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 9/21/2015 4486 rjpeter Initial creation.
#
#
#
from dynamicserialize.dstypes.com.raytheon.uf.common.time import FormattedDate
ClassAdapter = 'com.raytheon.uf.common.time.FormattedDate'
ClassAdapter = 'com.raytheon.uf.common.time.FormattedDate'
def serialize(context, date):
context.writeI64(date.getTime())
def deserialize(context):
result = FormattedDate()
result.setTime(context.readI64())
return result
return result

33
dynamicserialize/adapters/GeomDataRespAdapter.py
View file

@ -1,3 +1,24 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Efficient adapter for GetGeometryDataResponse
#
@ -11,9 +32,12 @@
#
#
from thrift.Thrift import TType
from dynamicserialize import SelfDescribingBinaryProtocol
from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.response import GeometryResponseData
from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.response import GetGeometryDataResponse
ClassAdapter = 'com.raytheon.uf.common.dataaccess.response.GetGeometryDataResponse'
@ -34,7 +58,7 @@ def serialize(context, resp):
context.writeObject(geo.getTime())
context.writeObject(geo.getLevel())
context.writeObject(geo.getLocationName())
context.writeObject(geo.getAttributes())
context.writeObject(geo.getAttributes())
# write data map
params = geo.getDataMap()
@ -49,17 +73,16 @@ def serialize(context, resp):
# unit
context.writeObject(value[2])
def deserialize(context):
size = context.readI32()
wkbs = []
for __ in range(size):
for i in range(size):
wkb = context.readBinary()
wkbs.append(wkb)
geoData = []
size = context.readI32()
for _ in range(size):
for i in range(size):
data = GeometryResponseData()
# wkb index
wkbIndex = context.readI32()
@ -77,7 +100,7 @@ def deserialize(context):
# parameters
paramSize = context.readI32()
paramMap = {}
for __ in range(paramSize):
for k in range(paramSize):
paramName = context.readString()
value = context.readObject()
tName = context.readString()

40
dynamicserialize/adapters/GeometryTypeAdapter.py
View file

@ -1,36 +1,56 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Adapter for com.vividsolutions.jts.geom.Polygon
#
#
# Adapter for org.locationtech.jts.geom.Polygon
#
#
# SOFTWARE HISTORY
#
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 01/20/11 dgilling Initial Creation.
#
#
#
import dynamicserialize
# TODO: Implement serialization/make deserialization useful.
# Deserialization was simply implemented to allow GridLocation objects to be
# passed through thrift, but the resulting Geometry object will not be transformed into
# useful data; the base byte array is passed to a worthless Geometry class.
from dynamicserialize.dstypes.com.vividsolutions.jts.geom import Geometry
from dynamicserialize.dstypes.org.locationtech.jts.geom import Geometry
# NOTE: At the moment, EDEX serializes Polygon, MultiPolygons, Points, and
# Geometrys with the tag of the base class Geometry. Java's serialization
# adapter is smarter and can determine the exact object by reading the binary
# data. This adapter doesn't need this _yet_, so it has not been implemented.
ClassAdapter = 'com.vividsolutions.jts.geom.Geometry'
ClassAdapter = 'org.locationtech.jts.geom.Geometry'
def serialize(context, coordinate):
raise dynamicserialize.SerializationException('Not implemented yet')
def deserialize(context):
data = context.readBinary()
geom = Geometry()
geom.setBinaryData(data)
return geom

33
dynamicserialize/adapters/GregorianCalendarAdapter.py
View file

@ -1,24 +1,45 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Adapter for java.util.Calendar
#
#
#
#
# SOFTWARE HISTORY
#
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 09/29/10 wldougher Initial Creation.
#
#
#
from dynamicserialize.dstypes.java.util import GregorianCalendar
ClassAdapter = 'java.util.GregorianCalendar'
def serialize(context, calendar):
calTiM = calendar.getTimeInMillis()
context.writeI64(calTiM)
def deserialize(context):
result = GregorianCalendar()
result.setTimeInMillis(context.readI64())

37
dynamicserialize/adapters/JTSEnvelopeAdapter.py
View file

@ -1,18 +1,39 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Adapter for com.vividsolutions.jts.geom.Envelope
#
#
# Adapter for org.locationtech.jts.geom.Envelope
#
#
# SOFTWARE HISTORY
#
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 05/29/13 2023 dgilling Initial Creation.
#
#
from dynamicserialize.dstypes.com.vividsolutions.jts.geom import Envelope
ClassAdapter = 'com.vividsolutions.jts.geom.Envelope'
from dynamicserialize.dstypes.org.locationtech.jts.geom import Envelope
ClassAdapter = 'org.locationtech.jts.geom.Envelope'
def serialize(context, envelope):
context.writeDouble(envelope.getMinX())
@ -20,7 +41,6 @@ def serialize(context, envelope):
context.writeDouble(envelope.getMinY())
context.writeDouble(envelope.getMaxY())
def deserialize(context):
env = Envelope()
env.setMinX(context.readDouble())
@ -28,3 +48,4 @@ def deserialize(context):
env.setMinY(context.readDouble())
env.setMaxY(context.readDouble())
return env

51
dynamicserialize/adapters/JobProgressAdapter.py Normal file
View file

@ -0,0 +1,51 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Adapter for com.raytheon.uf.common.dataplugin.gfe.svcbu.JobProgress
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 06/22/2015 4573 randerso Initial creation
#
#
#
from thrift.Thrift import TType
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.svcbu import JobProgress
ClassAdapter = 'com.raytheon.uf.common.dataplugin.gfe.svcbu.JobProgress'
def serialize(context, mode):
context.protocol.writeFieldBegin('__enumValue__', TType.STRING, 0)
context.writeString(mode.value)
def deserialize(context):
result = JobProgress()
# Read the TType.STRING, "__enumValue__", and id.
# We're not interested in any of those, so just discard them.
context.protocol.readFieldBegin()
# now get the actual enum value
result.value = context.readString()
return result

37
dynamicserialize/adapters/LocalizationLevelSerializationAdapter.py
View file

@ -1,14 +1,39 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Adapter for com.raytheon.uf.common.localization.LocalizationContext$LocalizationLevel
#
#
#
#
# SOFTWARE HISTORY
#
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 01/11/11 dgilling Initial Creation.
#
#
#
from dynamicserialize.dstypes.com.raytheon.uf.common.localization import LocalizationLevel
ClassAdapter = [
@ -16,12 +41,10 @@ ClassAdapter = [
'com.raytheon.uf.common.localization.LocalizationLevel'
]
def serialize(context, level):
context.writeString(level.getText())
context.writeI32(level.getOrder())
context.writeBool(level.isSystemLevel())
context.writeBool(level.isSystemLevel());
def deserialize(context):
text = context.readString()
@ -29,3 +52,5 @@ def deserialize(context):
systemLevel = context.readBool()
level = LocalizationLevel(text, order, systemLevel=systemLevel)
return level

38
dynamicserialize/adapters/LocalizationTypeSerializationAdapter.py
View file

@ -1,14 +1,39 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Adapter for com.raytheon.uf.common.localization.LocalizationContext$LocalizationType
#
#
#
#
# SOFTWARE HISTORY
#
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 01/11/11 dgilling Initial Creation.
#
#
#
from dynamicserialize.dstypes.com.raytheon.uf.common.localization import LocalizationType
ClassAdapter = [
@ -16,11 +41,10 @@ ClassAdapter = [
'com.raytheon.uf.common.localization.LocalizationType'
]
def serialize(context, ltype):
context.writeString(ltype.getText())
def serialize(context, type):
context.writeString(type.getText())
def deserialize(context):
typeString = context.readString()
return LocalizationType(typeString)

89
dynamicserialize/adapters/LockTableAdapter.py Normal file
View file

@ -0,0 +1,89 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Adapter for com.raytheon.uf.common.dataplugin.gfe.server.lock.LockTable
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------- -------- --------- ---------------------------------------------
# Apr 22, 2013 rjpeter Initial Creation.
# Jun 12, 2013 2099 dgilling Use new Lock constructor.
# Feb 06, 2017 5959 randerso Removed Java .toString() calls
#
##
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.server.lock import LockTable
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.server.lock import Lock
ClassAdapter = 'com.raytheon.uf.common.dataplugin.gfe.server.lock.LockTable'
def serialize(context, lockTable):
index=0
wsIds = {str(lockTable.getWsId()) : index}
index += 1
locks = lockTable.getLocks()
lockWsIdIndex = []
for lock in locks:
wsIdString = str(lock.getWsId())
if wsIdString in wsIds:
lockWsIdIndex.append(wsIds[wsIdString])
else:
lockWsIdIndex.append(index)
wsIds[wsIdString] = index
index += 1
context.writeObject(lockTable.getParmId())
context.writeI32(index)
for wsId in sorted(wsIds, key=wsIds.get):
context.writeObject(wsId)
context.writeI32(len(locks))
for lock, wsIndex in zip(locks, lockWsIdIndex):
serializer.writeI64(lock.getStartTime())
serializer.writeI64(lock.getEndTime())
serializer.writeI32(wsIndex)
def deserialize(context):
parmId = context.readObject()
numWsIds = context.readI32()
wsIds = []
for x in range(numWsIds):
wsIds.append(context.readObject())
numLocks = context.readI32()
locks = []
for x in range(numLocks):
startTime = context.readI64()
endTime = context.readI64()
wsId = wsIds[context.readI32()]
lock = Lock(parmId, wsId, startTime, endTime)
locks.append(lock)
lockTable = LockTable()
lockTable.setParmId(parmId)
lockTable.setWsId(wsIds[0])
lockTable.setLocks(locks)
return lockTable

37
dynamicserialize/adapters/ParmIDAdapter.py
View file

@ -1,23 +1,44 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Adapter for com.raytheon.uf.common.dataplugin.gfe.db.objects.ParmID
#
#
#
#
# SOFTWARE HISTORY
#
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 03/29/11 dgilling Initial Creation.
#
#
#
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import ParmID
ClassAdapter = 'com.raytheon.uf.common.dataplugin.gfe.db.objects.ParmID'
def serialize(context, parmId):
context.writeString(str(parmId))
def deserialize(context):
result = ParmID(context.readString())
return result
result = ParmID(context.readString())
return result

Some files were not shown because too many files have changed in this diff Show more