First step on bringing over v20 python code

- brought over all the pythonPackages from the following repos/packages:
  - awips2/pythonPackages
  - awips2/edexOsgi/com.raytheon.uf.common.alertviz/pythonPackages
  - awips2/edexOsgi/com.raytheon.uf.common.mpe/pythonPackages
  - awips2/edexOsgi/com.raytheon.uf.common.dataplugin.text/pythonPackages
  - awips2/edexOsgi/com.raytheon.uf.common.dataplugin.grid/pythonPackages
  - awips2/edexOsgi/com.raytheon.uf.common.activetable/pythonPackages
  - awips2/edexOsgi/com.raytheon.uf.common.management/pythonPackages
  - awips2/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/pythonPackages
  - awips2/edexOsgi/com.raytheon.uf.common.dataplugin.radar/pythonPackages
  - awips2/edexOsgi/com.raytheon.uf.common.site/pythonPackages
  - awips2-core/common/com.raytheon.uf.common.auth/pythonPackages
  - awips2-core/common/com.raytheon.uf.common.message/pythonPackages
  - awips2-core/common/com.raytheon.uf.common.localization/pythonPackages
  - awips2-core/common/com.raytheon.uf.common.datastorage/pythonPackages
  - awips2-core/common/com.raytheon.uf.common.pointdata/pythonPackages
  - awips2-core/common/com.raythoen.uf.common.pypies/pythonPackages
  - awips2-core/common/com.raytheon.uf.common.dataaccess/pythonPackages
  - awips2-core/common/com.raytheon.uf.common.dataplugin.level/pythonPackages
  - awips2-core/common/com.raytheon.uf.common.serialization/pythonPackages
  - awips2-core/common/com.raytheon.uf.common.time/pythonPackages
  - awips2-core/common/com.raytheon.uf.common.dataplugin/pythonPackages
  - awips2-core/common/com.raytheon.uf.common.dataquery/pythonPackages
- updated the _init_.py file in dynamicserialize/dstypes/com/raytheon/uf/common to have all the proper elements listed
- started to replace "ufpy" instances with "awips"
  - awips/test/testQpidTimeToLive.py
  - awips/test/dafTests/testWarning.py
  - awips/test/dafTests/testCombinedTimeQuery.py

- will continue the "ufpy" replacement in a separate commit for easier clarity

awips/AlertVizHandler.py

@@ -0,0 +1,87 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+#
+# Pure python logging mechanism for logging to AlertViz from
+# pure python (ie not JEP).  DO NOT USE IN PYTHON CALLED
+# FROM JAVA.
+#
+# Sends local-delivery messages only, but needs to know the EDEX host name in
+# order to forward the message. If the DEFAULT_HOST environment variable is
+# not set correctly then this will not work.
+#  
+#    
+#     SOFTWARE HISTORY
+#    
+#    Date          Ticket#     Engineer     Description
+#    ------------  ----------  -----------  --------------------------
+#    08/18/10                  njensen      Initial Creation.
+#    Apr 16, 2020  8144        tgurney      Reference AlertViz stomp port
+#                                           specified in NotificationMessage
+#    Aug 25, 2020  8220        tgurney      Change local-delivery strategy
+#    
+# 
+#
+
+import logging
+import os
+from . import NotificationMessage
+import socket
+    
+class AlertVizHandler(logging.Handler):
+
+    def __init__(self, host=None, port=None,
+                 category='LOCAL', source='ANNOUNCER', level=logging.NOTSET, filters=None):
+        logging.Handler.__init__(self, level)
+        if host is None:
+            host = os.getenv('DEFAULT_HOST', 'localhost')
+        if port is None:
+            port = os.getenv('DEFAULT_PORT', 9581)
+        self._category = category
+        self._host = host
+        self._port = port
+        self._source = source
+        self._filters = filters
+        filters['WORKSTATION'] = socket.getfqdn()
+        
+    
+    def emit(self, record):        
+        "Implements logging.Handler's interface.  Record argument is a logging.LogRecord."
+        priority = None
+        if record.levelno >= 50:
+            priority = 'CRITICAL'
+        elif record.levelno >= 40:
+            priority = 'SIGNIFICANT'
+        elif record.levelno >= 30:
+            priority = 'PROBLEM'
+        elif record.levelno >= 20:
+            priority = 'EVENTA'
+        elif record.levelno >= 10:
+            priority = 'EVENTB'
+        else:
+            priority = 'VERBOSE'
+
+        msg = self.format(record)
+
+        notify = NotificationMessage.NotificationMessage(self._host, self._port, msg, priority, self._category, self._source,
+                                                         filters=self._filters)
+        notify.send()
+

awips/ConfigFileUtil.py

@@ -0,0 +1,56 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+#
+# A set of utility functions for dealing with configuration files.
+#   
+#
+#    
+#     SOFTWARE HISTORY
+#    
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    09/27/10                      dgilling       Initial Creation.
+#    
+# 
+#
+
+
+def parseKeyValueFile(fileName):
+    propDict= dict()
+    
+    try:
+        propFile= open(fileName, "rU")
+        for propLine in propFile:
+            propDef= propLine.strip()
+            if len(propDef) == 0:
+                continue
+            if propDef[0] in ( '#' ):
+                continue
+            punctuation= [ propDef.find(c) for c in ':= ' ] + [ len(propDef) ]
+            found= min( [ pos for pos in punctuation if pos != -1 ] )
+            name= propDef[:found].rstrip()
+            value= propDef[found:].lstrip(":= ").rstrip()
+            propDict[name]= value
+        propFile.close()
+    except:
+        pass
+
+    return propDict

awips/DateTimeConverter.py

@@ -1,10 +1,30 @@
+# #
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+# #
+
 #
 # Functions for converting between the various "Java" dynamic serialize types
 # used by EDEX to the native python time datetime.
-#
+#  
-#
+#    
 #     SOFTWARE HISTORY
-#
+#    
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
 #    06/24/15         #4480        dgilling       Initial Creation.
@@ -17,22 +37,23 @@ from dynamicserialize.dstypes.java.util import Date
 from dynamicserialize.dstypes.java.sql import Timestamp
 from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange
 
+
 MAX_TIME = pow(2, 31) - 1
 MICROS_IN_SECOND = 1000000
 
 
 def convertToDateTime(timeArg):
-    """
+    """ 
-    Converts the given object to a python datetime object. Supports native
+    Converts the given object to a python datetime object. Supports native 
     python representations like datetime and struct_time, but also
     the dynamicserialize types like Date and Timestamp. Raises TypeError
     if no conversion can be performed.
-
+    
     Args:
             timeArg: a python object representing a date and time. Supported
             types include datetime, struct_time, float, int, long and the
             dynamicserialize types Date and Timestamp.
-
+    
     Returns:
         A datetime that represents the same date/time as the passed in object.
     """
@@ -56,7 +77,6 @@ def convertToDateTime(timeArg):
         objType = str(type(timeArg))
         raise TypeError("Cannot convert object of type " + objType + " to datetime.")
 
-
 def _convertSecsAndMicros(seconds, micros):
     if seconds < MAX_TIME:
         rval = datetime.datetime.utcfromtimestamp(seconds)
@@ -65,20 +85,19 @@ def _convertSecsAndMicros(seconds, micros):
         rval = datetime.datetime.utcfromtimestamp(MAX_TIME) + extraTime
     return rval.replace(microsecond=micros)
 
-
 def constructTimeRange(*args):
-    """
+    """ 
     Builds a python dynamicserialize TimeRange object from the given
     arguments.
-
+    
     Args:
-            args*: must be a TimeRange or a pair of objects that can be
+            args*: must be a TimeRange or a pair of objects that can be 
                    converted to a datetime via convertToDateTime().
-
+    
     Returns:
         A TimeRange.
     """
-
+    
     if len(args) == 1 and isinstance(args[0], TimeRange):
         return args[0]
     if len(args) != 2:

awips/NotificationMessage.py

@@ -0,0 +1,136 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+
+import socket
+import sys
+
+from . import ThriftClient
+from dynamicserialize.dstypes.com.raytheon.uf.common.alertviz import AlertVizRequest
+
+#
+# Provides a capability of constructing notification messages and sending 
+# them to a STOMP data source.
+#  
+#    
+#     SOFTWARE HISTORY
+#    
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    09/30/08                      chammack       Initial Creation.
+#    11/03/10        5849          cjeanbap       Moved to ufpy package from
+#                                                     com.raytheon.uf.tools.cli
+#    01/07/11        5645          cjeanbap       Added audio file to Status Message.
+#    05/27/11        3050          cjeanbap       Added if-statement to check Priority
+#                                                 value
+#    07/27/15        4654          skorolev       Added filters
+#    11/11/15        5120          rferrel        Cannot serialize empty filters.
+#    03/05/18        6899          dgilling       Update to latest version of stomp.py API.
+#    09/14/18        7464          dgilling       Only serialize audioFile if provided.
+#    Apr 16, 2020    8144          tgurney        Change AlertViz stomp port to
+#                                                 calculated value based on uid
+#    May 15, 2020    8144          tgurney        Remove local-delivery logic
+#                                                 (no longer works as of 19.3.4)
+
+class NotificationMessage:
+
+   priorityMap = { 
+             0: 'CRITICAL',
+             1: 'SIGNIFICANT',
+             2: 'PROBLEM',
+             3: 'EVENTA',
+             4: 'EVENTB',
+             5: 'VERBOSE'}
+
+   def __init__(self, host='localhost', port=9581, message='', priority='PROBLEM', category="LOCAL", source="ANNOUNCER", audioFile=None, filters=None):
+      self.host = host
+      self.port = port
+      self.message = message
+      self.audioFile = audioFile
+      self.source = source
+      self.category = category 
+      self.filters = filters
+
+      priorityInt = None
+
+      try:
+         priorityInt = int(priority)
+      except:
+         pass
+
+      if priorityInt is None:
+          #UFStatus.java contains mapping of Priority to Logging level mapping
+          if priority == 'CRITICAL' or priority == 'FATAL':
+            priorityInt = int(0)
+          elif priority == 'SIGNIFICANT' or priority == 'ERROR':
+              priorityInt = int(1)
+          elif priority == 'PROBLEM' or priority == 'WARN':
+              priorityInt = int(2)
+          elif priority == 'EVENTA' or priority == 'INFO':
+              priorityInt = int(3)
+          elif priority == 'EVENTB':
+              priorityInt = int(4)
+          elif priority == 'VERBOSE' or priority == 'DEBUG':
+              priorityInt = int(5)
+
+      if (priorityInt < 0 or priorityInt > 5):
+          print("Error occurred, supplied an invalid Priority value:", str(priorityInt))
+          print("Priority values are 0, 1, 2, 3, 4 and 5.")
+          sys.exit(1)
+
+      if priorityInt is not None:
+          self.priority = self.priorityMap[priorityInt]
+      else:
+          self.priority = priority
+
+   def send(self):
+        alertVizRequest = createRequest(self.message, self.priority, self.source, self.category, self.audioFile, self.filters)
+        thriftClient = ThriftClient.ThriftClient(self.host, self.port, "/services")
+
+        serverResponse = None
+        try:
+            serverResponse = thriftClient.sendRequest(alertVizRequest)
+        except Exception as ex:
+            print("Caught exception submitting AlertVizRequest:", str(ex))    
+
+        if (serverResponse != "None"):
+            print("Error occurred submitting Notification Message to AlertViz receiver:", serverResponse)
+            sys.exit(1)
+        else:
+            print("Response:", str(serverResponse))        
+
+def createRequest(message, priority, source, category, audioFile, filters):    
+    obj = AlertVizRequest()
+
+    obj.setMachine(socket.gethostname())    
+    obj.setPriority(priority)
+    obj.setCategory(category)
+    obj.setSourceKey(source)    
+    obj.setMessage(message)
+    if (audioFile is not None):
+        obj.setAudioFile(audioFile)
+    else:
+        obj.setAudioFile('\0')
+    obj.setFilters(filters)
+    return obj
+
+if __name__ == '__main__':
+    main()

awips/QpidSubscriber.py

@@ -1,103 +1,170 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
 #
 # Provides a Python-based interface for subscribing to qpid queues and topics.
+#   
+# SOFTWARE HISTORY
 #
+# Date          Ticket#  Engineer     Description
+# ------------- -------- ------------ --------------------------------------------
+# Nov 17, 2010           njensen      Initial Creation.
+# Aug 15, 2013  2169     bkowal       Optionally gzip decompress any data that is read.
+# Aug 04, 2016  2416     tgurney      Add queueStarted property
+# Feb 16, 2017  6084     bsteffen     Support ssl connections
+# Sep 07, 2017  6175     tgurney      Remove "decompressing" log message
+# Jul 23, 2019  7724     mrichardson  Upgrade Qpid to Qpid Proton
+# Nov 04, 2019  7724     tgurney      Fix topic creation
+# Jun 24, 2020  8187     randerso     Added qpid connection_id
 #
-#
+
-#     SOFTWARE HISTORY
+from __future__ import print_function
-#
+import logging
-#    Date            Ticket#       Engineer       Description
-#    ------------    ----------    -----------    --------------------------
-#    11/17/10                      njensen       Initial Creation.
-#    08/15/13        2169          bkowal        Optionally gzip decompress any data that is read.
-#    08/04/16        2416          tgurney       Add queueStarted property
-#    02/16/17        6084          bsteffen      Support ssl connections
-#    09/07/17        6175          tgurney       Remove "decompressing" log message
-#
-#
 
 import os
 import os.path
-import qpid
+import pwd
+import socket
 import zlib
+from ssl import SSLContext, PROTOCOL_TLS
 
-from Queue import Empty
+from proton import SSLDomain
-from qpid.exceptions import Closed
+from proton.handlers import MessagingHandler
+from proton.reactor import Container
 
+logging.basicConfig(level=logging.INFO, datefmt='%H:%M:%S',
+                    format='[%(process)s] %(asctime)s %(levelname)s: %(message)s')
+log = logging.getLogger('QpidSubscriber')
 
-class QpidSubscriber:
+SSL_PASSWORD = 'password'
+QPID_USERNAME = 'guest'
+QPID_PASSWORD = 'guest'
 
-    def __init__(self, host='127.0.0.1', port=5672, decompress=False, ssl=None):
+class QpidSubscriber(MessagingHandler):
+    
+    def __init__(self, host='127.0.0.1', port=5672, decompress=False, ssl=None, program="QpidSubscriber"):
+        super(QpidSubscriber, self).__init__(auto_accept=True)
+        #__init__ should only handle setting up properties;
+        # any connection and subscription actions should be handled
+        # by the reactor functions
+        
+        self.queues = {}
+        
+        self.scheme = 'amqp'
+        self.rest_scheme = 'https'
+        self.ssl_context = None
         self.host = host
         self.port = port
         self.decompress = decompress
-        socket = qpid.util.connect(host, port)
+        self.__queueStarted = False
+        self.__subscribed = False
+        
+        pwuid = pwd.getpwuid(os.getuid())
         if "QPID_SSL_CERT_DB" in os.environ:
-            certdb = os.environ["QPID_SSL_CERT_DB"]
+            certdbloc = os.environ["QPID_SSL_CERT_DB"]
         else:
-            certdb = os.path.expanduser("~/.qpid/")
+            certdbloc = pwuid.pw_dir + "/.qpid/"
         if "QPID_SSL_CERT_NAME" in os.environ:
             certname = os.environ["QPID_SSL_CERT_NAME"]
         else:
-            certname = "guest"
+            certname = QPID_USERNAME
-        certfile = os.path.join(certdb, certname + ".crt")
+        
-        if ssl or (ssl is None and os.path.exists(certfile)):
+        certfile = os.path.join(certdbloc, certname + ".crt")
-            keyfile = os.path.join(certdb, certname + ".key")
+        certkey = os.path.join(certdbloc, certname + ".key")
-            trustfile = os.path.join(certdb, "root.crt")
+        if ssl or (ssl is None and os.path.isfile(certfile) and os.path.isfile(certkey)):
-            socket = qpid.util.ssl(socket, keyfile=keyfile, certfile=certfile, ca_certs=trustfile)
+            self.scheme = "amqps"
-        self.__connection = qpid.connection.Connection(sock=socket, username='guest', password='guest')
+            self.rest_scheme = 'https'
-        self.__connection.start()
+            self.ssl_context = SSLContext(PROTOCOL_TLS)
-        self.__session = self.__connection.session(str(qpid.datatypes.uuid4()))
+            self.ssl_context.load_cert_chain(certfile, certkey)
-        self.subscribed = True
+            self.cert_file = certfile
-        self.__queueStarted = False
+            self.cert_key = certkey
+        self.url = '{}://{}:{}@{}:{}'.format(self.scheme, QPID_USERNAME, QPID_PASSWORD, self.host, self.port)
+        self.clientID = ":".join([
+            socket.gethostname(), 
+            pwuid.pw_name, 
+            program, 
+            str(os.getpid()), 
+        ])
 
     def topicSubscribe(self, topicName, callback):
+        self.topicName = topicName
+        self.callback = callback
+        Container(self).run()
+
+    def on_start(self, event):
+        '''
         # if the queue is edex.alerts, set decompress to true always for now to
         # maintain compatibility with existing python scripts.
-        if topicName == 'edex.alerts':
+        '''
+        if self.topicName == 'edex.alerts':
             self.decompress = True
 
-        print("Establishing connection to broker on", self.host)
+        self.container = event.container
-        queueName = topicName + self.__session.name
+        queueName = 'amq.topic/' + self.topicName
-        self.__session.queue_declare(queue=queueName, exclusive=True, auto_delete=True,
-                                     arguments={'qpid.max_count': 100, 'qpid.policy_type': 'ring'})
-        self.__session.exchange_bind(exchange='amq.topic', queue=queueName, binding_key=topicName)
-        self.__innerSubscribe(queueName, callback)
 
-    def __innerSubscribe(self, serverQueueName, callback):
+        self.ssl_domain = None
-        local_queue_name = 'local_queue_' + serverQueueName
+        if self.scheme == "amqps" and self.cert_file and self.cert_key:
-        queue = self.__session.incoming(local_queue_name)
+            self.ssl_domain = SSLDomain(mode=SSLDomain.MODE_CLIENT)
-        self.__session.message_subscribe(serverQueueName, destination=local_queue_name)
+            self.ssl_domain.set_credentials(self.cert_file, self.cert_key, SSL_PASSWORD)
-        queue.start()
+
-        print("Connection complete to broker on", self.host)
+        event.container.container_id = self.clientID
+        self.conn = event.container.connect(self.url, ssl_domain=self.ssl_domain)
+        self.receiver = event.container.create_receiver(self.conn, queueName)
         self.__queueStarted = True
+        self.__subscribed = True
 
-        while self.subscribed:
+    def on_message(self, event):
+        message = event.message
+        content = message.body
+        self.process_message(content)
+        if not self.__subscribed:
+            self.close()
+
+    def process_message(self, content):
+        if (self.decompress):
             try:
-                message = queue.get(timeout=10)
+                # http://stackoverflow.com/questions/2423866/python-decompressing-gzip-chunk-by-chunk
-                content = message.body
+                d = zlib.decompressobj(16 + zlib.MAX_WBITS)
-                self.__session.message_accept(qpid.datatypes.RangedSet(message.id))
+                content = d.decompress(content)
-                if self.decompress:
+            except Exception:
-                    try:
+                # decompression failed, return the original content
-                        # http://stackoverflow.com/questions/2423866/python-decompressing-gzip-chunk-by-chunk
-                        d = zlib.decompressobj(16+zlib.MAX_WBITS)
-                        content = d.decompress(content)
-                    except ValueError:
-                        # decompression failed, return the original content
-                        pass
-                callback(content)
-            except Empty:
                 pass
-            except Closed:
+        self.callback(content)
-                self.close()
 
     def close(self):
         self.__queueStarted = False
-        self.subscribed = False
+        self.unsubscribe()
         try:
-            self.__session.close(timeout=10)
+            self.receiver.close()
-        except ValueError:
+            self.conn.close()
+        except:
+            # already closed
             pass
 
     @property
     def queueStarted(self):
         return self.__queueStarted
+
+    @property
+    def subscribed(self):
+        return self.__subscribed
+
+    def unsubscribe(self):
+        self.__subscribed = False

awips/ThriftClient.py

@@ -1,80 +1,102 @@
-#
+##
-# Provides a Python-based interface for executing Thrift requests.
+# This software was developed and / or modified by Raytheon Company,
-#
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
-#
+# 
-#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
-#     SOFTWARE HISTORY
+# This software product contains export-restricted data whose
-#
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
-#    Date            Ticket#       Engineer       Description
+# to non-U.S. persons whether in the United States or abroad requires
-#    ------------    ----------    -----------    --------------------------
+# an export license or other authorization.
-#    09/20/10                      dgilling       Initial Creation.
+# 
-#
+# Contractor Name:        Raytheon Company
-#
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+import http.client
 
-try:
-    import http.client as httpcl
-except ImportError:
-    import httplib as httpcl
 from dynamicserialize import DynamicSerializationManager
 
 
-class ThriftClient:
+#
+# Provides a Python-based interface for executing Thrift requests.
+#   
+#
+#    
+#     SOFTWARE HISTORY
+#    
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    09/20/10                      dgilling       Initial Creation.
+#    
+# 
+#
 
+
+class ThriftClient:
+    
     # How to call this constructor:
-    #   1. Pass in all arguments separately (e.g.,
+    #   1. Pass in all arguments separately (e.g., 
     #      ThriftClient.ThriftClient("localhost", 9581, "/services"))
     #      will return a Thrift client pointed at http://localhost:9581/services.
-    #   2. Pass in all arguments through the host string (e.g.,
+    #   2. Pass in all arguments through the host string (e.g., 
     #      ThriftClient.ThriftClient("localhost:9581/services"))
     #      will return a Thrift client pointed at http://localhost:9581/services.
-    #   3. Pass in host/port arguments through the host string (e.g.,
+    #   3. Pass in host/port arguments through the host string (e.g., 
     #      ThriftClient.ThriftClient("localhost:9581", "/services"))
     #      will return a Thrift client pointed at http://localhost:9581/services.
     def __init__(self, host, port=9581, uri="/services"):
         hostParts = host.split("/", 1)
-        if len(hostParts) > 1:
+        if (len(hostParts) > 1):
             hostString = hostParts[0]
             self.__uri = "/" + hostParts[1]
-            self.__httpConn = httpcl.HTTPConnection(hostString)
+            self.__httpConn = http.client.HTTPConnection(hostString)
         else:
-            if port is None:
+            if (port is None):
-                self.__httpConn = httpcl.HTTPConnection(host)
+                self.__httpConn = http.client.HTTPConnection(host)
             else:
-                self.__httpConn = httpcl.HTTPConnection(host, port)
+                self.__httpConn = http.client.HTTPConnection(host, port)
-
+            
             self.__uri = uri
-
+        
         self.__dsm = DynamicSerializationManager.DynamicSerializationManager()
-
+        
     def sendRequest(self, request, uri="/thrift"):
         message = self.__dsm.serializeObject(request)
-
+        
         self.__httpConn.connect()
         self.__httpConn.request("POST", self.__uri + uri, message)
-
+        
         response = self.__httpConn.getresponse()
-        if response.status != 200:
+        if (response.status != 200):
             raise ThriftRequestException("Unable to post request to server")
-
+        
         rval = self.__dsm.deserializeBytes(response.read())
         self.__httpConn.close()
-
+        
         # let's verify we have an instance of ServerErrorResponse
         # IF we do, through an exception up to the caller along
         # with the original Java stack trace
         # ELSE: we have a valid response and pass it back
         try:
-            forceError = rval.getException()
+           forceError = rval.getException()          
-            raise ThriftRequestException(forceError)
+           raise ThriftRequestException(forceError)
         except AttributeError:
             pass
-
+        
         return rval
-
+    
-
+    
 class ThriftRequestException(Exception):
     def __init__(self, value):
         self.parameter = value
-
+    
     def __str__(self):
         return repr(self.parameter)
+    
+        

awips/TimeUtil.py

@@ -1,3 +1,22 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
 # ----------------------------------------------------------------------------
 # This software is in the public domain, furnished "as is", without technical
 # support, and with no warranty, express or implied, as to its usefulness for
@@ -9,9 +28,9 @@
 # Author: hansen/romberg
 # ----------------------------------------------------------------------------
 
-import string
 import time
 
+
 # Given the timeStr, return the offset (in seconds)
 # from the current time.
 # Also return the launchStr i.e. Programs launched from this
@@ -20,7 +39,7 @@ import time
 # negative for time in the past.
 #
 # May still want it to be normalized to the most recent midnight.
-#
+# 
 # NOTES about synchronizing:
 # --With synchronizing on, the "current time" for all processes started
 #   within a given hour will be the same.
@@ -33,7 +52,7 @@ import time
 #     For example, if someone starts the GFE at 12:59 and someone
 #     else starts it at 1:01, they will have different offsets and
 #     current times.
-# --With synchronizing off, when the process starts, the current time
+# --With synchronizing off, when the process starts, the current time 
 #   matches the drtTime in the command line.  However, with synchronizing
 #   on, the current time will be offset by the fraction of the hour at
 #   which the process was started. Examples:
@@ -44,15 +63,14 @@ import time
 #     Synchronizing on:
 #        GFE Spatial Editor at StartUp: 20040616_0030
 #
-
-
 def determineDrtOffset(timeStr):
     launchStr = timeStr
     # Check for time difference
-    if timeStr.find(",") >= 0:
+    if timeStr.find(",") >=0:
         times = timeStr.split(",")
         t1 = makeTime(times[0])
         t2 = makeTime(times[1])
+        #print "time offset", t1-t2, (t1-t2)/3600
         return t1-t2, launchStr
     # Check for synchronized mode
     synch = 0
@@ -60,28 +78,30 @@ def determineDrtOffset(timeStr):
         timeStr = timeStr[1:]
         synch = 1
     drt_t = makeTime(timeStr)
+    #print "input", year, month, day, hour, minute
     gm = time.gmtime()
     cur_t = time.mktime(gm)
-
+    
     # Synchronize to most recent hour
     # i.e. "truncate" cur_t to most recent hour.
+    #print "gmtime", gm
     if synch:
         cur_t = time.mktime((gm[0], gm[1], gm[2], gm[3], 0, 0, 0, 0, 0))
-        curStr = '%4s%2s%2s_%2s00\n' % (repr(gm[0]), repr(gm[1]),
+        curStr = time.strftime('%Y%m%d_%H00\n', gm)
-                                        repr(gm[2]), repr(gm[3]))
-        curStr = curStr.replace(' ', '0')
         launchStr = timeStr + "," + curStr
-
+    
-    offset = drt_t - cur_t
+    #print "drt, cur", drt_t, cur_t
+    offset = drt_t - cur_t    
+    #print "offset", offset, offset/3600, launchStr
     return int(offset), launchStr
 
-
 def makeTime(timeStr):
-    year = string.atoi(timeStr[0:4])
+    year = int(timeStr[0:4])
-    month = string.atoi(timeStr[4:6])
+    month = int(timeStr[4:6])
-    day = string.atoi(timeStr[6:8])
+    day = int(timeStr[6:8])
-    hour = string.atoi(timeStr[9:11])
+    hour = int(timeStr[9:11])
-    minute = string.atoi(timeStr[11:13])
+    minute = int(timeStr[11:13])
     # Do not use daylight savings because gmtime is not in daylight
     # savings time.
     return time.mktime((year, month, day, hour, minute, 0, 0, 0, 0))
+

awips/UsageArgumentParser.py

@@ -1,3 +1,22 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
 #
 # SOFTWARE HISTORY
 #
@@ -5,6 +24,7 @@
 # ------------- -------- --------- ---------------------------------------------
 # Feb 13, 2017  6092     randerso  Added StoreTimeAction
 #
+##
 
 import argparse
 import sys
@@ -15,7 +35,6 @@ from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects i
 
 TIME_FORMAT = "%Y%m%d_%H%M"
 
-
 class UsageArgumentParser(argparse.ArgumentParser):
     """
     A subclass of ArgumentParser that overrides error() to print the
@@ -26,8 +45,7 @@ class UsageArgumentParser(argparse.ArgumentParser):
         self.print_help()
         sys.exit(2)
 
-
+## Custom actions for ArgumentParser objects ##
-# Custom actions for ArgumentParser objects
 class StoreDatabaseIDAction(argparse.Action):
     def __call__(self, parser, namespace, values, option_string=None):
         did = DatabaseID(values)
@@ -36,19 +54,18 @@ class StoreDatabaseIDAction(argparse.Action):
         else:
             parser.error("DatabaseID [" + values + "] not a valid identifier")
 
-
 class AppendParmNameAndLevelAction(argparse.Action):
     def __call__(self, parser, namespace, values, option_string=None):
         tx = ParmID.parmNameAndLevel(values)
         comp = tx[0] + '_' + tx[1]
-        if (hasattr(namespace, self.dest)) and (getattr(namespace, self.dest) is not None):
+        if (hasattr(namespace, self.dest)) and \
+            (getattr(namespace, self.dest) is not None):
                 currentValues = getattr(namespace, self.dest)
                 currentValues.append(comp)
                 setattr(namespace, self.dest, currentValues)
         else:
             setattr(namespace, self.dest, [comp])
 
-
 class StoreTimeAction(argparse.Action):
     """
     argparse.Action subclass to validate GFE formatted time strings
@@ -57,6 +74,8 @@ class StoreTimeAction(argparse.Action):
     def __call__(self, parser, namespace, values, option_string=None):
         try:
             timeStruct = time.strptime(values, TIME_FORMAT)
-            setattr(namespace, self.dest, timeStruct)
+        except:
-        except ValueError:
             parser.error(str(values) + " is not a valid time string of the format YYYYMMDD_hhmm")
+
+        setattr(namespace, self.dest, timeStruct)
+

awips/UsageOptionParser.py

@@ -1,7 +1,26 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
 import sys
 from optparse import OptionParser
 
-
 class UsageOptionParser(OptionParser):
     """
     A subclass of OptionParser that prints that overrides error() to print the
@@ -16,3 +35,4 @@ class UsageOptionParser(OptionParser):
         sys.stderr.write(msg)
         sys.stderr.write("\n")
         sys.exit(2)
+

awips/__init__.py

@@ -1,3 +1,37 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
 
-__all__ = [
+
+#
+# __init__.py for ufpy package
+#  
+#    
+#     SOFTWARE HISTORY
+#    
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    09/21/10                      dgilling       Initial Creation.
+#    
+# 
+#
+
+
+__all__ = [           
            ]

awips/dataaccess/CombinedTimeQuery.py

@@ -1,23 +1,41 @@
+# #
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+# #
+
 #
 # Method for performing a DAF time query where all parameter/level/location
 # combinations must be available at the same time.
 #
-#
+#  
-#
+#    
 #     SOFTWARE HISTORY
-#
+#    
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
 #    06/22/16         #5591        bsteffen       Initial Creation.
 #
 
-from awips.dataaccess import DataAccessLayer
+from ufpy.dataaccess import DataAccessLayer
-
 
 def getAvailableTimes(request, refTimeOnly=False):
     return __getAvailableTimesForEachParameter(request, refTimeOnly)
 
-
 def __getAvailableTimesForEachParameter(request, refTimeOnly=False):
     parameters = request.getParameters()
     if parameters:
@@ -36,7 +54,6 @@ def __getAvailableTimesForEachParameter(request, refTimeOnly=False):
     else:
         return __getAvailableTimesForEachLevel(request, refTimeOnly)
 
-
 def __getAvailableTimesForEachLevel(request, refTimeOnly=False):
     levels = request.getLevels()
     if levels:
@@ -55,7 +72,6 @@ def __getAvailableTimesForEachLevel(request, refTimeOnly=False):
     else:
         return __getAvailableTimesForEachLocation(request, refTimeOnly)
 
-
 def __getAvailableTimesForEachLocation(request, refTimeOnly=False):
     locations = request.getLocationNames()
     if locations:
@@ -73,12 +89,12 @@ def __getAvailableTimesForEachLocation(request, refTimeOnly=False):
         return times
     else:
         return DataAccessLayer.getAvailableTimes(request, refTimeOnly)
-
+        
-
+        
 def __cloneRequest(request):
-    return DataAccessLayer.newDataRequest(datatype=request.getDatatype(),
+    return DataAccessLayer.newDataRequest(datatype      = request.getDatatype(),
-                                          parameters=request.getParameters(),
+                                          parameters    = request.getParameters(),
-                                          levels=request.getLevels(),
+                                          levels        = request.getLevels(),
-                                          locationNames=request.getLocationNames(),
+                                          locationNames = request.getLocationNames(),
-                                          envelope=request.getEnvelope(),
+                                          envelope      = request.getEnvelope(),
-                                          **request.getIdentifiers())
+                                          **request.getIdentifiers())

awips/dataaccess/DataAccessLayer.py

@@ -1,33 +1,62 @@
+# #
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
 #
-# Published interface for awips.dataaccess package
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+# #
+
+
+#
+# Published interface for ufpy.dataaccess package
 #
 #
 #     SOFTWARE HISTORY
 #
-#    Date           Ticket#  Engineer    Description
+#    Date            Ticket#       Engineer       Description
-#    ------------   -------  ----------  -------------------------
+#    ------------    ----------    -----------    --------------------------
-#    12/10/12                njensen     Initial Creation.
+#    12/10/12                      njensen        Initial Creation.
-#    Feb 14, 2013   1614     bsteffen    refactor data access framework to use single request.
+#    Feb 14, 2013    1614          bsteffen       refactor data access framework
-#    04/10/13       1871     mnash       move getLatLonCoords to JGridData and add default args
+#                                                 to use single request.
-#    05/29/13       2023     dgilling    Hook up ThriftClientRouter.
+#    04/10/13        1871          mnash          move getLatLonCoords to JGridData and add default args
-#    03/03/14       2673     bsteffen    Add ability to query only ref times.
+#    05/29/13        2023          dgilling       Hook up ThriftClientRouter.
-#    07/22/14       3185     njensen     Added optional/default args to newDataRequest
+#    03/03/14        2673          bsteffen       Add ability to query only ref times.
-#    07/30/14       3185     njensen     Renamed valid identifiers to optional
+#    07/22/14        3185          njensen        Added optional/default args to newDataRequest
-#    Apr 26, 2015   4259     njensen     Updated for new JEP API
+#    07/30/14        3185          njensen        Renamed valid identifiers to optional
-#    Apr 13, 2016   5379     tgurney     Add getIdentifierValues(), getRequiredIdentifiers(),
+#    Apr 26, 2015    4259          njensen        Updated for new JEP API
-#                                        and getOptionalIdentifiers()
+#    Apr 13, 2016    5379          tgurney        Add getIdentifierValues()
-#    Oct 07, 2016   ----     mjames@ucar Added getForecastRun
+#    Jun 01, 2016    5587          tgurney        Add new signatures for
-#    Oct 18, 2016   5916     bsteffen    Add setLazyLoadGridLatLon
+#                                                 getRequiredIdentifiers() and
-#    Oct 11, 2018   ----     mjames@ucar Added getMetarObs() getSynopticObs()
+#                                                 getOptionalIdentifiers()
+#    Oct 18, 2016    5916          bsteffen       Add setLazyLoadGridLatLon
+#
 #
 
+
 import sys
+import subprocess
 import warnings
 
-THRIFT_HOST = "edex"
+THRIFT_HOST = subprocess.check_output(
+                    "source /awips2/fxa/bin/setup.env; echo $DEFAULT_HOST",
+                    shell=True).decode().strip()
+
 
 USING_NATIVE_THRIFT = False
 
+
 if 'jep' in sys.modules:
     # intentionally do not catch if this fails to import, we want it to
     # be obvious that something is configured wrong when running from within
@@ -35,152 +64,11 @@ if 'jep' in sys.modules:
     import JepRouter
     router = JepRouter
 else:
-    from awips.dataaccess import ThriftClientRouter
+    from ufpy.dataaccess import ThriftClientRouter
     router = ThriftClientRouter.ThriftClientRouter(THRIFT_HOST)
     USING_NATIVE_THRIFT = True
 
 
-def getRadarProductIDs(availableParms):
-    """
-    Get only the numeric idetifiers for NEXRAD3 products.
-
-    Args:
-            availableParms: Full list of radar parameters
-
-    Returns:
-            List of filtered parameters
-    """
-    productIDs = []
-    for p in list(availableParms):
-        try:
-            if isinstance(int(p), int):
-                productIDs.append(str(p))
-        except ValueError:
-            pass
-
-    return productIDs
-
-
-def getRadarProductNames(availableParms):
-    """
-     Get only the named idetifiers for NEXRAD3 products.
-
-    Args:
-            availableParms: Full list of radar parameters
-
-    Returns:
-            List of filtered parameters
-    """
-    productNames = []
-    for p in list(availableParms):
-        if len(p) > 3:
-            productNames.append(p)
-
-    return productNames
-
-
-def getMetarObs(response):
-    """
-    Processes a DataAccessLayer "obs" response into a dictionary,
-    with special consideration for multi-value parameters
-    "presWeather", "skyCover", and "skyLayerBase".
-
-    Args:
-            response: DAL getGeometry() list
-
-    Returns:
-            A dictionary of METAR obs
-    """
-    from datetime import datetime
-    single_val_params = ["timeObs", "stationName", "longitude", "latitude",
-                         "temperature", "dewpoint", "windDir",
-                         "windSpeed", "seaLevelPress"]
-    multi_val_params = ["presWeather", "skyCover", "skyLayerBase"]
-    params = single_val_params + multi_val_params
-    station_names, pres_weather, sky_cov, sky_layer_base = [], [], [], []
-    obs = dict({params: [] for params in params})
-    for ob in response:
-        avail_params = ob.getParameters()
-        if "presWeather" in avail_params:
-            pres_weather.append(ob.getString("presWeather"))
-        elif "skyCover" in avail_params and "skyLayerBase" in avail_params:
-            sky_cov.append(ob.getString("skyCover"))
-            sky_layer_base.append(ob.getNumber("skyLayerBase"))
-        else:
-            # If we already have a record for this stationName, skip
-            if ob.getString('stationName') not in station_names:
-                station_names.append(ob.getString('stationName'))
-                for param in single_val_params:
-                    if param in avail_params:
-                        if param == 'timeObs':
-                            obs[param].append(datetime.fromtimestamp(ob.getNumber(param) / 1000.0))
-                        else:
-                            try:
-                                obs[param].append(ob.getNumber(param))
-                            except TypeError:
-                                obs[param].append(ob.getString(param))
-                    else:
-                        obs[param].append(None)
-
-                obs['presWeather'].append(pres_weather)
-                obs['skyCover'].append(sky_cov)
-                obs['skyLayerBase'].append(sky_layer_base)
-                pres_weather = []
-                sky_cov = []
-                sky_layer_base = []
-    return obs
-
-
-def getSynopticObs(response):
-    """
-    Processes a DataAccessLayer "sfcobs" response into a dictionary
-    of available parameters.
-
-    Args:
-            response: DAL getGeometry() list
-
-    Returns:
-            A dictionary of synop obs
-    """
-    from datetime import datetime
-    station_names = []
-    params = response[0].getParameters()
-    sfcobs = dict({params: [] for params in params})
-    for sfcob in response:
-        # If we already have a record for this stationId, skip
-        if sfcob.getString('stationId') not in station_names:
-            station_names.append(sfcob.getString('stationId'))
-            for param in params:
-                if param == 'timeObs':
-                    sfcobs[param].append(datetime.fromtimestamp(sfcob.getNumber(param) / 1000.0))
-                else:
-                    try:
-                        sfcobs[param].append(sfcob.getNumber(param))
-                    except TypeError:
-                        sfcobs[param].append(sfcob.getString(param))
-
-    return sfcobs
-
-
-def getForecastRun(cycle, times):
-    """
-    Get the latest forecast run (list of objects) from all
-    all cycles and times returned from DataAccessLayer "grid"
-    response.
-
-    Args:
-            cycle: Forecast cycle reference time
-            times: All available times/cycles
-
-    Returns:
-            DataTime array for a single forecast run
-    """
-    fcstRun = []
-    for t in times:
-        if str(t)[:19] == str(cycle):
-            fcstRun.append(t)
-    return fcstRun
-
 
 def getAvailableTimes(request, refTimeOnly=False):
     """
@@ -189,7 +77,7 @@ def getAvailableTimes(request, refTimeOnly=False):
     Args:
             request: the IDataRequest to get data for
             refTimeOnly: optional, use True if only unique refTimes should be
-                returned (without a forecastHr)
+                          returned (without a forecastHr)
 
     Returns:
             a list of DataTimes
@@ -206,7 +94,7 @@ def getGridData(request, times=[]):
     Args:
             request: the IDataRequest to get data for
             times: a list of DataTimes, a TimeRange, or None if the data is time
-                agnostic
+                    agnostic
 
     Returns:
             a list of IGridData
@@ -223,10 +111,10 @@ def getGeometryData(request, times=[]):
     Args:
             request: the IDataRequest to get data for
             times: a list of DataTimes, a TimeRange, or None if the data is time
-                agnostic
+                    agnostic
 
     Returns:
-       a list of IGeometryData
+            a list of IGeometryData
     """
     return router.getGeometryData(request, times)
 
@@ -319,9 +207,8 @@ def getIdentifierValues(request, identifierKey):
     """
     return router.getIdentifierValues(request, identifierKey)
 
-
 def newDataRequest(datatype=None, **kwargs):
-    """
+    """"
     Creates a new instance of IDataRequest suitable for the runtime environment.
     All args are optional and exist solely for convenience.
 
@@ -331,14 +218,13 @@ def newDataRequest(datatype=None, **kwargs):
             levels: a list of levels to set on the request
             locationNames: a list of locationNames to set on the request
             envelope: an envelope to limit the request
-            kwargs: any leftover kwargs will be set as identifiers
+            **kwargs: any leftover kwargs will be set as identifiers
 
     Returns:
             a new IDataRequest
     """
     return router.newDataRequest(datatype, **kwargs)
 
-
 def getSupportedDatatypes():
     """
     Gets the datatypes that are supported by the framework
@@ -356,7 +242,7 @@ def changeEDEXHost(newHostName):
     method will throw a TypeError.
 
     Args:
-            newHostName: the EDEX host to connect to
+            newHostHame: the EDEX host to connect to
     """
     if USING_NATIVE_THRIFT:
         global THRIFT_HOST
@@ -366,7 +252,6 @@ def changeEDEXHost(newHostName):
     else:
         raise TypeError("Cannot call changeEDEXHost when using JepRouter.")
 
-
 def setLazyLoadGridLatLon(lazyLoadGridLatLon):
     """
     Provide a hint to the Data Access Framework indicating whether to load the
@@ -379,7 +264,7 @@ def setLazyLoadGridLatLon(lazyLoadGridLatLon):
     set to False if it is guaranteed that all lat/lon information is needed and
     it would be better to get any performance overhead for generating the
     lat/lon data out of the way during the initial request.
-
+    
 
     Args:
             lazyLoadGridLatLon: Boolean value indicating whether to lazy load.
@@ -388,4 +273,4 @@ def setLazyLoadGridLatLon(lazyLoadGridLatLon):
         router.setLazyLoadGridLatLon(lazyLoadGridLatLon)
     except AttributeError:
         # The router is not required to support this capability.
-        pass
+        pass

awips/dataaccess/DataNotificationLayer.py

@@ -1,13 +1,34 @@
+# #
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
 #
-# Published interface for retrieving data updates via awips.dataaccess package
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
 #
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
 #
-#     SOFTWARE HISTORY
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+# #
+
 #
-#    Date            Ticket#       Engineer       Description
+# Published interface for retrieving data updates via ufpy.dataaccess package
-#    ------------    ----------    -----------    --------------------------
+#
-#    May 26, 2016    2416          rjpeter        Initial Creation.
+# SOFTWARE HISTORY
-#    Aug  1, 2016    2416          tgurney        Finish implementation
+#
+# Date          Ticket#  Engineer     Description
+# ------------- -------- ------------ --------------------------------------------
+# May 26, 2016  2416     rjpeter      Initial Creation.
+# Aug 01, 2016  2416     tgurney      Finish implementation
+# Nov 05, 2019  7884     tgurney      Python 3 fixes
+# Jun 24, 2020  8187     randerso     Added program for qpid connection_id
 #
 #
 
@@ -17,7 +38,7 @@ retrieval of new data as it is coming into the system.
 
 There are two ways to access this feature:
 
-1. The DataQueue module (awips.dataaccess.DataQueue) offers a collection that
+1. The DataQueue module (ufpy.dataaccess.DataQueue) offers a collection that
 automatically fills up with new data as it receives notifications. See that
 module for more information.
 
@@ -29,8 +50,8 @@ each time new data is received.
 Example code follows. This example prints temperature as observed from KOMA
 each time a METAR is received from there.
 
-  from awips.dataaccess import DataAccessLayer as DAL
+  from ufpy.dataaccess import DataAccessLayer as DAL
-  from awips.dataaccess import DataNotificationLayer as DNL
+  from ufpy.dataaccess import DataNotificationLayer as DNL
 
   def process_obs(list_of_data):
       for item in list_of_data:
@@ -39,25 +60,25 @@ each time a METAR is received from there.
   request = DAL.newDataRequest('obs')
   request.setParameters('temperature')
   request.setLocationNames('KOMA')
-
+  
   notifier = DNL.getGeometryDataUpdates(request)
   notifier.subscribe(process_obs)
   # process_obs will called with a list of data each time new data comes in
 
 """
 
-import re
 import sys
-from awips.dataaccess.PyGeometryNotification import PyGeometryNotification
+import subprocess
-from awips.dataaccess.PyGridNotification import PyGridNotification
+from ufpy.dataaccess.PyGeometryNotification import PyGeometryNotification
+from ufpy.dataaccess.PyGridNotification import PyGridNotification
 
 
-THRIFT_HOST = "edex"
+THRIFT_HOST = subprocess.check_output(
+                    "source /awips2/fxa/bin/setup.env; echo $DEFAULT_HOST",
+                    shell=True).decode().strip()
 
 USING_NATIVE_THRIFT = False
 
-JMS_HOST_PATTERN = re.compile('tcp://([^:]+):([0-9]+)')
-
 if 'jep' in sys.modules:
     # intentionally do not catch if this fails to import, we want it to
     # be obvious that something is configured wrong when running from within
@@ -65,7 +86,7 @@ if 'jep' in sys.modules:
     import JepRouter
     router = JepRouter
 else:
-    from awips.dataaccess import ThriftClientRouter
+    from ufpy.dataaccess import ThriftClientRouter
     router = ThriftClientRouter.ThriftClientRouter(THRIFT_HOST)
     USING_NATIVE_THRIFT = True
 
@@ -73,9 +94,9 @@ else:
 def _getJmsConnectionInfo(notifFilterResponse):
     serverString = notifFilterResponse.getJmsConnectionInfo()
     try:
-        host, port = JMS_HOST_PATTERN.match(serverString).groups()
+        host, port = serverString.split(':')
-    except AttributeError:
+    except Exception as e:
-        raise RuntimeError('Got bad JMS connection info from server: ' + serverString)
+        raise ValueError('Got bad JMS connection info from server: "' + serverString + '"') from e
     return {'host': host, 'port': port}
 
 
@@ -91,10 +112,9 @@ def getGridDataUpdates(request):
             calling its subscribe() method
     """
     response = router.getNotificationFilter(request)
-    notificationFilter = response.getNotificationFilter()
+    filter = response.getNotificationFilter()
     jmsInfo = _getJmsConnectionInfo(response)
-    notifier = PyGridNotification(request, notificationFilter,
+    notifier = PyGridNotification(request, filter, requestHost=THRIFT_HOST, program="daf-gridDataUpdates", **jmsInfo)
-                                  requestHost=THRIFT_HOST, **jmsInfo)
     return notifier
 
 
@@ -110,10 +130,9 @@ def getGeometryDataUpdates(request):
             calling its subscribe() method
     """
     response = router.getNotificationFilter(request)
-    notificationFilter = response.getNotificationFilter()
+    filter = response.getNotificationFilter()
     jmsInfo = _getJmsConnectionInfo(response)
-    notifier = PyGeometryNotification(request, notificationFilter,
+    notifier = PyGeometryNotification(request, filter, requestHost=THRIFT_HOST, program="daf-geometryDataUpdates", **jmsInfo)
-                                      requestHost=THRIFT_HOST, **jmsInfo)
     return notifier
 
 
@@ -124,7 +143,7 @@ def changeEDEXHost(newHostName):
     method will throw a TypeError.
 
     Args:
-            newHostName: the EDEX host to connect to
+            newHostHame: the EDEX host to connect to
     """
     if USING_NATIVE_THRIFT:
         global THRIFT_HOST

awips/dataaccess/DataQueue.py

@@ -1,3 +1,23 @@
+# #
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+# #
+
 #
 #   Convenience class for using the DAF's notifications feature. This is a
 #   collection that, once connected to EDEX by calling start(), fills with
@@ -13,25 +33,24 @@
 #    07/29/16        2416          tgurney        Initial creation
 #
 
-from awips.dataaccess import DataNotificationLayer as DNL
+from ufpy.dataaccess import DataNotificationLayer as DNL
 
 import time
 from threading import Thread
-import sys
 
 
-if sys.version_info.major == 2:
+from queue import Queue, Empty
-    from Queue import Queue, Empty
-else:  # Python 3 module renamed to 'queue'
-    from queue import Queue, Empty
 
-# Used to indicate a DataQueue that will produce geometry data.
+
+"""Used to indicate a DataQueue that will produce geometry data."""
 GEOMETRY = object()
 
-# Used to indicate a DataQueue that will produce grid data.
+
+"""Used to indicate a DataQueue that will produce grid data."""
 GRID = object()
 
-# Default maximum queue size.
+
+"""Default maximum queue size."""
 _DEFAULT_MAXSIZE = 100
 
 
@@ -128,12 +147,12 @@ class DataQueue(object):
             return self._queue.get(block, timeout)
         except Empty:
             return None
-
+        
     def get_all(self):
         """
         Get all data waiting for processing, in a single list. Always returns
         immediately. Returns an empty list if no data has arrived yet.
-
+        
         Returns:
             List of IData
         """
@@ -187,4 +206,4 @@ class DataQueue(object):
         return self
 
     def __exit__(self, *unused):
-        self.close()
+        self.close()

awips/dataaccess/PyData.py

@@ -1,23 +1,40 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
 #
 # Implements IData for use by native Python clients to the Data Access
 # Framework.
-#
+#  
-#
+#    
 #     SOFTWARE HISTORY
-#
+#    
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
 #    06/03/13                      dgilling      Initial Creation.
-#    10/05/18                      mjames@ucar   Encode/decode attribute names.
+#    
-#
 #
 
-from awips.dataaccess import IData
+from ufpy.dataaccess import IData
-import six
-
 
 class PyData(IData):
-
+    
     def __init__(self, dataRecord):
         self.__time = dataRecord.getTime()
         self.__level = dataRecord.getLevel()
@@ -26,19 +43,15 @@ class PyData(IData):
 
     def getAttribute(self, key):
         return self.__attributes[key]
-
+    
     def getAttributes(self):
-        return self.__attributes.keys()
+        return list(self.__attributes.keys())
-
+    
     def getDataTime(self):
         return self.__time
-
+    
     def getLevel(self):
-        if six.PY2:
-            return self.__level
-        if not isinstance(self.__level, str):
-            return self.__level.decode('utf-8')
         return self.__level
-
+    
     def getLocationName(self):
         return self.__locationName

awips/dataaccess/PyGeometryData.py

@@ -1,81 +1,76 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
 #
 # Implements IGeometryData for use by native Python clients to the Data Access
 # Framework.
-#
+#  
-#
+#    
 #     SOFTWARE HISTORY
-#
+#    
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
 #    06/03/13                      dgilling       Initial Creation.
 #    01/06/14        2537          bsteffen       Share geometry WKT.
 #    03/19/14        2882          dgilling       Raise an exception when getNumber()
-#                                                 is called for data that is not a
+#                                                 is called for data that is not a 
 #                                                 numeric Type.
 #    06/09/16        5574          mapeters       Handle 'SHORT' type in getNumber().
-#    10/05/18                      mjames@ucar    Encode/decode string, number val, and type
+#    
-#
 #
 
-from awips.dataaccess import IGeometryData
+from ufpy.dataaccess import IGeometryData
-from awips.dataaccess import PyData
+from ufpy.dataaccess import PyData
-import six
-
 
 class PyGeometryData(IGeometryData, PyData.PyData):
-
+    
     def __init__(self, geoDataRecord, geometry):
         PyData.PyData.__init__(self, geoDataRecord)
         self.__geometry = geometry
         self.__dataMap = {}
         tempDataMap = geoDataRecord.getDataMap()
-        for key, value in list(tempDataMap.items()):
+        for key, value in tempDataMap.items():
             self.__dataMap[key] = (value[0], value[1], value[2])
 
     def getGeometry(self):
         return self.__geometry
-
+    
-    def getParameters(self):
+    def getParameters(self):        
-        if six.PY2:
+        return list(self.__dataMap.keys())
-            return list(self.__dataMap.keys())
+    
-        else:
-            return [x.decode('utf-8') for x in list(self.__dataMap.keys())]
-
     def getString(self, param):
-        if six.PY2:
+        value = self.__dataMap[param][0]
-            return self.__dataMap[param][0]
-        value = self.__dataMap[param.encode('utf-8')][0]
-        if isinstance(value, bytes):
-            return str(value.decode('utf-8'))
         return str(value)
-
+    
-    def getNumber(self, param):
+    def getNumber(self, param):         
-        t = self.getType(param)
+        value = self.__dataMap[param][0]
-        if six.PY2:
+        t = self.getType(param)        
-            value = self.__dataMap[param][0]
+        if t in ('INT', 'SHORT', 'LONG'):
-        else:
-            value = self.__dataMap[param.encode('utf-8')][0]
-        if t == 'INT' or t == 'SHORT' or t == 'LONG':
             return int(value)
-        elif t == 'FLOAT':
+        elif t in ('DOUBLE', 'FLOAT'):
-            return float(value)
-        elif t == 'DOUBLE':
             return float(value)
         else:
             raise TypeError("Data for parameter " + param + " is not a numeric type.")
-
+    
     def getUnit(self, param):
-        if six.PY2:
+        return self.__dataMap[param][2]
-            return self.__dataMap[param][2]
+    
-        unit = self.__dataMap[param.encode('utf-8')][2]
-        if unit is not None:
-            return unit.decode('utf-8')
-        return unit
-
     def getType(self, param):
-        if six.PY2:
+        return self.__dataMap[param][1]
-            return self.__dataMap[param][1]
-        datatype = self.__dataMap[param.encode('utf-8')][1]
-        if datatype is not None:
-            return datatype.decode('utf-8')
-        return datatype

awips/dataaccess/PyGeometryNotification.py

@@ -1,19 +1,39 @@
+# #
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+# #
+
 #
 # Notification object that produces geometry data
-#
+#  
-#
+#    
 #     SOFTWARE HISTORY
-#
+#    
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
-#    07/22/16        2416          tgurney        Initial creation
+#    07/22/2016      2416          tgurney        Initial creation
-#    09/07/17        6175          tgurney        Override messageReceived
+#    09/07/2017      6175          tgurney        Override messageReceived
+#    11/05/2019      7884          tgurney        Add missing import
 #
 
-import traceback
 import dynamicserialize
-from awips.dataaccess.PyNotification import PyNotification
+import traceback
-
+from ufpy.dataaccess.PyNotification import PyNotification
 
 class PyGeometryNotification(PyNotification):
 
@@ -28,7 +48,7 @@ class PyGeometryNotification(PyNotification):
             try:
                 data = self.getData(self.request, list(dataTimes))
                 self.callback(data)
-            except ValueError:
+            except Exception as e:
                 traceback.print_exc()
 
     def getData(self, request, dataTimes):

awips/dataaccess/PyGridData.py

@@ -1,25 +1,45 @@
+# #
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+# #
+
 #
 # Implements IGridData for use by native Python clients to the Data Access
 # Framework.
-#
+#  
-#
+#    
 #     SOFTWARE HISTORY
-#
+#    
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
 #    06/03/13         #2023        dgilling      Initial Creation.
 #    10/13/16         #5916        bsteffen      Correct grid shape, allow lat/lon
 #    11/10/16         #5900        bsteffen      Correct grid shape
 #                                                to be requested by a delegate
+#    
 #
-#
+
 
 import numpy
 import warnings
-import six
 
-from awips.dataaccess import IGridData
+from ufpy.dataaccess import IGridData
-from awips.dataaccess import PyData
+from ufpy.dataaccess import PyData
 
 NO_UNIT_CONVERT_WARNING = """
 The ability to unit convert grid data is not currently available in this version of the Data Access Framework.
@@ -27,8 +47,8 @@ The ability to unit convert grid data is not currently available in this version
 
 
 class PyGridData(IGridData, PyData.PyData):
-
+    
-    def __init__(self, gridDataRecord, nx, ny, latLonGrid=None, latLonDelegate=None):
+    def __init__(self, gridDataRecord, nx, ny, latLonGrid = None, latLonDelegate = None):
         PyData.PyData.__init__(self, gridDataRecord)
         nx = nx
         ny = ny
@@ -38,16 +58,13 @@ class PyGridData(IGridData, PyData.PyData):
         self.__latLonGrid = latLonGrid
         self.__latLonDelegate = latLonDelegate
 
+    
     def getParameter(self):
         return self.__parameter
-
+    
     def getUnit(self):
-        if six.PY2:
-            return self.__unit
-        if self.__unit is not None and not isinstance(self.__unit, str):
-            return self.__unit.decode('utf-8')
         return self.__unit
-
+    
     def getRawData(self, unit=None):
         # TODO: Find a proper python library that deals will with numpy and
         # javax.measure style unit strings and hook it in to this method to
@@ -55,7 +72,7 @@ class PyGridData(IGridData, PyData.PyData):
         if unit is not None:
             warnings.warn(NO_UNIT_CONVERT_WARNING, stacklevel=2)
         return self.__gridData
-
+    
     def getLatLonCoords(self):
         if self.__latLonGrid is not None:
             return self.__latLonGrid

awips/dataaccess/PyGridNotification.py

@@ -1,3 +1,23 @@
+# #
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+# #
+
 #
 # Notification object that produces grid data
 #
@@ -6,14 +26,14 @@
 #
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
-#    06/03/16        2416          rjpeter        Initial Creation.
+#    06/03/2016      2416          rjpeter        Initial Creation.
-#    09/06/17        6175          tgurney        Override messageReceived
+#    09/06/2017      6175          tgurney        Override messageReceived
+#    11/05/2019      7884          tgurney        Add missing import
 #
 
 import dynamicserialize
 import traceback
-from awips.dataaccess.PyNotification import PyNotification
+from ufpy.dataaccess.PyNotification import PyNotification
-
 
 class PyGridNotification(PyNotification):
 
@@ -33,7 +53,7 @@ class PyGridNotification(PyNotification):
                 newReq.setParameters(self.request.getParameters())
                 data = self.getData(newReq, [])
                 self.callback(data)
-            except ValueError:
+            except Exception as e:
                 traceback.print_exc()
 
     def getData(self, request, dataTimes):

awips/dataaccess/PyNotification.py

@@ -1,39 +1,61 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
 #
 # Implements IData for use by native Python clients to the Data Access
 # Framework.
 #
+# SOFTWARE HISTORY
 #
-#     SOFTWARE HISTORY
+# Date          Ticket#  Engineer     Description
-#
+# ------------- -------- ------------ --------------------------------------------
-#    Date            Ticket#       Engineer      Description
+# Jun 22, 2016  2416     rjpeter      Initial creation
-#    ------------    ----------    -----------   --------------------------
+# Jul 22, 2016  2416     tgurney      Finish implementation
-#    Jun 22, 2016    2416          rjpeter       Initial creation
+# Sep 07, 2017  6175     tgurney      Override messageReceived in subclasses
-#    Jul 22, 2016    2416          tgurney       Finish implementation
+# Nov 05, 2019  7884     tgurney      Fix in subscribed()
-#    Sep 07, 2017    6175          tgurney       Override messageReceived in subclasses
+# Jun 24, 2020  8187     randerso     Added program for qpid connection_id
 #
 
-from six import with_metaclass
+
 import abc
 
-from awips.dataaccess import DataAccessLayer
+from ufpy.dataaccess import DataAccessLayer
-from awips.dataaccess import INotificationSubscriber
+from ufpy.dataaccess import INotificationSubscriber
-from awips.QpidSubscriber import QpidSubscriber
+from ufpy.QpidSubscriber import QpidSubscriber
 from dynamicserialize.dstypes.com.raytheon.uf.common.time import DataTime
 
 
-class PyNotification(with_metaclass(abc.ABCMeta, INotificationSubscriber)):
+class PyNotification(INotificationSubscriber, metaclass=abc.ABCMeta):
     """
     Receives notifications for new data and retrieves the data that meets
     specified filtering criteria.
     """
 
-    def __init__(self, request, notificationFilter, host='localhost',
+    def __init__(self, request, filter, host='localhost', port=5672, requestHost='localhost', program="PyNotification"):
-                 port=5672, requestHost='localhost'):
         self.DAL = DataAccessLayer
         self.DAL.changeEDEXHost(requestHost)
         self.request = request
-        self.notificationFilter = notificationFilter
+        self.notificationFilter = filter
-        self.__topicSubscriber = QpidSubscriber(host, port, decompress=True)
+        self.host = host
+        self.port = port
+        self.program=program
         self.__topicName = "edex.alerts"
         self.callback = None
 
@@ -47,12 +69,12 @@ class PyNotification(with_metaclass(abc.ABCMeta, INotificationSubscriber)):
         """
         assert hasattr(callback, '__call__'), 'callback arg must be callable'
         self.callback = callback
-        self.__topicSubscriber.topicSubscribe(self.__topicName, self.messageReceived)
+        self.qs = QpidSubscriber(host=self.host, port=self.port, decompress=True, program=self.program)
+        self.qs.topicSubscribe(self.__topicName, self.messageReceived)
         # Blocks here
 
     def close(self):
-        if self.__topicSubscriber.subscribed:
+        self.qs.close()
-            self.__topicSubscriber.close()
 
     def getDataTime(self, dataURI):
         dataTimeStr = dataURI.split('/')[2]
@@ -82,4 +104,7 @@ class PyNotification(with_metaclass(abc.ABCMeta, INotificationSubscriber)):
     @property
     def subscribed(self):
         """True if currently subscribed to notifications."""
-        return self.__topicSubscriber.queueStarted
+        try:
+            return self.qs.queueStarted
+        except AttributeError:
+            return False

awips/dataaccess/SoundingsSupport.py

@@ -0,0 +1,283 @@
+# #
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+# #
+
+#
+# Classes for retrieving soundings based on gridded data from the Data Access
+# Framework
+#
+#  
+#    
+#     SOFTWARE HISTORY
+#    
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    06/24/15         #4480        dgilling       Initial Creation.
+#
+
+from collections import defaultdict
+from shapely.geometry import Point
+
+from ufpy import DateTimeConverter
+from ufpy.dataaccess import DataAccessLayer
+
+from dynamicserialize.dstypes.com.raytheon.uf.common.time import DataTime
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.level import Level
+
+
+def getSounding(modelName, weatherElements, levels, samplePoint, refTime=None, timeRange=None):
+    """"
+    Performs a series of Data Access Framework requests to retrieve a sounding object
+    based on the specified request parameters.
+        
+    Args:
+            modelName: the grid model datasetid to use as the basis of the sounding.
+            weatherElements: a list of parameters to return in the sounding.
+            levels: a list of levels to sample the given weather elements at
+            samplePoint: a lat/lon pair to perform the sampling of data at.
+            refTime: (optional) the grid model reference time to use for the sounding. 
+                     If not specified, the latest ref time in the system will be used.
+            timeRange: (optional) a TimeRange to specify which forecast hours to use.
+                     If not specified, will default to all forecast hours.
+    
+    Returns:
+            A _SoundingCube instance, which acts a 3-tiered dictionary, keyed
+            by DataTime, then by level and finally by weather element. If no
+            data is available for the given request parameters, None is returned.
+    """
+    
+    (locationNames, parameters, levels, envelope, refTime, timeRange) = \
+        __sanitizeInputs(modelName, weatherElements, levels, samplePoint, refTime, timeRange)
+        
+    requestArgs = { 'datatype'      : 'grid',
+                    'locationNames' : locationNames,
+                    'parameters'    : parameters,
+                    'levels'        : levels,
+                    'envelope'      : envelope,
+                   }
+        
+    req = DataAccessLayer.newDataRequest(**requestArgs)
+    
+    forecastHours = __determineForecastHours(req, refTime, timeRange)
+    if not forecastHours:
+        return None
+
+    response = DataAccessLayer.getGeometryData(req, forecastHours)
+    soundingObject = _SoundingCube(response)
+    
+    return soundingObject
+
+def setEDEXHost(host):
+    """
+    Changes the EDEX host the Data Access Framework is communicating with.
+    
+    Args:
+            host: the EDEX host to connect to
+    """
+    
+    if host:
+        DataAccessLayer.changeEDEXHost(str(host))
+
+def __sanitizeInputs(modelName, weatherElements, levels, samplePoint, refTime, timeRange):
+    locationNames = [str(modelName)]
+    parameters = __buildStringList(weatherElements)
+    levels = __buildStringList(levels)
+    envelope = Point(samplePoint)
+    if refTime is not None:
+        refTime = DataTime(refTime=DateTimeConverter.convertToDateTime(refTime))
+    if timeRange is not None:
+        timeRange = DateTimeConverter.constructTimeRange(*timeRange)
+    return (locationNames, parameters, levels, envelope, refTime, timeRange)
+    
+def __determineForecastHours(request, refTime, timeRange):
+    dataTimes = DataAccessLayer.getAvailableTimes(request, False)
+    timesGen = [(DataTime(refTime=dataTime.getRefTime()), dataTime) for dataTime in dataTimes]
+    dataTimesMap = defaultdict(list)
+    for baseTime, dataTime in timesGen:
+        dataTimesMap[baseTime].append(dataTime)
+    
+    if refTime is None:
+        refTime = max(dataTimesMap.keys())
+    
+    forecastHours = dataTimesMap[refTime]
+    if timeRange is None:
+        return forecastHours
+    else:
+        return [forecastHour for forecastHour in forecastHours if timeRange.contains(forecastHour.getValidPeriod())]
+
+def __buildStringList(param):
+    if __notStringIter(param):
+        return [str(item) for item in param]
+    else:
+        return [str(param)]
+    
+def __notStringIter(iterable):
+    if not isinstance(iterable, str):
+        try:
+            iter(iterable)
+            return True
+        except TypeError:
+            return False
+
+
+
+class _SoundingCube(object):
+    """
+    The top-level sounding object returned when calling SoundingsSupport.getSounding.
+    
+    This object acts as a 3-tiered dict which is keyed by time then level 
+    then parameter name. Calling times() will return all valid keys into this 
+    object.
+    """
+    
+    def __init__(self, geometryDataObjects):
+        self._dataDict = {}
+        self._sortedTimes = []
+        if geometryDataObjects:
+            for geometryData in geometryDataObjects:
+                dataTime = geometryData.getDataTime()
+                level = geometryData.getLevel()
+                for parameter in geometryData.getParameters():
+                    self.__addItem(parameter, dataTime, level, geometryData.getNumber(parameter))
+    
+    def __addItem(self, parameter, dataTime, level, value):
+        timeLayer = self._dataDict.get(dataTime, _SoundingTimeLayer(dataTime))
+        self._dataDict[dataTime] = timeLayer
+        timeLayer._addItem(parameter, level, value)
+        if dataTime not in self._sortedTimes:
+            self._sortedTimes.append(dataTime)
+            self._sortedTimes.sort()
+    
+    def __getitem__(self, key):
+        return self._dataDict[key]
+    
+    def __len__(self):
+        return len(self._dataDict)
+        
+    def times(self):
+        """ 
+        Returns the valid times for this sounding.
+        
+        Returns:
+            A list containing the valid DataTimes for this sounding in order.
+        """
+        return self._sortedTimes
+
+ 
+class _SoundingTimeLayer(object):
+    """
+    The second-level sounding object returned when calling SoundingsSupport.getSounding.
+    
+    This object acts as a 2-tiered dict which is keyed by level then parameter
+    name. Calling levels() will return all valid keys into this 
+    object. Calling time() will return the DataTime for this particular layer.
+    """
+
+    def __init__(self, dataTime):
+        self._dataTime = dataTime
+        self._dataDict = {}
+        
+    def _addItem(self, parameter, level, value):
+        asString = str(level)
+        levelLayer = self._dataDict.get(asString, _SoundingTimeAndLevelLayer(self._dataTime, asString))
+        levelLayer._addItem(parameter, value)
+        self._dataDict[asString] = levelLayer
+    
+    def __getitem__(self, key):
+        asString = str(key)
+        if asString in self._dataDict:
+            return self._dataDict[asString]
+        else:
+            raise KeyError("Level " + str(key) + " is not a valid level for this sounding.")
+    
+    def __len__(self):
+        return len(self._dataDict)
+        
+    def time(self):
+        """ 
+        Returns the DataTime for this sounding cube layer.
+        
+        Returns:
+            The DataTime for this sounding layer.
+        """
+        return self._dataTime
+    
+    def levels(self):
+        """ 
+        Returns the valid levels for this sounding.
+        
+        Returns:
+            A list containing the valid levels for this sounding in order of
+            closest to surface to highest from surface.
+        """
+        sortedLevels = [Level(level) for level in list(self._dataDict.keys())]
+        sortedLevels.sort()
+        return [str(level) for level in sortedLevels]
+
+
+class _SoundingTimeAndLevelLayer(object):
+    """
+    The bottom-level sounding object returned when calling SoundingsSupport.getSounding.
+    
+    This object acts as a dict which is keyed by parameter name. Calling 
+    parameters() will return all valid keys into this object. Calling time()
+    will return the DataTime for this particular layer. Calling level() will
+    return the level for this layer.
+    """
+
+    def __init__(self, time, level):
+        self._time = time
+        self._level = level
+        self._parameters = {}
+        
+    def _addItem(self, parameter, value):
+        self._parameters[parameter] = value
+    
+    def __getitem__(self, key):
+        return self._parameters[key]
+    
+    def __len__(self):
+        return len(self._parameters)
+        
+    def level(self):
+        """ 
+        Returns the level for this sounding cube layer.
+        
+        Returns:
+            The level for this sounding layer.
+        """
+        return self._level
+
+    def parameters(self):
+        """ 
+        Returns the valid parameters for this sounding.
+        
+        Returns:
+            A list containing the valid parameter names.
+        """
+        return list(self._parameters.keys())
+
+    def time(self):
+        """ 
+        Returns the DataTime for this sounding cube layer.
+        
+        Returns:
+            The DataTime for this sounding layer.
+        """
+        return self._time

awips/dataaccess/ThriftClientRouter.py

@@ -1,7 +1,28 @@
+# #
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+# #
+
 #
 # Routes requests to the Data Access Framework through Python Thrift.
 #
 #
+#
 #    SOFTWARE HISTORY
 #
 #    Date            Ticket#       Engineer       Description
@@ -22,8 +43,8 @@
 #    10/26/16        5919          njensen        Speed up geometry creation in getGeometryData()
 #
 
+
 import numpy
-import six
 import shapely.wkb
 
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.impl import DefaultDataRequest
@@ -40,9 +61,9 @@ from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import G
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetSupportedDatatypesRequest
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetNotificationFilterRequest
 
-from awips import ThriftClient
+from ufpy import ThriftClient
-from awips.dataaccess import PyGeometryData
+from ufpy.dataaccess import PyGeometryData
-from awips.dataaccess import PyGridData
+from ufpy.dataaccess import PyGridData
 
 
 class LazyGridLatLon(object):
@@ -121,13 +142,7 @@ class ThriftClientRouter(object):
         retVal = []
         for gridDataRecord in response.getGridData():
             locationName = gridDataRecord.getLocationName()
-            if locationName is not None:
+            locData = locSpecificData[locationName]
-                if six.PY2:
-                    locData = locSpecificData[locationName]
-                else:
-                    locData = locSpecificData[locationName.encode('utf-8')]
-            else:
-                locData = locSpecificData[locationName]
             if self._lazyLoadGridLatLon:
                 retVal.append(PyGridData.PyGridData(gridDataRecord, locData[
                               0], locData[1], latLonDelegate=locData[2]))
@@ -165,20 +180,12 @@ class ThriftClientRouter(object):
         locNamesRequest = GetAvailableLocationNamesRequest()
         locNamesRequest.setRequestParameters(request)
         response = self._client.sendRequest(locNamesRequest)
-        if six.PY2:
-            return response
-        if response is not None:
-            return [x.decode('utf-8') for x in response]
         return response
 
     def getAvailableParameters(self, request):
         paramReq = GetAvailableParametersRequest()
         paramReq.setRequestParameters(request)
         response = self._client.sendRequest(paramReq)
-        if six.PY2:
-            return response
-        if response is not None:
-            return [x.decode('utf-8') for x in response]
         return response
 
     def getAvailableLevels(self, request):
@@ -194,10 +201,6 @@ class ThriftClientRouter(object):
         idReq = GetRequiredIdentifiersRequest()
         idReq.setRequest(request)
         response = self._client.sendRequest(idReq)
-        if six.PY2:
-            return response
-        if response is not None:
-            return [x.decode('utf-8') for x in response]
         return response
 
     def getOptionalIdentifiers(self, request):
@@ -207,10 +210,6 @@ class ThriftClientRouter(object):
         idReq = GetOptionalIdentifiersRequest()
         idReq.setRequest(request)
         response = self._client.sendRequest(idReq)
-        if six.PY2:
-            return response
-        if response is not None:
-            return [x.decode('utf-8') for x in response]
         return response
 
     def getIdentifierValues(self, request, identifierKey):
@@ -218,14 +217,9 @@ class ThriftClientRouter(object):
         idValReq.setIdentifierKey(identifierKey)
         idValReq.setRequestParameters(request)
         response = self._client.sendRequest(idValReq)
-        if six.PY2:
-            return response
-        if response is not None:
-            return [x.decode('utf-8') for x in response]
         return response
 
-    def newDataRequest(self, datatype, parameters=[], levels=[], locationNames=[],
+    def newDataRequest(self, datatype, parameters=[], levels=[], locationNames=[], envelope=None, **kwargs):
-                       envelope=None, **kwargs):
         req = DefaultDataRequest()
         if datatype:
             req.setDatatype(datatype)
@@ -244,10 +238,6 @@ class ThriftClientRouter(object):
 
     def getSupportedDatatypes(self):
         response = self._client.sendRequest(GetSupportedDatatypesRequest())
-        if six.PY2:
-            return response
-        if response is not None:
-            return [x.decode('utf-8') for x in response]
         return response
 
     def getNotificationFilter(self, request):

awips/dataaccess/__init__.py

@@ -1,5 +1,26 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
 #
-# __init__.py for awips.dataaccess package
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+#
+# __init__.py for ufpy.dataaccess package
 #
 #
 #     SOFTWARE HISTORY
@@ -18,20 +39,12 @@
 #
 
 __all__ = [
-    'IData',
+
-    'IDataRequest',
+           ]
-    'IGeometryData',
-    'IGridData',
-    'IGeometryData',
-    'INotificationFilter',
-    'INotificationSubscriber'
-]
 
 import abc
-from six import with_metaclass
 
-
+class IDataRequest(object, metaclass=abc.ABCMeta):
-class IDataRequest(with_metaclass(abc.ABCMeta, object)):
     """
     An IDataRequest to be submitted to the DataAccessLayer to retrieve data.
     """
@@ -151,7 +164,8 @@ class IDataRequest(with_metaclass(abc.ABCMeta, object)):
         return
 
 
-class IData(with_metaclass(abc.ABCMeta, object)):
+
+class IData(object, metaclass=abc.ABCMeta):
     """
     An IData representing data returned from the DataAccessLayer.
     """
@@ -211,6 +225,7 @@ class IData(with_metaclass(abc.ABCMeta, object)):
         return
 
 
+
 class IGridData(IData):
     """
     An IData representing grid data that is returned by the DataAccessLayer.
@@ -258,6 +273,7 @@ class IGridData(IData):
         return
 
 
+
 class IGeometryData(IData):
     """
     An IData representing geometry data that is returned by the DataAccessLayer.
@@ -336,7 +352,7 @@ class IGeometryData(IData):
         return
 
 
-class INotificationSubscriber(with_metaclass(abc.ABCMeta, object)):
+class INotificationSubscriber(object, metaclass=abc.ABCMeta):
     """
     An INotificationSubscriber representing a notification filter returned from
     the DataNotificationLayer.
@@ -359,8 +375,7 @@ class INotificationSubscriber(with_metaclass(abc.ABCMeta, object)):
         """Closes the notification subscriber"""
         pass
 
-
+class INotificationFilter(object, metaclass=abc.ABCMeta):
-class INotificationFilter(with_metaclass(abc.ABCMeta, object)):
     """
     Represents data required to filter a set of URIs and
     return a corresponding list of IDataRequest to retrieve data for.

awips/gfe/IFPClient.py

@@ -1,16 +1,25 @@
-#
+##
-# Provides a Python-based interface for executing GFE requests.
+# This software was developed and / or modified by Raytheon Company,
-#
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
-#
+# 
-#     SOFTWARE HISTORY
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
-#
+# This software product contains export-restricted data whose
-#    Date            Ticket#       Engineer       Description
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
-#    ------------    ----------    -----------    --------------------------
+# to non-U.S. persons whether in the United States or abroad requires
-#    07/26/12                      dgilling       Initial Creation.
+# an export license or other authorization.
-#
+# 
-#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+from ufpy import ThriftClient
 
-from awips import ThriftClient
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import DatabaseID
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import ParmID
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import CommitGridsRequest
@@ -23,6 +32,21 @@ from dynamicserialize.dstypes.com.raytheon.uf.common.site.requests import GetAct
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.server.message import ServerResponse
 
 
+#
+# Provides a Python-based interface for executing GFE requests.
+#   
+#
+#    
+#     SOFTWARE HISTORY
+#    
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    07/26/12                      dgilling       Initial Creation.
+#    
+# 
+#
+
+
 class IFPClient(object):
     def __init__(self, host, port, user, site=None, progName=None):
         self.__thrift = ThriftClient.ThriftClient(host, port)
@@ -33,15 +57,14 @@ class IFPClient(object):
             if len(sr.getPayload()) > 0:
                 site = sr.getPayload()[0]
         self.__siteId = site
-
+        
     def commitGrid(self, request):
-        if isinstance(request, CommitGridRequest):
+        if type(request) is CommitGridRequest:
             return self.__commitGrid([request])
         elif self.__isHomogenousIterable(request, CommitGridRequest):
             return self.__commitGrid([cgr for cgr in request])
-        raise TypeError("Invalid type: " + str(type(request)) +
+        raise TypeError("Invalid type: " + str(type(request)) + " specified to commitGrid(). Only accepts CommitGridRequest or lists of CommitGridRequest.")
-                        " for commitGrid(). Only accepts CommitGridRequest or lists of CommitGridRequest.")
+    
-
     def __commitGrid(self, requests):
         ssr = ServerResponse()
         request = CommitGridsRequest()
@@ -49,26 +72,25 @@ class IFPClient(object):
         sr = self.__makeRequest(request)
         ssr.setMessages(sr.getMessages())
         return ssr
-
+    
-    def getParmList(self, pid):
+    def getParmList(self, id):
-        argType = type(pid)
+        argType = type(id)
         if argType is DatabaseID:
-            return self.__getParmList([pid])
+            return self.__getParmList([id])
-        elif self.__isHomogenousIterable(pid, DatabaseID):
+        elif self.__isHomogenousIterable(id, DatabaseID):
-            return self.__getParmList([dbid for dbid in pid])
+            return self.__getParmList([dbid for dbid in id])
-        raise TypeError("Invalid type: " + str(argType) +
+        raise TypeError("Invalid type: " + str(argType) + " specified to getParmList(). Only accepts DatabaseID or lists of DatabaseID.")        
-                        " for getParmList(). Only accepts DatabaseID or lists of DatabaseID.")
+    
-
     def __getParmList(self, ids):
         ssr = ServerResponse()
         request = GetParmListRequest()
         request.setDbIds(ids)
         sr = self.__makeRequest(request)
         ssr.setMessages(sr.getMessages())
-        parmlist = sr.getPayload() if sr.getPayload() is not None else []
+        list = sr.getPayload() if sr.getPayload() is not None else []
-        ssr.setPayload(parmlist)
+        ssr.setPayload(list)
         return ssr
-
+    
     def __isHomogenousIterable(self, iterable, classType):
         try:
             iterator = iter(iterable)
@@ -78,23 +100,22 @@ class IFPClient(object):
         except TypeError:
             return False
         return True
-
+    
     def getGridInventory(self, parmID):
-        if isinstance(parmID, ParmID):
+        if type(parmID) is ParmID:
             sr = self.__getGridInventory([parmID])
-            inventoryList = []
+            list = []
             try:
-                inventoryList = sr.getPayload()[parmID]
+                list = sr.getPayload()[parmID]
             except KeyError:
                 # no-op, we've already default the TimeRange list to empty
                 pass
-            sr.setPayload(inventoryList)
+            sr.setPayload(list)
             return sr
         elif self.__isHomogenousIterable(parmID, ParmID):
-            return self.__getGridInventory([pid for pid in parmID])
+            return self.__getGridInventory([id for id in parmID])
-        raise TypeError("Invalid type: " + str(type(parmID)) +
+        raise TypeError("Invalid type: " + str(type(parmID)) + " specified to getGridInventory(). Only accepts ParmID or lists of ParmID.")
-                        " specified to getGridInventory(). Accepts ParmID or lists of ParmID.")
+    
-
     def __getGridInventory(self, parmIDs):
         ssr = ServerResponse()
         request = GetGridInventoryRequest()
@@ -104,7 +125,7 @@ class IFPClient(object):
         trs = sr.getPayload() if sr.getPayload() is not None else {}
         ssr.setPayload(trs)
         return ssr
-
+    
     def getSelectTR(self, name):
         request = GetSelectTimeRangeRequest()
         request.setName(name)
@@ -113,7 +134,7 @@ class IFPClient(object):
         ssr.setMessages(sr.getMessages())
         ssr.setPayload(sr.getPayload())
         return ssr
-
+        
     def getSiteID(self):
         ssr = ServerResponse()
         request = GetActiveSitesRequest()
@@ -122,7 +143,7 @@ class IFPClient(object):
         ids = sr.getPayload() if sr.getPayload() is not None else []
         sr.setPayload(ids)
         return sr
-
+    
     def __makeRequest(self, request):
         try:
             request.setSiteID(self.__siteId)
@@ -132,7 +153,7 @@ class IFPClient(object):
             request.setWorkstationID(self.__wsId)
         except AttributeError:
             pass
-
+        
         sr = ServerResponse()
         response = None
         try:
@@ -148,5 +169,5 @@ class IFPClient(object):
         except AttributeError:
             # not a server response, nothing else to do
             pass
-
+            
         return sr

awips/gfe/__init__.py

@@ -1,3 +1,37 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
 
-__all__ = [
+
+#
+# __init__.py for ufpy.gfe package
+#  
+#    
+#     SOFTWARE HISTORY
+#    
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    07/26/12                      dgilling       Initial Creation.
+#    
+# 
+#
+
+
+__all__ = [           
            ]

awips/ignite_password.py

@@ -0,0 +1,64 @@
+##############################################################################
+# Encryption/decryption for ignite passwords
+#
+# TODO RODO #8677: The ignite password encryption/decryption code in this and
+# associated files is based off similar JMS password code that exists in a
+# later version, so the similar code should be consolidated later on.
+#
+# SOFTWARE HISTORY
+#
+# Date          Ticket#  Engineer       Description
+# ------------- -------- -------------- -------------------------------------------
+# Oct 12, 2021  8667     mapeters       Initial version
+# Mar 03, 2022  8762     mapeters       Handle cache VM jars being moved under
+#                                       /awips2/ignite
+#
+##############################################################################
+
+import os
+import subprocess
+
+class IgniteConfigurationException(Exception):
+    """Exception subclass for ignite password errors."""
+    pass
+
+def updateIgnitePasswords(keystore_password, truststore_password, password_props_path):
+    igniteJar = __findPluginJar("com.raytheon.uf.common.datastore.ignite")
+    cryptoJar = __findPluginJar("com.raytheon.uf.common.security")
+    statusJar = __findPluginJar("com.raytheon.uf.common.status")
+    apacheJar = __findFossJar("org.apache.commons.codec")
+    classPath = ":".join([igniteJar, cryptoJar, statusJar, apacheJar])
+
+    passwords_dict = {'a2.ignite.keystore.password': keystore_password, 'a2.ignite.truststore.password': truststore_password}
+    for password_key, password in passwords_dict.items():
+        # need full java path since this is run as root, which doesn't have appropriate path vars set
+        process = subprocess.run(["/awips2/java/bin/java", "-cp", classPath, "com.raytheon.uf.common.datastore.ignite.IgnitePasswordUtils", "--update", password, password_key, password_props_path],
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.PIPE)
+        try:
+            process.check_returncode()
+        except subprocess.CalledProcessError:
+            raise IgniteConfigurationException(f"Failed to update {password_key}: {process.stderr.decode()}")
+
+def __findPluginJar(pluginName):
+    # for cache VMs
+    ignitePluginJar = f"/awips2/ignite/lib/plugins/{pluginName}.jar"
+    if os.path.isfile(ignitePluginJar):
+        return ignitePluginJar
+    # for edex VMs
+    edexPluginJar = f"/awips2/edex/lib/plugins/{pluginName}.jar"
+    if os.path.isfile(edexPluginJar):
+        return edexPluginJar
+    raise RuntimeError(f"Could not locate plugin {pluginName}.jar")
+
+
+def __findFossJar(libraryName):
+    # for cache VMs
+    igniteFossDir = f"/awips2/ignite/lib/dependencies/{libraryName}"
+    if os.path.isdir(igniteFossDir):
+        return f"{igniteFossDir}/*"
+    # for edex VMs
+    edexFossDir = f"/awips2/edex/lib/dependencies/{libraryName}"
+    if os.path.isdir(edexFossDir):
+        return f"{edexFossDir}/*"
+    raise RuntimeError(f"Could not locate plugin {libraryName}")

awips/localization/LocalizationFileManager.py

@@ -0,0 +1,477 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+#
+# Library for accessing localization files from python.
+#
+# SOFTWARE HISTORY
+#
+# Date      Ticket#  Engineer  Description
+# --------- -------- --------- --------------------------
+# 08/09/17  5731     bsteffen  Initial Creation.
+# 04/23/19  7756     mapeters  Make user context name determination work with IdM
+# 04/22/20  7883     tgurney   Python 3 string/bytes fixes
+# 01/10/22  8735     mapeters  Set Content-Type for PUTs
+
+
+import urllib.request, urllib.error, urllib.parse
+from json import load as loadjson
+from xml.etree.ElementTree import parse as parseXml
+from base64 import b64encode
+from io import BytesIO
+import dateutil.parser
+import contextlib
+import os
+from urllib.parse import urlunparse, urljoin
+
+from . import LocalizationUtil
+
+NON_EXISTENT_CHECKSUM = 'NON_EXISTENT_CHECKSUM'
+DIRECTORY_CHECKSUM = 'DIRECTORY_CHECKSUM'
+
+class LocalizationFileVersionConflictException(Exception):
+    pass
+
+class LocalizationFileDoesNotExistException(Exception):
+    pass
+
+class LocalizationFileIsNotDirectoryException(Exception):
+    pass
+
+class LocalizationContext(object):
+    """A localization context defines the scope of a localization file.
+    
+    For example the base localization context includes all the default files
+    installed with EDEX, while a particular user context has custom files for
+    that user.
+    
+    A localization context consists of a level and name. The level defines what
+    kind of entity this context is valid for, such as 'base', 'site', or 'user'.
+    The name identifies the specific entity, for example the name of a 'user'
+    level context is usually the username. The 'base' level does not have a name
+    because there cannot be only one 'base' context.
+    
+    Attributes:
+        level: the localization level
+        name: the context name
+    """
+    def __init__(self, level="base", name=None, localizationType="common_static"):
+        if level != "base":
+            assert name is not None
+        self.level = level
+        self.name = name
+        self.type = localizationType
+    def isBase(self):
+        return self.level == "base"
+    def _getUrlComponent(self):
+        if self.isBase():
+            return self.type + '/' + "base/"
+        else:
+            return self.type + '/' + self.level + '/' + self.name + '/'
+    def __str__(self):
+        if self.isBase():
+            return self.type + ".base"
+        else:
+            return self.type + "." + self.level + "." + self.name
+    def __eq__(self, other):
+        return self.level == other.level and \
+               self.name == other.name and \
+               self.type == other.type
+    def __hash__(self):
+        return hash((self.level, self.name, self.type))
+
+class _LocalizationOutput(BytesIO):
+    """A file-like object for writing a localization file.
+    
+    The contents being written are stored in memory and written to a
+    localization server only when the writing is finished.
+    
+    This object should be used as a context manager, a save operation will be
+    executed if the context exits with no errors. If errors occur the partial
+    contents are abandoned and the server is unchanged.
+    
+     It is also possible to save the contents to the server with the save()
+     method.
+    """
+    def __init__(self, manager, lFile):
+        super().__init__()
+        self._manager = manager
+        self._file = lFile
+    def save(self):
+        """Send the currently written contents to the server."""
+        request = self._manager._buildRequest(self._file.context, self._file.path, method="PUT")
+
+        request.data = self.getvalue()
+        request.add_header("If-Match", self._file.checksum)
+        # An empty file is created if Content-Type isn't specified (defaults to
+        # "application/x-www-form-urlencoded"). We aren't encoding the
+        # request.data bytes in any special way for either text files (e.g. .py
+        # or .xml) or binary files (e.g. .png), so this works for both.
+        request.add_header("Content-Type", "application/octet-stream")
+        try:
+            urllib.request.urlopen(request)
+        except urllib.error.HTTPError as e:
+            if e.code == 409:
+                raise LocalizationFileVersionConflictException(e.read())
+            else:
+                raise e
+    def __enter__(self):
+        return self
+    def __exit__(self, exc_type, exc_value, traceback):
+        if exc_type is None:
+            self.save()
+    def __str__(self):
+        return '<' + self.__class__.__name__ + " for " + str(self._file) + '>'
+
+class LocalizationFile(object):
+    """A specific file stored in localization.
+    
+    A localization file is uniquely defined by the context and path. There can
+    only be one valid file for that path and localization at a time. To access
+    the contents of the file use the open method.
+    
+    Attributes:
+        context: A LocalizationContext
+        path: A path to this file
+        checksum: A string representation of a checksum generated from the file contents.
+        timnestamp: A datetime.datetime object indicating when the file was last modified.
+    """
+    def __init__(self, manager, context, path, checksum, timestamp):
+        """Initialize a LocalizationFile with the given manager and attributes.
+        
+        Args:
+            manager: A LocalizationFileManager to assist with server communication
+            context: A LocalizationContext
+            path: A path to this file
+            checksum: A string representation of a checksum generated from the file contents.
+            timnestamp: A datetime.datetime object indicating when the file was last modified.
+        """
+        self._manager = manager
+        self.context = context
+        self.path = path
+        self.checksum = checksum
+        self.timestamp = timestamp
+    def open(self, mode='r'):
+        """Open the file.
+        
+        This should always be called as as part of a with statement. When
+        writing the content is not saved on the server until leaving the with
+        statement normally, if an error occurs the server is left unchanged.
+        
+        Example:
+            with locFile.open('w') as output:
+                output.write('some content')
+        
+        Args:
+            mode: 'r' for reading the file, 'w' for writing
+
+        Returns:
+            A file like object that can be used for reads or writes.
+        """
+        if mode == 'r':
+            request = self._manager._buildRequest(self.context, self.path)
+            response = urllib.request.urlopen(request)
+            # Not the recommended way of reading directories.
+            if not(self.isDirectory()):
+                checksum = response.headers["Content-MD5"]
+                if self.checksum != checksum:
+                    raise RuntimeError("Localization checksum mismatch " + self.checksum + " " + checksum)
+            return contextlib.closing(response)
+        elif mode == 'w':
+            return _LocalizationOutput(self._manager, self)
+        else:
+            raise ValueError("mode string must be 'r' or 'w' not " + str(r))
+    def delete(self):
+        """Delete this file from the server"""
+        request = self._manager._buildRequest(self.context, self.path, method='DELETE')
+        request.add_header("If-Match", self.checksum)
+        try:
+            urllib.request.urlopen(request)
+        except urllib.error.HTTPError as e:
+            if e.code == 409:
+                raise LocalizationFileVersionConflictException(e.read())
+            else:
+                raise e
+    def exists(self):
+        """Check if this file actually exists.
+        
+        Returns:
+            boolean indicating existence of this file
+        """
+        return self.checksum != NON_EXISTENT_CHECKSUM
+    def isDirectory(self):
+        """Check if this file is a directory.
+        
+        A file must exist to be considered a directory.
+        
+        Returns:
+            boolean indicating directorocity of this file
+        """
+        return self.checksum == DIRECTORY_CHECKSUM
+    def getCheckSum(self):
+        return self.checksum
+    def getContext(self):
+        return self.context
+    def getPath(self):
+        return self.path
+    def getTimeStamp(self):
+        return self.timestamp
+    def __str__(self):
+        return str(self.context) + "/" + self.path
+    def __eq__(self, other):
+        return self.context == other.context and \
+               self.path == other.path and \
+               self.checksum == other.checksum \
+               and self.timestamp == other.timestamp
+    def __hash__(self):
+        return hash((self.context, self.path, self.checksum, self.timestamp))
+
+def _getHost():
+    import subprocess
+    host = subprocess.check_output(
+                    "source /awips2/fxa/bin/setup.env; echo $DEFAULT_HOST",
+                    shell=True).strip().decode()
+    if host:
+        return host
+    return 'localhost'
+
+def _getSiteFromServer(host):
+    try:
+        from ufpy import ThriftClient
+        from dynamicserialize.dstypes.com.raytheon.uf.common.site.requests import GetPrimarySiteRequest
+        client = ThriftClient.ThriftClient(host)
+        return client.sendRequest(GetPrimarySiteRequest())
+    except:
+        # Servers that don't have GFE installed will not return a site
+        pass
+
+def _getSiteFromEnv():
+    site = os.environ.get('FXA_LOCAL_SITE')
+    if site is None:
+        site = os.environ.get('SITE_IDENTIFIER')
+    return site
+
+def _getSite(host):
+    site = _getSiteFromEnv()
+    if not(site):
+        site = _getSiteFromServer(host)
+    return site
+
+def _parseJsonList(manager, response, context, path):
+    fileList = []
+    jsonResponse = loadjson(response)
+    for name, jsonData in jsonResponse.items():
+        checksum = jsonData["checksum"]
+        timestampString = jsonData["timestamp"]
+        timestamp = dateutil.parser.parse(timestampString)
+        newpath = urljoin(path, name)
+        fileList.append(LocalizationFile(manager, context, newpath, checksum, timestamp))
+    return fileList
+
+def _parseXmlList(manager, response, context, path):
+    fileList = []
+    for xmlData in parseXml(response).getroot().findall('file'):
+        name = xmlData.get("name")
+        checksum = xmlData.get("checksum")
+        timestampString = xmlData.get("timestamp")
+        timestamp = dateutil.parser.parse(timestampString)
+        newpath = urljoin(path, name)
+        fileList.append(LocalizationFile(manager, context, newpath, checksum, timestamp))
+    return fileList
+
+class LocalizationFileManager(object):
+    """Connects to a server and retrieves LocalizationFiles."""
+    def __init__(self, host=None, port=9581, path="/services/localization/", contexts=None, site=None, localizationType="common_static"):
+        """Initializes a LocalizationFileManager with connection parameters and context information
+        
+        All arguments are optional and will use defaults or attempt to figure out appropriate values form the environment.
+        
+        Args:
+            host: A hostname of the localization server, such as 'ev'.
+            port: A port to use to connect to the localization server, usually 9581.
+            path: A path to reach the localization file service on the server.
+            contexts: A list of contexts to check for files, the order of the contexts will be used
+                    for the order of incremental results and the priority of absolute results. 
+            site: A site identifier to use for site specific contexts. This is only used if the contexts arg is None.
+            localizationType: A localization type for contexts. This is only used if the contexts arg is None.
+            
+        """
+        if host is None:
+            host = _getHost()
+        if contexts is None:
+            if site is None:
+                site = _getSite(host)
+            contexts = [LocalizationContext("base", None, localizationType)]
+            if site:
+                contexts.append(LocalizationContext("configured", site, localizationType))
+                contexts.append(LocalizationContext("site", site, localizationType))
+            contexts.append(LocalizationContext("user", LocalizationUtil.getUser(), localizationType))
+        netloc = host + ':' + str(port)
+        self._baseUrl = urlunparse(('http', netloc, path, None, None, None))
+        self._contexts = contexts
+    def _buildRequest(self, context, path, method='GET'):
+        url = urljoin(self._baseUrl, context._getUrlComponent())
+        url = urljoin(url, path)
+        request = urllib.request.Request(url, method=method)
+        username = LocalizationUtil.getUser()
+        # Currently password is ignored in the server
+        # this is the defacto standard for not providing one to this service.
+        password = username
+        base64string = b64encode(b'%s:%s' % (username.encode(), password.encode()))
+        request.add_header("Authorization", "Basic %s" % base64string.decode())
+        return request
+    def _normalizePath(self, path):
+        if path == '' or path == '/':
+            path = '.'
+        if path[0] == '/':
+            path = path[1:]
+        return path
+    def _list(self, path):
+        path = self._normalizePath(path)
+        if path[-1] != '/':
+            path += '/'
+        fileList = []
+        exists = False
+        for context in self._contexts:
+            try:
+                request = self._buildRequest(context, path)
+                request.add_header("Accept", "application/json, application/xml")
+                response = urllib.request.urlopen(request)
+                exists = True
+                if not(response.geturl().endswith("/")):
+                    # For ordinary files the server sends a redirect to remove the slash.
+                    raise LocalizationFileIsNotDirectoryException("Not a directory: " + path)
+                elif response.headers["Content-Type"] == "application/xml":
+                    fileList += _parseXmlList(self, response, context, path)
+                else:
+                    fileList += _parseJsonList(self, response, context, path)
+            except urllib.error.HTTPError as e:
+                if e.code != 404:
+                    raise e
+        if not(exists):
+            raise LocalizationFileDoesNotExistException("No such file or directory: " + path)
+        return fileList
+    def _get(self, context, path):
+        path = self._normalizePath(path)
+        try:
+            request = self._buildRequest(context, path, method='HEAD')
+            resp = urllib.request.urlopen(request)
+            if (resp.geturl().endswith("/")):
+                checksum = DIRECTORY_CHECKSUM
+            else:
+                if "Content-MD5" not in resp.headers:
+                    raise RuntimeError("Missing Content-MD5 header in response from " + resp.geturl())
+                checksum = resp.headers["Content-MD5"]
+                if "Last-Modified" not in resp.headers:
+                    raise RuntimeError("Missing Last-Modified header in response from " + resp.geturl())
+            timestamp = dateutil.parser.parse(resp.headers["Last-Modified"])
+            return LocalizationFile(self, context, path, checksum, timestamp)
+        except urllib.error.HTTPError as e:
+            if e.code != 404:
+                raise e
+            else:
+                return LocalizationFile(self, context, path, NON_EXISTENT_CHECKSUM, None)
+    def listAbsolute(self, path):
+        """List the files in a localization directory, only a single file is returned for each unique path.
+        
+        If a file exists in more than one context then the highest level(furthest from base) is used.
+        
+        Args:
+            path: A path to a directory that should be the root of the listing
+            
+        Returns:
+            A list of LocalizationFiles
+        """
+        merged = dict()
+        for lFile in self._list(path):
+            merged[lFile.path] = lFile
+        return sorted(merged.values(), key=lambda lFile: lFile.path)
+    def listIncremental(self, path):
+        """List the files in a localization directory, this includes all files for all contexts.
+        
+        Args:
+            path: A path to a directory that should be the root of the listing
+
+        Returns:
+            A list of tuples, each tuple will contain one or more files for the
+            same paths but different contexts. Each tuple will be ordered the
+            same as the contexts in this manager, generally with 'base' first
+            and 'user' last. 
+        """
+        merged = dict()
+        for lFile in self._list(path):
+            if lFile.path in merged:
+                merged[lFile.path] += (lFile,)
+            else:
+                merged[lFile.path] = (lFile,)
+        return sorted(merged.values(), key=lambda t: t[0].path)
+    def getAbsolute(self, path):
+        """Get a single localization file from the highest level context where it exists.
+        
+        Args:
+            path: A path to a localization file
+        
+        Returns:
+            A Localization File with the specified path or None if the file does not exist in any context.
+        
+        """
+        for context in reversed(self._contexts):
+            f = self._get(context, path)
+            if f.exists():
+                return f
+    def getIncremental(self, path):
+        """Get all the localization files that exist in any context for the provided path.
+
+        Args:
+            path: A path to a localization file
+        
+        Returns:
+            A tuple containing all the files that exist for this path in any context. The tuple
+            will be ordered the same as the contexts in this manager, generally with 'base' first
+            and 'user' last. 
+        """
+        result = ()
+        for context in self._contexts:
+            f = self._get(context, path)
+            if f.exists():
+                result += (f,)
+        return result
+    def getSpecific(self, level, path):
+        """Get a specific localization file at a given level, the file may not exist.
+        
+        The file is returned for whichever context is valid for the provided level in this manager.
+        
+        For writing new files this is the only way to get access to a file that
+        does not exist in order to create it.
+        
+        Args:
+            level: the name of a localization level, such as "base", "site", "user"
+            path: A path to a localization file
+        
+        Returns:
+            A Localization File with the specified path and a context for the specified level.
+        """
+        for context in self._contexts:
+            if context.level == level:
+                return self._get(context, path)
+        raise ValueError("No context defined for level " + level)
+    def __str__(self):
+        contextsStr = '[' + ' '.join((str(c) for c in self._contexts)) + ']'
+        return '<' + self.__class__.__name__ + " for " + self._baseUrl + ' ' + contextsStr + '>'

awips/localization/LocalizationUtil.py

@@ -0,0 +1,45 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+#
+# Utilities for localization.
+#
+# SOFTWARE HISTORY
+#
+# Date      Ticket#  Engineer  Description
+# --------- -------- --------- --------------------------
+# 04/23/19  7756     mapeters  Initial creation
+
+import getpass
+
+def getUser():
+    '''
+    Get the user context name.
+    '''
+    try:
+        # Match Java's way of determining the user if we have Jep access
+        from java.lang import System
+        user = System.getProperty('user.name')
+    except:
+        # Otherwise use built-in getpass module. With IdM, this can return
+        # user.name@REALM, so strip the @REALM portion if it exists.
+        user = getpass.getuser()
+        user = user.split('@')[0]
+    return user

awips/localization/__init__.py

@@ -0,0 +1,32 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+#
+# __init__.py for ufpy.localization package
+#
+#
+# SOFTWARE HISTORY
+#
+# Date      Ticket#  Engineer  Description
+# --------- -------- --------- --------------------------
+# 08/10/17  5731     bsteffen  Initial Creation.
+
+__all__ = [           
+           ]

awips/qpidingest.py

@@ -1,19 +1,10 @@
-# ===============================================================================
+#===============================================================================
 # qpidingest.py
 #
 # @author: Aaron Anderson
 # @organization: NOAA/WDTB OU/CIMMS
 # @version: 1.0 02/19/2010
-# @requires: QPID Python Client available from http://qpid.apache.org/download.html
+# @requires: awips2-python and awips2-qpid-proton-python RPMs
-#            The Python Client is located under Single Component Package/Client
-#
-#            From the README.txt Installation Instructions
-#                = INSTALLATION =
-#                Extract the release archive into a directory of your choice and set
-#                your PYTHONPATH accordingly:
-#
-#                tar -xzf qpid-python-<version>.tar.gz -C <install-prefix>
-#                export PYTHONPATH=<install-prefix>/qpid-<version>/python
 #
 #       ***EDEX and QPID must be running for this module to work***
 #
@@ -38,7 +29,7 @@
 # EXAMPLE:
 # Simple example program:
 #
-# ------------------------------------------------------------------------------
+#------------------------------------------------------------------------------
 # import qpidingest
 # #Tell EDEX to ingest a metar file from data_store. The filepath is
 # #/data_store/20100218/metar/00/standard/20100218_005920_SAUS46KSEW.metar
@@ -50,82 +41,102 @@
 #
 # conn.sendmessage('/data_store/20100218/metar/18/standard/20100218_185755_SAUS46KLOX.metar','SAUS46 KLOX')
 # conn.close()
-# -------------------------------------------------------------------------------
+#-------------------------------------------------------------------------------
 #
-#    SOFTWARE HISTORY
+# SOFTWARE HISTORY
 #
-#    Date            Ticket#       Engineer       Description
+# Date          Ticket#  Engineer     Description
-#    ------------    ----------    -----------    --------------------------
+# ------------- -------- ------------ ------------------------------------------
-#    ....
+# Jun 13, 2013  16242    D. Friedman  Add Qpid authentication info
-#    06/13/2013      DR 16242      D. Friedman    Add Qpid authentication info
+# Mar 06, 2014  17907    D. Friedman  Workaround for issue QPID-5569
-#    03/06/2014      DR 17907      D. Friedman    Workaround for issue QPID-5569
+# Feb 16, 2017  6084     bsteffen     Support ssl connections
-#    02/16/2017      DR 6084       bsteffen       Support ssl connections
+# Jun 14, 2019  7870     mrichardson  MHS env workaround
+# Jul 23, 2019  7724     mrichardson  Upgrade Qpid to Qpid Proton
+# Nov 06, 2019  7724     tgurney      Remove the unnecessary
+#                                     QpidQueueManager
+# Dec 12, 2019  7995     dgilling     Revert interface changes from #7724.
+# Jul 07, 2020  8187     randerso     Added qpid connection_id
 #
-# ===============================================================================
+#===============================================================================   
 
+import logging
 import os
+import pwd
 import os.path
+import socket
 
-import qpid
+import proton
-from qpid.util import connect
+import proton.utils
-from qpid.connection import Connection
+import proton.reactor
-from qpid.datatypes import Message, uuid4
 
-QPID_USERNAME = 'guest'
+log = logging.getLogger("qpidingest")
-QPID_PASSWORD = 'guest'
 
 
+class QpidIngestException(Exception):
+    """Exception subclass for broker communication exceptions."""
+    pass
+
 class IngestViaQPID:
-    def __init__(self, host='localhost', port=5672, ssl=None):
+    def __init__(self, host="localhost", port=5672, program="qpidingest"):
-        """
+        '''
         Connect to QPID and make bindings to route message to external.dropbox queue
         @param host: string hostname of computer running EDEX and QPID (default localhost)
         @param port: integer port used to connect to QPID (default 5672)
-        @param ssl: boolean to determine whether ssl is used, default value of None will use
+        '''
-            ssl only if a client certificate is found.
+
-        """
+        pwuid = pwd.getpwuid(os.getuid())
+        certdb = os.getenv("QPID_SSL_CERT_DB", os.path.join(pwuid.pw_dir, ".qpid"))
+        certname = os.getenv("QPID_SSL_CERT_NAME", "guest")
+        cert_password = os.getenv("QPID_SSL_CERT_PASSWORD", "password")
+        certfile = os.path.join(certdb, f"{certname}.crt")
+        keyfile = os.path.join(certdb, f"{certname}.key")
+
+        url = f"amqps://{host}:{port}"
+        ADDRESS = "external.dropbox"
+        ssl_domain = proton.SSLDomain(mode=proton.SSLDomain.MODE_CLIENT)
+        ssl_domain.set_credentials(certfile, keyfile, cert_password)
+        
+        clientID = ":".join([
+            socket.gethostname(), 
+            pwuid.pw_name, 
+            program, 
+            str(os.getpid()), 
+        ])
 
         try:
-            #
+            container = proton.reactor.Container()
-            socket = connect(host, port)
+            container.container_id = clientID 
-            if "QPID_SSL_CERT_DB" in os.environ:
+            self._conn = proton.utils.BlockingConnection(url, ssl_domain=ssl_domain)
-                certdb = os.environ["QPID_SSL_CERT_DB"]
+            self._sender = self._conn.create_sender(ADDRESS)
-            else:
+            log.debug("Connected to broker [%s], endpoint [%s].", url, ADDRESS)
-                certdb = os.path.expanduser("~/.qpid/")
+        except proton.ProtonException as e:
-            if "QPID_SSL_CERT_NAME" in os.environ:
+            log.exception("Failed to connect to broker [%s].", url)
-                certname = os.environ["QPID_SSL_CERT_NAME"]
+            raise QpidIngestException("Failed to connect to broker [{}].".format(url)) from e
-            else:
-                certname = QPID_USERNAME
-            certfile = os.path.join(certdb, certname + ".crt")
-            if ssl or (ssl is None and os.path.exists(certfile)):
-                keyfile = os.path.join(certdb, certname + ".key")
-                trustfile = os.path.join(certdb, "root.crt")
-                socket = qpid.util.ssl(socket, keyfile=keyfile, certfile=certfile, ca_certs=trustfile)
-            self.connection = Connection(sock=socket, username=QPID_USERNAME, password=QPID_PASSWORD)
-            self.connection.start()
-            self.session = self.connection.session(str(uuid4()))
-            self.session.exchange_bind(exchange='amq.direct', queue='external.dropbox', binding_key='external.dropbox')
-            print('Connected to Qpid')
-        except ValueError:
-            print('Unable to connect to Qpid')
 
     def sendmessage(self, filepath, header):
-        """
+        '''
         This function sends a message to the external.dropbox queue providing the path
         to the file to be ingested and a header to determine the plugin to be used to
         decode the file.
         @param filepath: string full path to file to be ingested
         @param header: string header used to determine plugin decoder to use
-        """
+        '''
-        props = self.session.delivery_properties(routing_key='external.dropbox')
+        try:
-        head = self.session.message_properties(application_headers={'header': header},
+            self._sender.send(proton.Message(body=filepath, subject=header))
-                                               user_id=QPID_USERNAME)
+        except proton.ProtonException as e:
-        self.session.message_transfer(destination='amq.direct', message=Message(props, head, filepath))
+            log.exception("Failed to send file [%s] to broker.", filepath)
+            raise QpidIngestException("Failed to send file [{}] to broker.".format(filepath)) from e 
 
     def close(self):
-        """
+        '''
         After all messages are sent call this function to close connection and make sure
         there are no threads left open
-        """
+        '''
-        self.session.close(timeout=10)
+        try:
-        print('Connection to Qpid closed')
+            self._sender.close()
+            self._conn.close()
+            log.debug("Disconnected from broker.")
+        except proton.ProtonException as e:
+            log.warning("Failed to disconnect from broker.", exc_info=True)
+            raise QpidIngestException("Failed to disconnect from broker.") from e 
+

awips/test/Record.py

@@ -1,12 +1,48 @@
-import sys
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
 
 
+#
+# Pure python logging mechanism for logging to AlertViz from
+# pure python (ie not JEP).  DO NOT USE IN PYTHON CALLED
+# FROM JAVA.
+#  
+#    
+#     SOFTWARE HISTORY
+#    
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    11/03/10        5849          cjeanbap       Initial Creation.
+#    
+# 
+#
+
+import os
+import sys
+
 class Record():
     def __init__(self, level=0, msg='Test Message'):
-        self.levelno = level
+        self.levelno=level
-        self.message = msg
+        self.message=msg
-        self.exc_info = sys.exc_info()
+        self.exc_info=sys.exc_info()
-        self.exc_text = "TEST"
+        self.exc_text="TEST"
-
+    
     def getMessage(self):
-        return self.message
+        return self.message

awips/test/Test

@@ -0,0 +1,48 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+#
+# Pure python logging mechanism for logging to AlertViz from
+# pure python (ie not JEP).  DO NOT USE IN PYTHON CALLED
+# FROM JAVA.
+#  
+#    
+#     SOFTWARE HISTORY
+#    
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    11/03/10        5849          cjeanbap       Initial Creation.
+#    
+# 
+#
+
+## to execute type python Test
+
+
+import os
+import logging
+from awips import AlertVizHandler
+import Record
+
+avh = AlertVizHandler.AlertVizHandler(host=os.getenv("BROKER_ADDR","localhost"), port=9581, category='LOCAL', source='ANNOUNCER', level=logging.NOTSET)
+record = Record.Record(10)
+avh.emit(record)
+        

awips/test/__init__.py

@@ -1,2 +1,34 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+#
+# __init__.py for ufpy package
+#  
+#    
+#     SOFTWARE HISTORY
+#    
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    11/03/10        5489          cjeanbap       Initial Creation.
+#    
+# 
+#
 
-__all__ = []

awips/test/dafTests/__init__.py

@@ -1,2 +1,36 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+#
+# __init__.py for ufpy.test.dafTests package
+#
+#
+#     SOFTWARE HISTORY
+#
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    02/09/2016      4795          mapeters       Initial creation.
+#    04/12/2016      5548          tgurney        Cleanup
+#
+#
+#
 
 __all__ = []

awips/test/dafTests/baseBufrMosTestCase.py

@@ -1,3 +1,28 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+from ufpy.dataaccess import DataAccessLayer as DAL
+
+from . import baseDafTestCase
+from . import params
+
 #
 # Base TestCase for BufrMos* tests.
 #
@@ -12,17 +37,12 @@
 #
 #
 
-from awips.dataaccess import DataAccessLayer as DAL
-
-from awips.test.dafTests import baseDafTestCase
-from awips.test.dafTests import params
-
 
 class BufrMosTestCase(baseDafTestCase.DafTestCase):
     """Base class for testing DAF support of bufrmos data"""
-
+    
     data_params = "temperature", "dewpoint"
-
+    
     def testGetAvailableParameters(self):
         req = DAL.newDataRequest(self.datatype)
         self.runParametersTest(req)

awips/test/dafTests/baseDafTestCase.py

@@ -1,3 +1,32 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+
+from ufpy.dataaccess import DataAccessLayer as DAL
+from ufpy.ThriftClient import ThriftRequestException
+
+import os
+import numpy
+import unittest
+
 #
 # Base TestCase for DAF tests. This class provides helper methods and
 # tests common to all DAF test cases.
@@ -24,16 +53,11 @@
 #                                                 time-agnostic data
 #    03/13/17        5981          tgurney        Do not check valid period on
 #                                                 data time
+#    04/14/22        8845          njensen        Add checks for NaNs in geometry
+#                                                 data tests
 #
 #
 
-from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
-from awips.ThriftClient import ThriftRequestException
-
-import os
-import unittest
-
 
 class DafTestCase(unittest.TestCase):
 
@@ -56,7 +80,7 @@ class DafTestCase(unittest.TestCase):
     def setUpClass(cls):
         host = os.environ.get('DAF_TEST_HOST')
         if host is None:
-            host = 'edex-cloud.unidata.ucar.edu'
+            host = 'localhost'
         DAL.changeEDEXHost(host)
 
     @staticmethod
@@ -68,13 +92,13 @@ class DafTestCase(unittest.TestCase):
         try:
             times = DAL.getAvailableTimes(req)
         except ThriftRequestException as e:
-            if 'TimeAgnosticDataException' not in str(e):
+            if not 'TimeAgnosticDataException' in str(e):
                 raise
         return times
 
     def testDatatypeIsSupported(self):
-        allSupported = DAL.getSupportedDatatypes()
+        allSupported = (item.lower() for item in DAL.getSupportedDatatypes())
-        self.assertIn(self.datatype, allSupported)
+        self.assertIn(self.datatype.lower(), allSupported)
 
     def testGetRequiredIdentifiers(self):
         req = DAL.newDataRequest(self.datatype)
@@ -89,21 +113,21 @@ class DafTestCase(unittest.TestCase):
         print("Optional identifiers:", optional)
 
     def runGetIdValuesTest(self, identifiers):
-        for identifier in identifiers:
+        for id in identifiers:
-            if identifier.lower() == 'datauri':
+            if id.lower() == 'datauri':
                 continue
             req = DAL.newDataRequest(self.datatype)
-            idValues = DAL.getIdentifierValues(req, identifier)
+            idValues = DAL.getIdentifierValues(req, id)
             self.assertTrue(hasattr(idValues, '__iter__'))
 
     def runInvalidIdValuesTest(self):
         badString = 'id from ' + self.datatype + '; select 1;'
-        with self.assertRaises(ThriftRequestException):
+        with self.assertRaises(ThriftRequestException) as cm:
             req = DAL.newDataRequest(self.datatype)
             DAL.getIdentifierValues(req, badString)
 
     def runNonexistentIdValuesTest(self):
-        with self.assertRaises(ThriftRequestException):
+        with self.assertRaises(ThriftRequestException) as cm:
             req = DAL.newDataRequest(self.datatype)
             DAL.getIdentifierValues(req, 'idthatdoesnotexist')
 
@@ -145,9 +169,22 @@ class DafTestCase(unittest.TestCase):
         times = DafTestCase.getTimesIfSupported(req)
         geomData = DAL.getGeometryData(req, times[:self.numTimesToLimit])
         self.assertIsNotNone(geomData)
+        if times:
+            self.assertNotEqual(len(geomData), 0)
         if not geomData:
             raise unittest.SkipTest("No data available")
         print("Number of geometry records: " + str(len(geomData)))
+        print("Sample geometry data:")
+        for record in geomData[:self.sampleDataLimit]:
+            if (checkDataTimes and times and
+                    "PERIOD_USED" not in record.getDataTime().getUtilityFlags()):
+                self.assertIn(record.getDataTime(), times[:self.numTimesToLimit])
+            print("geometry=" + str(record.getGeometry()), end="")
+            for p in req.getParameters():
+                print(" " + p + "=" + record.getString(p), end="")
+                if record.getType(p) in ['FLOAT', 'DOUBLE']:
+                    self.assertFalse(numpy.isnan(record.getNumber(p)))
+            print()
         return geomData
 
     def runGeometryDataTestWithTimeRange(self, req, timeRange):
@@ -160,6 +197,16 @@ class DafTestCase(unittest.TestCase):
         if not geomData:
             raise unittest.SkipTest("No data available")
         print("Number of geometry records: " + str(len(geomData)))
+        print("Sample geometry data:")
+        for record in geomData[:self.sampleDataLimit]:
+            self.assertGreaterEqual(record.getDataTime().getRefTime().getTime(), timeRange.getStartInMillis())
+            self.assertLessEqual(record.getDataTime().getRefTime().getTime(), timeRange.getEndInMillis())
+            print("geometry=" + str(record.getGeometry()), end="")
+            for p in req.getParameters():
+                print(" " + p + "=" + record.getString(p), end="")
+                if record.getType(p) in ['FLOAT', 'DOUBLE']:
+                    self.assertFalse(numpy.isnan(record.getNumber(p)))
+            print()
         return geomData
 
     def runGridDataTest(self, req, testSameShape=True):
@@ -168,7 +215,6 @@ class DafTestCase(unittest.TestCase):
         request.
 
         Args:
-            req: the grid request
             testSameShape: whether or not to verify that all the retrieved data
                            have the same shape (most data don't change shape)
         """

awips/test/dafTests/baseRadarTestCase.py

@@ -1,3 +1,30 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+from ufpy.dataaccess import DataAccessLayer as DAL
+from ufpy.ThriftClient import ThriftRequestException
+
+from . import baseDafTestCase
+from . import params
+
 #
 # Tests common to all radar factories
 #
@@ -23,13 +50,6 @@
 #
 #
 
-from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
-from awips.ThriftClient import ThriftRequestException
-
-from awips.test.dafTests import baseDafTestCase
-from awips.test.dafTests import params
-
 
 class BaseRadarTestCase(baseDafTestCase.DafTestCase):
     """Tests common to all radar factories"""
@@ -50,7 +70,7 @@ class BaseRadarTestCase(baseDafTestCase.DafTestCase):
     def testGetAvailableLevels(self):
         req = DAL.newDataRequest(self.datatype)
         self.runLevelsTest(req)
-
+        
     def testGetAvailableLevelsWithInvalidLevelIdentifierThrowsException(self):
         req = DAL.newDataRequest(self.datatype)
         req.addIdentifier('level.one.field', 'invalidLevelField')
@@ -88,11 +108,6 @@ class BaseRadarTestCase(baseDafTestCase.DafTestCase):
         for record in gridData:
             self.assertEqual(record.getAttribute('icao'), 1000)
 
-    def testGetDataWithEqualsLong(self):
-        gridData = self.runConstraintTest('icao', '=', 1000)
-        for record in gridData:
-            self.assertEqual(record.getAttribute('icao'), 1000)
-
     def testGetDataWithEqualsFloat(self):
         gridData = self.runConstraintTest('icao', '=', 1.0)
         for record in gridData:

awips/test/dafTests/params.py

@@ -1,3 +1,24 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
 #
 # Site-specific parameters for DAF tests
 #
@@ -7,17 +28,18 @@
 #    ------------    ----------    -----------    --------------------------
 #    12/07/16        5981          tgurney        Initial creation
 #    12/15/16        5981          tgurney        Add ENVELOPE
+#    04/14/22        8845          njensen        Add POINT
 #
 #
 
-from shapely.geometry import box
+from shapely.geometry import box, Point
-
 
 AIRPORT = 'OMA'
 OBS_STATION = 'KOMA'
 SITE_ID = 'OAX'
 STATION_ID = '72558'
 RADAR = 'KOAX'
+POINT = Point(-96.25, 41.16)
 SAMPLE_AREA = (-97.0, 41.0, -96.0, 42.0)
 
-ENVELOPE = box(*SAMPLE_AREA)
+ENVELOPE = box(*SAMPLE_AREA)

awips/test/dafTests/testAcars.py

@@ -1,3 +1,28 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+from ufpy.dataaccess import DataAccessLayer as DAL
+
+from . import baseDafTestCase
+
 #
 # Test DAF support for ACARS data
 #
@@ -11,10 +36,6 @@
 #
 #
 
-from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
-from awips.test.dafTests import baseDafTestCase
-
 
 class AcarsTestCase(baseDafTestCase.DafTestCase):
     """Test DAF support for ACARS data"""

awips/test/dafTests/testAirep.py

@@ -1,3 +1,29 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+from ufpy.dataaccess import DataAccessLayer as DAL
+
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
+from . import baseDafTestCase
+
 #
 # Test DAF support for airep data
 #
@@ -14,12 +40,6 @@
 #
 #
 
-from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
-
-from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
-from awips.test.dafTests import baseDafTestCase
-
 
 class AirepTestCase(baseDafTestCase.DafTestCase):
     """Test DAF support for airep data"""

awips/test/dafTests/testBinLightning.py

@@ -1,3 +1,31 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+from ufpy.dataaccess import DataAccessLayer as DAL
+from ufpy.ThriftClient import ThriftRequestException
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
+
+
+from . import baseDafTestCase
+
 #
 # Test DAF support for binlightning data
 #
@@ -18,20 +46,14 @@
 #    06/13/16        5574          tgurney        Typo
 #    06/30/16        5725          tgurney        Add test for NOT IN
 #    11/08/16        5985          tgurney        Do not check data times
+#    
 #
-#
-from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
-from awips.ThriftClient import ThriftRequestException
-from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
-from awips.test.dafTests import baseDafTestCase
 
 
 class BinLightningTestCase(baseDafTestCase.DafTestCase):
     """Test DAF support for binlightning data"""
 
     datatype = "binlightning"
-    source = "GLMfl"
 
     def testGetAvailableParameters(self):
         req = DAL.newDataRequest(self.datatype)
@@ -39,18 +61,18 @@ class BinLightningTestCase(baseDafTestCase.DafTestCase):
 
     def testGetAvailableTimes(self):
         req = DAL.newDataRequest(self.datatype)
-        req.addIdentifier('source', self.source)
+        req.addIdentifier("source", "NLDN")
         self.runTimesTest(req)
 
     def testGetGeometryDataSingleSourceSingleParameter(self):
         req = DAL.newDataRequest(self.datatype)
-        req.addIdentifier('source', self.source)
+        req.addIdentifier("source", "NLDN")
         req.setParameters('intensity')
         self.runGeometryDataTest(req, checkDataTimes=False)
 
     def testGetGeometryDataInvalidParamRaisesIncompatibleRequestException(self):
         req = DAL.newDataRequest(self.datatype)
-        req.addIdentifier('source', self.source)
+        req.addIdentifier("source", "NLDN")
         req.setParameters('blahblahblah')
         with self.assertRaises(ThriftRequestException) as cm:
             self.runGeometryDataTest(req)
@@ -58,7 +80,7 @@ class BinLightningTestCase(baseDafTestCase.DafTestCase):
 
     def testGetGeometryDataSingleSourceAllParameters(self):
         req = DAL.newDataRequest(self.datatype)
-        req.addIdentifier('source', self.source)
+        req.addIdentifier("source", "NLDN")
         req.setParameters(*DAL.getAvailableParameters(req))
         self.runGeometryDataTest(req, checkDataTimes=False)
 
@@ -82,20 +104,15 @@ class BinLightningTestCase(baseDafTestCase.DafTestCase):
         return self.runGeometryDataTest(req, checkDataTimes=False)
 
     def testGetDataWithEqualsString(self):
-        geomData = self._runConstraintTest('source', '=', self.source)
+        geomData = self._runConstraintTest('source', '=', 'NLDN')
         for record in geomData:
-            self.assertEqual(record.getAttribute('source'), self.source)
+            self.assertEqual(record.getAttribute('source'), 'NLDN')
 
     def testGetDataWithEqualsInt(self):
         geomData = self._runConstraintTest('source', '=', 1000)
         for record in geomData:
             self.assertEqual(record.getAttribute('source'), 1000)
 
-    def testGetDataWithEqualsLong(self):
-        geomData = self._runConstraintTest('source', '=', 1000)
-        for record in geomData:
-            self.assertEqual(record.getAttribute('source'), 1000)
-
     def testGetDataWithEqualsFloat(self):
         geomData = self._runConstraintTest('source', '=', 1.0)
         for record in geomData:
@@ -107,9 +124,9 @@ class BinLightningTestCase(baseDafTestCase.DafTestCase):
             self.assertIsNone(record.getAttribute('source'))
 
     def testGetDataWithNotEquals(self):
-        geomData = self._runConstraintTest('source', '!=', self.source)
+        geomData = self._runConstraintTest('source', '!=', 'NLDN')
         for record in geomData:
-            self.assertNotEqual(record.getAttribute('source'), self.source)
+            self.assertNotEqual(record.getAttribute('source'), 'NLDN')
 
     def testGetDataWithNotEqualsNone(self):
         geomData = self._runConstraintTest('source', '!=', None)
@@ -117,49 +134,49 @@ class BinLightningTestCase(baseDafTestCase.DafTestCase):
             self.assertIsNotNone(record.getAttribute('source'))
 
     def testGetDataWithGreaterThan(self):
-        geomData = self._runConstraintTest('source', '>', self.source)
+        geomData = self._runConstraintTest('source', '>', 'NLDN')
         for record in geomData:
-            self.assertGreater(record.getAttribute('source'), self.source)
+            self.assertGreater(record.getAttribute('source'), 'NLDN')
 
     def testGetDataWithLessThan(self):
-        geomData = self._runConstraintTest('source', '<', self.source)
+        geomData = self._runConstraintTest('source', '<', 'NLDN')
         for record in geomData:
-            self.assertLess(record.getAttribute('source'), self.source)
+            self.assertLess(record.getAttribute('source'), 'NLDN')
 
     def testGetDataWithGreaterThanEquals(self):
-        geomData = self._runConstraintTest('source', '>=', self.source)
+        geomData = self._runConstraintTest('source', '>=', 'NLDN')
         for record in geomData:
-            self.assertGreaterEqual(record.getAttribute('source'), self.source)
+            self.assertGreaterEqual(record.getAttribute('source'), 'NLDN')
 
     def testGetDataWithLessThanEquals(self):
-        geomData = self._runConstraintTest('source', '<=', self.source)
+        geomData = self._runConstraintTest('source', '<=', 'NLDN')
         for record in geomData:
-            self.assertLessEqual(record.getAttribute('source'), self.source)
+            self.assertLessEqual(record.getAttribute('source'), 'NLDN')
 
     def testGetDataWithInTuple(self):
-        geomData = self._runConstraintTest('source', 'in', (self.source, 'GLMev'))
+        geomData = self._runConstraintTest('source', 'in', ('NLDN', 'ENTLN'))
         for record in geomData:
-            self.assertIn(record.getAttribute('source'), (self.source, 'GLMev'))
+            self.assertIn(record.getAttribute('source'), ('NLDN', 'ENTLN'))
 
     def testGetDataWithInList(self):
-        geomData = self._runConstraintTest('source', 'in', [self.source, 'GLMev'])
+        geomData = self._runConstraintTest('source', 'in', ['NLDN', 'ENTLN'])
         for record in geomData:
-            self.assertIn(record.getAttribute('source'), (self.source, 'GLMev'))
+            self.assertIn(record.getAttribute('source'), ('NLDN', 'ENTLN'))
 
     def testGetDataWithInGenerator(self):
-        generator = (item for item in (self.source, 'GLMev'))
+        generator = (item for item in ('NLDN', 'ENTLN'))
         geomData = self._runConstraintTest('source', 'in', generator)
         for record in geomData:
-            self.assertIn(record.getAttribute('source'), (self.source, 'GLMev'))
+            self.assertIn(record.getAttribute('source'), ('NLDN', 'ENTLN'))
 
     def testGetDataWithNotInList(self):
-        geomData = self._runConstraintTest('source', 'not in', [self.source, 'blah'])
+        geomData = self._runConstraintTest('source', 'not in', ['NLDN', 'blah'])
         for record in geomData:
-            self.assertNotIn(record.getAttribute('source'), (self.source, 'blah'))
+            self.assertNotIn(record.getAttribute('source'), ('NLDN', 'blah'))
 
     def testGetDataWithInvalidConstraintTypeThrowsException(self):
         with self.assertRaises(ValueError):
-            self._runConstraintTest('source', 'junk', self.source)
+            self._runConstraintTest('source', 'junk', 'NLDN')
 
     def testGetDataWithInvalidConstraintValueThrowsException(self):
         with self.assertRaises(TypeError):

awips/test/dafTests/testBufrMosAvn.py

@@ -0,0 +1,44 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+
+from . import baseBufrMosTestCase
+
+#
+# Test DAF support for bufrmosAVN data
+#
+#     SOFTWARE HISTORY
+#
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    01/19/16        4795          mapeters       Initial Creation.
+#    04/11/16        5548          tgurney        Cleanup
+#    04/18/16        5548          tgurney        More cleanup
+#
+#
+
+
+class BufrMosAvnTestCase(baseBufrMosTestCase.BufrMosTestCase):
+    """Test DAF support for bufrmosAVN data"""
+
+    datatype = "bufrmosAVN"
+
+    # All tests inherited from superclass

awips/test/dafTests/testBufrMosEta.py

@@ -0,0 +1,44 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+
+from . import baseBufrMosTestCase
+
+#
+# Test DAF support for bufrmosETA data
+#
+#     SOFTWARE HISTORY
+#
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    01/19/16        4795          mapeters       Initial Creation.
+#    04/11/16        5548          tgurney        Cleanup
+#    04/18/16        5548          tgurney        More cleanup
+#
+#
+
+
+class BufrMosEtaTestCase(baseBufrMosTestCase.BufrMosTestCase):
+    """Test DAF support for bufrmosETA data"""
+
+    datatype = "bufrmosETA"
+
+    # All tests inherited from superclass

awips/test/dafTests/testBufrMosGfs.py

@@ -1,3 +1,27 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+
+from . import baseBufrMosTestCase
+
 #
 # Test DAF support for bufrmosGFS data
 #
@@ -11,8 +35,6 @@
 #
 #
 
-from awips.test.dafTests import baseBufrMosTestCase
-
 
 class BufrMosGfsTestCase(baseBufrMosTestCase.BufrMosTestCase):
     """Test DAF support for bufrmosGFS data"""

awips/test/dafTests/testBufrMosHpc.py

@@ -0,0 +1,46 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+from . import baseBufrMosTestCase
+
+#
+# Test DAF support for bufrmosHPC data
+#
+#     SOFTWARE HISTORY
+#
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    01/19/16        4795          mapeters       Initial Creation.
+#    04/11/16        5548          tgurney        Cleanup
+#    04/18/16        5548          tgurney        More cleanup
+#    12/07/16        5981          tgurney        Parameterize
+#    12/20/16        5981          tgurney        Inherit all tests
+#
+#
+
+
+class BufrMosHpcTestCase(baseBufrMosTestCase.BufrMosTestCase):
+    """Test DAF support for bufrmosHPC data"""
+
+    datatype = "bufrmosHPC"
+    data_params = "forecastHr", "maxTemp24Hour"
+
+    # All tests inherited from superclass

awips/test/dafTests/testBufrMosLamp.py

@@ -0,0 +1,44 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+
+from . import baseBufrMosTestCase
+
+#
+# Test DAF support for bufrmosLAMP data
+#
+#     SOFTWARE HISTORY
+#
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    01/19/16        4795          mapeters       Initial Creation.
+#    04/11/16        5548          tgurney        Cleanup
+#    04/18/16        5548          tgurney        More cleanup
+#
+#
+
+
+class BufrMosLampTestCase(baseBufrMosTestCase.BufrMosTestCase):
+    """Test DAF support for bufrmosLAMP data"""
+
+    datatype = "bufrmosLAMP"
+
+    # All tests inherited from superclass

awips/test/dafTests/testBufrMosMrf.py

@@ -0,0 +1,45 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+from . import baseBufrMosTestCase
+
+#
+# Test DAF support for bufrmosMRF data
+#
+#     SOFTWARE HISTORY
+#
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    01/19/16        4795          mapeters       Initial Creation.
+#    04/11/16        5548          tgurney        Cleanup
+#    04/18/16        5548          tgurney        More cleanup
+#    12/07/16        5981          tgurney        Parameterize
+#    12/20/16        5981          tgurney        Inherit all tests
+#
+#
+
+
+class BufrMosMrfTestCase(baseBufrMosTestCase.BufrMosTestCase):
+    """Test DAF support for bufrmosMRF data"""
+
+    datatype = "bufrmosMRF"
+    data_params = "forecastHr", "maxTempDay"
+
+    # All tests inherited from superclass

awips/test/dafTests/testBufrUa.py

@@ -1,9 +1,30 @@
-from __future__ import print_function
+# #
-from awips.dataaccess import DataAccessLayer as DAL
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+# #
+
+
+from ufpy.dataaccess import DataAccessLayer as DAL
 
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
-from awips.test.dafTests import baseDafTestCase
+from . import baseDafTestCase
-from awips.test.dafTests import params
+from . import params
+import unittest
 
 #
 # Test DAF support for bufrua data
@@ -110,11 +131,6 @@ class BufrUaTestCase(baseDafTestCase.DafTestCase):
         for record in geometryData:
             self.assertEqual(record.getString('rptType'), '2022')
 
-    def testGetDataWithEqualsLong(self):
-        geometryData = self._runConstraintTest('reportType', '=', 2022)
-        for record in geometryData:
-            self.assertEqual(record.getString('rptType'), '2022')
-
     # No float test because no float identifiers are available
 
     def testGetDataWithEqualsNone(self):

awips/test/dafTests/testClimate.py

@@ -0,0 +1,419 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+import datetime
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
+from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange
+from ufpy.dataaccess import DataAccessLayer as DAL
+from ufpy.ThriftClient import ThriftRequestException
+
+from . import baseDafTestCase
+from . import params
+
+#
+# Test DAF support for climate data
+#
+#     SOFTWARE HISTORY
+#
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    01/19/16        4795          mapeters       Initial Creation.
+#    04/11/16        5548          tgurney        Cleanup
+#    04/18/16        5548          tgurney        More cleanup
+#    04/26/16        5587          tgurney        Add identifier values tests
+#    06/09/16        5574          mapeters       Add advanced query tests, Short parameter test
+#    06/13/16        5574          tgurney        Fix checks for None
+#    06/21/16        5548          tgurney        Skip tests that cause errors
+#    06/30/16        5725          tgurney        Add test for NOT IN
+#    10/06/16        5926          dgilling       Add additional time and location tests.
+#    12/07/16        5981          tgurney        Parameterize
+#    12/20/16        5981          tgurney        Add envelope test
+#    08/16/17        6388          tgurney        Test for duplicate data
+#
+#
+
+
+class ClimateTestCase(baseDafTestCase.DafTestCase):
+    """Test DAF support for climate data"""
+
+    datatype = 'climate'
+    obsStation = params.OBS_STATION
+
+    def testGetAvailableParameters(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.cli_asos_monthly')
+        self.runParametersTest(req)
+
+    def testGetAvailableLocations(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.cli_asos_monthly')
+        self.runLocationsTest(req)
+
+    def testGetAvailableLocationsForRptTable(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.rpt')
+        self.runLocationsTest(req)
+
+    def testGetAvailableLocationsForStationId(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.day_climate_norm')
+        self.runLocationsTest(req)
+
+    def testGetAvailableLocationsForInformId(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.cli_mon_season_yr')
+        self.runLocationsTest(req)
+
+    def testGetAvailableLocationsWithConstraints(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.cli_asos_monthly')
+        req.addIdentifier('maxtemp_mon', RequestConstraint.new('>', 95))
+        self.runLocationsTest(req)
+
+    def testGetAvailableLocationsWithInvalidTable(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.boolean_values')
+        with self.assertRaises(ThriftRequestException) as cm:
+            DAL.getAvailableLocationNames(req)
+        self.assertIn('IncompatibleRequestException', str(cm.exception))
+
+    def testGetAvailableTimes(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.cli_asos_monthly')
+        req.setParameters('maxtemp_mon', 'min_sea_press')
+        self.runTimesTest(req)
+
+    def testGetAvailableTimesWithLocationNamesForYearMonth(self):
+        """
+        Test retrieval of times for a climo table that uses year and 
+        month columns to build DataTimes.
+        """
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.cli_asos_monthly')
+        req.setLocationNames(self.obsStation, 'KABR', 'KDMO')
+        req.setParameters('maxtemp_mon', 'min_sea_press')
+        self.runTimesTest(req)
+        
+    def testGetAvailableTimesWithLocationNamesForYearDayOfYear(self):
+        """
+        Test retrieval of times for a climo table that uses year and 
+        day_of_year columns to build DataTimes.
+        """
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.cli_asos_daily')
+        req.setLocationNames(self.obsStation, 'KABR', 'KDMO')
+        req.setParameters('maxtemp_cal', 'min_press')
+        self.runTimesTest(req)
+
+    def testGetAvailableTimesWithLocationNamesForPeriod(self):
+        """
+        Test retrieval of times for a climo table that uses 
+        period_start and period_end columns to build DataTimes.
+        """
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.cli_mon_season_yr')
+        req.setLocationNames(self.obsStation, 'KABR', 'KDMO')
+        req.setParameters('max_temp', 'precip_total')
+        self.runTimesTest(req)
+
+    def testGetAvailableTimesWithLocationNamesForDate(self):
+        """
+        Test retrieval of times for a climo table that uses a date 
+        column to build DataTimes.
+        """
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.daily_climate')
+        req.setLocationNames(self.obsStation, 'KABR', 'KDMO')
+        req.setParameters('max_temp', 'precip', 'avg_wind_speed')
+        self.runTimesTest(req)
+
+    def testGetAvailableTimesWithConstraint(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.cli_asos_monthly')
+        req.addIdentifier('maxtemp_mon', RequestConstraint.new('<', 75))
+        req.setParameters('maxtemp_mon', 'min_sea_press')
+        self.runTimesTest(req)
+
+    def testGetGeometryData(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.cli_asos_monthly')
+        req.setLocationNames('KFNB')
+        req.setParameters('maxtemp_mon', 'min_sea_press')
+        self.runGeometryDataTest(req)
+
+    def testGetGeometryDataWithEnvelopeThrowsException(self):
+        # Envelope is not used
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.cli_asos_monthly')
+        req.setParameters('maxtemp_mon', 'min_sea_press')
+        req.setEnvelope(params.ENVELOPE)
+        with self.assertRaises(Exception):
+            self.runGeometryDataTest(req)
+
+    def testGetGeometryDataForYearAndDayOfYearTable(self):
+        """
+        Test retrieval of data for a climo table that uses year and 
+        day_of_year columns to build DataTimes.
+        """
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.cli_asos_daily')
+        req.setLocationNames('KFNB')
+        req.setParameters('maxtemp_cal', 'min_press')
+        self.runGeometryDataTest(req)
+
+    def testGetGeometryDataForPeriodTable(self):
+        """
+        Test retrieval of data for a climo table that uses a period_start and 
+        period_end columns to build DataTimes.
+        """
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.cli_mon_season_yr')
+        req.setLocationNames('KFNB')
+        req.setParameters('max_temp', 'precip_total')
+        self.runGeometryDataTest(req)
+
+    def testGetGeometryDataForDateTable(self):
+        """
+        Test retrieval of data for a climo table that uses a date column to 
+        build DataTimes.
+        """
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.daily_climate')
+        req.setLocationNames('KFNB')
+        req.setParameters('max_temp', 'precip', 'avg_wind_speed')
+        self.runGeometryDataTest(req)
+
+    def testGetGeometryDataWithShortParameter(self):
+        """
+        Test that a parameter that is stored in Java as a Short is correctly
+        retrieved as a number.
+        """
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'cli_asos_monthly')
+        req.setParameters('month')
+        geometryData = self.runGeometryDataTest(req)
+        for record in geometryData:
+            self.assertIsNotNone(record.getNumber('month'))
+
+    def testGetTableIdentifierValues(self):
+        self.runGetIdValuesTest(['table'])
+
+    def testGetColumnIdValuesWithTable(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.cli_asos_monthly')
+        idValues = DAL.getIdentifierValues(req, 'year')
+        self.assertTrue(hasattr(idValues, '__iter__'))
+
+    def testGetColumnIdValuesWithoutTableThrowsException(self):
+        req = DAL.newDataRequest(self.datatype)
+        with self.assertRaises(ThriftRequestException):
+            DAL.getIdentifierValues(req, 'year')
+
+    def testGetInvalidIdentifierValuesThrowsException(self):
+        self.runInvalidIdValuesTest()
+
+    def testGetNonexistentIdentifierValuesThrowsException(self):
+        self.runNonexistentIdValuesTest()
+
+    def _runConstraintTest(self, key, operator, value):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'cli_asos_monthly')
+        constraint = RequestConstraint.new(operator, value)
+        req.addIdentifier(key, constraint)
+        req.setParameters('station_code', 'avg_daily_max')
+        return self.runGeometryDataTest(req)
+
+    def testGetDataWithEqualsString(self):
+        geometryData = self._runConstraintTest('station_code', '=', self.obsStation)
+        for record in geometryData:
+            self.assertEqual(record.getString('station_code'), self.obsStation)
+
+    def testGetDataWithEqualsInt(self):
+        geometryData = self._runConstraintTest('avg_daily_max', '=', 70)
+        for record in geometryData:
+            self.assertEqual(record.getNumber('avg_daily_max'), 70)
+
+    def testGetDataWithEqualsFloat(self):
+        geometryData = self._runConstraintTest('avg_daily_max', '=', 69.2)
+        for record in geometryData:
+            self.assertEqual(round(record.getNumber('avg_daily_max'), 1), 69.2)
+
+    def testGetDataWithEqualsNone(self):
+        geometryData = self._runConstraintTest('station_code', '=', None)
+        self.assertEqual(len(geometryData), 0)
+
+    def testGetDataWithNotEquals(self):
+        geometryData = self._runConstraintTest('station_code', '!=', self.obsStation)
+        for record in geometryData:
+            self.assertNotEqual(record.getString('station_code'), self.obsStation)
+
+    def testGetDataWithNotEqualsNone(self):
+        geometryData = self._runConstraintTest('station_code', '!=', None)
+        for record in geometryData:
+            self.assertNotEqual(record.getType('station_code'), 'NULL')
+
+    def testGetDataWithGreaterThan(self):
+        geometryData = self._runConstraintTest('avg_daily_max', '>', 70)
+        for record in geometryData:
+            self.assertGreater(record.getNumber('avg_daily_max'), 70)
+
+    def testGetDataWithLessThan(self):
+        geometryData = self._runConstraintTest('avg_daily_max', '<', 70)
+        for record in geometryData:
+            self.assertLess(record.getNumber('avg_daily_max'), 70)
+
+    def testGetDataWithGreaterThanEquals(self):
+        geometryData = self._runConstraintTest('avg_daily_max', '>=', 70)
+        for record in geometryData:
+            self.assertGreaterEqual(record.getNumber('avg_daily_max'), 70)
+
+    def testGetDataWithLessThanEquals(self):
+        geometryData = self._runConstraintTest('avg_daily_max', '<=', 70)
+        for record in geometryData:
+            self.assertLessEqual(record.getNumber('avg_daily_max'), 70)
+
+    def testGetDataWithInTuple(self):
+        collection = (self.obsStation, 'KABR')
+        geometryData = self._runConstraintTest('station_code', 'in', collection)
+        for record in geometryData:
+            self.assertIn(record.getString('station_code'), collection)
+
+    def testGetDataWithInList(self):
+        collection = [self.obsStation, 'KABR']
+        geometryData = self._runConstraintTest('station_code', 'in', collection)
+        for record in geometryData:
+            self.assertIn(record.getString('station_code'), collection)
+
+    def testGetDataWithInGenerator(self):
+        collection = (self.obsStation, 'KABR')
+        generator = (item for item in collection)
+        geometryData = self._runConstraintTest('station_code', 'in', generator)
+        for record in geometryData:
+            self.assertIn(record.getString('station_code'), collection)
+
+    def testGetDataWithNotInList(self):
+        collection = ['KORD', 'KABR']
+        geometryData = self._runConstraintTest('station_code', 'not in', collection)
+        for record in geometryData:
+            self.assertNotIn(record.getString('station_code'), collection)
+
+    def testGetDataWithInvalidConstraintTypeThrowsException(self):
+        with self.assertRaises(ValueError):
+            self._runConstraintTest('station_code', 'junk', self.obsStation)
+
+    def testGetDataWithInvalidConstraintValueThrowsException(self):
+        with self.assertRaises(TypeError):
+            self._runConstraintTest('station_code', '=', {})
+
+    def testGetDataWithEmptyInConstraintThrowsException(self):
+        with self.assertRaises(ValueError):
+            self._runConstraintTest('station_code', 'in', [])
+
+    def testGetDataWithTimeRangeWithYearAndMonth1(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.cli_asos_monthly')
+        req.setLocationNames('KFNB')
+        req.setParameters('maxtemp_mon', 'min_sea_press')
+        startTime = datetime.datetime(2009, 1, 1)
+        endTime = datetime.datetime(2009, 12, 31)
+        tr = TimeRange(startTime, endTime)
+        self.runGeometryDataTestWithTimeRange(req, tr)
+
+    def testGetDataWithTimeRangeWithYearAndMonth2(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.cli_asos_monthly')
+        req.setLocationNames('KFNB')
+        req.setParameters('maxtemp_mon', 'min_sea_press')
+        startTime = datetime.datetime(2008, 1, 1)
+        endTime = datetime.datetime(2009, 3, 31)
+        tr = TimeRange(startTime, endTime)
+        self.runGeometryDataTestWithTimeRange(req, tr)
+
+    def testGetDataWithTimeRangeWithYearAndMonth3(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.cli_asos_monthly')
+        req.setLocationNames('KFNB')
+        req.setParameters('maxtemp_mon', 'min_sea_press')
+        startTime = datetime.datetime(2007, 7, 1)
+        endTime = datetime.datetime(2009, 3, 31)
+        tr = TimeRange(startTime, endTime)
+        self.runGeometryDataTestWithTimeRange(req, tr)
+
+    def testGetDataWithTimeRangeWithYearAndDayOfYear1(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.cli_asos_daily')
+        req.setLocationNames('KFNB')
+        req.setParameters('maxtemp_cal', 'min_press')
+        startTime = datetime.datetime(2009, 1, 1)
+        endTime = datetime.datetime(2009, 7, 31)
+        tr = TimeRange(startTime, endTime)
+        self.runGeometryDataTestWithTimeRange(req, tr)
+
+    def testGetDataWithTimeRangeWithYearAndDayOfYear2(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.cli_asos_daily')
+        req.setLocationNames('KFNB')
+        req.setParameters('maxtemp_cal', 'min_press')
+        startTime = datetime.datetime(2008, 7, 1)
+        endTime = datetime.datetime(2009, 3, 31)
+        tr = TimeRange(startTime, endTime)
+        self.runGeometryDataTestWithTimeRange(req, tr)
+
+    def testGetDataWithTimeRangeWithYearAndDayOfYear3(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.cli_asos_daily')
+        req.setLocationNames('KFNB')
+        req.setParameters('maxtemp_cal', 'min_press')
+        startTime = datetime.datetime(2007, 7, 1)
+        endTime = datetime.datetime(2009, 3, 31)
+        tr = TimeRange(startTime, endTime)
+        self.runGeometryDataTestWithTimeRange(req, tr)
+
+    def testGetDataWithTimeRangeWithPeriodTable(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.cli_mon_season_yr')
+        req.setLocationNames('KFNB')
+        req.setParameters('max_temp', 'precip_total')
+        startTime = datetime.datetime(2007, 7, 1)
+        endTime = datetime.datetime(2009, 3, 31)
+        tr = TimeRange(startTime, endTime)
+        self.runGeometryDataTestWithTimeRange(req, tr)
+
+    def testGetDataWithTimeRangeWithForDateTable(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.daily_climate')
+        req.setLocationNames('KFNB')
+        req.setParameters('max_temp', 'precip', 'avg_wind_speed')
+        startTime = datetime.datetime(2007, 7, 1)
+        endTime = datetime.datetime(2009, 3, 31)
+        tr = TimeRange(startTime, endTime)
+        self.runGeometryDataTestWithTimeRange(req, tr)
+
+    def testNoDuplicateData(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.cli_asos_monthly')
+        req.setLocationNames('KOMA')
+        req.setParameters('maxtemp_day1')
+        rows = DAL.getGeometryData(req, DAL.getAvailableTimes(req)[0:5])
+        for i in range(len(rows)):
+            for j in range(len(rows)):
+                if i != j:
+                    self.assertNotEqual(rows[i].__dict__, rows[j].__dict__)

awips/test/dafTests/testCombinedTimeQuery.py

@@ -1,3 +1,30 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+from awips.dataaccess import DataAccessLayer as DAL
+
+from awips.dataaccess import CombinedTimeQuery as CTQ
+
+import unittest
+import os
+
 #
 # Test the CombinedTimedQuery module
 #
@@ -11,39 +38,30 @@
 #
 #
 
-from awips.dataaccess import DataAccessLayer as DAL
-from awips.dataaccess import CombinedTimeQuery as CTQ
-
-import unittest
-import os
-
-
 class CombinedTimeQueryTestCase(unittest.TestCase):
 
-    modelName = "RAP13"
-
     @classmethod
     def setUp(cls):
         host = os.environ.get('DAF_TEST_HOST')
         if host is None:
-            host = 'edex-cloud.unidata.ucar.edu'
+            host = 'localhost'
         DAL.changeEDEXHost(host)
 
     def testSuccessfulQuery(self):
         req = DAL.newDataRequest('grid')
-        req.setLocationNames(self.modelName)
+        req.setLocationNames('RUC130')
-        req.setParameters('T', 'GH')
+        req.setParameters('T','GH')
-        req.setLevels('300MB', '500MB', '700MB')
+        req.setLevels('300MB', '500MB','700MB')
-        times = CTQ.getAvailableTimes(req)
+        times = CTQ.getAvailableTimes(req);
         self.assertNotEqual(len(times), 0)
-
+    
     def testNonIntersectingQuery(self):
         """
         Test that when a parameter is only available on one of the levels that no times are returned.
         """
         req = DAL.newDataRequest('grid')
-        req.setLocationNames(self.modelName)
+        req.setLocationNames('RUC130')
-        req.setParameters('T', 'GH', 'LgSP1hr')
+        req.setParameters('T','GH', 'LgSP1hr')
-        req.setLevels('300MB', '500MB', '700MB', '0.0SFC')
+        req.setLevels('300MB', '500MB','700MB','0.0SFC')
-        times = CTQ.getAvailableTimes(req)
+        times = CTQ.getAvailableTimes(req);
         self.assertEqual(len(times), 0)

awips/test/dafTests/testCommonObsSpatial.py

@@ -1,3 +1,30 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+from ufpy.dataaccess import DataAccessLayer as DAL
+
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
+from . import baseDafTestCase
+from . import params
+
 #
 # Test DAF support for common_obs_spatial data
 #
@@ -18,12 +45,6 @@
 #    01/06/17        5981          tgurney        Do not check data times
 #
 
-from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
-from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
-from awips.test.dafTests import baseDafTestCase
-from awips.test.dafTests import params
-
 
 class CommonObsSpatialTestCase(baseDafTestCase.DafTestCase):
     """Test DAF support for common_obs_spatial data"""
@@ -69,11 +90,6 @@ class CommonObsSpatialTestCase(baseDafTestCase.DafTestCase):
         for record in geometryData:
             self.assertEqual(record.getNumber('catalogtype'), 32)
 
-    def testGetDataWithEqualsLong(self):
-        geometryData = self._runConstraintTest('elevation', '=', 0)
-        for record in geometryData:
-            self.assertEqual(record.getNumber('elevation'), 0)
-
     # No float test since there are no float identifiers available. Attempting
     # to filter a non-float identifier on a float value raises an exception.
 

awips/test/dafTests/testDataTime.py

@@ -1,3 +1,27 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+from dynamicserialize.dstypes.com.raytheon.uf.common.time import DataTime
+
+import unittest
+
 #
 # Unit tests for Python implementation of RequestConstraint
 #
@@ -9,10 +33,6 @@
 #
 #
 
-from dynamicserialize.dstypes.com.raytheon.uf.common.time import DataTime
-
-import unittest
-
 
 class DataTimeTestCase(unittest.TestCase):
 
@@ -37,14 +57,14 @@ class DataTimeTestCase(unittest.TestCase):
         self.assertEqual(expected, str(DataTime(s)))
         s = s.replace(' ', '_')
         self.assertEqual(expected, str(DataTime(s)))
-
+    
     def testFromStrWithFcstTimeHr(self):
         s = '2016-08-02 01:23:45 (17)'
         expected = s
         self.assertEqual(expected, str(DataTime(s)))
         s = s.replace(' ', '_')
         self.assertEqual(expected, str(DataTime(s)))
-
+        
     def testFromStrWithFcstTimeHrZeroMillis(self):
         s = '2016-08-02 01:23:45.0 (17)'
         expected = '2016-08-02 01:23:45 (17)'
@@ -58,7 +78,7 @@ class DataTimeTestCase(unittest.TestCase):
         self.assertEqual(expected, str(DataTime(s)))
         s = s.replace(' ', '_')
         self.assertEqual(expected, str(DataTime(s)))
-
+        
     def testFromStrWithFcstTimeHrMin(self):
         s = '2016-08-02 01:23:45 (17:34)'
         expected = s
@@ -72,28 +92,28 @@ class DataTimeTestCase(unittest.TestCase):
         self.assertEqual(expected, str(DataTime(s)))
         s = s.replace(' ', '_')
         self.assertEqual(expected, str(DataTime(s)))
-
+        
     def testFromStrWithPeriod(self):
         s = '2016-08-02 01:23:45[2016-08-02 02:34:45--2016-08-02 03:45:56]'
         expected = s
         self.assertEqual(expected, str(DataTime(s)))
         s = s.replace(' ', '_')
         self.assertEqual(expected, str(DataTime(s)))
-
+    
     def testFromStrWithPeriodZeroMillis(self):
         s = '2016-08-02 01:23:45.0[2016-08-02 02:34:45.0--2016-08-02 03:45:56.0]'
         expected = '2016-08-02 01:23:45[2016-08-02 02:34:45--2016-08-02 03:45:56]'
         self.assertEqual(expected, str(DataTime(s)))
         s = s.replace(' ', '_')
         self.assertEqual(expected, str(DataTime(s)))
-
+    
     def testFromStrWithEverything(self):
         s = '2016-08-02 01:23:45.0_(17:34)[2016-08-02 02:34:45.0--2016-08-02 03:45:56.0]'
         expected = '2016-08-02 01:23:45 (17:34)[2016-08-02 02:34:45--2016-08-02 03:45:56]'
         self.assertEqual(expected, str(DataTime(s)))
         s = s.replace(' ', '_')
         self.assertEqual(expected, str(DataTime(s)))
-
+        
     def testDataTimeReconstructItselfFromString(self):
         times = [
              '2016-08-02 01:23:45',
@@ -111,4 +131,4 @@ class DataTimeTestCase(unittest.TestCase):
              '2016-08-02 01:23:45.456_(17:34)[2016-08-02_02:34:45.0--2016-08-02_03:45:56.0]'
              ]
         for time in times:
-            self.assertEqual(DataTime(time), DataTime(str(DataTime(time))), time)
+            self.assertEqual(DataTime(time), DataTime(str(DataTime(time))), time)

awips/test/dafTests/testFfmp.py

@@ -0,0 +1,222 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
+from ufpy.dataaccess import DataAccessLayer as DAL
+
+from . import baseDafTestCase
+from . import params
+
+#
+# Test DAF support for ffmp data
+#
+#     SOFTWARE HISTORY
+#
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    01/19/16        4795          mapeters       Initial Creation.
+#    04/11/16        5548          tgurney        Cleanup
+#    04/18/16        5548          tgurney        More cleanup
+#    04/18/16        5587          tgurney        Add test for sane handling of
+#                                                 zero records returned
+#    06/20/16        5587          tgurney        Add identifier values tests
+#    07/01/16        5728          mapeters       Add advanced query tests,
+#                                                 include huc and accumHrs in
+#                                                 id values tests, test that
+#                                                 accumHrs id is never required
+#    08/03/16        5728          mapeters       Fixed minor bugs, replaced
+#                                                 PRTM parameter since it isn't
+#                                                 configured for ev-oma
+#    11/08/16        5985          tgurney        Do not check data times
+#    12/07/16        5981          tgurney        Parameterize
+#    12/20/16        5981          tgurney        Do not check data times
+#
+#
+
+
+class FfmpTestCase(baseDafTestCase.DafTestCase):
+    """Test DAF support for ffmp data"""
+
+    datatype = 'ffmp'
+    location = params.RADAR.lower()
+
+    @staticmethod
+    def addIdentifiers(req):
+        req.addIdentifier('wfo', params.SITE_ID)
+        req.addIdentifier('siteKey', 'hpe')
+        req.addIdentifier('dataKey', 'hpe')
+        req.addIdentifier('huc', 'ALL')
+
+    def testGetAvailableParameters(self):
+        req = DAL.newDataRequest(self.datatype)
+        self.runParametersTest(req)
+
+    def testGetAvailableLocations(self):
+        req = DAL.newDataRequest(self.datatype)
+        self.addIdentifiers(req)
+        self.runLocationsTest(req)
+
+    def testGetAvailableTimes(self):
+        req = DAL.newDataRequest(self.datatype)
+        self.addIdentifiers(req)
+        req.setParameters('DHRMOSAIC')
+        self.runTimesTest(req)
+
+    def testGetGeometryData(self):
+        req = DAL.newDataRequest(self.datatype)
+        self.addIdentifiers(req)
+        req.setParameters('DHRMOSAIC')
+        self.runGeometryDataTest(req, checkDataTimes=False)
+
+    def testGetGeometryDataEmptyResult(self):
+        req = DAL.newDataRequest(self.datatype)
+        self.addIdentifiers(req)
+        req.setParameters('blah blah blah') # force 0 records returned
+        result = self.runGeometryDataTest(req, checkDataTimes=False)
+        self.assertEqual(len(result), 0)
+
+    def testGetIdentifierValues(self):
+        req = DAL.newDataRequest(self.datatype)
+        optionalIds = set(DAL.getOptionalIdentifiers(req))
+        requiredIds = set(DAL.getRequiredIdentifiers(req))
+        ids = requiredIds | optionalIds
+        for id in ids:
+            req = DAL.newDataRequest(self.datatype)
+            if id == 'accumHrs':
+                req.setParameters('ARI6H2YR')
+                req.addIdentifier('wfo', params.SITE_ID)
+                req.addIdentifier('siteKey', self.location)
+                req.addIdentifier('huc', 'ALL')
+            idValues = DAL.getIdentifierValues(req, id)
+            self.assertTrue(hasattr(idValues, '__iter__'))
+            print(id + " values: " + str(idValues))
+
+    def testGetInvalidIdentifierValuesThrowsException(self):
+        self.runInvalidIdValuesTest()
+
+    def testGetNonexistentIdentifierValuesThrowsException(self):
+        self.runNonexistentIdValuesTest()
+
+    def _runConstraintTest(self, key, operator, value):
+        req = DAL.newDataRequest(self.datatype)
+        constraint = RequestConstraint.new(operator, value)
+        req.addIdentifier(key, constraint)
+        req.addIdentifier('wfo', params.SITE_ID)
+        req.addIdentifier('huc', 'ALL')
+        req.setParameters('QPFSCAN')
+        return self.runGeometryDataTest(req, checkDataTimes=False)
+
+    def testGetDataWithEqualsString(self):
+        geometryData = self._runConstraintTest('siteKey', '=', self.location)
+        for record in geometryData:
+            self.assertEqual(record.getAttribute('siteKey'), self.location)
+
+    # No numeric tests since no numeric identifiers are available that support
+    # RequestConstraints.
+
+    def testGetDataWithEqualsNone(self):
+        geometryData = self._runConstraintTest('siteKey', '=', None)
+        for record in geometryData:
+            self.assertIsNone(record.getAttribute('siteKey'))
+
+    def testGetDataWithNotEquals(self):
+        geometryData = self._runConstraintTest('siteKey', '!=', self.location)
+        for record in geometryData:
+            self.assertNotEqual(record.getAttribute('siteKey'), self.location)
+
+    def testGetDataWithNotEqualsNone(self):
+        geometryData = self._runConstraintTest('siteKey', '!=', None)
+        for record in geometryData:
+            self.assertIsNotNone(record.getAttribute('siteKey'))
+
+    def testGetDataWithGreaterThan(self):
+        geometryData = self._runConstraintTest('siteKey', '>', self.location)
+        for record in geometryData:
+            self.assertGreater(record.getAttribute('siteKey'), self.location)
+
+    def testGetDataWithLessThan(self):
+        geometryData = self._runConstraintTest('siteKey', '<', self.location)
+        for record in geometryData:
+            self.assertLess(record.getAttribute('siteKey'), self.location)
+
+    def testGetDataWithGreaterThanEquals(self):
+        geometryData = self._runConstraintTest('siteKey', '>=', self.location)
+        for record in geometryData:
+            self.assertGreaterEqual(record.getAttribute('siteKey'), self.location)
+
+    def testGetDataWithLessThanEquals(self):
+        geometryData = self._runConstraintTest('siteKey', '<=', self.location)
+        for record in geometryData:
+            self.assertLessEqual(record.getAttribute('siteKey'), self.location)
+
+    def testGetDataWithInList(self):
+        collection = [self.location, 'kuex']
+        geometryData = self._runConstraintTest('siteKey', 'in', collection)
+        for record in geometryData:
+            self.assertIn(record.getAttribute('siteKey'), collection)
+
+    def testGetDataWithNotInList(self):
+        collection = [self.location, 'kuex']
+        geometryData = self._runConstraintTest('siteKey', 'not in', collection)
+        for record in geometryData:
+            self.assertNotIn(record.getAttribute('siteKey'), collection)
+
+    def testGetDataWithInvalidConstraintTypeThrowsException(self):
+        with self.assertRaises(ValueError):
+            self._runConstraintTest('siteKey', 'junk', self.location)
+
+    def testGetDataWithInvalidConstraintValueThrowsException(self):
+        with self.assertRaises(TypeError):
+            self._runConstraintTest('siteKey', '=', {})
+
+    def testGetDataWithEmptyInConstraintThrowsException(self):
+        with self.assertRaises(ValueError):
+            self._runConstraintTest('siteKey', 'in', [])
+
+    def testGetDataWithSiteKeyAndDataKeyConstraints(self):
+        siteKeys = [self.location, 'hpe']
+        dataKeys = ['kuex', 'kdmx']
+
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('wfo', params.SITE_ID)
+        req.addIdentifier('huc', 'ALL')
+
+        siteKeysConstraint = RequestConstraint.new('in', siteKeys)
+        req.addIdentifier('siteKey', siteKeysConstraint)
+        dataKeysConstraint = RequestConstraint.new('in', dataKeys)
+        req.addIdentifier('dataKey', dataKeysConstraint)
+
+        req.setParameters('QPFSCAN')
+        geometryData = self.runGeometryDataTest(req, checkDataTimes=False)
+        for record in geometryData:
+            self.assertIn(record.getAttribute('siteKey'), siteKeys)
+            # dataKey attr. is comma-separated list of dataKeys that had data
+            for dataKey in record.getAttribute('dataKey').split(','):
+                self.assertIn(dataKey, dataKeys)
+
+    def testGetGuidanceDataWithoutAccumHrsIdentifierSet(self):
+        # Test that accumHrs identifier is not required for guidance data
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('wfo', params.SITE_ID)
+        req.addIdentifier('siteKey', self.location)
+        req.addIdentifier('huc', 'ALL')
+        req.setParameters('FFG0124hr')
+        self.runGeometryDataTest(req, checkDataTimes=False)

awips/test/dafTests/testGfe.py

@@ -1,3 +1,32 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
+from ufpy.dataaccess import DataAccessLayer as DAL
+from shapely.geometry import box, Point
+
+from . import baseDafTestCase
+from . import params
+import unittest
+
 #
 # Test DAF support for GFE data
 #
@@ -23,15 +52,6 @@
 #
 #
 
-from __future__ import print_function
-from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
-from awips.dataaccess import DataAccessLayer as DAL
-from shapely.geometry import box, Point
-
-from awips.test.dafTests import baseDafTestCase
-from awips.test.dafTests import params
-import unittest
-
 
 class GfeTestCase(baseDafTestCase.DafTestCase):
     """Test DAF support for GFE data"""
@@ -78,7 +98,7 @@ class GfeTestCase(baseDafTestCase.DafTestCase):
         # Ensure all points are within one degree of the original box
         # to allow slight margin of error for reprojection distortion.
         testEnv = box(params.ENVELOPE.bounds[0] - 1, params.ENVELOPE.bounds[1] - 1,
-                      params.ENVELOPE.bounds[2] + 1, params.ENVELOPE.bounds[3] + 1)
+                      params.ENVELOPE.bounds[2] + 1, params.ENVELOPE.bounds[3] + 1 )
 
         for i in range(len(lons)):
             self.assertTrue(testEnv.contains(Point(lons[i], lats[i])))
@@ -89,17 +109,17 @@ class GfeTestCase(baseDafTestCase.DafTestCase):
         req.addIdentifier('parmId.dbId.siteId', params.SITE_ID)
         req.setParameters('Wind')
         times = DAL.getAvailableTimes(req)
-        if not times:
+        if not(times):
             raise unittest.SkipTest('No Wind Data available for testing')
         gridData = DAL.getGridData(req, [times[0]])
         rawWind = None
         rawDir = None
         for grid in gridData:
             if grid.getParameter() == 'Wind':
-                self.assertEqual(grid.getUnit(), 'kts')
+                self.assertEqual(grid.getUnit(),'kts')
                 rawWind = grid.getRawData()
             elif grid.getParameter() == 'WindDirection':
-                self.assertEqual(grid.getUnit(), 'deg')
+                self.assertEqual(grid.getUnit(),'deg')
                 rawDir = grid.getRawData()
         self.assertIsNotNone(rawWind, 'Wind Magnitude grid is not present')
         self.assertIsNotNone(rawDir, 'Wind Direction grid is not present')
@@ -107,7 +127,7 @@ class GfeTestCase(baseDafTestCase.DafTestCase):
         self.assertTrue((rawWind >= 0).all(), 'Wind Speed should not contain negative values')
         self.assertTrue((rawDir >= 0).all(), 'Wind Direction should not contain negative values')
         self.assertTrue((rawDir <= 360).all(), 'Wind Direction should be less than or equal to 360')
-        self.assertFalse((rawDir == rawWind).all(), 'Wind Direction should be different from Wind Speed')
+        self.assertFalse((rawDir == rawWind).all(), 'Wind Direction should be different from Wind Speed')        
 
     def testGetIdentifierValues(self):
         req = DAL.newDataRequest(self.datatype)
@@ -192,3 +212,4 @@ class GfeTestCase(baseDafTestCase.DafTestCase):
     def testGetDataWithEmptyInConstraintThrowsException(self):
         with self.assertRaises(ValueError):
             self._runConstraintTest('parmId.dbId.modelName', 'in', [])
+

awips/test/dafTests/testGfeEditArea.py

@@ -0,0 +1,215 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
+from ufpy.dataaccess import DataAccessLayer as DAL
+from ufpy.ThriftClient import ThriftRequestException
+
+from . import baseDafTestCase
+from . import params
+
+#
+# Test DAF support for GFE edit area data
+#
+#     SOFTWARE HISTORY
+#
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    06/08/17        6298          mapeters       Initial Creation.
+#    09/27/17        6463          tgurney        Remove GID site identifier
+#
+#
+
+
+class GfeEditAreaTestCase(baseDafTestCase.DafTestCase):
+    """Test DAF support for GFE edit area data"""
+
+    datatype = 'gfeEditArea'
+
+    siteIdKey = 'siteId'
+
+    editAreaNames = ['ISC_NHA', 'SDZ066', 'StormSurgeWW_EditArea']
+
+    groupKey = 'group'
+
+    groups = ['ISC', 'WFOs', 'FIPS_' + params.SITE_ID]
+
+    def testGetAvailableParameters(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier(self.siteIdKey, params.SITE_ID)
+        with self.assertRaises(ThriftRequestException):
+            self.runParametersTest(req)
+
+    def testGetAvailableLocations(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier(self.siteIdKey, params.SITE_ID)
+        self.runLocationsTest(req)
+
+    def testGetAvailableTimes(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier(self.siteIdKey, params.SITE_ID)
+        with self.assertRaises(ThriftRequestException):
+            self.runTimesTest(req)
+
+    def testGetGeometryDataWithoutSiteIdThrowsException(self):
+        req = DAL.newDataRequest(self.datatype)
+        with self.assertRaises(ThriftRequestException):
+            self.runGeometryDataTest(req)
+
+    def testGetGeometryData(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier(self.siteIdKey, params.SITE_ID)
+        data = self.runGeometryDataTest(req)
+        for item in data:
+            self.assertEqual(params.SITE_ID, item.getAttribute(self.siteIdKey))
+
+    def testGetGeometryDataWithLocNames(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier(self.siteIdKey, params.SITE_ID)
+        req.setLocationNames(*self.editAreaNames)
+        data = self.runGeometryDataTest(req)
+        for item in data:
+            self.assertEqual(params.SITE_ID, item.getAttribute(self.siteIdKey))
+            self.assertIn(item.getLocationName(), self.editAreaNames)
+
+    def testGetGeometryDataWithGroups(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier(self.siteIdKey, params.SITE_ID)
+        req.addIdentifier(self.groupKey, RequestConstraint.new('in', self.groups))
+        data = self.runGeometryDataTest(req)
+        for item in data:
+            self.assertEqual(params.SITE_ID, item.getAttribute(self.siteIdKey))
+            self.assertIn(item.getAttribute(self.groupKey), self.groups)
+
+    def testGetGeometryDataWithLocNamesAndGroupsThrowException(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier(self.siteIdKey, params.SITE_ID)
+        req.setLocationNames(*self.editAreaNames)
+        req.addIdentifier(self.groupKey, RequestConstraint.new('in', self.groups))
+        with self.assertRaises(ThriftRequestException):
+            self.runGeometryDataTest(req)
+
+    def testGetGeometryDataWithEnvelope(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier(self.siteIdKey, params.SITE_ID)
+        req.setEnvelope(params.ENVELOPE)
+        data = self.runGeometryDataTest(req)
+        for item in data:
+            self.assertEqual(params.SITE_ID, item.getAttribute(self.siteIdKey))
+            self.assertTrue(params.ENVELOPE.intersects(item.getGeometry()))
+
+    def testGetIdentifierValues(self):
+        req = DAL.newDataRequest(self.datatype)
+        optionalIds = set(DAL.getOptionalIdentifiers(req))
+        requiredIds = set(DAL.getRequiredIdentifiers(req))
+        self.runGetIdValuesTest(optionalIds | requiredIds)
+
+    def testGetInvalidIdentifierValuesThrowsException(self):
+        self.runInvalidIdValuesTest()
+
+    def testGetNonexistentIdentifierValuesThrowsException(self):
+        self.runNonexistentIdValuesTest()
+
+    def _runConstraintTest(self, key, operator, value):
+        req = DAL.newDataRequest(self.datatype)
+        constraint = RequestConstraint.new(operator, value)
+        req.addIdentifier(key, constraint)
+        req.setLocationNames(*self.editAreaNames)
+        return self.runGeometryDataTest(req)
+
+    def testGetDataWithEqualsString(self):
+        geomData = self._runConstraintTest(self.siteIdKey, '=', params.SITE_ID)
+        for record in geomData:
+            self.assertEqual(record.getAttribute(self.siteIdKey), params.SITE_ID)
+
+    # No numeric tests since no numeric identifiers are available.
+
+    def testGetDataWithEqualsNone(self):
+        geomData = self._runConstraintTest(self.siteIdKey, '=', None)
+        for record in geomData:
+            self.assertIsNone(record.getAttribute(self.siteIdKey))
+
+    def testGetDataWithNotEquals(self):
+        geomData = self._runConstraintTest(self.siteIdKey, '!=', params.SITE_ID)
+        for record in geomData:
+            self.assertNotEqual(record.getAttribute(self.siteIdKey), params.SITE_ID)
+
+    def testGetDataWithNotEqualsNone(self):
+        geomData = self._runConstraintTest(self.siteIdKey, '!=', None)
+        for record in geomData:
+            self.assertIsNotNone(record.getAttribute(self.siteIdKey))
+
+    def testGetDataWithGreaterThan(self):
+        geomData = self._runConstraintTest(self.siteIdKey, '>', params.SITE_ID)
+        for record in geomData:
+            self.assertGreater(record.getAttribute(self.siteIdKey), params.SITE_ID)
+
+    def testGetDataWithLessThan(self):
+        geomData = self._runConstraintTest(self.siteIdKey, '<', params.SITE_ID)
+        for record in geomData:
+            self.assertLess(record.getAttribute(self.siteIdKey), params.SITE_ID)
+
+    def testGetDataWithGreaterThanEquals(self):
+        geomData = self._runConstraintTest(self.siteIdKey, '>=', params.SITE_ID)
+        for record in geomData:
+            self.assertGreaterEqual(record.getAttribute(self.siteIdKey), params.SITE_ID)
+
+    def testGetDataWithLessThanEquals(self):
+        geomData = self._runConstraintTest(self.siteIdKey, '<=', params.SITE_ID)
+        for record in geomData:
+            self.assertLessEqual(record.getAttribute(self.siteIdKey), params.SITE_ID)
+
+    def testGetDataWithInTuple(self):
+        collection = (params.SITE_ID,)
+        geomData = self._runConstraintTest(self.siteIdKey, 'in', collection)
+        for record in geomData:
+            self.assertIn(record.getAttribute(self.siteIdKey), collection)
+
+    def testGetDataWithInList(self):
+        collection = [params.SITE_ID,]
+        geomData = self._runConstraintTest(self.siteIdKey, 'in', collection)
+        for record in geomData:
+            self.assertIn(record.getAttribute(self.siteIdKey), collection)
+
+    def testGetDataWithInGenerator(self):
+        collection = (params.SITE_ID,)
+        generator = (item for item in collection)
+        geomData = self._runConstraintTest(self.siteIdKey, 'in', generator)
+        for record in geomData:
+            self.assertIn(record.getAttribute(self.siteIdKey), collection)
+
+    def testGetDataWithNotInList(self):
+        collection = [params.SITE_ID,]
+        geomData = self._runConstraintTest(self.siteIdKey, 'not in', collection)
+        for record in geomData:
+            self.assertNotIn(record.getAttribute(self.siteIdKey), collection)
+
+    def testGetDataWithInvalidConstraintTypeThrowsException(self):
+        with self.assertRaises(ValueError):
+            self._runConstraintTest(self.siteIdKey, 'junk', params.SITE_ID)
+
+    def testGetDataWithInvalidConstraintValueThrowsException(self):
+        with self.assertRaises(TypeError):
+            self._runConstraintTest(self.siteIdKey, '=', {})
+
+    def testGetDataWithEmptyInConstraintThrowsException(self):
+        with self.assertRaises(ValueError):
+            self._runConstraintTest(self.siteIdKey, 'in', [])

awips/test/dafTests/testGrid.py

@@ -1,3 +1,33 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
+from shapely.geometry import box, Point
+from ufpy.dataaccess import DataAccessLayer as DAL
+from ufpy.ThriftClient import ThriftRequestException
+
+from . import baseDafTestCase
+from . import params
+import unittest
+
 #
 # Test DAF support for grid data
 #
@@ -18,18 +48,9 @@
 #    12/07/16        5981          tgurney        Parameterize
 #    01/06/17        5981          tgurney        Skip envelope test when no
 #                                                 data is available
+#    04/14/22        8845          njensen        Added testGetDataAtPoint
 #
 
-from __future__ import print_function
-from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
-from awips.dataaccess import DataAccessLayer as DAL
-from awips.ThriftClient import ThriftRequestException
-from shapely.geometry import box, Point
-
-from awips.test.dafTests import baseDafTestCase
-from awips.test.dafTests import params
-import unittest
-
 
 class GridTestCase(baseDafTestCase.DafTestCase):
     """Test DAF support for grid data"""
@@ -86,6 +107,7 @@ class GridTestCase(baseDafTestCase.DafTestCase):
     def testGetNonexistentIdentifierValuesThrowsException(self):
         self.runNonexistentIdValuesTest()
 
+        
     def testGetDataWithEnvelope(self):
         req = DAL.newDataRequest(self.datatype)
         req.addIdentifier('info.datasetId', self.model)
@@ -98,15 +120,16 @@ class GridTestCase(baseDafTestCase.DafTestCase):
         lons, lats = gridData[0].getLatLonCoords()
         lons = lons.reshape(-1)
         lats = lats.reshape(-1)
-
+        
         # Ensure all points are within one degree of the original box
         # to allow slight margin of error for reprojection distortion.
         testEnv = box(params.ENVELOPE.bounds[0] - 1, params.ENVELOPE.bounds[1] - 1,
-                      params.ENVELOPE.bounds[2] + 1, params.ENVELOPE.bounds[3] + 1)
+                      params.ENVELOPE.bounds[2] + 1, params.ENVELOPE.bounds[3] + 1 )
-
+        
         for i in range(len(lons)):
             self.assertTrue(testEnv.contains(Point(lons[i], lats[i])))
 
+
     def _runConstraintTest(self, key, operator, value):
         req = DAL.newDataRequest(self.datatype)
         constraint = RequestConstraint.new(operator, value)
@@ -127,11 +150,6 @@ class GridTestCase(baseDafTestCase.DafTestCase):
         for record in gridData:
             self.assertEqual(record.getAttribute('info.level.levelonevalue'), 2000)
 
-    def testGetDataWithEqualsLong(self):
-        gridData = self._runConstraintTest('info.level.levelonevalue', '=', 2000)
-        for record in gridData:
-            self.assertEqual(record.getAttribute('info.level.levelonevalue'), 2000)
-
     def testGetDataWithEqualsFloat(self):
         gridData = self._runConstraintTest('info.level.levelonevalue', '=', 2000.0)
         for record in gridData:
@@ -259,3 +277,11 @@ class GridTestCase(baseDafTestCase.DafTestCase):
             self.runGridDataTest(req)
         self.assertIn('IncompatibleRequestException', str(cm.exception))
         self.assertIn('info.level.masterLevel.name', str(cm.exception))
+
+    def testGetDataAtPoint(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('info.datasetId', self.model)
+        req.setLevels('2FHAG')
+        req.setParameters('T')
+        req.setEnvelope(params.POINT)
+        self.runGeometryDataTest(req)

awips/test/dafTests/testHydro.py

@@ -0,0 +1,243 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+import datetime
+from ufpy.dataaccess import DataAccessLayer as DAL
+from ufpy.ThriftClient import ThriftRequestException
+
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
+from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange
+from . import baseDafTestCase
+
+#
+# Test DAF support for hydro data
+#
+#     SOFTWARE HISTORY
+#
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    01/19/16        4795          mapeters       Initial Creation.
+#    04/11/16        5548          tgurney        Cleanup
+#    04/18/16        5548          tgurney        More cleanup
+#    04/21/16        5596          tgurney        Add tests to verify #5596
+#    04/26/16        5587          tgurney        Add identifier values tests
+#    06/09/16        5574          tgurney        Add advanced query tests
+#    06/13/16        5574          tgurney        Fix checks for None
+#    06/21/16        5548          tgurney        Skip tests that cause errors
+#    06/30/16        5725          tgurney        Add test for NOT IN
+#    10/06/16        5926          dgilling       Add additional location tests.
+#
+#
+
+
+class HydroTestCase(baseDafTestCase.DafTestCase):
+    """Test DAF support for hydro data"""
+
+    datatype = 'hydro'
+
+    def testGetAvailableParameters(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'height')
+        self.runParametersTest(req)
+
+    def testGetAvailableParametersFullyQualifiedTable(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.height')
+        self.runParametersTest(req)
+
+    def testGetAvailableParamsNoTableThrowsInvalidIdentifiersException(self):
+        req = DAL.newDataRequest(self.datatype)
+        with self.assertRaises(ThriftRequestException) as cm:
+            self.runParametersTest(req)
+        self.assertIn('InvalidIdentifiersException', str(cm.exception))
+
+    def testGetAvailableLocations(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'height')
+        self.runLocationsTest(req)
+
+    def testGetAvailableLocationsWithConstraint(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'height')
+        req.addIdentifier('value', RequestConstraint.new('>', 5.0))
+        self.runLocationsTest(req)
+
+    def testGetAvailableLocationsWithInvalidTable(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'city')
+        with self.assertRaises(ThriftRequestException) as cm:
+            DAL.getAvailableLocationNames(req)
+        self.assertIn('IncompatibleRequestException', str(cm.exception))
+
+    def testGetAvailableTimes(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'height')
+        req.setParameters('lid', 'quality_code')
+        self.runTimesTest(req)
+
+    def testGetGeometryDataWithoutLocationSpecified(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'height')
+        req.setParameters('lid', 'quality_code')
+        self.runGeometryDataTest(req)
+
+    def testGetGeometryDataWithLocationSpecified(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'fcstheight')
+        locs = DAL.getAvailableLocationNames(req)
+        if locs:
+            req.setLocationNames(locs[0])
+            req.setParameters('probability', 'value')
+            data = self.runGeometryDataTest(req)
+            self.assertNotEqual(len(data), 0)
+
+    def testGetTableIdentifierValues(self):
+        self.runGetIdValuesTest(['table'])
+
+    def testGetColumnIdValuesWithTable(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'height')
+        idValues = DAL.getIdentifierValues(req, 'lid')
+        self.assertTrue(hasattr(idValues, '__iter__'))
+
+    def testGetColumnIdValuesWithoutTableThrowsException(self):
+        req = DAL.newDataRequest(self.datatype)
+        with self.assertRaises(ThriftRequestException):
+            DAL.getIdentifierValues(req, 'lid')
+
+    def testGetInvalidIdentifierValuesThrowsException(self):
+        self.runInvalidIdValuesTest()
+
+    def testGetNonexistentIdentifierValuesThrowsException(self):
+        self.runNonexistentIdValuesTest()
+
+    def _runConstraintTest(self, key, operator, value):
+        req = DAL.newDataRequest(self.datatype)
+        constraint = RequestConstraint.new(operator, value)
+        req.addIdentifier(key, constraint)
+        req.addIdentifier('table', 'height')
+        req.addIdentifier('ts', 'RG')
+        req.setParameters('value', 'lid', 'quality_code')
+        return self.runGeometryDataTest(req)
+
+    def testGetDataWithEqualsString(self):
+        geometryData = self._runConstraintTest('value', '=', '3')
+        for record in geometryData:
+            self.assertEqual(record.getNumber('value'), 3)
+
+    def testGetDataWithEqualsInt(self):
+        geometryData = self._runConstraintTest('value', '=', 3)
+        for record in geometryData:
+            self.assertEqual(record.getNumber('value'), 3)
+
+    def testGetDataWithEqualsFloat(self):
+        geometryData = self._runConstraintTest('value', '=', 3.0)
+        for record in geometryData:
+            self.assertEqual(round(record.getNumber('value'), 1), 3.0)
+
+    def testGetDataWithEqualsNone(self):
+        geometryData = self._runConstraintTest('value', '=', None)
+        self.assertEqual(len(geometryData), 0)
+
+    def testGetDataWithNotEquals(self):
+        geometryData = self._runConstraintTest('value', '!=', 3)
+        for record in geometryData:
+            self.assertNotEqual(record.getNumber('value'), '3')
+
+    def testGetDataWithNotEqualsNone(self):
+        geometryData = self._runConstraintTest('value', '!=', None)
+        self.assertNotEqual(len(geometryData), 0)
+        for record in geometryData:
+            self.assertNotEqual(record.getType('value'), 'NULL')
+
+    def testGetDataWithGreaterThan(self):
+        geometryData = self._runConstraintTest('value', '>', 3)
+        for record in geometryData:
+            self.assertGreater(record.getNumber('value'), 3)
+
+    def testGetDataWithLessThan(self):
+        geometryData = self._runConstraintTest('value', '<', 3)
+        for record in geometryData:
+            self.assertLess(record.getNumber('value'), 3)
+
+    def testGetDataWithGreaterThanEquals(self):
+        geometryData = self._runConstraintTest('value', '>=', 3)
+        for record in geometryData:
+            self.assertGreaterEqual(record.getNumber('value'), 3)
+
+    def testGetDataWithLessThanEquals(self):
+        geometryData = self._runConstraintTest('value', '<=', 3)
+        for record in geometryData:
+            self.assertLessEqual(record.getNumber('value'), 3)
+
+    def testGetDataWithInTuple(self):
+        collection = (3, 4)
+        geometryData = self._runConstraintTest('value', 'in', collection)
+        for record in geometryData:
+            self.assertIn(record.getNumber('value'), collection)
+
+    def testGetDataWithInList(self):
+        collection = [3, 4]
+        geometryData = self._runConstraintTest('value', 'in', collection)
+        for record in geometryData:
+            self.assertIn(record.getNumber('value'), collection)
+
+    def testGetDataWithInGenerator(self):
+        collection = (3, 4)
+        generator = (item for item in collection)
+        geometryData = self._runConstraintTest('value', 'in', generator)
+        for record in geometryData:
+            self.assertIn(record.getNumber('value'), collection)
+
+    def testGetDataWithNotInList(self):
+        collection = [3, 4]
+        geometryData = self._runConstraintTest('value', 'not in', collection)
+        for record in geometryData:
+            self.assertNotIn(record.getNumber('value'), collection)
+
+    def testGetDataWithTimeRange(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'height')
+        req.addIdentifier('ts', 'RG')
+        req.setParameters('value', 'lid', 'quality_code')
+        times = DAL.getAvailableTimes(req)
+        limitTimes = times[-self.numTimesToLimit:]
+        startTime = datetime.datetime.utcfromtimestamp(limitTimes[0].getRefTime().getTime()/1000)
+        endTime = datetime.datetime.utcnow()
+        tr = TimeRange(startTime, endTime)
+        self.runGeometryDataTestWithTimeRange(req, tr)
+
+    def testGetDataWithInvalidConstraintTypeThrowsException(self):
+        with self.assertRaises(ValueError):
+            self._runConstraintTest('value', 'junk', 3)
+
+    def testGetDataWithInvalidConstraintValueThrowsException(self):
+        with self.assertRaises(TypeError):
+            self._runConstraintTest('value', '=', {})
+
+    def testGetDataWithEmptyInConstraintThrowsException(self):
+        with self.assertRaises(ValueError):
+            self._runConstraintTest('value', 'in', [])
+
+    def testGetDataWithNestedInConstraintThrowsException(self):
+        collection = ('3', '4', ())
+        with self.assertRaises(TypeError):
+            self._runConstraintTest('value', 'in', collection)

awips/test/dafTests/testLdadMesonet.py

@@ -0,0 +1,84 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+from shapely.geometry import Polygon
+from ufpy.dataaccess import DataAccessLayer as DAL
+
+from . import baseDafTestCase
+
+#
+# Test DAF support for ldadmesonet data
+#
+#     SOFTWARE HISTORY
+#
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    01/19/16        4795          mapeters       Initial Creation.
+#    04/11/16        5548          tgurney        Cleanup
+#    04/18/16        5548          tgurney        More cleanup
+#    01/20/17        6095          tgurney        Add null identifiers test
+#
+#
+
+
+class LdadMesonetTestCase(baseDafTestCase.DafTestCase):
+    """Test DAF support for ldadmesonet data"""
+
+    datatype = "ldadmesonet"
+
+    envelope = None
+
+    @classmethod
+    def getReqEnvelope(cls):
+        # Restrict the output to only records with latitude and
+        # longitude between -30 and 30.
+        if not cls.envelope:
+            vertices = [(-30, -30), (-30, 30), (30, 30), (30, -30)]
+            polygon = Polygon(vertices)
+            cls.envelope = polygon.envelope
+        return cls.envelope
+
+    def testGetAvailableParameters(self):
+        req = DAL.newDataRequest(self.datatype)
+        self.runParametersTest(req)
+
+    def testGetAvailableLocations(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.setEnvelope(self.getReqEnvelope())
+        self.runLocationsTest(req)
+
+    def testGetAvailableTimes(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.setEnvelope(self.getReqEnvelope())
+        self.runTimesTest(req)
+
+    def testGetGeometryData(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.setParameters("highLevelCloud", "pressure")
+        req.setEnvelope(self.getReqEnvelope())
+        self.runGeometryDataTest(req)
+
+    def testGetGeometryDataNullIdentifiers(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.setParameters("highLevelCloud", "pressure")
+        req.setEnvelope(self.getReqEnvelope())
+        req.identifiers = None
+        self.runGeometryDataTest(req)

awips/test/dafTests/testMaps.py

@@ -1,3 +1,30 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
+from ufpy.dataaccess import DataAccessLayer as DAL
+from ufpy.ThriftClient import ThriftRequestException
+
+from . import baseDafTestCase
+
 #
 # Test DAF support for maps data
 #
@@ -16,13 +43,6 @@
 #
 #
 
-from __future__ import print_function
-from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
-from awips.dataaccess import DataAccessLayer as DAL
-from awips.ThriftClient import ThriftRequestException
-
-from awips.test.dafTests import baseDafTestCase
-
 
 class MapsTestCase(baseDafTestCase.DafTestCase):
     """Test DAF support for maps data"""
@@ -110,11 +130,6 @@ class MapsTestCase(baseDafTestCase.DafTestCase):
         for record in geometryData:
             self.assertEqual(record.getNumber('reservoir'), 1)
 
-    def testGetDataWithEqualsLong(self):
-        geometryData = self._runConstraintTest('reservoir', '=', 1)
-        for record in geometryData:
-            self.assertEqual(record.getNumber('reservoir'), 1)
-
     def testGetDataWithEqualsFloat(self):
         geometryData = self._runConstraintTest('area_sq_mi', '=', 5.00)
         for record in geometryData:

awips/test/dafTests/testModelSounding.py

@@ -1,3 +1,31 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+from ufpy.dataaccess import DataAccessLayer as DAL
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
+
+from . import baseDafTestCase
+from . import params
+import unittest
+
 #
 # Test DAF support for modelsounding data
 #
@@ -19,39 +47,30 @@
 #
 #
 
-from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
-from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
-
-from awips.test.dafTests import baseDafTestCase
-from awips.test.dafTests import params
-
 
 class ModelSoundingTestCase(baseDafTestCase.DafTestCase):
     """Test DAF support for modelsounding data"""
 
     datatype = "modelsounding"
 
-    reporttype = "ETA"
-
     def testGetAvailableParameters(self):
         req = DAL.newDataRequest(self.datatype)
         self.runParametersTest(req)
 
     def testGetAvailableLocations(self):
         req = DAL.newDataRequest(self.datatype)
-        req.addIdentifier("reportType", self.reporttype)
+        req.addIdentifier("reportType", "ETA")
         self.runLocationsTest(req)
 
     def testGetAvailableTimes(self):
         req = DAL.newDataRequest(self.datatype)
-        req.addIdentifier("reportType", self.reporttype)
+        req.addIdentifier("reportType", "ETA")
         req.setLocationNames(params.OBS_STATION)
         self.runTimesTest(req)
 
     def testGetGeometryData(self):
         req = DAL.newDataRequest(self.datatype)
-        req.addIdentifier("reportType", self.reporttype)
+        req.addIdentifier("reportType", "ETA")
         req.setLocationNames(params.OBS_STATION)
         req.setParameters("temperature", "pressure", "specHum", "sfcPress", "temp2", "q2")
         print("Testing getGeometryData()")
@@ -81,7 +100,7 @@ class ModelSoundingTestCase(baseDafTestCase.DafTestCase):
 
     def testGetGeometryDataWithEnvelope(self):
         req = DAL.newDataRequest(self.datatype)
-        req.addIdentifier("reportType", self.reporttype)
+        req.addIdentifier("reportType", "ETA")
         req.setEnvelope(params.ENVELOPE)
         req.setParameters("temperature", "pressure", "specHum", "sfcPress", "temp2", "q2")
         print("Testing getGeometryData()")

awips/test/dafTests/testObs.py

@@ -1,3 +1,30 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+from ufpy.dataaccess import DataAccessLayer as DAL
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
+
+from . import baseDafTestCase
+from . import params
+
 #
 # Test DAF support for obs data
 #
@@ -16,13 +43,6 @@
 #
 #
 
-from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
-from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
-
-from awips.test.dafTests import baseDafTestCase
-from awips.test.dafTests import params
-
 
 class ObsTestCase(baseDafTestCase.DafTestCase):
     """Test DAF support for obs data"""

awips/test/dafTests/testPirep.py

@@ -0,0 +1,91 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+from ufpy.dataaccess import DataAccessLayer as DAL
+
+from . import baseDafTestCase
+from . import params
+import unittest
+
+#
+# Test DAF support for pirep data
+#
+#     SOFTWARE HISTORY
+#
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    01/19/16        4795          mapeters       Initial Creation.
+#    04/11/16        5548          tgurney        Cleanup
+#    04/18/16        5548          tgurney        More cleanup
+#    12/07/16        5981          tgurney        Parameterize
+#    12/20/16        5981          tgurney        Add envelope test
+#
+#
+
+
+class PirepTestCase(baseDafTestCase.DafTestCase):
+    """Test DAF support for pirep data"""
+
+    datatype = "pirep"
+
+    def testGetAvailableParameters(self):
+        req = DAL.newDataRequest(self.datatype)
+        self.runParametersTest(req)
+
+    def testGetAvailableLocations(self):
+        req = DAL.newDataRequest(self.datatype)
+        self.runLocationsTest(req)
+
+    def testGetAvailableTimes(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.setLocationNames(params.AIRPORT)
+        self.runTimesTest(req)
+
+    def testGetGeometryData(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.setLocationNames(params.AIRPORT)
+        req.setParameters("temperature", "windSpeed", "hazardType", "turbType")
+        print("Testing getGeometryData()")
+        geomData = DAL.getGeometryData(req)
+        self.assertIsNotNone(geomData)
+        print("Number of geometry records: " + str(len(geomData)))
+        print("Sample geometry data:")
+        for record in geomData[:self.sampleDataLimit]:
+            print("level=", record.getLevel(), end="")
+            # One dimensional parameters are reported on the 0.0UNKNOWN level.
+            # 2D parameters are reported on MB levels from pressure.
+            if record.getLevel() == "0.0UNKNOWN":
+                print(" temperature=" + record.getString("temperature") + record.getUnit("temperature"), end="")
+                print(" windSpeed=" + record.getString("windSpeed") + record.getUnit("windSpeed"), end="")
+            else:
+                print(" hazardType=" + record.getString("hazardType"), end="")
+                print(" turbType=" + record.getString("turbType"), end="")
+            print(" geometry=", record.getGeometry())
+        print("getGeometryData() complete\n")
+
+    def testGetGeometryDataWithEnvelope(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.setParameters("temperature", "windSpeed", "hazardType", "turbType")
+        req.setEnvelope(params.ENVELOPE)
+        print("Testing getGeometryData()")
+        data = DAL.getGeometryData(req)
+        for item in data:
+            self.assertTrue(params.ENVELOPE.contains(item.getGeometry()))

awips/test/dafTests/testPracticeWarning.py

@@ -0,0 +1,44 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+from . import testWarning
+
+#
+# Test DAF support for practicewarning data
+#
+#     SOFTWARE HISTORY
+#
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    01/19/16        4795          mapeters       Initial Creation.
+#    04/11/16        5548          tgurney        Cleanup
+#    04/18/16        5548          tgurney        More cleanup
+#    06/10/16        5548          tgurney        Inherit all tests from
+#                                                 warning
+#
+
+
+class PracticeWarningTestCase(testWarning.WarningTestCase):
+    """Test DAF support for practicewarning data"""
+
+    datatype = "practicewarning"
+
+    # All tests taken from testWarning

awips/test/dafTests/testRadarGraphics.py

@@ -1,3 +1,30 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
+from ufpy.dataaccess import DataAccessLayer as DAL
+
+from . import baseRadarTestCase
+from . import params
+
+
 #
 # Test DAF support for radar graphics data
 #
@@ -15,14 +42,6 @@
 #                                                 returned data
 #
 #
-
-from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
-from awips.dataaccess import DataAccessLayer as DAL
-
-from awips.test.dafTests import baseRadarTestCase
-from awips.test.dafTests import params
-
-
 class RadarGraphicsTestCase(baseRadarTestCase.BaseRadarTestCase):
     """Test DAF support for radar data"""
 

awips/test/dafTests/testRadarGrid.py

@@ -1,3 +1,28 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+from ufpy.dataaccess import DataAccessLayer as DAL
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
+
+from . import baseRadarTestCase
+from . import params
 #
 # Test DAF support for radar grid data
 #
@@ -9,12 +34,6 @@
 #
 #
 
-from awips.dataaccess import DataAccessLayer as DAL
-from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
-
-from awips.test.dafTests import baseRadarTestCase
-from awips.test.dafTests import params
-
 
 class RadarTestCase(baseRadarTestCase.BaseRadarTestCase):
     """Test DAF support for radar data"""

awips/test/dafTests/testRadarSpatial.py

@@ -1,3 +1,30 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+from ufpy.dataaccess import DataAccessLayer as DAL
+
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
+from . import baseDafTestCase
+from . import params
+
 #
 # Test DAF support for radar_spatial data
 #
@@ -18,13 +45,6 @@
 #
 #
 
-from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
-
-from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
-from awips.test.dafTests import baseDafTestCase
-from awips.test.dafTests import params
-
 
 class RadarSpatialTestCase(baseDafTestCase.DafTestCase):
     """Test DAF support for radar_spatial data"""
@@ -70,11 +90,6 @@ class RadarSpatialTestCase(baseDafTestCase.DafTestCase):
         for record in geometryData:
             self.assertEqual(record.getNumber('immutablex'), 57)
 
-    def testGetDataWithEqualsLong(self):
-        geometryData = self._runConstraintTest('immutablex', '=', 57)
-        for record in geometryData:
-            self.assertEqual(record.getNumber('immutablex'), 57)
-
     def testGetDataWithEqualsFloat(self):
         geometryData = self._runConstraintTest('immutablex', '=', 57.0)
         for record in geometryData:
@@ -88,7 +103,7 @@ class RadarSpatialTestCase(baseDafTestCase.DafTestCase):
     def testGetDataWithNotEquals(self):
         geometryData = self._runConstraintTest('wfo_id', '!=', params.SITE_ID)
         for record in geometryData:
-            self.assertNotEquals(record.getString('wfo_id'), params.SITE_ID)
+            self.assertNotEqual(record.getString('wfo_id'), params.SITE_ID)
 
     def testGetDataWithNotEqualsNone(self):
         geometryData = self._runConstraintTest('wfo_id', '!=', None)

awips/test/dafTests/testRequestConstraint.py

@@ -1,3 +1,27 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
+
+import unittest
+
 #
 # Unit tests for Python implementation of RequestConstraint
 #
@@ -9,10 +33,6 @@
 #
 #
 
-from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
-
-import unittest
-
 
 class RequestConstraintTestCase(unittest.TestCase):
 
@@ -27,9 +47,7 @@ class RequestConstraintTestCase(unittest.TestCase):
         self.assertTrue(new('=', 3).evaluate(3))
         self.assertTrue(new('=', 3).evaluate('3'))
         self.assertTrue(new('=', '3').evaluate(3))
-        self.assertTrue(new('=', 12345).evaluate(12345))
         self.assertTrue(new('=', 'a').evaluate('a'))
-        self.assertTrue(new('=', 'a').evaluate(u'a'))
         self.assertTrue(new('=', 1.0001).evaluate(2.0 - 0.999999))
         self.assertTrue(new('=', 1.00001).evaluate(1))
         self.assertFalse(new('=', 'a').evaluate(['a']))
@@ -48,9 +66,7 @@ class RequestConstraintTestCase(unittest.TestCase):
         self.assertFalse(new('!=', 3).evaluate('3'))
         self.assertFalse(new('!=', '3').evaluate(3))
         self.assertFalse(new('!=', 3).evaluate(3))
-        self.assertFalse(new('!=', 12345).evaluate(12345))
         self.assertFalse(new('!=', 'a').evaluate('a'))
-        self.assertFalse(new('!=', 'a').evaluate(u'a'))
         self.assertFalse(new('!=', 1.0001).evaluate(2.0 - 0.9999))
 
     def testEvaluateGreaterThan(self):
@@ -59,7 +75,6 @@ class RequestConstraintTestCase(unittest.TestCase):
         self.assertTrue(new('>', 'a').evaluate('b'))
         self.assertTrue(new('>', 3).evaluate(4))
         self.assertFalse(new('>', 20).evaluate(3))
-        self.assertFalse(new('>', 12345).evaluate(12345))
         self.assertFalse(new('>', 'a').evaluate('a'))
         self.assertFalse(new('>', 'z').evaluate('a'))
         self.assertFalse(new('>', 4).evaluate(3))
@@ -67,7 +82,6 @@ class RequestConstraintTestCase(unittest.TestCase):
     def testEvaluateGreaterThanEquals(self):
         new = RequestConstraint.new
         self.assertTrue(new('>=', 3).evaluate(3))
-        self.assertTrue(new('>=', 12345).evaluate(12345))
         self.assertTrue(new('>=', 'a').evaluate('a'))
         self.assertTrue(new('>=', 1.0001).evaluate(1.0002))
         self.assertTrue(new('>=', 'a').evaluate('b'))
@@ -81,7 +95,6 @@ class RequestConstraintTestCase(unittest.TestCase):
         self.assertTrue(new('<', 'z').evaluate('a'))
         self.assertTrue(new('<', 30).evaluate(4))
         self.assertFalse(new('<', 3).evaluate(3))
-        self.assertFalse(new('<', 12345).evaluate(12345))
         self.assertFalse(new('<', 'a').evaluate('a'))
         self.assertFalse(new('<', 1.0001).evaluate(1.0002))
         self.assertFalse(new('<', 'a').evaluate('b'))
@@ -92,7 +105,6 @@ class RequestConstraintTestCase(unittest.TestCase):
         self.assertTrue(new('<=', 'z').evaluate('a'))
         self.assertTrue(new('<=', 20).evaluate(3))
         self.assertTrue(new('<=', 3).evaluate(3))
-        self.assertTrue(new('<=', 12345).evaluate(12345))
         self.assertTrue(new('<=', 'a').evaluate('a'))
         self.assertFalse(new('<=', 1.0001).evaluate(1.0002))
         self.assertFalse(new('<=', 'a').evaluate('b'))
@@ -134,6 +146,73 @@ class RequestConstraintTestCase(unittest.TestCase):
         self.assertFalse(new('not in', 'a').evaluate('a'))
         self.assertFalse(new('not in', [1.0001, 2, 3]).evaluate(2.0 - 0.9999))
 
+    def testEvaluateLike(self):
+        # cannot make "like" with RequestConstraint.new()
+        new = self._newRequestConstraint
+        self.assertTrue(new('LIKE', 'a').evaluate('a'))
+        self.assertTrue(new('LIKE', 'a%').evaluate('a'))
+        self.assertTrue(new('LIKE', 'a%').evaluate('abcd'))
+        self.assertTrue(new('LIKE', '%a').evaluate('a'))
+        self.assertTrue(new('LIKE', '%a').evaluate('bcda'))
+        self.assertTrue(new('LIKE', '%').evaluate(''))
+        self.assertTrue(new('LIKE', '%').evaluate('anything'))
+        self.assertTrue(new('LIKE', 'a%d').evaluate('ad'))
+        self.assertTrue(new('LIKE', 'a%d').evaluate('abcd'))
+        self.assertTrue(new('LIKE', 'aa.()!{[]^%$').evaluate('aa.()!{[]^zzz$'))
+        self.assertTrue(new('LIKE', 'a__d%').evaluate('abcdefg'))
+        self.assertFalse(new('LIKE', 'a%').evaluate('b'))
+        self.assertFalse(new('LIKE', 'a%').evaluate('ba'))
+        self.assertFalse(new('LIKE', '%a').evaluate('b'))
+        self.assertFalse(new('LIKE', '%a').evaluate('ab'))
+        self.assertFalse(new('LIKE', 'a%').evaluate('A'))
+        self.assertFalse(new('LIKE', 'A%').evaluate('a'))
+        self.assertFalse(new('LIKE', 'a%d').evaluate('da'))
+        self.assertFalse(new('LIKE', 'a__d%').evaluate('abccdefg'))
+        self.assertFalse(new('LIKE', '....').evaluate('aaaa'))
+        self.assertFalse(new('LIKE', '.*').evaluate('anything'))
+
+    def testEvaluateILike(self):
+        # cannot make "ilike" with RequestConstraint.new()
+        new = self._newRequestConstraint
+        self.assertTrue(new('ILIKE', 'a').evaluate('a'))
+        self.assertTrue(new('ILIKE', 'a%').evaluate('a'))
+        self.assertTrue(new('ILIKE', 'a%').evaluate('abcd'))
+        self.assertTrue(new('ILIKE', '%a').evaluate('a'))
+        self.assertTrue(new('ILIKE', '%a').evaluate('bcda'))
+        self.assertTrue(new('ILIKE', '%').evaluate(''))
+        self.assertTrue(new('ILIKE', '%').evaluate('anything'))
+        self.assertTrue(new('ILIKE', 'a%d').evaluate('ad'))
+        self.assertTrue(new('ILIKE', 'a%d').evaluate('abcd'))
+        self.assertTrue(new('ILIKE', 'a').evaluate('A'))
+        self.assertTrue(new('ILIKE', 'a%').evaluate('A'))
+        self.assertTrue(new('ILIKE', 'a%').evaluate('ABCD'))
+        self.assertTrue(new('ILIKE', '%a').evaluate('A'))
+        self.assertTrue(new('ILIKE', '%a').evaluate('BCDA'))
+        self.assertTrue(new('ILIKE', '%').evaluate(''))
+        self.assertTrue(new('ILIKE', '%').evaluate('anything'))
+        self.assertTrue(new('ILIKE', 'a%d').evaluate('AD'))
+        self.assertTrue(new('ILIKE', 'a%d').evaluate('ABCD'))
+        self.assertTrue(new('ILIKE', 'A').evaluate('a'))
+        self.assertTrue(new('ILIKE', 'A%').evaluate('a'))
+        self.assertTrue(new('ILIKE', 'A%').evaluate('abcd'))
+        self.assertTrue(new('ILIKE', '%A').evaluate('a'))
+        self.assertTrue(new('ILIKE', '%A').evaluate('bcda'))
+        self.assertTrue(new('ILIKE', '%').evaluate(''))
+        self.assertTrue(new('ILIKE', '%').evaluate('anything'))
+        self.assertTrue(new('ILIKE', 'A%D').evaluate('ad'))
+        self.assertTrue(new('ILIKE', 'A%D').evaluate('abcd'))
+        self.assertTrue(new('ILIKE', 'aa.()!{[]^%$').evaluate('AA.()!{[]^zzz$'))
+        self.assertTrue(new('ILIKE', 'a__d%').evaluate('abcdefg'))
+        self.assertTrue(new('ILIKE', 'a__d%').evaluate('ABCDEFG'))
+        self.assertFalse(new('ILIKE', 'a%').evaluate('b'))
+        self.assertFalse(new('ILIKE', 'a%').evaluate('ba'))
+        self.assertFalse(new('ILIKE', '%a').evaluate('b'))
+        self.assertFalse(new('ILIKE', '%a').evaluate('ab'))
+        self.assertFalse(new('ILIKE', 'a%d').evaluate('da'))
+        self.assertFalse(new('ILIKE', 'a__d%').evaluate('abccdefg'))
+        self.assertFalse(new('ILIKE', '....').evaluate('aaaa'))
+        self.assertFalse(new('ILIKE', '.*').evaluate('anything'))
+
     def testEvaluateBetween(self):
         # cannot make "between" with RequestConstraint.new()
         new = self._newRequestConstraint

awips/test/dafTests/testSatellite.py

@@ -1,3 +1,30 @@
+#!/awips2/python/bin/python3
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+from ufpy.dataaccess import DataAccessLayer as DAL
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
+
+from . import baseDafTestCase
+
 #
 # Test DAF support for satellite data
 #
@@ -17,12 +44,6 @@
 #
 #
 
-from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
-from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
-
-from awips.test.dafTests import baseDafTestCase
-
 
 class SatelliteTestCase(baseDafTestCase.DafTestCase):
     """Test DAF support for satellite data"""
@@ -78,11 +99,6 @@ class SatelliteTestCase(baseDafTestCase.DafTestCase):
         for record in gridData:
             self.assertEqual(record.getAttribute('creatingEntity'), 1000)
 
-    def testGetDataWithEqualsLong(self):
-        gridData = self._runConstraintTest('creatingEntity', '=', 1000)
-        for record in gridData:
-            self.assertEqual(record.getAttribute('creatingEntity'), 1000)
-
     def testGetDataWithEqualsFloat(self):
         gridData = self._runConstraintTest('creatingEntity', '=', 1.0)
         for record in gridData:

awips/test/dafTests/testSfcObs.py

@@ -1,3 +1,29 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+from ufpy.dataaccess import DataAccessLayer as DAL
+
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
+from . import baseDafTestCase
+
 #
 # Test DAF support for sfcobs data
 #
@@ -15,12 +41,6 @@
 #
 #
 
-from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
-from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
-
-from awips.test.dafTests import baseDafTestCase
-
 
 class SfcObsTestCase(baseDafTestCase.DafTestCase):
     """Test DAF support for sfcobs data"""
@@ -81,11 +101,6 @@ class SfcObsTestCase(baseDafTestCase.DafTestCase):
         for record in geometryData:
             self.assertEqual(record.getString('reportType'), '1004')
 
-    def testGetDataWithEqualsLong(self):
-        geometryData = self._runConstraintTest('reportType', '=', 1004)
-        for record in geometryData:
-            self.assertEqual(record.getString('reportType'), '1004')
-
     # No float test because no float identifiers are available
 
     def testGetDataWithEqualsNone(self):

awips/test/dafTests/testTopo.py

@@ -1,3 +1,30 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+from ufpy.dataaccess import DataAccessLayer as DAL
+from ufpy.ThriftClient import ThriftRequestException
+
+from . import baseDafTestCase
+import shapely.geometry
+
 #
 # Test DAF support for topo data
 #
@@ -12,15 +39,10 @@
 #                                                 getIdentifierValues()
 #    06/01/16        5587          tgurney        Update testGetIdentifierValues
 #    07/18/17        6253          randerso       Removed referenced to GMTED
+#    02/20/18        7220          mapeters       Added tests for getting filtered
+#                                                 group/dataset identifier values
 #
 
-from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
-from awips.ThriftClient import ThriftRequestException
-import shapely.geometry
-
-from awips.test.dafTests import baseDafTestCase
-
 
 class TopoTestCase(baseDafTestCase.DafTestCase):
     """Test DAF support for topo data"""
@@ -49,6 +71,7 @@ class TopoTestCase(baseDafTestCase.DafTestCase):
             print("Sample grid data shape:\n" + str(gridData[0].getRawData().shape) + "\n")
             print("Sample grid data:\n" + str(gridData[0].getRawData()) + "\n")
 
+
     def testRequestingTooMuchDataThrowsResponseTooLargeException(self):
         req = DAL.newDataRequest(self.datatype)
         req.addIdentifier("group", "/")
@@ -67,6 +90,18 @@ class TopoTestCase(baseDafTestCase.DafTestCase):
         requiredIds = set(DAL.getRequiredIdentifiers(req))
         self.runGetIdValuesTest(optionalIds | requiredIds)
 
+    def testGetFilteredDatasetValues(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('group', '/')
+        datasetVals = DAL.getIdentifierValues(req, 'dataset')
+        self.assertSequenceEqual(datasetVals, ['full'])
+
+    def testGetFilteredGroupValues(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('dataset', '1')
+        groupVals = DAL.getIdentifierValues(req, 'group')
+        self.assertSequenceEqual(groupVals, ['/interpolated'])
+
     def testGetInvalidIdentifierValuesThrowsException(self):
         self.runInvalidIdValuesTest()
 

awips/test/dafTests/testWarning.py

@@ -1,3 +1,30 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+from awips.dataaccess import DataAccessLayer as DAL
+
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
+from . import baseDafTestCase
+import unittest
+
 #
 # Test DAF support for warning data
 #
@@ -16,15 +43,9 @@
 #    06/21/16        5548          tgurney        Skip tests that cause errors
 #    06/30/16        5725          tgurney        Add test for NOT IN
 #    12/12/16        5981          tgurney        Improve test performance
+#    02/20/18        7220          mapeters       Added test for getting filtered
+#                                                 column identifier values
 #
-#
-
-from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
-from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
-
-from awips.test.dafTests import baseDafTestCase
-import unittest
 
 
 class WarningTestCase(baseDafTestCase.DafTestCase):
@@ -95,13 +116,12 @@ class WarningTestCase(baseDafTestCase.DafTestCase):
     def testGetColumnIdentifierValues(self):
         self.runGetIdValuesTest(['act'])
 
-    @unittest.skip('avoid EDEX error')
+    def testGetFilteredColumnIdentifierValues(self):
-    def testGetInvalidIdentifierValuesThrowsException(self):
+        req = DAL.newDataRequest(self.datatype)
-        self.runInvalidIdValuesTest()
+        req.addIdentifier('sig', 'W')
-
+        phensigs = DAL.getIdentifierValues(req, 'phensig')
-    @unittest.skip('avoid EDEX error')
+        for phensig in phensigs:
-    def testGetNonexistentIdentifierValuesThrowsException(self):
+            self.assertTrue(phensig.endswith('.W'))
-        self.runNonexistentIdValuesTest()
 
     def _runConstraintTest(self, key, operator, value):
         req = DAL.newDataRequest(self.datatype)
@@ -120,11 +140,6 @@ class WarningTestCase(baseDafTestCase.DafTestCase):
         for record in geometryData:
             self.assertEqual(record.getString('etn'), '1000')
 
-    def testGetDataWithEqualsLong(self):
-        geometryData = self._runConstraintTest('etn', '=', 1000)
-        for record in geometryData:
-            self.assertEqual(record.getString('etn'), '1000')
-
     def testGetDataWithEqualsFloat(self):
         geometryData = self._runConstraintTest('etn', '=', 1.0)
         for record in geometryData:

awips/test/localization/__init__.py

@@ -0,0 +1,32 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+#
+# __init__.py for ufpy.test.localization package
+#
+#
+# SOFTWARE HISTORY
+#
+# Date      Ticket#  Engineer  Description
+# --------- -------- --------- --------------------------
+# 08/07/17  5731     bsteffen  Initial Creation.
+
+__all__ = []

awips/test/localization/testLF.py

@@ -0,0 +1,91 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     2120 South 72nd Street, Suite 900
+#                         Omaha, NE 68124
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+#
+# Tests for the lf utility script
+#
+# SOFTWARE HISTORY
+#
+# Date      Ticket#  Engineer  Description
+# --------- -------- --------- --------------------------
+# 01/11/22  8735     mapeters  Initial creation
+
+import os
+import subprocess
+import unittest
+
+class TxtFileTestCase(unittest.TestCase):
+    def setUp(self):
+        """Test file needs to not exist at the start of the test. Delete it,
+        suppressing any errors (an error would be printed if it doesn't exist).
+        """
+        self.locDir = "lfTestDir"
+        self.locFile = os.path.join(self.locDir, "TestFile.txt")
+        subprocess.run(["lf", "rm", "-o", self.locFile], stderr=subprocess.DEVNULL)
+
+    def test_file_operations(self):
+        """Run through a typical set of interactions with a .txt file and
+        verify everything works correctly.
+        """
+        subprocess.run(["lf", "write", self.locFile], input=b"Test Data", check=True)
+
+        proc = subprocess.run(["lf", "ls", "-l", self.locDir], stdout=subprocess.PIPE, check=True)
+        self.assertIn(self.locFile, proc.stdout.decode())
+
+        proc = subprocess.run(["lf", "read", self.locFile], stdout=subprocess.PIPE, check=True)
+        self.assertEqual(proc.stdout.decode(), "Test Data")
+
+        subprocess.run(["lf", "rm", "-o", self.locFile], check=True)
+
+        proc = subprocess.run(["lf", "ls", "-l", self.locDir], stdout=subprocess.PIPE, check=True)
+        self.assertNotIn(self.locFile, proc.stdout.decode())
+
+class PngFileTestCase(unittest.TestCase):
+    def setUp(self):
+        """Test file needs to not exist at the start of the test. Delete it,
+        suppressing any errors (an error would be printed if it doesn't exist).
+        """
+        self.locDir = "lfTestDir"
+        self.locFile = os.path.join(self.locDir, "TestImage.png")
+        subprocess.run(["lf", "rm", "-o", self.locFile], stderr=subprocess.DEVNULL)
+
+    def test_file_operations(self):
+        """Run through a typical set of interactions with a .png file and
+        verify everything works correctly.
+        """
+        cwdPath = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+        localPngPath = os.path.join(cwdPath, "smallTestImage.png")
+        with open(localPngPath, "rb") as f:
+            subprocess.run(["lf", "write", self.locFile], stdin=f, check=True)
+
+        proc = subprocess.run(["lf", "ls", self.locDir], stdout=subprocess.PIPE, check=True)
+        self.assertIn(self.locFile, proc.stdout.decode().splitlines())
+
+        proc = subprocess.run(["lf", "read", self.locFile], stdout=subprocess.PIPE, check=True)
+        with open(localPngPath, "rb") as f:
+            localPngBytes = f.read()
+        self.assertEqual(proc.stdout, localPngBytes)
+
+        subprocess.run(["lf", "rm", "-o", self.locFile], check=True)
+
+        proc = subprocess.run(["lf", "ls", self.locDir], stdout=subprocess.PIPE, check=True)
+        self.assertNotIn(self.locFile, proc.stdout.decode().splitlines())
+
+if __name__ == '__main__':
+    unittest.main()

awips/test/localization/testLocalizationFileManager.py

@@ -0,0 +1,173 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+#
+# Tests for the LocalizationFileManager
+#
+# SOFTWARE HISTORY
+#
+# Date      Ticket#  Engineer  Description
+# --------- -------- --------- --------------------------
+# 08/09/17  5731     bsteffen  Initial Creation.
+# 01/06/22  8735     mapeters  Python 3 fix
+
+import unittest
+
+from ufpy.localization.LocalizationFileManager import (LocalizationFileManager,
+                                                       LocalizationFileVersionConflictException,
+                                                       LocalizationContext,
+                                                       LocalizationFileIsNotDirectoryException,
+                                                       LocalizationFileDoesNotExistException)
+
+testFile = "purge/defaultPurgeRules.xml"
+testContent = b"<purgeRuleSet><defaultRule><period>05-05:05:05</period></defaultRule></purgeRuleSet>"
+testDir = "purge/"
+testNewFile = "purge/testPurgeRules.xml"
+
+class ContextTestCase(unittest.TestCase):
+    def test_eq(self):
+        c1 = LocalizationContext()
+        c2 = LocalizationContext()
+        self.assertEqual(c1, c2)
+        c3 = LocalizationContext("site", "test")
+        c4 = LocalizationContext("site", "test")
+        self.assertEqual(c3, c4)
+        self.assertNotEqual(c1, c3)
+
+    def test_hash(self):
+        c1 = LocalizationContext()
+        c2 = LocalizationContext()
+        self.assertEqual(hash(c1), hash(c2))
+        c3 = LocalizationContext("site", "test")
+        c4 = LocalizationContext("site", "test")
+        self.assertEqual(hash(c3), hash(c4))
+
+class LFMTestCase(unittest.TestCase):
+    def setUp(self):
+        self.manager = LocalizationFileManager()
+        userFile = self.manager.getSpecific("user", testFile)
+        if userFile.exists():
+            userFile.delete()
+        newFile = self.manager.getSpecific("user", testNewFile)
+        if newFile.exists():
+            newFile.delete()
+    def test_gets(self):
+        startingIncremental = self.manager.getIncremental(testFile)
+        baseFile = self.manager.getSpecific("base", testFile)
+        self.assertEqual(baseFile, startingIncremental[0])
+        self.assertTrue(baseFile.exists())
+        self.assertFalse(baseFile.isDirectory())
+        userFile = self.manager.getSpecific("user", testFile)
+        self.assertFalse(userFile.exists())
+        with userFile.open("w") as stream:
+            stream.write(testContent)
+        userFile = self.manager.getSpecific("user", testFile)
+        self.assertTrue(userFile.exists())
+        with userFile.open('r') as stream:
+            self.assertEqual(stream.read(), testContent)
+        absFile = self.manager.getAbsolute(testFile)
+        self.assertEqual(absFile, userFile)
+        endingIncremental = self.manager.getIncremental(testFile)
+        self.assertEqual(len(startingIncremental) + 1, len(endingIncremental))
+        self.assertEqual(userFile, endingIncremental[-1])
+        self.assertEqual(baseFile, endingIncremental[0])
+
+
+        userFile.delete()
+        userFile = self.manager.getSpecific("user", testFile)
+        self.assertFalse(userFile.exists())
+
+    def test_concurrent_edit(self):
+        userFile1 = self.manager.getSpecific("user", testFile)
+        userFile2 = self.manager.getSpecific("user", testFile)
+        self.assertFalse(userFile1.exists())
+        self.assertFalse(userFile2.exists())
+        with self.assertRaises(LocalizationFileVersionConflictException):
+            with userFile1.open("w") as stream1:
+                stream1.write(testContent)
+                with userFile2.open("w") as stream2:
+                    stream2.write(testContent)
+
+        userFile = self.manager.getSpecific("user", testFile)
+        userFile.delete()
+
+    def test_dir(self):
+        locDir = self.manager.getAbsolute(testDir)
+        self.assertTrue(locDir.isDirectory())
+        with self.assertRaises(Exception):
+            locDir.delete()
+
+    def test_list(self):
+        abs1 = self.manager.listAbsolute(testDir)
+        inc1 = self.manager.listIncremental(testDir)
+        self.assertEqual(len(abs1), len(inc1))
+        for i in range(len(abs1)):
+            self.assertEqual(abs1[i], inc1[i][-1])
+
+        userFile = self.manager.getSpecific("user", testNewFile)
+        self.assertNotIn(userFile, abs1)
+
+        with userFile.open("w") as stream:
+            stream.write(testContent)
+        userFile = self.manager.getSpecific("user", testNewFile)
+
+
+        abs2 = self.manager.listAbsolute(testDir)
+        inc2 = self.manager.listIncremental(testDir)
+        self.assertEqual(len(abs2), len(inc2))
+        for i in range(len(abs2)):
+            self.assertEqual(abs2[i], inc2[i][-1])
+
+        self.assertEqual(len(abs1) + 1, len(abs2))
+        self.assertIn(userFile, abs2)
+
+        userFile.delete()
+
+    def test_list_file(self):
+        with self.assertRaises(LocalizationFileIsNotDirectoryException):
+            self.manager.listIncremental(testFile)
+
+    def test_list_nonexistant(self):
+        with self.assertRaises(LocalizationFileDoesNotExistException):
+            self.manager.listIncremental('dontNameYourDirectoryThis')
+
+    def test_root_variants(self):
+        list1 = self.manager.listAbsolute(".")
+        list2 = self.manager.listAbsolute("")
+        list3 = self.manager.listAbsolute("/")
+        self.assertEqual(list1, list2)
+        self.assertEqual(list2, list3)
+
+    def test_slashiness(self):
+        raw = testDir
+        if raw[0] == '/':
+            raw = raw[1:]
+        if raw[-1] == '/':
+            raw = raw[:-1]
+        list1 = self.manager.listAbsolute(raw)
+        list2 = self.manager.listAbsolute(raw + "/")
+        list3 = self.manager.listAbsolute("/" + raw)
+        self.assertEqual(list1, list2)
+        self.assertEqual(list2, list3)
+
+
+
+if __name__ == '__main__':
+    unittest.main()

awips/test/localization/testLocalizationRest.py

@@ -0,0 +1,365 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+import unittest
+import urllib.request, urllib.error, urllib.parse
+
+from html.parser import HTMLParser
+from xml.etree.ElementTree import parse as parseXml
+from json import load as loadjson
+from urllib.parse import urljoin
+from base64 import b64encode
+
+#
+# Test the localization REST service.
+#
+# SOFTWARE HISTORY
+#
+# Date      Ticket#  Engineer  Description
+# --------- -------- --------- --------------------------
+# 08/07/17  5731     bsteffen  Initial Creation.
+# 01/06/22  8735     mapeters  Python 3 fixes, test configured level instead of
+#                              site since it more reliably has menus/ dir
+
+baseURL = "http://localhost:9581/services/localization/"
+testSite = "OAX"
+testDir = "menus"
+testFile = "test.xml"
+username = "test"
+password = username
+
+base64string = b64encode(b'%s:%s' % (username.encode(), password.encode()))
+authString = "Basic %s" % base64string.decode()
+
+class ValidHTMLParser(HTMLParser):
+    """Simple HTML parser that performs very minimal validation.
+    
+    This ensures that all start and end tags match, and also that there are
+    some tags. It also accumulates the text of all links in the html file
+    in the link_texts attribute, which can be used for further validation.
+    """
+
+    def __init__(self, testcase):
+        HTMLParser.__init__(self)
+        self._testcase = testcase
+        self._tags = []
+        self._any = False
+        self.link_texts = []
+
+    def handle_starttag(self, tag, attrs): # NOSONAR
+        """Suppressing Pylint:W0613 "Unused argument" on above line since this
+        is a method override.
+        """
+        self._tags.append(tag)
+        self._any = True
+
+    def handle_endtag(self, tag):
+        self._testcase.assertNotEqual([], self._tags, "Unstarted end tag " + tag)
+        self._testcase.assertEqual(tag, self._tags.pop())
+
+    def handle_data(self, data):
+        if self._tags[-1] == "a":
+            self.link_texts.append(data)
+
+    def close(self):
+        HTMLParser.close(self)
+        self._testcase.assertTrue(self._any)
+        self._testcase.assertEqual([], self._tags)
+
+
+
+class AbstractListingTestCase():
+    """Base test case for testing listings, retrieves data as html, xml, and json.
+    
+    Sub classes should implement assertValidHtml, assertValidXml, and
+    assertValidJson to ensure that the content returned matches what was
+    expected.
+    """
+
+    def assertRequestGetsHtml(self, request):
+        response = urllib.request.urlopen(request)
+        self.assertEqual(response.headers["Content-Type"], "text/html")
+        body = response.read().decode()
+        parser = ValidHTMLParser(self)
+        parser.feed(body)
+        parser.close()
+        self.assertValidHtml(parser)
+
+    def assertValidHtml(self, parser):
+        """Intended to be overriden by subclasses to validate HTML content.
+    
+        The argument is a populated instance of ValidHTMLParser.
+        """
+        pass
+
+    def test_default(self):
+        request = urllib.request.Request(self.url)
+        self.assertRequestGetsHtml(request)
+
+    def test_last_slash(self):
+        if self.url.endswith("/"):
+            request = urllib.request.Request(self.url[:-1])
+        else:
+            request = urllib.request.Request(self.url + "/")
+        self.assertRequestGetsHtml(request)
+
+    def test_wild_mime(self):
+        request = urllib.request.Request(self.url)
+        request.add_header("Accept", "*/*")
+        self.assertRequestGetsHtml(request)
+        request.add_header("Accept", "text/*")
+        self.assertRequestGetsHtml(request)
+
+    def test_html(self):
+        request = urllib.request.Request(self.url)
+        request.add_header("Accept", "text/html")
+        self.assertRequestGetsHtml(request)
+
+    def test_json(self):
+        request = urllib.request.Request(self.url)
+        request.add_header("Accept", "application/json")
+        response = urllib.request.urlopen(request)
+        self.assertEqual(response.headers["Content-Type"], "application/json")
+        jsonData = loadjson(response)
+        self.assertValidJson(jsonData)
+
+
+    def assertValidJson(self, jsonData):
+        """Intended to be overriden by subclasses to validate JSON content.
+    
+        The argument is a python object as returned from json.load
+        """
+        pass
+
+    def test_xml(self):
+        request = urllib.request.Request(self.url)
+        request.add_header("Accept", "application/xml")
+        response = urllib.request.urlopen(request)
+        self.assertEqual(response.headers["Content-Type"], "application/xml")
+        xmlData = parseXml(response)
+        self.assertValidXml(xmlData)
+
+    def assertValidXml(self, xmlData):
+        """Intended to be overriden by subclasses to validate XML content.
+    
+        The argument is an ElementTree
+        """
+        pass
+
+    def test_delete(self):
+        request = urllib.request.Request(self.url, method="DELETE")
+        with self.assertRaises(urllib.error.HTTPError) as cm:
+            urllib.request.urlopen(request)
+        self.assertEqual(405, cm.exception.code)
+
+    def test_put(self):
+        request = urllib.request.Request(self.url, method="PUT")
+        request.data = b"Test Data"
+        with self.assertRaises(urllib.error.HTTPError) as cm:
+            urllib.request.urlopen(request)
+        self.assertEqual(405, cm.exception.code)
+
+    def test_unacceptable(self):
+        request = urllib.request.Request(self.url)
+        request.add_header("Accept", "application/fakemimetype")
+        with self.assertRaises(urllib.error.HTTPError) as cm:
+            urllib.request.urlopen(request)
+        self.assertEqual(406, cm.exception.code)
+        request.add_header("Accept", "fakemimetype/*")
+        with self.assertRaises(urllib.error.HTTPError) as cm:
+            urllib.request.urlopen(request)
+        self.assertEqual(406, cm.exception.code)
+
+    def test_accept_quality_factor(self):
+        request = urllib.request.Request(self.url)
+        request.add_header("Accept", "application/xml; q=0.8, application/json; q=0.2")
+        response = urllib.request.urlopen(request)
+        self.assertEqual(response.headers["Content-Type"], "application/xml")
+        xmlData = parseXml(response)
+        self.assertValidXml(xmlData)
+
+        request.add_header("Accept", "application/xml; q=0.2, application/json; q=0.8")
+        response = urllib.request.urlopen(request)
+        self.assertEqual(response.headers["Content-Type"], "application/json")
+        jsonData = loadjson(response)
+        self.assertValidJson(jsonData)
+
+        request.add_header("Accept", "application/xml, application/json; q=0.8")
+        response = urllib.request.urlopen(request)
+        self.assertEqual(response.headers["Content-Type"], "application/xml")
+        xmlData = parseXml(response)
+        self.assertValidXml(xmlData)
+
+        request.add_header("Accept", "application/fakemimetype, application/json; q=0.8")
+        response = urllib.request.urlopen(request)
+        self.assertEqual(response.headers["Content-Type"], "application/json")
+        jsonData = loadjson(response)
+        self.assertValidJson(jsonData)
+
+class RootTestCase(AbstractListingTestCase, unittest.TestCase):
+    """Test that the root of the localization service returns listing of localization types."""
+    def setUp(self):
+        self.url = baseURL
+    def assertValidHtml(self, parser):
+        self.assertIn("common_static/", parser.link_texts)
+    def assertValidJson(self, jsonData):
+        self.assertIn("common_static/", jsonData)
+    def assertValidXml(self, xmlData):
+        root = xmlData.getroot()
+        self.assertEqual(root.tag, "entries")
+        names = [e.text for e in root.findall("entry")]
+        self.assertIn("common_static/", names)
+
+class TypeTestCase(AbstractListingTestCase, unittest.TestCase):
+    """Test that common_static will list context levels."""
+    def setUp(self):
+        self.url = urljoin(baseURL, "common_static/")
+    def assertValidHtml(self, parser):
+        self.assertIn("base/", parser.link_texts)
+        self.assertIn("site/", parser.link_texts)
+    def assertValidJson(self, jsonData):
+        self.assertIn("base/", jsonData)
+        self.assertIn("site/", jsonData)
+    def assertValidXml(self, xmlData):
+        root = xmlData.getroot()
+        self.assertEqual(root.tag, "entries")
+        names = [e.text for e in root.findall("entry")]
+        self.assertIn("base/", names)
+        self.assertIn("site/", names)
+
+class LevelTestCase(AbstractListingTestCase, unittest.TestCase):
+    """Test that common_static/site will list sites."""
+    def setUp(self):
+        self.url = urljoin(baseURL, "common_static/site/")
+    def assertValidHtml(self, parser):
+        self.assertIn(testSite + "/", parser.link_texts)
+    def assertValidJson(self, jsonData):
+        self.assertIn(testSite + "/", jsonData)
+    def assertValidXml(self, xmlData):
+        root = xmlData.getroot()
+        self.assertEqual(root.tag, "entries")
+        names = [e.text for e in root.findall("entry")]
+        self.assertIn(testSite + "/", names)
+
+class AbstractFileListingTestCase(AbstractListingTestCase):
+    """Base test case for a file listing"""
+
+    def assertValidHtml(self, parser):
+        self.assertIn(testDir + "/", parser.link_texts)
+        self.assertEqual(parser.link_texts, sorted(parser.link_texts))
+    def assertValidJson(self, jsonData):
+        self.assertIn(testDir + "/", jsonData)
+    def assertValidXml(self, xmlData):
+        root = xmlData.getroot()
+        self.assertEqual(root.tag, "files")
+        names = [e.get("name") for e in root.findall("file")]
+        self.assertIn(testDir + "/", names)
+        self.assertEqual(names, sorted(names))
+
+class BaseFileListingTestCase(AbstractFileListingTestCase, unittest.TestCase):
+    """Test that common_static/base lists files"""
+    def setUp(self):
+        self.url = urljoin(baseURL, "common_static/base/")
+
+class ConfiguredFileListingTestCase(AbstractFileListingTestCase, unittest.TestCase):
+    """Test that common_static/site/<testSite>/ lists files"""
+    def setUp(self):
+        self.url = urljoin(baseURL, "common_static/configured/" + testSite + "/")
+
+class FileTestCase(unittest.TestCase):
+    """Test retrieval, modification and deletion of an individual."""
+    def setUp(self):
+        self.url = urljoin(baseURL, "common_static/user/" + username + "/" + testFile)
+        # The file should not exist before the test, but if it does then delete it
+        # This is some of the same functionality we are testing so if setup fails
+        # then the test would probably fail anyway
+        try:
+            request = urllib.request.Request(self.url)
+            response = urllib.request.urlopen(request)
+            request = urllib.request.Request(self.url, method="DELETE")
+            request.add_header("Authorization", authString)
+            request.add_header("If-Match", response.headers["Content-MD5"])
+            response = urllib.request.urlopen(request)
+        except urllib.error.HTTPError as e:
+            if e.code != 404:
+                raise e
+    def test_file_operations(self):
+        """Run through a typical set of file interactions and verify everything works correctly."""
+        request = urllib.request.Request(self.url, method="PUT")
+        request.data = b"Test Data"
+        with self.assertRaises(urllib.error.HTTPError) as cm:
+            response = urllib.request.urlopen(request)
+        self.assertEqual(401, cm.exception.code)
+
+        request = urllib.request.Request(self.url, method="PUT")
+        request.data = b"Test Data"
+        request.add_header("Authorization", authString)
+        with self.assertRaises(urllib.error.HTTPError) as cm:
+            response = urllib.request.urlopen(request)
+        self.assertEqual(409, cm.exception.code)
+
+        request = urllib.request.Request(self.url, method="PUT")
+        request.data = b"Test Data"
+        request.add_header("Authorization", authString)
+        request.add_header("If-Match", "NON_EXISTENT_CHECKSUM")
+        request.add_header("Content-Type", "text/plain")
+        response = urllib.request.urlopen(request)
+
+
+        request = urllib.request.Request(self.url)
+        response = urllib.request.urlopen(request)
+        self.assertEqual(response.read().decode(), "Test Data")
+
+        request = urllib.request.Request(self.url + "/")
+        response = urllib.request.urlopen(request)
+        self.assertEqual(response.read().decode(), "Test Data")
+
+        request = urllib.request.Request(self.url, method="PUT")
+        request.data = b"Test Data2"
+        request.add_header("If-Match", response.headers["Content-MD5"])
+        request.add_header("Authorization", authString)
+        request.add_header("Content-Type", "text/plain")
+        response = urllib.request.urlopen(request)
+        checksum = response.headers["Content-MD5"]
+
+        request = urllib.request.Request(self.url)
+        response = urllib.request.urlopen(request)
+        self.assertEqual(response.read().decode(), "Test Data2")
+
+        request = urllib.request.Request(self.url, method="DELETE")
+        with self.assertRaises(urllib.error.HTTPError) as cm:
+            response = urllib.request.urlopen(request)
+        self.assertEqual(401, cm.exception.code)
+
+        request.add_header("Authorization", authString)
+        with self.assertRaises(urllib.error.HTTPError) as cm:
+            response = urllib.request.urlopen(request)
+        self.assertEqual(409, cm.exception.code)
+
+        request.add_header("If-Match", checksum)
+        response = urllib.request.urlopen(request)
+
+        request = urllib.request.Request(self.url)
+        with self.assertRaises(urllib.error.HTTPError) as cm:
+            response = urllib.request.urlopen(request)
+        self.assertEqual(404, cm.exception.code)
+
+if __name__ == '__main__':
+    unittest.main()

awips/test/testQpidTimeToLive.py

@@ -0,0 +1,98 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+#
+# SOFTWARE HISTORY
+#
+# Date          Ticket#  Engineer     Description
+# ------------- -------- ------------ --------------------------------------------
+# Mar 09, 2011           njensen      Initial Creation.
+# Aug 15, 2013  2169     bkowal       Decompress data read from the queue
+# Jun 24, 2020  8187     randerso     Added program for qpid connection_id
+# 
+#
+
+import time
+import threading
+
+
+TIME_TO_SLEEP = 300
+
+class ListenThread(threading.Thread):
+
+    def __init__(self, hostname, portNumber, topicName):
+        self.hostname = hostname
+        self.portNumber = portNumber
+        self.topicName = topicName
+        self.nMessagesReceived = 0
+        self.waitSecond = 0
+        self.stopped = False
+        threading.Thread.__init__(self)
+    
+    def run(self):
+        from awips import QpidSubscriber
+        self.qs = QpidSubscriber.QpidSubscriber(host=self.hostname, port=self.portNumber, decompress=True, program="testQpidTimeToLive")        
+        self.qs.topicSubscribe(self.topicName, self.receivedMessage)
+    
+    def receivedMessage(self, msg):
+        print("Received message")
+        self.nMessagesReceived += 1
+        if self.waitSecond == 0:
+            fmsg = open('/tmp/rawMessage', 'w')
+            fmsg.write(msg)
+            fmsg.close()
+        
+        while self.waitSecond < TIME_TO_SLEEP and not self.stopped:
+            if self.waitSecond % 60 == 0:
+                print(time.strftime('%H:%M:%S'), "Sleeping and stuck in not so infinite while loop")
+            self.waitSecond += 1
+            time.sleep(1)
+        
+        print(time.strftime('%H:%M:%S'), "Received", self.nMessagesReceived, "messages")                     
+        
+    def stop(self):
+        print("Stopping")
+        self.stopped = True
+        self.qs.close()
+        
+        
+
+def main():
+    print("Starting up at", time.strftime('%H:%M:%S'))
+    
+    topic = 'edex.alerts'
+    host = 'localhost'
+    port = 5672
+    
+    thread = ListenThread(host, port, topic)    
+    try:
+        thread.start()
+        while True:
+            time.sleep(3)
+    except KeyboardInterrupt:
+        pass
+    finally:        
+        thread.stop()
+    
+    
+if __name__ == '__main__':
+    main()
+    
+        
+        

dynamicserialize/DynamicSerializationManager.py

@@ -1,45 +1,69 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
 #
 # A port of the Java DynamicSerializeManager.  Should be used to read/write
 # DynamicSerialize binary data.
 #
 #
+#  
+#    
 #     SOFTWARE HISTORY
-#
+#    
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
 #    06/09/10                      njensen       Initial Creation.
+#    
+# 
 #
 
 from thrift.transport import TTransport
 from . import SelfDescribingBinaryProtocol, ThriftSerializationContext
 
-
 class DynamicSerializationManager:
-
+    
     def __init__(self):
-        self.transport = None
+        self.transport = None    
-
+    
     def _deserialize(self, ctx):
         return ctx.deserializeMessage()
-
+    
-    def deserializeBytes(self, sbytes):
+    def deserializeBytes(self, bytes):
-        ctx = self._buildSerializationContext(sbytes)
+        ctx = self._buildSerializationContext(bytes)
         ctx.readMessageStart()
         obj = self._deserialize(ctx)
         ctx.readMessageEnd()
         return obj
-
+    
-    def _buildSerializationContext(self, sbytes=None):
+    def _buildSerializationContext(self, bytes=None):        
-        self.transport = TTransport.TMemoryBuffer(sbytes)
+        self.transport = TTransport.TMemoryBuffer(bytes)
         protocol = SelfDescribingBinaryProtocol.SelfDescribingBinaryProtocol(self.transport)
         return ThriftSerializationContext.ThriftSerializationContext(self, protocol)
-
+    
     def serializeObject(self, obj):
         ctx = self._buildSerializationContext()
         ctx.writeMessageStart("dynamicSerialize")
         self._serialize(ctx, obj)
         ctx.writeMessageEnd()
-        return self.transport.getvalue()
+        return self.transport.getvalue() 
-
+    
     def _serialize(self, ctx, obj):
-        ctx.serializeMessage(obj)
+        ctx.serializeMessage(obj)

dynamicserialize/SelfDescribingBinaryProtocol.py

@@ -1,114 +1,151 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
 #
 # Partially compatible AWIPS-II Thrift Binary Protocol
 #
-#
+# <B>Missing functionality:</B>
+# <UL>
+# <LI> Custom Serializers
+# <LI> Inheritance
+# </UL>
+#    
 #     SOFTWARE HISTORY
-#
+#    
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
-#    11/11/09                      chammack        Initial Creation.
+#    11/11/09                      chammack       Initial Creation.
-#    06/09/10                      njensen            Added float, list methods
+#    06/09/10                      njensen        Added float, list methods
-#    Apr 24, 2015    4425          nabowle         Add F64List support.
+#    Apr 24, 2015    4425          nabowle        Add F64List support. 
-#
+#    Jun 25, 2019    7821          tgurney        Replace numpy.getbuffer
+#                                                 with memoryview
+#    
+# 
 #
 
+from thrift.protocol.TProtocol import *
+from thrift.protocol.TBinaryProtocol import *
+
 import struct
 import numpy
-from thrift.protocol.TProtocol import *
+
-from thrift.protocol.TBinaryProtocol import *
+######################################
+# TODO Remove after switch to Python 3
+# This is necessary because some APIs in Python 2 expect a buffer and not a
+# memoryview.
+import sys
+if sys.version_info.major < 3:
+    memoryview = buffer
+######################################
 
 FLOAT = 64
 
 intList = numpy.dtype(numpy.int32).newbyteorder('>')
 floatList = numpy.dtype(numpy.float32).newbyteorder('>')
-longList = numpy.dtype(numpy.int64).newbyteorder('>')
+longList = numpy.dtype(numpy.int64).newbyteorder('>')  
 shortList = numpy.dtype(numpy.int16).newbyteorder('>')
 byteList = numpy.dtype(numpy.int8).newbyteorder('>')
 doubleList = numpy.dtype(numpy.float64).newbyteorder('>')
 
-
 class SelfDescribingBinaryProtocol(TBinaryProtocol):
+    
+  def readFieldBegin(self):
+    type = self.readByte()
+    if type == TType.STOP:
+      return (None, type, 0)
+    name = self.readString()
+    id = self.readI16()
+    return (name, type, id)
 
-    def readFieldBegin(self):
+  def readStructBegin(self):
-        ftype = self.readByte()
+     return self.readString()
-        if ftype == TType.STOP:
-            return None, ftype, 0
-        name = self.readString()
-        fid = self.readI16()
-        return name, ftype, fid
 
-    def readStructBegin(self):
+  def writeStructBegin(self, name):
-        return self.readString()
+     self.writeString(name)
 
-    def writeStructBegin(self, name):
+  def writeFieldBegin(self, name, type, id):
-        self.writeString(name)
+     self.writeByte(type)
+     self.writeString(name)
+     self.writeI16(id)
 
-    def writeFieldBegin(self, name, ftype, fid):
+  def readFloat(self):
-        self.writeByte(ftype)
+      d = self.readI32()
-        self.writeString(name)
+      dAsBytes = struct.pack('i', d)
-        self.writeI16(fid)
+      f = struct.unpack('f', dAsBytes)
+      return f[0]
 
-    def readFloat(self):
+  def writeFloat(self, f):
-        d = self.readI32()
+      dAsBytes = struct.pack('f', f)
-        dAsBytes = struct.pack('i', d)
+      i = struct.unpack('i', dAsBytes)
-        f = struct.unpack('f', dAsBytes)
+      self.writeI32(i[0])
-        return f[0]
 
-    def writeFloat(self, f):
+  def readI32List(self, sz):
-        dAsBytes = struct.pack('f', f)
+      buff = self.trans.readAll(4*sz)
-        i = struct.unpack('i', dAsBytes)
+      val = numpy.frombuffer(buff, dtype=intList, count=sz)
-        self.writeI32(i[0])
+      return val
 
-    def readI32List(self, sz):
+  def readF32List(self, sz):
-        buff = self.trans.readAll(4*sz)
+      buff = self.trans.readAll(4*sz)
-        val = numpy.frombuffer(buff, dtype=intList, count=sz)
+      val = numpy.frombuffer(buff, dtype=floatList, count=sz)
-        return val
+      return val
 
-    def readF32List(self, sz):
+  def readF64List(self, sz):
-        buff = self.trans.readAll(4*sz)
+      buff = self.trans.readAll(8*sz)
-        val = numpy.frombuffer(buff, dtype=floatList, count=sz)
+      val = numpy.frombuffer(buff, dtype=doubleList, count=sz)
-        return val
+      return val
 
-    def readF64List(self, sz):
+  def readI64List(self, sz):
-        buff = self.trans.readAll(8*sz)
+      buff = self.trans.readAll(8*sz)
-        val = numpy.frombuffer(buff, dtype=doubleList, count=sz)
+      val = numpy.frombuffer(buff, dtype=longList, count=sz)
-        return val
+      return val
 
-    def readI64List(self, sz):
+  def readI16List(self, sz):
-        buff = self.trans.readAll(8*sz)
+      buff = self.trans.readAll(2*sz)
-        val = numpy.frombuffer(buff, dtype=longList, count=sz)
+      val = numpy.frombuffer(buff, dtype=shortList, count=sz)
-        return val
+      return val
 
-    def readI16List(self, sz):
+  def readI8List(self, sz):
-        buff = self.trans.readAll(2*sz)
+      buff = self.trans.readAll(sz)
-        val = numpy.frombuffer(buff, dtype=shortList, count=sz)
+      val = numpy.frombuffer(buff, dtype=byteList, count=sz)
-        return val
+      return val
 
-    def readI8List(self, sz):
+  def writeI32List(self, buff):
-        buff = self.trans.readAll(sz)
+      b = numpy.asarray(buff, intList)
-        val = numpy.frombuffer(buff, dtype=byteList, count=sz)
+      self.trans.write(memoryview(b))
-        return val
 
-    def writeI32List(self, buff):
+  def writeF32List(self, buff):
-        b = numpy.asarray(buff, intList)
+      b = numpy.asarray(buff, floatList)
-        self.trans.write(numpy.getbuffer(b))
+      self.trans.write(memoryview(b))
 
-    def writeF32List(self, buff):
+  def writeF64List(self, buff):
-        b = numpy.asarray(buff, floatList)
+      b = numpy.asarray(buff, doubleList)
-        self.trans.write(numpy.getbuffer(b))
+      self.trans.write(memoryview(b))
 
-    def writeF64List(self, buff):
+  def writeI64List(self, buff):
-        b = numpy.asarray(buff, doubleList)
+      b = numpy.asarray(buff, longList)
-        self.trans.write(numpy.getbuffer(b))
+      self.trans.write(memoryview(b))
 
-    def writeI64List(self, buff):
+  def writeI16List(self, buff):
-        b = numpy.asarray(buff, longList)
+      b = numpy.asarray(buff, shortList)
-        self.trans.write(numpy.getbuffer(b))
+      self.trans.write(memoryview(b))
 
-    def writeI16List(self, buff):
+  def writeI8List(self, buff):
-        b = numpy.asarray(buff, shortList)
+      b = numpy.asarray(buff, byteList)
-        self.trans.write(numpy.getbuffer(b))
+      self.trans.write(memoryview(b))
-
-    def writeI8List(self, buff):
-        b = numpy.asarray(buff, byteList)
-        self.trans.write(numpy.getbuffer(b))

dynamicserialize/ThriftSerializationContext.py

@@ -1,3 +1,24 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
 #
 # A port of the Java ThriftSerializationContext, used for reading/writing
 # DynamicSerialize objects to/from thrift.
@@ -17,19 +38,22 @@
 #                                                writeObject().
 #    Apr 24, 2015    4425          nabowle       Add Double support
 #    Oct 17, 2016    5919          njensen       Optimized for speed
-#    Sep 06, 2018                  mjames@ucar   Python3 compliance
+#    Oct 11, 2018    7306          dgilling      Fix handling of ndarrays in 
+#                                                _serializeArray.
+#    Apr 30, 2019    7814          dgilling      Serialize python unicode strings.
+#    Jun 26, 2019    7888          tgurney       Python 3 fixes
+#    Jul 23, 2019    7888          tgurney       Fix handling of bytes as strings
+#    Sep 13, 2019    7888          tgurney       Fix serialization of bytes
 #
 #
 
+from thrift.Thrift import TType
 import inspect
 import sys
-import types
-import six
-import numpy
-from thrift.Thrift import TType
 import dynamicserialize
 from dynamicserialize import dstypes, adapters
-from dynamicserialize import SelfDescribingBinaryProtocol
+from . import SelfDescribingBinaryProtocol
+import numpy
 
 DS_LEN = len('dynamicserialize.dstypes.')
 
@@ -49,61 +73,39 @@ def buildObjMap(module):
         tname = tname[DS_LEN:]
         dsObjTypes[tname] = clz
 
-
 buildObjMap(dstypes)
 
-if six.PY2:
+pythonToThriftMap = {
-    pythonToThriftMap = {
+    bytes: TType.LIST,
-        types.StringType: TType.STRING,
+    str: TType.STRING,
-        types.IntType: TType.I32,
+    int: TType.I64,
-        types.LongType: TType.I64,
+    list: TType.LIST,
-        types.ListType: TType.LIST,
+    dict: TType.MAP,
-        unicode: TType.STRING,
+    set: TType.SET,
-        types.DictionaryType: TType.MAP,
+    float: SelfDescribingBinaryProtocol.FLOAT,
-        type(set([])): TType.SET,
+    # types.FloatType: TType.DOUBLE,
-        types.FloatType: SelfDescribingBinaryProtocol.FLOAT,
+    bool: TType.BOOL,
-        # types.FloatType: TType.DOUBLE,
+    object: TType.STRUCT,
-        types.BooleanType: TType.BOOL,
+    type(None): TType.VOID,
-        types.InstanceType: TType.STRUCT,
+    numpy.float32: SelfDescribingBinaryProtocol.FLOAT,
-        types.NoneType: TType.VOID,
+    numpy.int32: TType.I32,
-        numpy.float32: SelfDescribingBinaryProtocol.FLOAT,
+    numpy.ndarray: TType.LIST,
-        numpy.int32: TType.I32,
+    numpy.object_: TType.STRING,  # making an assumption here
-        numpy.ndarray: TType.LIST,
+    numpy.string_: TType.STRING,
-        numpy.object_: TType.STRING,  # making an assumption here
+    # numpy.bytes_ is the same as numpy.string_
-        numpy.string_: TType.STRING,
+    numpy.unicode_: TType.STRING,
-        numpy.float64: TType.DOUBLE,
+    # numpy.str_ is the same as numpy.unicode_
-        numpy.int16: TType.I16,
+    numpy.float64: TType.DOUBLE,
-        numpy.int8: TType.BYTE,
+    numpy.int16: TType.I16,
-        numpy.int64: TType.I64
+    numpy.int8: TType.BYTE,
-    }
+    numpy.int64: TType.I64
-else:
+}
-    pythonToThriftMap = {
-        bytes: TType.STRING,
-        int: TType.I32,
-        int: TType.I64,
-        list: TType.LIST,
-        dict: TType.MAP,
-        type(set([])): TType.SET,
-        float: SelfDescribingBinaryProtocol.FLOAT,
-        # types.FloatType: TType.DOUBLE,
-        bool: TType.BOOL,
-        object: TType.STRUCT,
-        str: TType.STRING,
-        type(None): TType.VOID,
-        numpy.float32: SelfDescribingBinaryProtocol.FLOAT,
-        numpy.int32: TType.I32,
-        numpy.ndarray: TType.LIST,
-        numpy.object_: TType.STRING,  # making an assumption here
-        numpy.string_: TType.STRING,
-        numpy.float64: TType.DOUBLE,
-        numpy.int16: TType.I16,
-        numpy.int8: TType.BYTE,
-        numpy.int64: TType.I64
-    }
 
 primitiveSupport = (TType.BYTE, TType.I16, TType.I32, TType.I64,
                     SelfDescribingBinaryProtocol.FLOAT, TType.DOUBLE)
 
+BYTE_STRING_TYPES = {numpy.string_, numpy.bytes_, bytes}
+
 
 class ThriftSerializationContext(object):
 
@@ -126,7 +128,7 @@ class ThriftSerializationContext(object):
             TType.VOID: lambda: None
         }
         self.typeSerializationMethod = {
-            TType.STRING: self.protocol.writeString,
+            TType.STRING: self.writeString,
             TType.I16: self.protocol.writeI16,
             TType.I32: self.protocol.writeI32,
             TType.LIST: self._serializeArray,
@@ -166,19 +168,18 @@ class ThriftSerializationContext(object):
 
     def deserializeMessage(self):
         name = self.protocol.readStructBegin()
-        name = name.decode('cp437')
-        name = name.replace('_', '.')
         if name.isdigit():
             obj = self._deserializeType(int(name))
             return obj
+        name = name.replace('_', '.')
         if name in adapters.classAdapterRegistry:
             return adapters.classAdapterRegistry[name].deserialize(self)
         elif '$' in name:
             # it's an inner class, we're going to hope it's an enum, treat it
             # special
             fieldName, fieldType, fieldId = self.protocol.readFieldBegin()
-            if fieldName.decode('utf8') != '__enumValue__':
+            if fieldName != '__enumValue__':
-                raise dynamicserialize.SerializationException(
+                raise dynamiceserialize.SerializationException(
                     "Expected to find enum payload.  Found: " + fieldName)
             obj = self.protocol.readString()
             self.protocol.readFieldEnd()
@@ -187,7 +188,7 @@ class ThriftSerializationContext(object):
             clz = dsObjTypes[name]
             obj = clz()
 
-        while self._deserializeField(obj):
+        while self._deserializeField(name, obj):
             pass
 
         self.protocol.readStructEnd()
@@ -200,19 +201,18 @@ class ThriftSerializationContext(object):
             raise dynamicserialize.SerializationException(
                 "Unsupported type value " + str(b))
 
-    def _deserializeField(self, obj):
+    def _deserializeField(self, structname, obj):
         fieldName, fieldType, fieldId = self.protocol.readFieldBegin()
         if fieldType == TType.STOP:
             return False
         elif fieldType != TType.VOID:
             result = self._deserializeType(fieldType)
-            fn_str = bytes.decode(fieldName)
+            lookingFor = "set" + fieldName[0].upper() + fieldName[1:]
-            lookingFor = "set" + fn_str[0].upper() + fn_str[1:]
 
             try:
                 setMethod = getattr(obj, lookingFor)
-                setMethod(result)
+                setMethod(result)                
-            except ValueError:
+            except:
                 raise dynamicserialize.SerializationException(
                     "Couldn't find setter method " + lookingFor)
 
@@ -225,7 +225,7 @@ class ThriftSerializationContext(object):
         if size:
             if listType not in primitiveSupport:
                 m = self.typeDeserializationMethod[listType]
-                result = [m() for __ in range(size)]
+                result = [m() for n in range(size)]
             else:
                 result = self.listDeserializationMethod[listType](size)
         self.protocol.readListEnd()
@@ -234,7 +234,7 @@ class ThriftSerializationContext(object):
     def _deserializeMap(self):
         keyType, valueType, size = self.protocol.readMapBegin()
         result = {}
-        for __ in range(size):
+        for n in range(size):
             # can't go off the type, due to java generics limitations dynamic serialize is
             # serializing keys and values as void
             key = self.typeDeserializationMethod[TType.STRUCT]()
@@ -246,7 +246,7 @@ class ThriftSerializationContext(object):
     def _deserializeSet(self):
         setType, setSize = self.protocol.readSetBegin()
         result = set([])
-        for __ in range(setSize):
+        for n in range(setSize):
             result.add(self.typeDeserializationMethod[TType.STRUCT]())
         self.protocol.readSetEnd()
         return result
@@ -256,11 +256,10 @@ class ThriftSerializationContext(object):
         if pyt in pythonToThriftMap:
             return pythonToThriftMap[pyt]
         elif pyt.__module__[:DS_LEN - 1] == ('dynamicserialize.dstypes'):
-            if six.PY2:
-                return pythonToThriftMap[types.InstanceType]
             return pythonToThriftMap[object]
-        raise dynamicserialize.SerializationException(
+        else:
-            "Don't know how to serialize object of type: " + str(pyt))
+            raise dynamicserialize.SerializationException(
+                "Don't know how to serialize object of type: " + str(pyt))
 
     def serializeMessage(self, obj):
         tt = self._lookupType(obj)
@@ -314,29 +313,34 @@ class ThriftSerializationContext(object):
 
     def _serializeArray(self, obj):
         size = len(obj)
-        if size:
+        objType = type(obj)
-            if isinstance(obj, numpy.ndarray):
+        if objType is numpy.ndarray:
-                t = pythonToThriftMap[obj.dtype.type]
+            t = pythonToThriftMap[obj.dtype.type]
-                size = obj.size
+            size = obj.size
-            else:
+        elif objType is bytes:
-                t = self._lookupType(obj[0])
+            t = TType.BYTE
+            obj = list(obj)
+        elif size:
+            t = self._lookupType(obj[0])
         else:
             t = TType.STRUCT
         self.protocol.writeListBegin(t, size)
         if t == TType.STRING:
-            if isinstance(obj, numpy.ndarray):
+            # For TType.STRING we always assume that if the objType is bytes we
+            # want to decode it into a str
+            if objType is numpy.ndarray:
                 if len(obj.shape) == 1:
                     for x in obj:
-                        s = str(x).strip()
+                        s = self._decodeString(x).strip()
                         self.typeSerializationMethod[t](s)
                 else:
                     for x in obj:
                         for y in x:
-                            s = str(y).strip()
+                            s = self._decodeString(y).strip()
                             self.typeSerializationMethod[t](s)
             else:
                 for x in obj:
-                    s = str(x)
+                    s = self._decodeString(x)
                     self.typeSerializationMethod[t](s)
         elif t not in primitiveSupport:
             for x in obj:
@@ -348,9 +352,9 @@ class ThriftSerializationContext(object):
     def _serializeMap(self, obj):
         size = len(obj)
         self.protocol.writeMapBegin(TType.VOID, TType.VOID, size)
-        for k in list(obj.keys()):
+        for (key, value) in obj.items():
-            self.typeSerializationMethod[TType.STRUCT](k)
+            self.typeSerializationMethod[TType.STRUCT](key)
-            self.typeSerializationMethod[TType.STRUCT](obj[k])
+            self.typeSerializationMethod[TType.STRUCT](value)
         self.protocol.writeMapEnd()
 
     def _serializeSet(self, obj):
@@ -360,6 +364,13 @@ class ThriftSerializationContext(object):
             self.typeSerializationMethod[TType.STRUCT](x)
         self.protocol.writeSetEnd()
 
+    def _decodeString(self, s):
+        """If s is a byte string, return s.decode(). Otherwise return str(s)"""
+        if type(s) in BYTE_STRING_TYPES:
+            return s.decode()
+        else:
+            return str(s)
+
     def writeMessageStart(self, name):
         self.protocol.writeMessageBegin(name, TType.VOID, 0)
 
@@ -412,6 +423,7 @@ class ThriftSerializationContext(object):
         return self.protocol.readString()
 
     def writeString(self, s):
+        s = self._decodeString(s)
         self.protocol.writeString(s)
 
     def readBinary(self):

dynamicserialize/__init__.py

@@ -1,35 +1,58 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
 #
+# TODO
+#  
+#    
 #     SOFTWARE HISTORY
-#
+#    
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
 #    08/20/10                      njensen       Initial Creation.
-#
+#    
+# 
 #
 
-__all__ = ['SerializationException']
+__all__ = [
+           ]
 
 from . import dstypes, adapters
 from . import DynamicSerializationManager
 
-
 class SerializationException(Exception):
-
+    
     def __init__(self, message=None):
         self.message = message
-
+    
     def __str__(self):
         if self.message:
-            return self.message
+            return self.message 
         else:
             return ""
-
+        
-
 def serialize(obj):
     dsm = DynamicSerializationManager.DynamicSerializationManager()
     return dsm.serializeObject(obj)
 
-
+def deserialize(bytes):
-def deserialize(objbytes):
     dsm = DynamicSerializationManager.DynamicSerializationManager()
-    return dsm.deserializeBytes(objbytes)
+    return dsm.deserializeBytes(bytes)

dynamicserialize/adapters/ActiveTableModeAdapter.py

@@ -0,0 +1,51 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+#
+# Adapter for com.raytheon.uf.common.activetable.ActiveTableMode
+#  
+#    
+#     SOFTWARE HISTORY
+#    
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    09/29/10                      wldougher     Initial Creation.
+#    
+# 
+#
+
+from thrift.Thrift import TType
+from dynamicserialize.dstypes.com.raytheon.uf.common.activetable import ActiveTableMode
+
+ClassAdapter = 'com.raytheon.uf.common.activetable.ActiveTableMode'
+
+def serialize(context, mode):
+    context.protocol.writeFieldBegin('__enumValue__', TType.STRING, 0)
+    context.writeString(mode.value)
+    
+def deserialize(context):
+    result = ActiveTableMode()
+    # Read the TType.STRING, "__enumValue__", and id.
+    # We're not interested in any of those, so just discard them.
+    context.protocol.readFieldBegin()
+    # now get the actual enum value
+    result.value = context.readString()
+    return result

dynamicserialize/adapters/ByteBufferAdapter.py

@@ -1,21 +1,46 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
 #
 # Adapter for java.nio.ByteBuffer
-#
+#  
-#
+#    
 #     SOFTWARE HISTORY
-#
+#    
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
 #    08/03/11                      dgilling       Initial Creation.
+#    
+# 
 #
 
-ClassAdapter = ['java.nio.ByteBuffer', 'java.nio.HeapByteBuffer']
+ClassAdapter = ['java.nio.ByteBuffer', 'java.nio.HeapByteBuffer'] 
 
 
-def serialize(context, bufferset):
+def serialize(context, set):
     raise NotImplementedError("Serialization of ByteBuffers is not supported.")
 
-
 def deserialize(context):
     byteBuf = context.readBinary()
     return byteBuf
+    
+    
+    

dynamicserialize/adapters/CalendarAdapter.py

@@ -1,24 +1,45 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
 #
 # Adapter for java.util.Calendar
-#
+#  
-#
+#    
 #     SOFTWARE HISTORY
-#
+#    
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
 #    09/29/10                      wldougher     Initial Creation.
+#    
+# 
 #
 
 from dynamicserialize.dstypes.java.util import Calendar
 
 ClassAdapter = 'java.util.Calendar'
 
-
 def serialize(context, calendar):
     calTiM = calendar.getTimeInMillis()
     context.writeI64(calTiM)
-
+    
-
 def deserialize(context):
     result = Calendar()
     result.setTimeInMillis(context.readI64())

dynamicserialize/adapters/CommutativeTimestampAdapter.py

@@ -1,25 +1,46 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
 #
 # Adapter for CommutativeTimestamp
-#
+#  
-#
+#    
 #     SOFTWARE HISTORY
-#
+#    
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
 #    9/21/2015       4486          rjpeter        Initial creation.
 #    Jun 23, 2016    5696          rjpeter        Handle CommutativeTimestamp.
+# 
 #
 
 from dynamicserialize.dstypes.com.raytheon.uf.common.time import CommutativeTimestamp
 
-ClassAdapter = 'com.raytheon.uf.common.time.CommutativeTimestamp'
 
+ClassAdapter = 'com.raytheon.uf.common.time.CommutativeTimestamp'
 
 def serialize(context, date):
     context.writeI64(date.getTime())
-
+    
-
 def deserialize(context):
     result = CommutativeTimestamp()
     result.setTime(context.readI64())
-    return result
+    return result

dynamicserialize/adapters/CoordAdapter.py

@@ -1,24 +1,45 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
 #
-# Adapter for com.vividsolutions.jts.geom.Coordinate
+# Adapter for org.locationtech.jts.geom.Coordinate
-#
+#  
-#
+#    
 #     SOFTWARE HISTORY
-#
+#    
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
 #    01/20/11                      dgilling      Initial Creation.
+#    
+# 
 #
 
-from dynamicserialize.dstypes.com.vividsolutions.jts.geom import Coordinate
+from dynamicserialize.dstypes.org.locationtech.jts.geom import Coordinate
-
-ClassAdapter = 'com.vividsolutions.jts.geom.Coordinate'
 
+ClassAdapter = 'org.locationtech.jts.geom.Coordinate'
 
 def serialize(context, coordinate):
     context.writeDouble(coordinate.getX())
     context.writeDouble(coordinate.getY())
 
-
 def deserialize(context):
     x = context.readDouble()
     y = context.readDouble()
@@ -26,3 +47,4 @@ def deserialize(context):
     coord.setX(x)
     coord.setY(y)
     return coord
+

dynamicserialize/adapters/DatabaseIDAdapter.py

@@ -1,23 +1,44 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
 #
 # Adapter for com.raytheon.uf.common.dataplugin.gfe.db.objects.DatabaseID
-#
+#  
-#
+#    
 #     SOFTWARE HISTORY
-#
+#    
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
 #    03/29/11                      dgilling      Initial Creation.
+#    
+# 
 #
 
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import DatabaseID
 
 ClassAdapter = 'com.raytheon.uf.common.dataplugin.gfe.db.objects.DatabaseID'
 
-
 def serialize(context, dbId):
     context.writeString(str(dbId))
-
+    
-
 def deserialize(context):
-    result = DatabaseID(context.readString())
+    result = DatabaseID(context.readString()) 
-    return result
+    return result

dynamicserialize/adapters/DateAdapter.py

@@ -1,24 +1,45 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
 #
 # Adapter for java.util.Date
-#
+#  
-#
+#    
 #     SOFTWARE HISTORY
-#
+#    
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
 #    12/06/10                      dgilling      Initial Creation.
+#    
+# 
 #
 
 from dynamicserialize.dstypes.java.util import Date
 
 ClassAdapter = 'java.util.Date'
 
-
 def serialize(context, date):
     context.writeI64(date.getTime())
-
+    
-
 def deserialize(context):
     result = Date()
     result.setTime(context.readI64())
-    return result
+    return result

dynamicserialize/adapters/EnumSetAdapter.py

@@ -1,32 +1,57 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
 #
 # Adapter for java.util.EnumSet
-#
+#  
-#
+#    
 #     SOFTWARE HISTORY
-#
+#    
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
 #    07/28/11                      dgilling       Initial Creation.
 #    12/02/13        2537          bsteffen       Serialize empty enum sets.
+#    
+# 
 #
 
+
+
 from dynamicserialize.dstypes.java.util import EnumSet
 
-ClassAdapter = ['java.util.EnumSet', 'java.util.RegularEnumSet']
+ClassAdapter = ['java.util.EnumSet', 'java.util.RegularEnumSet'] 
 
 
-def serialize(context, bufferset):
+def serialize(context, set):
-    setSize = len(bufferset)
+    setSize = len(set)
     context.writeI32(setSize)
-    context.writeString(bufferset.getEnumClass())
+    context.writeString(set.getEnumClass())
-    for val in bufferset:
+    for val in set:
         context.writeString(val)
-
+    
 
 def deserialize(context):
     setSize = context.readI32()
     enumClassName = context.readString()
     valList = []
-    for __ in range(setSize):
+    for i in range(setSize):
         valList.append(context.readString())
-    return EnumSet(enumClassName, valList)
+    return EnumSet(enumClassName, valList)     

dynamicserialize/adapters/FloatBufferAdapter.py

@@ -1,21 +1,46 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
 #
 # Adapter for java.nio.FloatBuffer
-#
+#  
-#
+#    
 #     SOFTWARE HISTORY
-#
+#    
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
 #    08/01/11                      dgilling       Initial Creation.
+#    
+# 
 #
 
-ClassAdapter = ['java.nio.FloatBuffer', 'java.nio.HeapFloatBuffer']
+ClassAdapter = ['java.nio.FloatBuffer', 'java.nio.HeapFloatBuffer'] 
 
 
-def serialize(context, bufferset):
+def serialize(context, set):
     raise NotImplementedError("Serialization of FloatBuffers is not supported.")
 
-
 def deserialize(context):
     floatBuf = context.readFloatArray()
     return floatBuf
+    
+    
+    

dynamicserialize/adapters/FormattedDateAdapter.py

@@ -1,24 +1,46 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
 #
 # Adapter for FormattedDate
-#
+#  
-#
+#    
 #     SOFTWARE HISTORY
-#
+#    
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
 #    9/21/2015       4486          rjpeter        Initial creation.
+#    
+# 
 #
 
 from dynamicserialize.dstypes.com.raytheon.uf.common.time import FormattedDate
 
-ClassAdapter = 'com.raytheon.uf.common.time.FormattedDate'
 
+ClassAdapter = 'com.raytheon.uf.common.time.FormattedDate'
 
 def serialize(context, date):
     context.writeI64(date.getTime())
-
+    
-
 def deserialize(context):
     result = FormattedDate()
     result.setTime(context.readI64())
-    return result
+    return result

dynamicserialize/adapters/GeomDataRespAdapter.py

@@ -1,3 +1,24 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
 #
 #    Efficient adapter for GetGeometryDataResponse
 #
@@ -11,9 +32,12 @@
 #
 #
 
+from thrift.Thrift import TType
+from dynamicserialize import SelfDescribingBinaryProtocol
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.response import GeometryResponseData
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.response import GetGeometryDataResponse
 
+
 ClassAdapter = 'com.raytheon.uf.common.dataaccess.response.GetGeometryDataResponse'
 
 
@@ -34,7 +58,7 @@ def serialize(context, resp):
         context.writeObject(geo.getTime())
         context.writeObject(geo.getLevel())
         context.writeObject(geo.getLocationName())
-        context.writeObject(geo.getAttributes())
+        context.writeObject(geo.getAttributes())        
 
         # write data map
         params = geo.getDataMap()
@@ -49,17 +73,16 @@ def serialize(context, resp):
             # unit
             context.writeObject(value[2])
 
-
 def deserialize(context):
     size = context.readI32()
     wkbs = []
-    for __ in range(size):
+    for i in range(size):
         wkb = context.readBinary()
         wkbs.append(wkb)
 
     geoData = []
     size = context.readI32()
-    for _ in range(size):
+    for i in range(size):
         data = GeometryResponseData()
         # wkb index
         wkbIndex = context.readI32()
@@ -77,7 +100,7 @@ def deserialize(context):
         # parameters
         paramSize = context.readI32()
         paramMap = {}
-        for __ in range(paramSize):
+        for k in range(paramSize):
             paramName = context.readString()
             value = context.readObject()
             tName = context.readString()

dynamicserialize/adapters/GeometryTypeAdapter.py

@@ -1,36 +1,56 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
 #
-# Adapter for com.vividsolutions.jts.geom.Polygon
+# Adapter for org.locationtech.jts.geom.Polygon
-#
+#  
-#
+#    
 #     SOFTWARE HISTORY
-#
+#    
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
 #    01/20/11                      dgilling      Initial Creation.
+#    
+# 
 #
 
-import dynamicserialize
-
 # TODO: Implement serialization/make deserialization useful.
 # Deserialization was simply implemented to allow GridLocation objects to be
 # passed through thrift, but the resulting Geometry object will not be transformed into
 # useful data; the base byte array is passed to a worthless Geometry class.
 
-from dynamicserialize.dstypes.com.vividsolutions.jts.geom import Geometry
+from dynamicserialize.dstypes.org.locationtech.jts.geom import Geometry
 
 # NOTE: At the moment, EDEX serializes Polygon, MultiPolygons, Points, and
 # Geometrys with the tag of the base class Geometry. Java's serialization
 # adapter is smarter and can determine the exact object by reading the binary
 # data. This adapter doesn't need this _yet_, so it has not been implemented.
-ClassAdapter = 'com.vividsolutions.jts.geom.Geometry'
+ClassAdapter = 'org.locationtech.jts.geom.Geometry'
-
 
 def serialize(context, coordinate):
     raise dynamicserialize.SerializationException('Not implemented yet')
 
-
 def deserialize(context):
     data = context.readBinary()
     geom = Geometry()
     geom.setBinaryData(data)
     return geom
+

dynamicserialize/adapters/GregorianCalendarAdapter.py

@@ -1,24 +1,45 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
 #
 # Adapter for java.util.Calendar
-#
+#  
-#
+#    
 #     SOFTWARE HISTORY
-#
+#    
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
 #    09/29/10                      wldougher     Initial Creation.
+#    
+# 
 #
 
 from dynamicserialize.dstypes.java.util import GregorianCalendar
 
 ClassAdapter = 'java.util.GregorianCalendar'
 
-
 def serialize(context, calendar):
     calTiM = calendar.getTimeInMillis()
     context.writeI64(calTiM)
-
+    
-
 def deserialize(context):
     result = GregorianCalendar()
     result.setTimeInMillis(context.readI64())

dynamicserialize/adapters/JTSEnvelopeAdapter.py

@@ -1,18 +1,39 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
 #
-# Adapter for com.vividsolutions.jts.geom.Envelope
+# Adapter for org.locationtech.jts.geom.Envelope
-#
+#  
-#
+#    
 #     SOFTWARE HISTORY
-#
+#    
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
 #    05/29/13         2023         dgilling       Initial Creation.
 #
+#
 
-from dynamicserialize.dstypes.com.vividsolutions.jts.geom import Envelope
+from dynamicserialize.dstypes.org.locationtech.jts.geom import Envelope
-
-ClassAdapter = 'com.vividsolutions.jts.geom.Envelope'
 
+ClassAdapter = 'org.locationtech.jts.geom.Envelope'
 
 def serialize(context, envelope):
     context.writeDouble(envelope.getMinX())
@@ -20,7 +41,6 @@ def serialize(context, envelope):
     context.writeDouble(envelope.getMinY())
     context.writeDouble(envelope.getMaxY())
 
-
 def deserialize(context):
     env = Envelope()
     env.setMinX(context.readDouble())
@@ -28,3 +48,4 @@ def deserialize(context):
     env.setMinY(context.readDouble())
     env.setMaxY(context.readDouble())
     return env
+

dynamicserialize/adapters/JobProgressAdapter.py

@@ -0,0 +1,51 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+#
+# Adapter for com.raytheon.uf.common.dataplugin.gfe.svcbu.JobProgress
+#  
+#    
+#     SOFTWARE HISTORY
+#    
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    06/22/2015       4573         randerso       Initial creation
+#    
+# 
+#
+
+from thrift.Thrift import TType
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.svcbu import JobProgress
+
+ClassAdapter = 'com.raytheon.uf.common.dataplugin.gfe.svcbu.JobProgress'
+
+def serialize(context, mode):
+    context.protocol.writeFieldBegin('__enumValue__', TType.STRING, 0)
+    context.writeString(mode.value)
+    
+def deserialize(context):
+    result = JobProgress()
+    # Read the TType.STRING, "__enumValue__", and id.
+    # We're not interested in any of those, so just discard them.
+    context.protocol.readFieldBegin()
+    # now get the actual enum value
+    result.value = context.readString()
+    return result

dynamicserialize/adapters/LocalizationLevelSerializationAdapter.py

@@ -1,14 +1,39 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
 #
 # Adapter for com.raytheon.uf.common.localization.LocalizationContext$LocalizationLevel
-#
+#  
-#
+#    
 #     SOFTWARE HISTORY
-#
+#    
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
 #    01/11/11                      dgilling       Initial Creation.
+#    
+# 
 #
 
+
+
 from dynamicserialize.dstypes.com.raytheon.uf.common.localization import LocalizationLevel
 
 ClassAdapter = [
@@ -16,12 +41,10 @@ ClassAdapter = [
                  'com.raytheon.uf.common.localization.LocalizationLevel'
                 ]
 
-
 def serialize(context, level):
     context.writeString(level.getText())
     context.writeI32(level.getOrder())
-    context.writeBool(level.isSystemLevel())
+    context.writeBool(level.isSystemLevel());
-
 
 def deserialize(context):
     text = context.readString()
@@ -29,3 +52,5 @@ def deserialize(context):
     systemLevel = context.readBool()
     level = LocalizationLevel(text, order, systemLevel=systemLevel)
     return level
+    
+

dynamicserialize/adapters/LocalizationTypeSerializationAdapter.py

@@ -1,14 +1,39 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
 #
 # Adapter for com.raytheon.uf.common.localization.LocalizationContext$LocalizationType
-#
+#  
-#
+#    
 #     SOFTWARE HISTORY
-#
+#    
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
 #    01/11/11                      dgilling       Initial Creation.
+#    
+# 
 #
 
+
+
 from dynamicserialize.dstypes.com.raytheon.uf.common.localization import LocalizationType
 
 ClassAdapter = [
@@ -16,11 +41,10 @@ ClassAdapter = [
                 'com.raytheon.uf.common.localization.LocalizationType'
                 ]
 
-
+def serialize(context, type):
-def serialize(context, ltype):
+    context.writeString(type.getText())
-    context.writeString(ltype.getText())
-
 
 def deserialize(context):
     typeString = context.readString()
     return LocalizationType(typeString)
+

dynamicserialize/adapters/LockTableAdapter.py

@@ -0,0 +1,89 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+#
+# Adapter for com.raytheon.uf.common.dataplugin.gfe.server.lock.LockTable
+#  
+#
+# SOFTWARE HISTORY
+#
+# Date          Ticket#  Engineer  Description
+# ------------- -------- --------- ---------------------------------------------
+# Apr 22, 2013           rjpeter   Initial Creation.
+# Jun 12, 2013  2099     dgilling  Use new Lock constructor.
+# Feb 06, 2017  5959     randerso  Removed Java .toString() calls 
+# 
+##
+
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.server.lock import LockTable
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.server.lock import Lock
+
+ClassAdapter = 'com.raytheon.uf.common.dataplugin.gfe.server.lock.LockTable'
+
+def serialize(context, lockTable):
+    index=0
+    wsIds = {str(lockTable.getWsId()) : index}
+    index += 1
+    locks = lockTable.getLocks()
+    lockWsIdIndex = []
+    for lock in locks:
+        wsIdString = str(lock.getWsId())
+        
+        if wsIdString in wsIds:
+            lockWsIdIndex.append(wsIds[wsIdString])
+        else:
+            lockWsIdIndex.append(index)
+            wsIds[wsIdString] = index
+            index += 1
+
+    context.writeObject(lockTable.getParmId())
+    
+    context.writeI32(index)
+    for wsId in sorted(wsIds, key=wsIds.get):
+        context.writeObject(wsId)
+        
+    context.writeI32(len(locks))
+    for lock, wsIndex in zip(locks, lockWsIdIndex):
+        serializer.writeI64(lock.getStartTime())
+        serializer.writeI64(lock.getEndTime())
+        serializer.writeI32(wsIndex)
+    
+def deserialize(context):
+    parmId = context.readObject()
+    numWsIds = context.readI32()
+    wsIds = []
+    for x in range(numWsIds):
+        wsIds.append(context.readObject())
+        
+    numLocks = context.readI32()
+    locks = []
+    for x in range(numLocks):
+        startTime = context.readI64()
+        endTime = context.readI64()
+        wsId = wsIds[context.readI32()]
+        lock = Lock(parmId, wsId, startTime, endTime)
+        locks.append(lock)
+        
+    lockTable = LockTable()
+    lockTable.setParmId(parmId)
+    lockTable.setWsId(wsIds[0])
+    lockTable.setLocks(locks)
+
+    return lockTable

dynamicserialize/adapters/ParmIDAdapter.py

@@ -1,23 +1,44 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
 #
 # Adapter for com.raytheon.uf.common.dataplugin.gfe.db.objects.ParmID
-#
+#  
-#
+#    
 #     SOFTWARE HISTORY
-#
+#    
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
 #    03/29/11                      dgilling      Initial Creation.
+#    
+# 
 #
 
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import ParmID
 
 ClassAdapter = 'com.raytheon.uf.common.dataplugin.gfe.db.objects.ParmID'
 
-
 def serialize(context, parmId):
     context.writeString(str(parmId))
-
+    
-
 def deserialize(context):
-    result = ParmID(context.readString())
+    result = ParmID(context.readString()) 
-    return result
+    return result