xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Transition from ufpy to awips python package

Browse source 
-Had to update the awips-ade and awips-devel dockers
-Update rpm building to include awips2-python-awips (which includes awips, dynamicserialize, thrift packages), and remove rpm install dependencies on ufpy, dynamicserialize, and thrift
-Update LDM scripts and edexBridge to use awips instead of ufpy
-Update multiple python scripts across several repos to use awips instead of ufpy
  -awips2
  -awips2-core
  -awips2-nativelib
  -awips2-ncep
  -awips2-rpm
This commit is contained in:
ucar-tmeyer 2024-01-16 16:10:09 +00:00
parent ba4f5ea96a
commit ddb3fd58dc
149 changed files with 50008 additions and 59016 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
build/awips-ade/Dockerfile.awips-ade-20.3.2-2.el7 Normal file
View file

@ -0,0 +1,23 @@
FROM tiffanym13/awips-devel-20.3.2-2:el7
ENV VERSION 20.3.2
ENV RELEASE 2
MAINTAINER Tiffany Meyer<tiffanym@ucar.edu>
USER root
COPY el7-dev.repo /etc/yum.repos.d/awips2.repo
RUN groupadd fxalpha && useradd -G fxalpha awips
RUN mkdir -p /home/awips/dev/unidata_20.3.2/awips2/dist/el7-dev-20231212/
ADD el7-dev-20231212 /home/awips/dev/unidata_20.3.2/awips2/dist/el7-dev-20231212
RUN yum -y clean all
RUN yum groupinstall awips2-ade -y
RUN mkdir -p /awips2/jenkins/buildspace/workspace/AWIPS2-UPC_build/baseline && mkdir -p /awips2/jenkins/buildspace/workspace/tmp
RUN mkdir -p /awips2/jenkins/build/rpms/awips2_latest/{x86_64,noarch}/
RUN chown -R awips:fxalpha /awips2/jenkins/
ENTRYPOINT ["/bin/bash"]

22
build/awips-ade/Dockerfile.awips-devel-20.3.2-2.el7 Normal file
View file

@ -0,0 +1,22 @@
FROM centos:7
ENV VERSION 20.3.2-2
ENV RELEASE 2
MAINTAINER Tiffany Meyer<tiffanym@ucar.edu>
USER root
RUN yum update yum -y
RUN yum groupinstall "Development tools" -y
RUN yum install epel-release -y
RUN yum clean all -y
ENV systemDeps="wget rsync git net-tools gzip libtool"
ENV rpmDeps="gcc-c++ gcc-gfortran rpm-build createrepo expat-devel lua-devel cyrus-sasl-devel cyrus-sasl-plain cyrus-sasl-md5 nss-devel nspr-devel libxml2-devel openldap-devel cmake"
ENV pythonDeps="tk-devel tcl-devel readline-devel bzip2-devel openssl-devel compat-libf2c-34"
ENV awipsDeps="netcdf netcdf-devel"
RUN yum install $systemDeps $rpmDeps $pythonDeps $awipsDeps -y
RUN yum update -y
ENTRYPOINT ["/bin/bash"]

19
build/awips-ade/awips-ade.sh
View file

@ -10,16 +10,17 @@ if [ -z "$1" ]; then
fi
os_version=$1
existing=$(sudo docker images |grep awips-ade | grep $1 | awk '{ print $3 }')
existing=$(docker images |grep awips-ade | grep $1 | awk '{ print $3 }')
if [ ! -z "$existing" ]; then
sudo docker rmi $existing
docker rmi $existing
fi
img="20.3.2-1"
img="20.3.2-2"
pushd /awips2/repo/awips2-builds/build/awips-ade
sudo docker build -t tiffanym13/awips-ade-${img} -f Dockerfile.awips-ade-${img}.${os_version} .
dockerID=$(sudo docker images | grep awips-ade | grep latest | awk '{print $3}' | head -1 )
#sudo docker tag $dockerID unidata/awips-ade:${AWIPSII_VERSION}-${os_version}
sudo docker tag $dockerID tiffanym13/awips-ade-${img}:${AWIPSII_VERSION}-${os_version}
sudo docker rmi tiffanym13/awips-ade-${img}:latest
sudo docker push tiffanym13/awips-ade-${img}:${AWIPSII_VERSION}-${os_version}
docker build -t tiffanym13/awips-ade-${img} -f Dockerfile.awips-ade-${img}.${os_version} .
dockerID=$(docker images | grep awips-ade | awk '{print $3}' | head -1 )
#docker tag $dockerID unidata/awips-ade:${AWIPSII_VERSION}-${os_version}
docker tag $dockerID tiffanym13/awips-ade-${img}:${AWIPSII_VERSION}-${os_version}
docker rmi tiffanym13/awips-ade-${img}:latest
#docker rmi tiffanym13/awips-ade-${img}:${AWIPSII_VERSION}-${os_version}
docker push tiffanym13/awips-ade-${img}:${AWIPSII_VERSION}-${os_version}

14
build/awips-ade/awips-devel.sh
View file

@ -2,7 +2,7 @@
dir="$( cd "$(dirname "$0")" ; pwd -P )"
pushd $dir
. ../buildEnvironment.sh
img="awips-devel-20.3.2-1"
img="awips-devel-20.3.2-2"
if [ -z "$1" ]; then
@ -13,11 +13,11 @@ os_version=$1
existing=$(sudo docker images |grep ${img} | grep $1 | awk '{ print $3 }')
if [ ! -z "$existing" ]; then
sudo docker rmi $existing
docker rmi $existing
fi
pushd /awips2/repo/awips2-builds/build/awips-ade
sudo docker build -t tiffanym13/${img} -f Dockerfile.${img}.${os_version} .
dockerID=$(sudo docker images | grep ${img} | grep latest | awk '{print $3}' | head -1 )
sudo docker tag $dockerID tiffanym13/${img}:${os_version}
sudo docker rmi tiffanym13/${img}:latest
sudo docker push tiffanym13/${img}:${os_version}
docker build -t tiffanym13/${img} -f Dockerfile.${img}.${os_version} .
dockerID=$(docker images | grep ${img} | grep latest | awk '{print $3}' | head -1 )
docker tag $dockerID tiffanym13/${img}:${os_version}
docker rmi tiffanym13/${img}:latest
docker push tiffanym13/${img}:${os_version}

3
build/awips-ade/el7-dev.repo
View file

@ -1,7 +1,8 @@
[awips2repo]
name=AWIPS II Repository
#baseurl=http://www.unidata.ucar.edu/repos/yum/18.2.1-ade
baseurl=file:///home/awips/dev/build/rpmbuild/RPMS
#baseurl=file:///home/awips/dev/build/rpmbuild/RPMS
baseurl=file:///home/awips/dev/unidata_20.3.2/awips2/dist/el7-dev-20231212
enabled=1
protect=0
gpgcheck=0

1
build/rsync.dirs
View file

@ -32,3 +32,4 @@ build/deploy.ignite.awips2
../awips2-ogc/foss/*
../awips2-ogc/edex/*
../awips2-ogc/features/*
../python-awips

12
build/setup.sh
View file

@ -52,7 +52,7 @@ fi
#
imgname=tiffanym13/awips-ade
imgvers=20.3.2
sudo docker run --entrypoint=/bin/bash --privileged -d -ti -e "container=docker" $dirs $imgname-$imgvers-1:$imgvers-$os_version
sudo docker run --entrypoint=/bin/bash --privileged -d -ti -e "container=docker" $dirs $imgname-$imgvers-2:$imgvers-$os_version
dockerID=$(sudo docker ps | grep awips-ade | awk '{print $1}' | head -1 )
sudo docker logs $dockerID
sudo docker exec -ti $dockerID /bin/bash -xec "/awips2/repo/awips2-builds/build/build_rpms.sh $os_version $rpmname";
@ -77,11 +77,13 @@ if [[ $(whoami) == "awips" ]]; then # local build
sudo mv dist/${os_version}-dev dist/${os_version}-dev-${date}
sudo su - -c "createrepo -g /awips2/repo/awips2/dist/comps.xml /awips2/repo/awips2/dist/${os_version}-dev-${date}/"
sudo chown -R awips:fxalpha dist/${os_version}-dev-${date}
echo "rsync -aP dist/${os_version}-dev-${date}"
#echo "rsync -aP dist/${os_version}-dev-${date} tiffanym@fserv:/share/awips2/${AWIPSII_VERSION}/linux/"
#rsync -aP dist/${os_version}-dev-${date} tiffanym@fserv:/share/awips2/${AWIPSII_VERSION}/linux/
echo "rsync -aP dist/${os_version}-dev-${date} tiffanym@fserv:/share/awips2/${AWIPSII_VERSION}/linux/"
rsync -aP dist/${os_version}-dev-${date} tiffanym@fserv:/share/awips2/${AWIPSII_VERSION}/linux/
cmd="cd /share/awips2/${AWIPSII_VERSION}/linux ; find ${os_version}-dev-${date} -type f | ../../git_nexus_tool/nexus-tools/bash/nexus-upload.sh -t downloads -u tiffanym -o awips2 -v ${AWIPSII_VERSION}/linux/rpms/"
echo "Need to run ssh@tiffanym '${cmd}' and provide -p [password]"
#rsync -aP dist/${os_version}-dev-${date} awips@edex3:/awips2/dev
rsync -aP dist/${os_version}-dev-${date} awips@hardy:/awips2/dev
#rsync -aP dist/${os_version}-dev-${date} awips@hardy:/awips2/dev
#repomanage -k1 --old dist/${os_version}-dev | xargs rm -f
#
# Push to web server

8
cave/com.raytheon.viz.gfe/GFESuite/src/gfeClient/gfeClient.py
View file

@ -35,10 +35,10 @@ import argparse
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request.GfeClientRequest import GfeClientRequest
from dynamicserialize.dstypes.java.util import Date
from ufpy import ThriftClient
from ufpy import UsageArgumentParser
from ufpy.UsageArgumentParser import StoreTimeAction
from ufpy.UsageArgumentParser import TIME_FORMAT
from awips import ThriftClient
from awips import UsageArgumentParser
from awips.UsageArgumentParser import StoreTimeAction
from awips.UsageArgumentParser import TIME_FORMAT
def validateArgs(args=None):

4
cave/com.raytheon.viz.gfe/GFESuite/src/ifpimage/PngWriter.py
View file

@ -474,8 +474,8 @@ def validateArgs(args=None, parents=[]):
# imports required for this method must be here so it can be invoked
# from gfeClient.py
############################################################################
from ufpy import UsageArgumentParser
from ufpy.UsageArgumentParser import StoreTimeAction
from awips import UsageArgumentParser
from awips.UsageArgumentParser import StoreTimeAction
global DEFAULT_OUTPUT_DIR
DEFAULT_OUTPUT_DIR = '../products/IMAGE'

10
cave/com.raytheon.viz.gfe/GFESuite/src/runifptext/runIFPText.py
View file

@ -55,7 +55,7 @@ def runFormatter(args):
from com.raytheon.viz.gfe.core import DataManagerUIFactory
from com.raytheon.viz.gfe.core import DataManager
from ufpy.UsageArgumentParser import TIME_FORMAT
from awips.UsageArgumentParser import TIME_FORMAT
from com.raytheon.viz.gfe.core import DataManagerFactory
LOGGER.info("TextFormatter Starting")
@ -88,10 +88,10 @@ def validateArgs(args=None, parents=[]):
# imports required for this method must be here so it can be invoked
# from gfeClient.py
############################################################################
from ufpy import UsageArgumentParser
from ufpy.UsageArgumentParser import StoreDatabaseIDAction
from ufpy.UsageArgumentParser import StoreTimeAction
from ufpy.UsageArgumentParser import TIME_FORMAT
from awips import UsageArgumentParser
from awips.UsageArgumentParser import StoreDatabaseIDAction
from awips.UsageArgumentParser import StoreTimeAction
from awips.UsageArgumentParser import TIME_FORMAT
import time
parser = UsageArgumentParser.UsageArgumentParser(conflict_handler="resolve",

4
cave/com.raytheon.viz.gfe/GFESuite/src/runprocedure/runProcedure.py
View file

@ -170,8 +170,8 @@ def validateArgs(args=None, parents=[]):
# imports required for this method must be here so it can be invoked
# from gfeClient.py
############################################################################
from ufpy import UsageArgumentParser
from ufpy.UsageArgumentParser import StoreTimeAction
from awips import UsageArgumentParser
from awips.UsageArgumentParser import StoreTimeAction
parser = UsageArgumentParser.UsageArgumentParser(conflict_handler="resolve",
parents=parents,

2
cave/com.raytheon.viz.gfe/localization/gfe/userPython/textUtilities/offsetTime.py
View file

@ -41,7 +41,7 @@
import time, string
import logging
from ufpy import TimeUtil
from awips import TimeUtil
offset = 0
timeStr = ""

2
cave/com.raytheon.viz.gfe/localization/gfe/userPython/utilities/TropicalUtility.py
View file

@ -55,7 +55,7 @@ import errno
import os
import re
from ufpy.dataaccess import DataAccessLayer
from awips.dataaccess import DataAccessLayer
import GridManipulation
import HazardUtils

145
dist/comps.xml vendored
View file

@ -2,6 +2,46 @@
<!-- <meta> -->
<!-- Meta Information Will Go Here Eventually -->
<!-- </meta> -->
<group>
<id>awips2-ade</id>
<name>AWIPS Development</name>
<default>true</default>
<description>This Will Install All Of The AWIPS Components That Are Required For Deploying in Eclipse (non DB)</description>
<uservisible>true</uservisible>
<packagelist>
<packagereq type="default">awips2</packagereq>
<packagereq type="default">awips2-ant</packagereq>
<packagereq type="default">awips2-eclipse</packagereq>
<packagereq type="default">awips2-hdf5-devel</packagereq>
<packagereq type="default">awips2-maven</packagereq>
<packagereq type="default">awips2-python-cheroot</packagereq>
<packagereq type="default">awips2-python-contextlib2</packagereq>
<packagereq type="default">awips2-python-cython</packagereq>
<packagereq type="default">awips2-python-jaraco.functools</packagereq>
<packagereq type="default">awips2-python-more-itertools</packagereq>
<packagereq type="default">awips2-python-pkgconfig</packagereq>
<packagereq type="default">awips2-python-portend</packagereq>
<packagereq type="default">awips2-python-pycairo</packagereq>
<packagereq type="default">awips2-python-pygobject</packagereq>
<packagereq type="default">awips2-python-setuptools_scm_git_archive</packagereq>
<packagereq type="default">awips2-python-setuptools_scm</packagereq>
<packagereq type="default">awips2-python-tempora</packagereq>
<packagereq type="default">awips2-python-zc.lockfile</packagereq>
<packagereq type="default">awips2-python-numpy</packagereq>
<packagereq type="default">awips2-python-dateutil</packagereq>
<packagereq type="default">awips2-python-pyparsing</packagereq>
<packagereq type="default">awips2-python-pbr</packagereq>
<packagereq type="default">awips2-python-mock</packagereq>
<packagereq type="default">awips2-python-numexpr</packagereq>
<packagereq type="default">awips2-python-thrift</packagereq>
<packagereq type="default">awips2-python-setuptools</packagereq>
<packagereq type="default">awips2-hdf5</packagereq>
<packagereq type="default">awips2-python-six</packagereq>
<packagereq type="default">awips2-python-pytz</packagereq>
<packagereq type="default">awips2-netcdf-devel</packagereq>
<packagereq type="default">awips2-qpid-proton</packagereq>
</packagelist>
</group>
<group>
<id>awips2-server</id>
<name>AWIPS EDEX Server</name>
@ -126,7 +166,6 @@
<packagereq type="mandatory">awips2-data.gfe</packagereq>
<packagereq type="mandatory">awips2-aviation-shared</packagereq>
<packagereq type="mandatory">awips2-python-dynamicserialize</packagereq>
<packagereq type="mandatory">awips2-python-h5py</packagereq>
<packagereq type="mandatory">awips2-python-cycler</packagereq>
<packagereq type="mandatory">awips2-python-kiwisolver</packagereq>
@ -137,9 +176,8 @@
<packagereq type="mandatory">awips2-qpid-proton-python</packagereq>
<packagereq type="mandatory">awips2-python-scipy</packagereq>
<packagereq type="mandatory">awips2-python-tables</packagereq>
<packagereq type="mandatory">awips2-python-thrift</packagereq>
<packagereq type="mandatory">awips2-python-tpg</packagereq>
<packagereq type="mandatory">awips2-python-ufpy</packagereq>
<packagereq type="mandatory">awips2-python-awips</packagereq>
<packagereq type="mandatory">awips2-python-werkzeug</packagereq>
<packagereq type="mandatory">awips2-python-shapely</packagereq>
<packagereq type="mandatory">awips2-python-dateutil</packagereq>
@ -270,7 +308,6 @@
<packagereq type="mandatory">awips2-rcm</packagereq>
<packagereq type="mandatory">awips2-aviation-shared</packagereq>
<packagereq type="mandatory">awips2-python-dynamicserialize</packagereq>
<packagereq type="mandatory">awips2-python-h5py</packagereq>
<packagereq type="mandatory">awips2-python-cycler</packagereq>
<packagereq type="mandatory">awips2-python-kiwisolver</packagereq>
@ -281,9 +318,8 @@
<packagereq type="mandatory">awips2-qpid-proton-python</packagereq>
<packagereq type="mandatory">awips2-python-scipy</packagereq>
<packagereq type="mandatory">awips2-python-tables</packagereq>
<packagereq type="mandatory">awips2-python-thrift</packagereq>
<packagereq type="mandatory">awips2-python-tpg</packagereq>
<packagereq type="mandatory">awips2-python-ufpy</packagereq>
<packagereq type="mandatory">awips2-python-awips</packagereq>
<packagereq type="mandatory">awips2-python-werkzeug</packagereq>
<packagereq type="mandatory">awips2-python-shapely</packagereq>
<packagereq type="mandatory">awips2-python-dateutil</packagereq>
@ -325,7 +361,6 @@
<packagereq type="mandatory">awips2-notification</packagereq>
<packagereq type="mandatory">awips2-qpid-proton</packagereq>
<packagereq type="mandatory">awips2-python-dynamicserialize</packagereq>
<packagereq type="mandatory">awips2-python-h5py</packagereq>
<packagereq type="mandatory">awips2-python-cycler</packagereq>
<packagereq type="mandatory">awips2-python-kiwisolver</packagereq>
@ -336,9 +371,8 @@
<packagereq type="mandatory">awips2-qpid-proton-python</packagereq>
<packagereq type="mandatory">awips2-python-scipy</packagereq>
<packagereq type="mandatory">awips2-python-tables</packagereq>
<packagereq type="mandatory">awips2-python-thrift</packagereq>
<packagereq type="mandatory">awips2-python-tpg</packagereq>
<packagereq type="mandatory">awips2-python-ufpy</packagereq>
<packagereq type="mandatory">awips2-python-awips</packagereq>
<packagereq type="mandatory">awips2-python-werkzeug</packagereq>
<packagereq type="mandatory">awips2-python-shapely</packagereq>
<packagereq type="mandatory">awips2-python-dateutil</packagereq>
@ -379,7 +413,6 @@
<packagereq type="mandatory">awips2-cli</packagereq>
<packagereq type="mandatory">awips2-notification</packagereq>
<packagereq type="mandatory">awips2-python-dynamicserialize</packagereq>
<packagereq type="mandatory">awips2-python-h5py</packagereq>
<packagereq type="mandatory">awips2-python-cycler</packagereq>
<packagereq type="mandatory">awips2-python-kiwisolver</packagereq>
@ -390,9 +423,8 @@
<packagereq type="mandatory">awips2-qpid-proton-python</packagereq>
<packagereq type="mandatory">awips2-python-scipy</packagereq>
<packagereq type="mandatory">awips2-python-tables</packagereq>
<packagereq type="mandatory">awips2-python-thrift</packagereq>
<packagereq type="mandatory">awips2-python-tpg</packagereq>
<packagereq type="mandatory">awips2-python-ufpy</packagereq>
<packagereq type="mandatory">awips2-python-awips</packagereq>
<packagereq type="mandatory">awips2-python-werkzeug</packagereq>
<packagereq type="mandatory">awips2-python-shapely</packagereq>
<packagereq type="mandatory">awips2-python-dateutil</packagereq>
@ -521,7 +553,6 @@
<packagereq type="mandatory">awips2-notification</packagereq>
<packagereq type="mandatory">awips2-aviation-shared</packagereq>
<packagereq type="mandatory">awips2-python-dynamicserialize</packagereq>
<packagereq type="mandatory">awips2-python-h5py</packagereq>
<packagereq type="mandatory">awips2-python-cycler</packagereq>
<packagereq type="mandatory">awips2-python-kiwisolver</packagereq>
@ -532,9 +563,8 @@
<packagereq type="mandatory">awips2-qpid-proton-python</packagereq>
<packagereq type="mandatory">awips2-python-scipy</packagereq>
<packagereq type="mandatory">awips2-python-tables</packagereq>
<packagereq type="mandatory">awips2-python-thrift</packagereq>
<packagereq type="mandatory">awips2-python-tpg</packagereq>
<packagereq type="mandatory">awips2-python-ufpy</packagereq>
<packagereq type="mandatory">awips2-python-awips</packagereq>
<packagereq type="mandatory">awips2-python-werkzeug</packagereq>
<packagereq type="mandatory">awips2-python-shapely</packagereq>
<packagereq type="mandatory">awips2-python-dateutil</packagereq>
@ -579,7 +609,6 @@
<packagereq type="mandatory">awips2-notification</packagereq>
<packagereq type="mandatory">awips2-edex-hazards-scripts</packagereq>
<packagereq type="mandatory">awips2-python-dynamicserialize</packagereq>
<packagereq type="mandatory">awips2-python-h5py</packagereq>
<packagereq type="mandatory">awips2-python-cycler</packagereq>
<packagereq type="mandatory">awips2-python-kiwisolver</packagereq>
@ -590,9 +619,8 @@
<packagereq type="mandatory">awips2-qpid-proton-python</packagereq>
<packagereq type="mandatory">awips2-python-scipy</packagereq>
<packagereq type="mandatory">awips2-python-tables</packagereq>
<packagereq type="mandatory">awips2-python-thrift</packagereq>
<packagereq type="mandatory">awips2-python-tpg</packagereq>
<packagereq type="mandatory">awips2-python-ufpy</packagereq>
<packagereq type="mandatory">awips2-python-awips</packagereq>
<packagereq type="mandatory">awips2-python-werkzeug</packagereq>
<packagereq type="mandatory">awips2-python-shapely</packagereq>
<packagereq type="mandatory">awips2-python-dateutil</packagereq>
@ -629,7 +657,6 @@
<packagereq type="mandatory">awips2-pypies</packagereq>
<packagereq type="mandatory">awips2-data.hdf5-topo</packagereq>
<packagereq type="mandatory">awips2-python-dynamicserialize</packagereq>
<packagereq type="mandatory">awips2-python-h5py</packagereq>
<packagereq type="mandatory">awips2-python-cycler</packagereq>
<packagereq type="mandatory">awips2-python-kiwisolver</packagereq>
@ -640,9 +667,8 @@
<packagereq type="mandatory">awips2-qpid-proton-python</packagereq>
<packagereq type="mandatory">awips2-python-scipy</packagereq>
<packagereq type="mandatory">awips2-python-tables</packagereq>
<packagereq type="mandatory">awips2-python-thrift</packagereq>
<packagereq type="mandatory">awips2-python-tpg</packagereq>
<packagereq type="mandatory">awips2-python-ufpy</packagereq>
<packagereq type="mandatory">awips2-python-awips</packagereq>
<packagereq type="mandatory">awips2-python-werkzeug</packagereq>
<packagereq type="mandatory">awips2-python-shapely</packagereq>
<packagereq type="mandatory">awips2-python-dateutil</packagereq>
@ -674,7 +700,6 @@
<packagereq type="mandatory">awips2-python-jep</packagereq>
<packagereq type="mandatory">awips2-java</packagereq>
<packagereq type="mandatory">awips2-qpid-broker-j</packagereq>
<packagereq type="mandatory">awips2-python-dynamicserialize</packagereq>
<packagereq type="mandatory">awips2-python-h5py</packagereq>
<packagereq type="mandatory">awips2-python-cycler</packagereq>
<packagereq type="mandatory">awips2-python-kiwisolver</packagereq>
@ -685,56 +710,8 @@
<packagereq type="mandatory">awips2-qpid-proton-python</packagereq>
<packagereq type="mandatory">awips2-python-scipy</packagereq>
<packagereq type="mandatory">awips2-python-tables</packagereq>
<packagereq type="mandatory">awips2-python-thrift</packagereq>
<packagereq type="mandatory">awips2-python-tpg</packagereq>
<packagereq type="mandatory">awips2-python-ufpy</packagereq>
<packagereq type="mandatory">awips2-python-werkzeug</packagereq>
<packagereq type="mandatory">awips2-python-shapely</packagereq>
<packagereq type="mandatory">awips2-python-dateutil</packagereq>
<packagereq type="mandatory">awips2-python-pyparsing</packagereq>
<packagereq type="mandatory">awips2-python-pytz</packagereq>
<packagereq type="mandatory">awips2-python-six</packagereq>
<packagereq type="mandatory">awips2-python-stomp.py</packagereq>
<packagereq type="mandatory">awips2-python-cftime</packagereq>
<packagereq type="mandatory">awips2-python-netcdf4</packagereq>
<packagereq type="mandatory">awips2-hdf5</packagereq>
<packagereq type="mandatory">awips2-netcdf</packagereq>
<packagereq type="mandatory">awips2-netcdf-devel</packagereq>
<packagereq type="mandatory">awips2-localapps-environment</packagereq>
<packagereq type="mandatory">awips2-watchdog</packagereq>
</packagelist>
</group>
<group>
<id>awips2-ldm-server</id>
<name>AWIPS II LDM Server</name>
<default>true</default>
<description>This Will Install The AWIPS II LDM Server.</description>
<uservisible>true</uservisible>
<packagelist>
<packagereq type="mandatory">awips2</packagereq>
<packagereq type="mandatory">awips2-version</packagereq>
<packagereq type="mandatory">awips2-python</packagereq>
<packagereq type="mandatory">awips2-python-jep</packagereq>
<packagereq type="mandatory">awips2-java</packagereq>
<packagereq type="mandatory">awips2-psql</packagereq>
<packagereq type="mandatory">awips2-ldm</packagereq>
<packagereq type="mandatory">awips2-cli</packagereq>
<packagereq type="mandatory">awips2-python-dynamicserialize</packagereq>
<packagereq type="mandatory">awips2-python-h5py</packagereq>
<packagereq type="mandatory">awips2-python-cycler</packagereq>
<packagereq type="mandatory">awips2-python-kiwisolver</packagereq>
<packagereq type="mandatory">awips2-python-backports-lru_cache</packagereq>
<packagereq type="mandatory">awips2-python-matplotlib</packagereq>
<packagereq type="mandatory">awips2-python-setuptools</packagereq>
<packagereq type="mandatory">awips2-python-numpy</packagereq>
<packagereq type="mandatory">awips2-qpid-proton-python</packagereq>
<packagereq type="mandatory">awips2-python-scipy</packagereq>
<packagereq type="mandatory">awips2-python-tables</packagereq>
<packagereq type="mandatory">awips2-python-thrift</packagereq>
<packagereq type="mandatory">awips2-python-tpg</packagereq>
<packagereq type="mandatory">awips2-python-ufpy</packagereq>
<packagereq type="mandatory">awips2-python-awips</packagereq>
<packagereq type="mandatory">awips2-python-werkzeug</packagereq>
<packagereq type="mandatory">awips2-python-shapely</packagereq>
<packagereq type="mandatory">awips2-python-dateutil</packagereq>
@ -770,7 +747,6 @@
<packagereq type="default">awips2-cli</packagereq>
<packagereq type="default">awips2-notification</packagereq>
<packagereq type="default">awips2-python-dynamicserialize</packagereq>
<packagereq type="default">awips2-python-h5py</packagereq>
<packagereq type="default">awips2-python-matplotlib</packagereq>
<packagereq type="default">awips2-python-setuptools</packagereq>
@ -778,9 +754,8 @@
<packagereq type="default">awips2-qpid-proton-python</packagereq>
<packagereq type="default">awips2-python-scipy</packagereq>
<packagereq type="default">awips2-python-tables</packagereq>
<packagereq type="default">awips2-python-thrift</packagereq>
<packagereq type="default">awips2-python-tpg</packagereq>
<packagereq type="default">awips2-python-ufpy</packagereq>
<packagereq type="default">awips2-python-awips</packagereq>
<packagereq type="default">awips2-python-werkzeug</packagereq>
<packagereq type="default">awips2-python-shapely</packagereq>
<packagereq type="default">awips2-python-dateutil</packagereq>
@ -878,7 +853,6 @@
<packagereq type="mandatory">awips2-postgresql</packagereq>
<packagereq type="mandatory">awips2-psql</packagereq>
<packagereq type="mandatory">awips2-python-dateutil</packagereq>
<packagereq type="mandatory">awips2-python-dynamicserialize</packagereq>
<packagereq type="mandatory">awips2-python-h5py</packagereq>
<packagereq type="mandatory">awips2-python-jep</packagereq>
<packagereq type="mandatory">awips2-python-cycler</packagereq>
@ -898,9 +872,8 @@
<packagereq type="mandatory">awips2-python-cftime</packagereq>
<packagereq type="mandatory">awips2-python-netcdf4</packagereq>
<packagereq type="mandatory">awips2-python-tables</packagereq>
<packagereq type="mandatory">awips2-python-thrift</packagereq>
<packagereq type="mandatory">awips2-python-tpg</packagereq>
<packagereq type="mandatory">awips2-python-ufpy</packagereq>
<packagereq type="mandatory">awips2-python-awips</packagereq>
<packagereq type="mandatory">awips2-python-werkzeug</packagereq>
<packagereq type="mandatory">awips2-hdf5</packagereq>
<packagereq type="mandatory">awips2-watchdog</packagereq>
@ -1013,7 +986,6 @@
<packagereq type="default">awips2-notification</packagereq>
<packagereq type="default">awips2-aviation-shared</packagereq>
<packagereq type="default">awips2-python-dynamicserialize</packagereq>
<packagereq type="default">awips2-python-h5py</packagereq>
<packagereq type="default">awips2-python-matplotlib</packagereq>
<packagereq type="default">awips2-python-setuptools</packagereq>
@ -1021,9 +993,8 @@
<packagereq type="default">awips2-qpid-proton-python</packagereq>
<packagereq type="default">awips2-python-scipy</packagereq>
<packagereq type="default">awips2-python-tables</packagereq>
<packagereq type="default">awips2-python-thrift</packagereq>
<packagereq type="default">awips2-python-tpg</packagereq>
<packagereq type="default">awips2-python-ufpy</packagereq>
<packagereq type="default">awips2-python-awips</packagereq>
<packagereq type="default">awips2-python-werkzeug</packagereq>
<packagereq type="default">awips2-python-shapely</packagereq>
<packagereq type="default">awips2-python-dateutil</packagereq>
@ -1145,7 +1116,6 @@
<packagereq type="mandatory">awips2-notification</packagereq>
<packagereq type="mandatory">awips2-aviation-shared</packagereq>
<packagereq type="mandatory">awips2-python-dynamicserialize</packagereq>
<packagereq type="mandatory">awips2-python-h5py</packagereq>
<packagereq type="mandatory">awips2-python-cycler</packagereq>
<packagereq type="mandatory">awips2-python-kiwisolver</packagereq>
@ -1156,9 +1126,8 @@
<packagereq type="mandatory">awips2-qpid-proton-python</packagereq>
<packagereq type="mandatory">awips2-python-scipy</packagereq>
<packagereq type="mandatory">awips2-python-tables</packagereq>
<packagereq type="mandatory">awips2-python-thrift</packagereq>
<packagereq type="mandatory">awips2-python-tpg</packagereq>
<packagereq type="mandatory">awips2-python-ufpy</packagereq>
<packagereq type="mandatory">awips2-python-awips</packagereq>
<packagereq type="mandatory">awips2-python-werkzeug</packagereq>
<packagereq type="mandatory">awips2-python-shapely</packagereq>
<packagereq type="mandatory">awips2-python-dateutil</packagereq>
@ -1257,7 +1226,6 @@
<packagereq type="mandatory">awips2-qpid-proton</packagereq>
<packagereq type="mandatory">awips2-hydroapps-shared</packagereq>
<packagereq type="mandatory">awips2-python-dynamicserialize</packagereq>
<packagereq type="mandatory">awips2-python-h5py</packagereq>
<packagereq type="mandatory">awips2-python-cycler</packagereq>
<packagereq type="mandatory">awips2-python-kiwisolver</packagereq>
@ -1268,9 +1236,8 @@
<packagereq type="mandatory">awips2-qpid-proton-python</packagereq>
<packagereq type="mandatory">awips2-python-scipy</packagereq>
<packagereq type="mandatory">awips2-python-tables</packagereq>
<packagereq type="mandatory">awips2-python-thrift</packagereq>
<packagereq type="mandatory">awips2-python-tpg</packagereq>
<packagereq type="mandatory">awips2-python-ufpy</packagereq>
<packagereq type="mandatory">awips2-python-awips</packagereq>
<packagereq type="mandatory">awips2-python-werkzeug</packagereq>
<packagereq type="mandatory">awips2-python-shapely</packagereq>
<packagereq type="mandatory">awips2-python-dateutil</packagereq>
@ -1349,13 +1316,11 @@
<packagereq type="mandatory">awips2-bmh</packagereq>
<packagereq type="mandatory">awips2-edex-bmh</packagereq>
<packagereq type="mandatory">awips2-neospeech</packagereq>
<packagereq type="mandatory">awips2-python-dynamicserialize</packagereq>
<packagereq type="mandatory">awips2-python-numpy</packagereq>
<packagereq type="mandatory">awips2-qpid-proton-python</packagereq>
<packagereq type="mandatory">awips2-python-scipy</packagereq>
<packagereq type="mandatory">awips2-python-thrift</packagereq>
<packagereq type="mandatory">awips2-python-tpg</packagereq>
<packagereq type="mandatory">awips2-python-ufpy</packagereq>
<packagereq type="mandatory">awips2-python-awips</packagereq>
<packagereq type="mandatory">awips2-python-werkzeug</packagereq>
<packagereq type="mandatory">awips2-python-shapely</packagereq>
<packagereq type="mandatory">awips2-python-dateutil</packagereq>
@ -1428,7 +1393,7 @@
<packagereq type="mandatory">awips2-version</packagereq>
<packagereq type="mandatory">awips2-ignite</packagereq>
<packagereq type="mandatory">awips2-java</packagereq>
<packagereq type="mandatory">awips2-python-ufpy</packagereq>
<packagereq type="mandatory">awips2-python-awips</packagereq>
<packagereq type="mandatory">awips2-watchdog</packagereq>
</packagelist>
</group>

2
dist/getRPMOutput.pl vendored
View file

@ -1,6 +1,6 @@
#!/usr/bin/perl
$date="20221018";
$date="20240108";
$path="/awips2/repo/awips2/dist/el7-dev-$date";

98188
dist/rpmOutput vendored
View file

File diff suppressed because it is too large Load diff

2
edexOsgi/com.raytheon.edex.plugin.gfe/utility/common_static/base/gfe/python/isc/IrtServer.py
View file

@ -229,7 +229,7 @@ def sendWfoMessage(siteID, msgFile):
logEvent("Message received from site: %s\n%s" % (siteID, message))
# send to AlertViz
from ufpy import NotificationMessage
from awips import NotificationMessage
msg = NotificationMessage.NotificationMessage(port='9581', message=message,
category='GFE', priority='SIGNIFICANT', source='GFE')
msg.send()

2
edexOsgi/com.raytheon.edex.plugin.gfe/utility/common_static/base/gfe/textproducts/configureTextProducts.py
View file

@ -46,7 +46,7 @@ import sys
from os.path import dirname
from os.path import abspath
from ufpy import ThriftClient
from awips import ThriftClient
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import ConfigureTextProductsRequest
SCRIPT_DIR = abspath(dirname(sys.argv[0]))

2
edexOsgi/com.raytheon.edex.plugin.warning/WarningDecoder.py
View file

@ -65,7 +65,7 @@
import calendar
import sys, os, time, re, getopt
from ufpy import TimeUtil
from awips import TimeUtil
from com.raytheon.uf.common.wmo import WMOTimeParser
from com.raytheon.uf.edex.decodertools.time import TimeTools

2
edexOsgi/com.raytheon.uf.common.localization.python/utility/common_static/base/python/PythonOverriderPure.py
View file

@ -42,7 +42,7 @@
import os, tempfile, shutil
import numpy
import PythonOverriderCore
from ufpy import ThriftClient
from awips import ThriftClient
from dynamicserialize.dstypes.com.raytheon.uf.common.auth.resp import SuccessfulExecution
from dynamicserialize.dstypes.com.raytheon.uf.common.localization import LocalizationContext
from dynamicserialize.dstypes.com.raytheon.uf.common.localization.msgs import ListUtilityCommand

2
edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JData.py
View file

@ -37,7 +37,7 @@
from ufpy.dataaccess import IData
from awips.dataaccess import IData
import JUtil, DataTime
class JData(IData, JUtil.JavaWrapperClass):

2
edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JDataRequest.py
View file

@ -41,7 +41,7 @@
from ufpy.dataaccess import IDataRequest
from awips.dataaccess import IDataRequest
from com.raytheon.uf.common.dataplugin.level import Level
import JUtil
import jep

2
edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JGeometryData.py
View file

@ -41,7 +41,7 @@
from ufpy.dataaccess import IGeometryData
from awips.dataaccess import IGeometryData
import JData
class JGeometryData(IGeometryData, JData.JData):

2
edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JGridData.py
View file

@ -41,7 +41,7 @@
from ufpy.dataaccess import IGridData
from awips.dataaccess import IGridData
import JData
import numpy as np

2
edexOsgi/com.raytheon.uf.tools.cli/impl/a2pgca
View file

@ -1697,7 +1697,7 @@ def refresh_impl(args, dry_run=False, rsync_dry_run=False,
continue
dst_path = join(path, 'passwords.properties')
# need full python path since this is run as root, which doesn't have appropriate path vars set
cmd = ['ssh', host, f"/awips2/python/bin/python -c \"from ufpy import ignite_password; ignite_password.updateIgnitePasswords('{keystore_password}', '{truststore_password}', '{dst_path}')\""]
cmd = ['ssh', host, f"/awips2/python/bin/python -c \"from awips import ignite_password; ignite_password.updateIgnitePasswords('{keystore_password}', '{truststore_password}', '{dst_path}')\""]
if not dry_run:
try:
run(cmd, echo_stdout=verbose, echo_stderr_filtered=True)

2
edexOsgi/com.raytheon.uf.tools.cli/impl/importAdaptivePlot.py
View file

@ -111,7 +111,7 @@ if exportFileName is None:
if idx > -1:
exportFileName = exportFileName[0:idx]
from ufpy.ThriftClient import *
from awips.ThriftClient import *
client = ThriftClient(server)
from dynamicserialize.dstypes.com.raytheon.uf.common.pointdata.requests import *

2
edexOsgi/com.raytheon.uf.tools.cli/impl/lf
View file

@ -34,7 +34,7 @@
import sys
import argparse
from ufpy.localization.LocalizationFileManager import (LocalizationFileManager,
from awips.localization.LocalizationFileManager import (LocalizationFileManager,
NON_EXISTENT_CHECKSUM,
LocalizationFileIsNotDirectoryException)

2
edexOsgi/com.raytheon.uf.tools.cli/impl/src/convertcdl2xml/convertCDL2XML.py
View file

@ -43,7 +43,7 @@ import sys
import xml.dom.minidom as minidom
import xml.etree.ElementTree as ET
from ufpy import UsageArgumentParser
from awips import UsageArgumentParser
import netCDF4

2
edexOsgi/com.raytheon.uf.tools.cli/impl/src/createusertopo/createUserTopoFile.py
View file

@ -41,7 +41,7 @@ import numpy as np
from scipy import ndimage
import netCDF4
from ufpy import UsageArgumentParser
from awips import UsageArgumentParser

2
edexOsgi/com.raytheon.uf.tools.cli/impl/src/msg/fxaAnnounce.py
View file

@ -48,7 +48,7 @@ import os
# local-delivery messages
#
from ufpy import NotificationMessage
from awips import NotificationMessage
def usage():
print('')

2
edexOsgi/com.raytheon.uf.tools.cli/impl/src/msg/sendNotificationMsg.py
View file

@ -47,7 +47,7 @@ import traceback
# May 25, 2022 DR 23144 aghanava Modify workstation filter to use the short name.
#
from ufpy import NotificationMessage
from awips import NotificationMessage
class PrintHelpOnErrorParser(ArgumentParser):
def error(self, message):

2
edexOsgi/com.raytheon.uf.tools.cli/impl/src/pgen/ActivityUtil.py
View file

@ -16,7 +16,7 @@
import os
import re
import xml.etree.ElementTree as ET
from ufpy import ThriftClient
from awips import ThriftClient
from dynamicserialize.dstypes.gov.noaa.nws.ncep.common.dataplugin.pgen.request import RetrieveActivityMapRequest
class ActivityUtil:

2
edexOsgi/com.raytheon.uf.tools.cli/impl/src/pgen/ProductRetriever.py
View file

@ -1,5 +1,5 @@
import os
from ufpy import ThriftClient
from awips import ThriftClient
from dynamicserialize.dstypes.com.raytheon.uf.common.datastorage.records import StringDataRecord
from dynamicserialize.dstypes.com.raytheon.uf.common.datastorage.records import ByteDataRecord
from dynamicserialize.dstypes.gov.noaa.nws.ncep.common.dataplugin.pgen.request import RetrieveAllProductsRequest

2
edexOsgi/com.raytheon.uf.tools.cli/impl/src/pgen/ProductStorer.py
View file

@ -1,5 +1,5 @@
import os
from ufpy import ThriftClient
from awips import ThriftClient
#from dynamicserialize.dstypes.com.raytheon.uf.common.datastorage.records import ByteDataRecord
from dynamicserialize.dstypes.gov.noaa.nws.ncep.common.dataplugin.pgen.request import StoreActivityRequest
from dynamicserialize.dstypes.gov.noaa.nws.ncep.common.dataplugin.pgen import ResponseMessageValidate

2
edexOsgi/com.raytheon.uf.tools.cli/impl/src/pgen/retrieveActivity.py
View file

@ -23,7 +23,7 @@
import logging
from tkinter import *
from ufpy import UsageArgumentParser
from awips import UsageArgumentParser
import ProductRetriever
import ActivityUtil

2
edexOsgi/com.raytheon.uf.tools.cli/impl/src/pgen/storeActivity.py
View file

@ -9,7 +9,7 @@ import os
import logging
import xml.etree.ElementTree as ET
from ufpy import UsageArgumentParser
from awips import UsageArgumentParser
from dynamicserialize.dstypes.gov.noaa.nws.ncep.common.dataplugin.pgen import ActivityInfo
import ProductStorer

4
edexOsgi/com.raytheon.uf.tools.cli/impl/src/purgeallmodeldata/purgeAllModelData.py
View file

@ -37,8 +37,8 @@
import logging
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.grid.request import DeleteAllGridDataRequest
from ufpy import ThriftClient
from ufpy import UsageArgumentParser
from awips import ThriftClient
from awips import UsageArgumentParser
logger = None
def __initLogger():

2
edexOsgi/com.raytheon.uf.tools.cli/impl/src/qpidNotify/qpidNotify.py
View file

@ -50,7 +50,7 @@ import sys
import time
from lib.Util import time_limit, TimeoutException
from ufpy import qpidingest
from awips import qpidingest
logging.basicConfig(level=logging.INFO, datefmt='%H:%M:%S',
format="[%(process)s] %(asctime)s %(levelname)s: %(message)s")

2
edexOsgi/com.raytheon.uf.tools.cli/impl/src/subscription/SubscriptionManager.py
View file

@ -27,7 +27,7 @@ import lib.Util as util
import conf.SMConfig as config
import collections
from ufpy import ThriftClient
from awips import ThriftClient
from dynamicserialize.dstypes.com.raytheon.uf.common.message import Message, Header
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.text.subscription.request import SubscriptionRequest
##############################################################################

2
edexOsgi/com.raytheon.uf.tools.cli/impl/src/textdb/TextDB.py
View file

@ -39,7 +39,7 @@ import subscription.SubscriptionManager as SM
import conf.TDBConfig as config
import collections
from ufpy import ThriftClient
from awips import ThriftClient
from dynamicserialize.dstypes.com.raytheon.uf.common.message import Message, Header
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.text.dbsrv import TextDBRequest

2
edexOsgi/com.raytheon.uf.tools.gfesuite.servicebackup/svcBackup/ServiceBackup/scripts/receive_configuration.py
View file

@ -25,7 +25,7 @@ import os
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import ProcessReceivedConfRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.message import WsId
from ufpy import ThriftClient
from awips import ThriftClient
#
# TODO: ADD DESCRIPTION

2
edexOsgi/com.raytheon.uf.tools.gfesuite.servicebackup/svcBackup/ServiceBackup/scripts/receive_grids.py
View file

@ -25,7 +25,7 @@ import os
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import ProcessReceivedDigitalDataRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.message import WsId
from ufpy import ThriftClient
from awips import ThriftClient
#
# TODO: ADD DESCRIPTION

2
edexOsgi/com.raytheon.uf.tools.gfesuite.servicebackup/svcBackup/bin/src/iscdatarec/iscDataRec.py
View file

@ -24,7 +24,7 @@ import os
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import IscDataRecRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.message import WsId
from ufpy import ThriftClient
from awips import ThriftClient
#
# CLI tool to process incoming ISC requests. Receives incoming request via MHS

2
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/getNotify.py
View file

@ -42,7 +42,7 @@ import dynamicserialize
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.server.notify import *
from dynamicserialize.dstypes.com.raytheon.uf.common.activetable import VTECTableChangeNotification
from ufpy import QpidSubscriber
from awips import QpidSubscriber
printoutMap = OrderedDict([
('L', 'LOCK'),

4
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/VTECDecoder/VTECDecoder.py
View file

@ -9,8 +9,8 @@ import os
import sys
from dynamicserialize.dstypes.com.raytheon.uf.common.activetable import SendPracticeProductRequest
from ufpy import ThriftClient
from ufpy import TimeUtil
from awips import ThriftClient
from awips import TimeUtil
logging.basicConfig(level=logging.INFO)

6
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/activeTable/MergeVTEC.py
View file

@ -45,9 +45,9 @@ import sys
from dynamicserialize.dstypes.com.raytheon.uf.common.activetable.request import MergeActiveTableRequest
from ufpy import ThriftClient
from ufpy import TimeUtil
from ufpy import UsageArgumentParser
from awips import ThriftClient
from awips import TimeUtil
from awips import UsageArgumentParser
logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s: %(message)s",

2
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/activeTable/dumpAT.py
View file

@ -22,7 +22,7 @@
import getopt, sys
import logging, time, traceback, gzip
from dynamicserialize.dstypes.com.raytheon.uf.common.activetable import DumpActiveTableRequest
from ufpy import ThriftClient
from awips import ThriftClient
## sorting function
#def sortFunc(r1, r2):

2
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/activeTable/getAT.py
View file

@ -20,7 +20,7 @@
import os
import logging
from dynamicserialize.dstypes.com.raytheon.uf.common.activetable import GetActiveTableDictRequest
from ufpy import ThriftClient
from awips import ThriftClient
from getVtecAttribute import getVtecAttribute
logging.basicConfig(level=logging.INFO)

2
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/activeTable/getFourCharSites.py
View file

@ -20,7 +20,7 @@
import logging
from dynamicserialize.dstypes.com.raytheon.uf.common.activetable import GetFourCharSitesRequest
from ufpy import ThriftClient
from awips import ThriftClient
##
# Convert a list with 3-char site IDs to a list of 4-char site IDs.

2
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/activeTable/getVtecAttribute.py
View file

@ -21,7 +21,7 @@
import os
import logging
from dynamicserialize.dstypes.com.raytheon.uf.common.activetable import GetVtecAttributeRequest
from ufpy import ThriftClient
from awips import ThriftClient
##
# Ask the server for an attribute from the VTECPartners script for a site.

4
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/activeTable/ingestAT.py
View file

@ -39,8 +39,8 @@ import logging
import os
import MergeVTEC
from ufpy import TimeUtil
from ufpy import UsageArgumentParser
from awips import TimeUtil
from awips import UsageArgumentParser

4
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/activeTable/requestAT.py
View file

@ -36,8 +36,8 @@ import sys
from dynamicserialize.dstypes.com.raytheon.uf.common.activetable.request import RetrieveRemoteActiveTableRequest
from ufpy import ThriftClient
from ufpy import UsageArgumentParser
from awips import ThriftClient
from awips import UsageArgumentParser
logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s: %(message)s",

4
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/activeTable/sendAT.py
View file

@ -41,8 +41,8 @@ import logging
import sys
from dynamicserialize.dstypes.com.raytheon.uf.common.activetable.request import SendActiveTableRequest
from ufpy import ThriftClient
from ufpy import UsageArgumentParser
from awips import ThriftClient
from awips import UsageArgumentParser
logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s: %(message)s",

4
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/ifpBreakAllLocks/ifpBreakAllLocks.py
View file

@ -30,8 +30,8 @@ from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.server.reque
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import DatabaseID
from dynamicserialize.dstypes.com.raytheon.uf.common.site.requests import GetActiveSitesRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.message import WsId
from ufpy import ThriftClient
from ufpy import UsageArgumentParser
from awips import ThriftClient
from awips import UsageArgumentParser
#
# Provides a command-line utility to break all locks.

6
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/ifpag/ifpAG.py
View file

@ -32,9 +32,9 @@ from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request impo
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import SaveASCIIGridsRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange
from dynamicserialize.dstypes.com.raytheon.uf.common.message import WsId
from ufpy import ThriftClient
from ufpy import UsageArgumentParser
from ufpy.localization import LocalizationUtil
from awips import ThriftClient
from awips import UsageArgumentParser
from awips.localization import LocalizationUtil
##

4
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/ifpinit/ifpInit.py
View file

@ -24,8 +24,8 @@ import os
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import SmartInitRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.message import WsId
from ufpy import ThriftClient
from ufpy import UsageArgumentParser
from awips import ThriftClient
from awips import UsageArgumentParser
##

8
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/ifpnetcdf/ifpnetCDF.py
View file

@ -41,10 +41,10 @@ import sys
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import ExecuteIfpNetCDFGridRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.message import WsId
from ufpy import ThriftClient
from ufpy import UsageArgumentParser
from ufpy.UsageArgumentParser import StoreDatabaseIDAction as StoreDatabaseIDAction
from ufpy.UsageArgumentParser import AppendParmNameAndLevelAction as AppendParmNameAndLevelAction
from awips import ThriftClient
from awips import UsageArgumentParser
from awips.UsageArgumentParser import StoreDatabaseIDAction as StoreDatabaseIDAction
from awips.UsageArgumentParser import AppendParmNameAndLevelAction as AppendParmNameAndLevelAction
RETRY_ATTEMPTS = 3

6
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/ifpservertext/ifpServerText.py
View file

@ -38,9 +38,9 @@ from dynamicserialize.dstypes.com.raytheon.uf.common.localization import Localiz
from dynamicserialize.dstypes.com.raytheon.uf.common.localization import LocalizationType
from dynamicserialize.dstypes.com.raytheon.uf.common.message import WsId
from dynamicserialize.dstypes.com.raytheon.uf.common.site.requests import GetActiveSitesRequest
from ufpy import ThriftClient
from ufpy.localization import LocalizationUtil
from ufpy.localization.LocalizationFileManager import LocalizationFileVersionConflictException
from awips import ThriftClient
from awips.localization import LocalizationUtil
from awips.localization.LocalizationFileManager import LocalizationFileVersionConflictException
#
# The ifpServerText program. Stores, deletes, gets, and inventories text.

8
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/iscmosaic/iscMosaic.py
View file

@ -22,10 +22,10 @@ import logging
import os
import sys
from ufpy import ThriftClient
from ufpy import UsageArgumentParser
from ufpy.UsageArgumentParser import StoreDatabaseIDAction as StoreDatabaseIDAction
from ufpy.UsageArgumentParser import AppendParmNameAndLevelAction as AppendParmNameAndLevelAction
from awips import ThriftClient
from awips import UsageArgumentParser
from awips.UsageArgumentParser import StoreDatabaseIDAction as StoreDatabaseIDAction
from awips.UsageArgumentParser import AppendParmNameAndLevelAction as AppendParmNameAndLevelAction
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import ExecuteIscMosaicRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.message import WsId

8
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/movegfedata/moveGFEData.py
View file

@ -31,10 +31,10 @@ import urllib.error
import urllib.parse
import urllib.request
from ufpy import ThriftClient
from ufpy import UsageArgumentParser
from ufpy.localization import LocalizationUtil
from ufpy.localization.LocalizationFileManager import LocalizationFileVersionConflictException
from awips import ThriftClient
from awips import UsageArgumentParser
from awips.localization import LocalizationUtil
from awips.localization.LocalizationFileManager import LocalizationFileVersionConflictException
from dynamicserialize.dstypes.com.raytheon.uf.common.site.requests import GetActiveSitesRequest

4
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/publishgfe/publishGFE.py
View file

@ -23,8 +23,8 @@
import logging
import sys
from ufpy import UsageArgumentParser
from ufpy.gfe import IFPClient
from awips import UsageArgumentParser
from awips.gfe import IFPClient
from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import DatabaseID

6
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/purgeallgrids/purgeAllGrids.py
View file

@ -40,9 +40,9 @@ import sys
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import PurgeGfeGridsRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.message import WsId
from ufpy import ThriftClient
from ufpy import UsageArgumentParser
from ufpy.UsageArgumentParser import StoreDatabaseIDAction as StoreDatabaseIDAction
from awips import ThriftClient
from awips import UsageArgumentParser
from awips.UsageArgumentParser import StoreDatabaseIDAction as StoreDatabaseIDAction
logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s: %(message)s",

4
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/rsyncGridsToCWF/rsyncGridsToCWF.py
View file

@ -28,8 +28,8 @@
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import RsyncGridsToCWFRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.message import WsId
from ufpy import ThriftClient
from ufpy import UsageArgumentParser
from awips import ThriftClient
from awips import UsageArgumentParser
import os

2
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/sendgfemessage/sendGfeMessage.py
View file

@ -21,7 +21,7 @@
import shlex
import subprocess
from ufpy import UsageArgumentParser
from awips import UsageArgumentParser
#

4
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/sendgridstondfd/SendGridsToNDFD.py
View file

@ -40,8 +40,8 @@ import argparse
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import ExportGridsRequest
from ufpy import ThriftClient
from ufpy import UsageArgumentParser
from awips import ThriftClient
from awips import UsageArgumentParser
class SendGridsToNDFD:

2
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/siteActivation/ActivationTopicListener.py
View file

@ -37,7 +37,7 @@ import traceback
import dynamicserialize
from dynamicserialize.dstypes.com.raytheon.uf.common.site.notify import ClusterActivationNotification
from ufpy import QpidSubscriber
from awips import QpidSubscriber
class ActivationTopicListener(threading.Thread):
def __init__(self, host='localHost', port='5762', topic="edex.alerts.siteActivate", program="siteActivation" ):

4
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/siteActivation/activateSite.py
View file

@ -35,8 +35,8 @@ import sys
import time
from dynamicserialize.dstypes.com.raytheon.uf.common.site.requests import ActivateSiteRequest
from ufpy import ThriftClient
from ufpy import UsageArgumentParser
from awips import ThriftClient
from awips import UsageArgumentParser
from ActivationTopicListener import ActivationTopicListener

4
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/siteActivation/deactivateSite.py
View file

@ -34,8 +34,8 @@ import time
from dynamicserialize.dstypes.com.raytheon.uf.common.site.requests import DeactivateSiteRequest
from ufpy import ThriftClient
from ufpy import UsageArgumentParser
from awips import ThriftClient
from awips import UsageArgumentParser
from ActivationTopicListener import ActivationTopicListener

4
edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/validateconfig/validateConfig.py
View file

@ -41,8 +41,8 @@ import sys
import traceback
from dynamicserialize.dstypes.com.raytheon.uf.common.site.requests import ValidateConfigRequest
from ufpy import ThriftClient
from ufpy import UsageArgumentParser
from awips import ThriftClient
from awips import UsageArgumentParser
def validate_args():

87
pythonPackages/ufpy/AlertVizHandler.py
View file

@ -1,87 +0,0 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Pure python logging mechanism for logging to AlertViz from
# pure python (ie not JEP). DO NOT USE IN PYTHON CALLED
# FROM JAVA.
#
# Sends local-delivery messages only, but needs to know the EDEX host name in
# order to forward the message. If the DEFAULT_HOST environment variable is
# not set correctly then this will not work.
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 08/18/10 njensen Initial Creation.
# Apr 16, 2020 8144 tgurney Reference AlertViz stomp port
# specified in NotificationMessage
# Aug 25, 2020 8220 tgurney Change local-delivery strategy
#
#
#
import logging
import os
from . import NotificationMessage
import socket
class AlertVizHandler(logging.Handler):
def __init__(self, host=None, port=None,
category='LOCAL', source='ANNOUNCER', level=logging.NOTSET, filters=None):
logging.Handler.__init__(self, level)
if host is None:
host = os.getenv('DEFAULT_HOST', 'localhost')
if port is None:
port = os.getenv('DEFAULT_PORT', 9581)
self._category = category
self._host = host
self._port = port
self._source = source
self._filters = filters
filters['WORKSTATION'] = socket.getfqdn()
def emit(self, record):
"Implements logging.Handler's interface. Record argument is a logging.LogRecord."
priority = None
if record.levelno >= 50:
priority = 'CRITICAL'
elif record.levelno >= 40:
priority = 'SIGNIFICANT'
elif record.levelno >= 30:
priority = 'PROBLEM'
elif record.levelno >= 20:
priority = 'EVENTA'
elif record.levelno >= 10:
priority = 'EVENTB'
else:
priority = 'VERBOSE'
msg = self.format(record)
notify = NotificationMessage.NotificationMessage(self._host, self._port, msg, priority, self._category, self._source,
filters=self._filters)
notify.send()

56
pythonPackages/ufpy/ConfigFileUtil.py
View file

@ -1,56 +0,0 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# A set of utility functions for dealing with configuration files.
#
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 09/27/10 dgilling Initial Creation.
#
#
#
def parseKeyValueFile(fileName):
propDict= dict()
try:
propFile= open(fileName, "rU")
for propLine in propFile:
propDef= propLine.strip()
if len(propDef) == 0:
continue
if propDef[0] in ( '#' ):
continue
punctuation= [ propDef.find(c) for c in ':= ' ] + [ len(propDef) ]
found= min( [ pos for pos in punctuation if pos != -1 ] )
name= propDef[:found].rstrip()
value= propDef[found:].lstrip(":= ").rstrip()
propDict[name]= value
propFile.close()
except:
pass
return propDict

107
pythonPackages/ufpy/DateTimeConverter.py
View file

@ -1,107 +0,0 @@
# #
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
# #
#
# Functions for converting between the various "Java" dynamic serialize types
# used by EDEX to the native python time datetime.
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 06/24/15 #4480 dgilling Initial Creation.
#
import datetime
import time
from dynamicserialize.dstypes.java.util import Date
from dynamicserialize.dstypes.java.sql import Timestamp
from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange
MAX_TIME = pow(2, 31) - 1
MICROS_IN_SECOND = 1000000
def convertToDateTime(timeArg):
"""
Converts the given object to a python datetime object. Supports native
python representations like datetime and struct_time, but also
the dynamicserialize types like Date and Timestamp. Raises TypeError
if no conversion can be performed.
Args:
timeArg: a python object representing a date and time. Supported
types include datetime, struct_time, float, int, long and the
dynamicserialize types Date and Timestamp.
Returns:
A datetime that represents the same date/time as the passed in object.
"""
if isinstance(timeArg, datetime.datetime):
return timeArg
elif isinstance(timeArg, time.struct_time):
return datetime.datetime(*timeArg[:6])
elif isinstance(timeArg, float):
# seconds as float, should be avoided due to floating point errors
totalSecs = int(timeArg)
micros = int((timeArg - totalSecs) * MICROS_IN_SECOND)
return _convertSecsAndMicros(totalSecs, micros)
elif isinstance(timeArg, int):
# seconds as integer
totalSecs = timeArg
return _convertSecsAndMicros(totalSecs, 0)
elif isinstance(timeArg, (Date, Timestamp)):
totalSecs = timeArg.getTime()
return _convertSecsAndMicros(totalSecs, 0)
else:
objType = str(type(timeArg))
raise TypeError("Cannot convert object of type " + objType + " to datetime.")
def _convertSecsAndMicros(seconds, micros):
if seconds < MAX_TIME:
rval = datetime.datetime.utcfromtimestamp(seconds)
else:
extraTime = datetime.timedelta(seconds=(seconds - MAX_TIME))
rval = datetime.datetime.utcfromtimestamp(MAX_TIME) + extraTime
return rval.replace(microsecond=micros)
def constructTimeRange(*args):
"""
Builds a python dynamicserialize TimeRange object from the given
arguments.
Args:
args*: must be a TimeRange or a pair of objects that can be
converted to a datetime via convertToDateTime().
Returns:
A TimeRange.
"""
if len(args) == 1 and isinstance(args[0], TimeRange):
return args[0]
if len(args) != 2:
raise TypeError("constructTimeRange takes exactly 2 arguments, " + str(len(args)) + " provided.")
startTime = convertToDateTime(args[0])
endTime = convertToDateTime(args[1])
return TimeRange(startTime, endTime)

136
pythonPackages/ufpy/NotificationMessage.py
View file

@ -1,136 +0,0 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
import socket
import sys
from . import ThriftClient
from dynamicserialize.dstypes.com.raytheon.uf.common.alertviz import AlertVizRequest
#
# Provides a capability of constructing notification messages and sending
# them to a STOMP data source.
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 09/30/08 chammack Initial Creation.
# 11/03/10 5849 cjeanbap Moved to ufpy package from
# com.raytheon.uf.tools.cli
# 01/07/11 5645 cjeanbap Added audio file to Status Message.
# 05/27/11 3050 cjeanbap Added if-statement to check Priority
# value
# 07/27/15 4654 skorolev Added filters
# 11/11/15 5120 rferrel Cannot serialize empty filters.
# 03/05/18 6899 dgilling Update to latest version of stomp.py API.
# 09/14/18 7464 dgilling Only serialize audioFile if provided.
# Apr 16, 2020 8144 tgurney Change AlertViz stomp port to
# calculated value based on uid
# May 15, 2020 8144 tgurney Remove local-delivery logic
# (no longer works as of 19.3.4)
class NotificationMessage:
priorityMap = {
0: 'CRITICAL',
1: 'SIGNIFICANT',
2: 'PROBLEM',
3: 'EVENTA',
4: 'EVENTB',
5: 'VERBOSE'}
def __init__(self, host='localhost', port=9581, message='', priority='PROBLEM', category="LOCAL", source="ANNOUNCER", audioFile=None, filters=None):
self.host = host
self.port = port
self.message = message
self.audioFile = audioFile
self.source = source
self.category = category
self.filters = filters
priorityInt = None
try:
priorityInt = int(priority)
except:
pass
if priorityInt is None:
#UFStatus.java contains mapping of Priority to Logging level mapping
if priority == 'CRITICAL' or priority == 'FATAL':
priorityInt = int(0)
elif priority == 'SIGNIFICANT' or priority == 'ERROR':
priorityInt = int(1)
elif priority == 'PROBLEM' or priority == 'WARN':
priorityInt = int(2)
elif priority == 'EVENTA' or priority == 'INFO':
priorityInt = int(3)
elif priority == 'EVENTB':
priorityInt = int(4)
elif priority == 'VERBOSE' or priority == 'DEBUG':
priorityInt = int(5)
if (priorityInt < 0 or priorityInt > 5):
print("Error occurred, supplied an invalid Priority value:", str(priorityInt))
print("Priority values are 0, 1, 2, 3, 4 and 5.")
sys.exit(1)
if priorityInt is not None:
self.priority = self.priorityMap[priorityInt]
else:
self.priority = priority
def send(self):
alertVizRequest = createRequest(self.message, self.priority, self.source, self.category, self.audioFile, self.filters)
thriftClient = ThriftClient.ThriftClient(self.host, self.port, "/services")
serverResponse = None
try:
serverResponse = thriftClient.sendRequest(alertVizRequest)
except Exception as ex:
print("Caught exception submitting AlertVizRequest:", str(ex))
if (serverResponse != "None"):
print("Error occurred submitting Notification Message to AlertViz receiver:", serverResponse)
sys.exit(1)
else:
print("Response:", str(serverResponse))
def createRequest(message, priority, source, category, audioFile, filters):
obj = AlertVizRequest()
obj.setMachine(socket.gethostname())
obj.setPriority(priority)
obj.setCategory(category)
obj.setSourceKey(source)
obj.setMessage(message)
if (audioFile is not None):
obj.setAudioFile(audioFile)
else:
obj.setAudioFile('\0')
obj.setFilters(filters)
return obj
if __name__ == '__main__':
main()

170
pythonPackages/ufpy/QpidSubscriber.py
View file

@ -1,170 +0,0 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Provides a Python-based interface for subscribing to qpid queues and topics.
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------- -------- ------------ --------------------------------------------
# Nov 17, 2010 njensen Initial Creation.
# Aug 15, 2013 2169 bkowal Optionally gzip decompress any data that is read.
# Aug 04, 2016 2416 tgurney Add queueStarted property
# Feb 16, 2017 6084 bsteffen Support ssl connections
# Sep 07, 2017 6175 tgurney Remove "decompressing" log message
# Jul 23, 2019 7724 mrichardson Upgrade Qpid to Qpid Proton
# Nov 04, 2019 7724 tgurney Fix topic creation
# Jun 24, 2020 8187 randerso Added qpid connection_id
#
from __future__ import print_function
import logging
import os
import os.path
import pwd
import socket
import zlib
from ssl import SSLContext, PROTOCOL_TLS
from proton import SSLDomain
from proton.handlers import MessagingHandler
from proton.reactor import Container
logging.basicConfig(level=logging.INFO, datefmt='%H:%M:%S',
format='[%(process)s] %(asctime)s %(levelname)s: %(message)s')
log = logging.getLogger('QpidSubscriber')
SSL_PASSWORD = 'password'
QPID_USERNAME = 'guest'
QPID_PASSWORD = 'guest'
class QpidSubscriber(MessagingHandler):
def __init__(self, host='127.0.0.1', port=5672, decompress=False, ssl=None, program="QpidSubscriber"):
super(QpidSubscriber, self).__init__(auto_accept=True)
#__init__ should only handle setting up properties;
# any connection and subscription actions should be handled
# by the reactor functions
self.queues = {}
self.scheme = 'amqp'
self.rest_scheme = 'https'
self.ssl_context = None
self.host = host
self.port = port
self.decompress = decompress
self.__queueStarted = False
self.__subscribed = False
pwuid = pwd.getpwuid(os.getuid())
if "QPID_SSL_CERT_DB" in os.environ:
certdbloc = os.environ["QPID_SSL_CERT_DB"]
else:
certdbloc = pwuid.pw_dir + "/.qpid/"
if "QPID_SSL_CERT_NAME" in os.environ:
certname = os.environ["QPID_SSL_CERT_NAME"]
else:
certname = QPID_USERNAME
certfile = os.path.join(certdbloc, certname + ".crt")
certkey = os.path.join(certdbloc, certname + ".key")
if ssl or (ssl is None and os.path.isfile(certfile) and os.path.isfile(certkey)):
self.scheme = "amqps"
self.rest_scheme = 'https'
self.ssl_context = SSLContext(PROTOCOL_TLS)
self.ssl_context.load_cert_chain(certfile, certkey)
self.cert_file = certfile
self.cert_key = certkey
self.url = '{}://{}:{}@{}:{}'.format(self.scheme, QPID_USERNAME, QPID_PASSWORD, self.host, self.port)
self.clientID = ":".join([
socket.gethostname(),
pwuid.pw_name,
program,
str(os.getpid()),
])
def topicSubscribe(self, topicName, callback):
self.topicName = topicName
self.callback = callback
Container(self).run()
def on_start(self, event):
'''
# if the queue is edex.alerts, set decompress to true always for now to
# maintain compatibility with existing python scripts.
'''
if self.topicName == 'edex.alerts':
self.decompress = True
self.container = event.container
queueName = 'amq.topic/' + self.topicName
self.ssl_domain = None
if self.scheme == "amqps" and self.cert_file and self.cert_key:
self.ssl_domain = SSLDomain(mode=SSLDomain.MODE_CLIENT)
self.ssl_domain.set_credentials(self.cert_file, self.cert_key, SSL_PASSWORD)
event.container.container_id = self.clientID
self.conn = event.container.connect(self.url, ssl_domain=self.ssl_domain)
self.receiver = event.container.create_receiver(self.conn, queueName)
self.__queueStarted = True
self.__subscribed = True
def on_message(self, event):
message = event.message
content = message.body
self.process_message(content)
if not self.__subscribed:
self.close()
def process_message(self, content):
if (self.decompress):
try:
# http://stackoverflow.com/questions/2423866/python-decompressing-gzip-chunk-by-chunk
d = zlib.decompressobj(16 + zlib.MAX_WBITS)
content = d.decompress(content)
except Exception:
# decompression failed, return the original content
pass
self.callback(content)
def close(self):
self.__queueStarted = False
self.unsubscribe()
try:
self.receiver.close()
self.conn.close()
except:
# already closed
pass
@property
def queueStarted(self):
return self.__queueStarted
@property
def subscribed(self):
return self.__subscribed
def unsubscribe(self):
self.__subscribed = False

102
pythonPackages/ufpy/ThriftClient.py
View file

@ -1,102 +0,0 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
import http.client
from dynamicserialize import DynamicSerializationManager
#
# Provides a Python-based interface for executing Thrift requests.
#
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 09/20/10 dgilling Initial Creation.
#
#
#
class ThriftClient:
# How to call this constructor:
# 1. Pass in all arguments separately (e.g.,
# ThriftClient.ThriftClient("localhost", 9581, "/services"))
# will return a Thrift client pointed at http://localhost:9581/services.
# 2. Pass in all arguments through the host string (e.g.,
# ThriftClient.ThriftClient("localhost:9581/services"))
# will return a Thrift client pointed at http://localhost:9581/services.
# 3. Pass in host/port arguments through the host string (e.g.,
# ThriftClient.ThriftClient("localhost:9581", "/services"))
# will return a Thrift client pointed at http://localhost:9581/services.
def __init__(self, host, port=9581, uri="/services"):
hostParts = host.split("/", 1)
if (len(hostParts) > 1):
hostString = hostParts[0]
self.__uri = "/" + hostParts[1]
self.__httpConn = http.client.HTTPConnection(hostString)
else:
if (port is None):
self.__httpConn = http.client.HTTPConnection(host)
else:
self.__httpConn = http.client.HTTPConnection(host, port)
self.__uri = uri
self.__dsm = DynamicSerializationManager.DynamicSerializationManager()
def sendRequest(self, request, uri="/thrift"):
message = self.__dsm.serializeObject(request)
self.__httpConn.connect()
self.__httpConn.request("POST", self.__uri + uri, message)
response = self.__httpConn.getresponse()
if (response.status != 200):
raise ThriftRequestException("Unable to post request to server")
rval = self.__dsm.deserializeBytes(response.read())
self.__httpConn.close()
# let's verify we have an instance of ServerErrorResponse
# IF we do, through an exception up to the caller along
# with the original Java stack trace
# ELSE: we have a valid response and pass it back
try:
forceError = rval.getException()
raise ThriftRequestException(forceError)
except AttributeError:
pass
return rval
class ThriftRequestException(Exception):
def __init__(self, value):
self.parameter = value
def __str__(self):
return repr(self.parameter)

107
pythonPackages/ufpy/TimeUtil.py
View file

@ -1,107 +0,0 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
# ----------------------------------------------------------------------------
# This software is in the public domain, furnished "as is", without technical
# support, and with no warranty, express or implied, as to its usefulness for
# any purpose.
#
# offsetTime.py
# Handles Displaced Real Time for various applications
#
# Author: hansen/romberg
# ----------------------------------------------------------------------------
import time
# Given the timeStr, return the offset (in seconds)
# from the current time.
# Also return the launchStr i.e. Programs launched from this
# offset application will use the launchStr as the -z argument.
# The offset will be positive for time in the future,
# negative for time in the past.
#
# May still want it to be normalized to the most recent midnight.
#
# NOTES about synchronizing:
# --With synchronizing on, the "current time" for all processes started
# within a given hour will be the same.
# This guarantees that GFE's have the same current time and ISC grid
# time stamps are syncrhonized and can be exchanged.
# Formatters launched from the GFE in this mode will be synchronized as
# well by setting the launchStr to use the time difference format
# (YYYYMMDD_HHMM,YYYYMMDD_HHMM).
# --This does not solve the problem in the general case.
# For example, if someone starts the GFE at 12:59 and someone
# else starts it at 1:01, they will have different offsets and
# current times.
# --With synchronizing off, when the process starts, the current time
# matches the drtTime in the command line. However, with synchronizing
# on, the current time will be offset by the fraction of the hour at
# which the process was started. Examples:
# Actual Starting time: 20040617_1230
# drtTime 20040616_0000
# Synchronizing off:
# GFE Spatial Editor at StartUp: 20040616_0000
# Synchronizing on:
# GFE Spatial Editor at StartUp: 20040616_0030
#
def determineDrtOffset(timeStr):
launchStr = timeStr
# Check for time difference
if timeStr.find(",") >=0:
times = timeStr.split(",")
t1 = makeTime(times[0])
t2 = makeTime(times[1])
#print "time offset", t1-t2, (t1-t2)/3600
return t1-t2, launchStr
# Check for synchronized mode
synch = 0
if timeStr[0] == "S":
timeStr = timeStr[1:]
synch = 1
drt_t = makeTime(timeStr)
#print "input", year, month, day, hour, minute
gm = time.gmtime()
cur_t = time.mktime(gm)
# Synchronize to most recent hour
# i.e. "truncate" cur_t to most recent hour.
#print "gmtime", gm
if synch:
cur_t = time.mktime((gm[0], gm[1], gm[2], gm[3], 0, 0, 0, 0, 0))
curStr = time.strftime('%Y%m%d_%H00\n', gm)
launchStr = timeStr + "," + curStr
#print "drt, cur", drt_t, cur_t
offset = drt_t - cur_t
#print "offset", offset, offset/3600, launchStr
return int(offset), launchStr
def makeTime(timeStr):
year = int(timeStr[0:4])
month = int(timeStr[4:6])
day = int(timeStr[6:8])
hour = int(timeStr[9:11])
minute = int(timeStr[11:13])
# Do not use daylight savings because gmtime is not in daylight
# savings time.
return time.mktime((year, month, day, hour, minute, 0, 0, 0, 0))

81
pythonPackages/ufpy/UsageArgumentParser.py
View file

@ -1,81 +0,0 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------- -------- --------- ---------------------------------------------
# Feb 13, 2017 6092 randerso Added StoreTimeAction
#
##
import argparse
import sys
import time
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import DatabaseID
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import ParmID
TIME_FORMAT = "%Y%m%d_%H%M"
class UsageArgumentParser(argparse.ArgumentParser):
"""
A subclass of ArgumentParser that overrides error() to print the
whole help text, rather than just the usage string.
"""
def error(self, message):
sys.stderr.write('%s: error: %s\n' % (self.prog, message))
self.print_help()
sys.exit(2)
## Custom actions for ArgumentParser objects ##
class StoreDatabaseIDAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
did = DatabaseID(values)
if did.isValid():
setattr(namespace, self.dest, did)
else:
parser.error("DatabaseID [" + values + "] not a valid identifier")
class AppendParmNameAndLevelAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
tx = ParmID.parmNameAndLevel(values)
comp = tx[0] + '_' + tx[1]
if (hasattr(namespace, self.dest)) and \
(getattr(namespace, self.dest) is not None):
currentValues = getattr(namespace, self.dest)
currentValues.append(comp)
setattr(namespace, self.dest, currentValues)
else:
setattr(namespace, self.dest, [comp])
class StoreTimeAction(argparse.Action):
"""
argparse.Action subclass to validate GFE formatted time strings
and parse them to time.struct_time
"""
def __call__(self, parser, namespace, values, option_string=None):
try:
timeStruct = time.strptime(values, TIME_FORMAT)
except:
parser.error(str(values) + " is not a valid time string of the format YYYYMMDD_hhmm")
setattr(namespace, self.dest, timeStruct)

38
pythonPackages/ufpy/UsageOptionParser.py
View file

@ -1,38 +0,0 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
import sys
from optparse import OptionParser
class UsageOptionParser(OptionParser):
"""
A subclass of OptionParser that prints that overrides error() to print the
whole help text, rather than just the usage string.
"""
def error(self, msg):
"""
Print the help text and exit.
"""
self.print_help(sys.stderr)
sys.stderr.write("\n")
sys.stderr.write(msg)
sys.stderr.write("\n")
sys.exit(2)

37
pythonPackages/ufpy/__init__.py
View file

@ -1,37 +0,0 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# __init__.py for ufpy package
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 09/21/10 dgilling Initial Creation.
#
#
#
__all__ = [
]

100
pythonPackages/ufpy/dataaccess/CombinedTimeQuery.py
View file

@ -1,100 +0,0 @@
# #
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
# #
#
# Method for performing a DAF time query where all parameter/level/location
# combinations must be available at the same time.
#
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 06/22/16 #5591 bsteffen Initial Creation.
#
from ufpy.dataaccess import DataAccessLayer
def getAvailableTimes(request, refTimeOnly=False):
return __getAvailableTimesForEachParameter(request, refTimeOnly)
def __getAvailableTimesForEachParameter(request, refTimeOnly=False):
parameters = request.getParameters()
if parameters:
times = None
for parameter in parameters:
specificRequest = __cloneRequest(request)
specificRequest.setParameters(parameter)
specificTimes = __getAvailableTimesForEachLevel(specificRequest, refTimeOnly)
if times is None:
times = specificTimes
else:
times.intersection_update(specificTimes)
if not times:
break
return times
else:
return __getAvailableTimesForEachLevel(request, refTimeOnly)
def __getAvailableTimesForEachLevel(request, refTimeOnly=False):
levels = request.getLevels()
if levels:
times = None
for level in levels:
specificRequest = __cloneRequest(request)
specificRequest.setLevels(level)
specificTimes = __getAvailableTimesForEachLocation(specificRequest, refTimeOnly)
if times is None:
times = specificTimes
else:
times.intersection_update(specificTimes)
if not times:
break
return times
else:
return __getAvailableTimesForEachLocation(request, refTimeOnly)
def __getAvailableTimesForEachLocation(request, refTimeOnly=False):
locations = request.getLocationNames()
if locations:
times = None
for location in locations:
specificRequest = __cloneRequest(request)
specificRequest.setLocationNames(location)
specificTimes = DataAccessLayer.getAvailableTimes(specificRequest, refTimeOnly)
if times is None:
times = set(specificTimes)
else:
times.intersection_update(specificTimes)
if not times:
break
return times
else:
return DataAccessLayer.getAvailableTimes(request, refTimeOnly)
def __cloneRequest(request):
return DataAccessLayer.newDataRequest(datatype = request.getDatatype(),
parameters = request.getParameters(),
levels = request.getLevels(),
locationNames = request.getLocationNames(),
envelope = request.getEnvelope(),
**request.getIdentifiers())

276
pythonPackages/ufpy/dataaccess/DataAccessLayer.py
View file

@ -1,276 +0,0 @@
# #
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
# #
#
# Published interface for ufpy.dataaccess package
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 12/10/12 njensen Initial Creation.
# Feb 14, 2013 1614 bsteffen refactor data access framework
# to use single request.
# 04/10/13 1871 mnash move getLatLonCoords to JGridData and add default args
# 05/29/13 2023 dgilling Hook up ThriftClientRouter.
# 03/03/14 2673 bsteffen Add ability to query only ref times.
# 07/22/14 3185 njensen Added optional/default args to newDataRequest
# 07/30/14 3185 njensen Renamed valid identifiers to optional
# Apr 26, 2015 4259 njensen Updated for new JEP API
# Apr 13, 2016 5379 tgurney Add getIdentifierValues()
# Jun 01, 2016 5587 tgurney Add new signatures for
# getRequiredIdentifiers() and
# getOptionalIdentifiers()
# Oct 18, 2016 5916 bsteffen Add setLazyLoadGridLatLon
#
#
import sys
import subprocess
import warnings
THRIFT_HOST = subprocess.check_output(
"source /awips2/fxa/bin/setup.env; echo $DEFAULT_HOST",
shell=True).decode().strip()
USING_NATIVE_THRIFT = False
if 'jep' in sys.modules:
# intentionally do not catch if this fails to import, we want it to
# be obvious that something is configured wrong when running from within
# Java instead of allowing false confidence and fallback behavior
import JepRouter
router = JepRouter
else:
from ufpy.dataaccess import ThriftClientRouter
router = ThriftClientRouter.ThriftClientRouter(THRIFT_HOST)
USING_NATIVE_THRIFT = True
def getAvailableTimes(request, refTimeOnly=False):
"""
Get the times of available data to the request.
Args:
request: the IDataRequest to get data for
refTimeOnly: optional, use True if only unique refTimes should be
returned (without a forecastHr)
Returns:
a list of DataTimes
"""
return router.getAvailableTimes(request, refTimeOnly)
def getGridData(request, times=[]):
"""
Gets the grid data that matches the request at the specified times. Each
combination of parameter, level, and dataTime will be returned as a
separate IGridData.
Args:
request: the IDataRequest to get data for
times: a list of DataTimes, a TimeRange, or None if the data is time
agnostic
Returns:
a list of IGridData
"""
return router.getGridData(request, times)
def getGeometryData(request, times=[]):
"""
Gets the geometry data that matches the request at the specified times.
Each combination of geometry, level, and dataTime will be returned as a
separate IGeometryData.
Args:
request: the IDataRequest to get data for
times: a list of DataTimes, a TimeRange, or None if the data is time
agnostic
Returns:
a list of IGeometryData
"""
return router.getGeometryData(request, times)
def getAvailableLocationNames(request):
"""
Gets the available location names that match the request without actually
requesting the data.
Args:
request: the request to find matching location names for
Returns:
a list of strings of available location names.
"""
return router.getAvailableLocationNames(request)
def getAvailableParameters(request):
"""
Gets the available parameters names that match the request without actually
requesting the data.
Args:
request: the request to find matching parameter names for
Returns:
a list of strings of available parameter names.
"""
return router.getAvailableParameters(request)
def getAvailableLevels(request):
"""
Gets the available levels that match the request without actually
requesting the data.
Args:
request: the request to find matching levels for
Returns:
a list of strings of available levels.
"""
return router.getAvailableLevels(request)
def getRequiredIdentifiers(request):
"""
Gets the required identifiers for this request. These identifiers
must be set on a request for the request of this datatype to succeed.
Args:
request: the request to find required identifiers for
Returns:
a list of strings of required identifiers
"""
if str(request) == request:
warnings.warn("Use getRequiredIdentifiers(IDataRequest) instead",
DeprecationWarning)
return router.getRequiredIdentifiers(request)
def getOptionalIdentifiers(request):
"""
Gets the optional identifiers for this request.
Args:
request: the request to find optional identifiers for
Returns:
a list of strings of optional identifiers
"""
if str(request) == request:
warnings.warn("Use getOptionalIdentifiers(IDataRequest) instead",
DeprecationWarning)
return router.getOptionalIdentifiers(request)
def getIdentifierValues(request, identifierKey):
"""
Gets the allowed values for a particular identifier on this datatype.
Args:
request: the request to find identifier values for
identifierKey: the identifier to find values for
Returns:
a list of strings of allowed values for the specified identifier
"""
return router.getIdentifierValues(request, identifierKey)
def newDataRequest(datatype=None, **kwargs):
""""
Creates a new instance of IDataRequest suitable for the runtime environment.
All args are optional and exist solely for convenience.
Args:
datatype: the datatype to create a request for
parameters: a list of parameters to set on the request
levels: a list of levels to set on the request
locationNames: a list of locationNames to set on the request
envelope: an envelope to limit the request
**kwargs: any leftover kwargs will be set as identifiers
Returns:
a new IDataRequest
"""
return router.newDataRequest(datatype, **kwargs)
def getSupportedDatatypes():
"""
Gets the datatypes that are supported by the framework
Returns:
a list of strings of supported datatypes
"""
return router.getSupportedDatatypes()
def changeEDEXHost(newHostName):
"""
Changes the EDEX host the Data Access Framework is communicating with. Only
works if using the native Python client implementation, otherwise, this
method will throw a TypeError.
Args:
newHostHame: the EDEX host to connect to
"""
if USING_NATIVE_THRIFT:
global THRIFT_HOST
THRIFT_HOST = newHostName
global router
router = ThriftClientRouter.ThriftClientRouter(THRIFT_HOST)
else:
raise TypeError("Cannot call changeEDEXHost when using JepRouter.")
def setLazyLoadGridLatLon(lazyLoadGridLatLon):
"""
Provide a hint to the Data Access Framework indicating whether to load the
lat/lon data for a grid immediately or wait until it is needed. This is
provided as a performance tuning hint and should not affect the way the
Data Access Framework is used. Depending on the internal implementation of
the Data Access Framework this hint might be ignored. Examples of when this
should be set to True are when the lat/lon information is not used or when
it is used only if certain conditions within the data are met. It could be
set to False if it is guaranteed that all lat/lon information is needed and
it would be better to get any performance overhead for generating the
lat/lon data out of the way during the initial request.
Args:
lazyLoadGridLatLon: Boolean value indicating whether to lazy load.
"""
try:
router.setLazyLoadGridLatLon(lazyLoadGridLatLon)
except AttributeError:
# The router is not required to support this capability.
pass

154
pythonPackages/ufpy/dataaccess/DataNotificationLayer.py
View file

@ -1,154 +0,0 @@
# #
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
# #
#
# Published interface for retrieving data updates via ufpy.dataaccess package
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------- -------- ------------ --------------------------------------------
# May 26, 2016 2416 rjpeter Initial Creation.
# Aug 01, 2016 2416 tgurney Finish implementation
# Nov 05, 2019 7884 tgurney Python 3 fixes
# Jun 24, 2020 8187 randerso Added program for qpid connection_id
#
#
"""
Interface for the DAF's data notification feature, which allows continuous
retrieval of new data as it is coming into the system.
There are two ways to access this feature:
1. The DataQueue module (ufpy.dataaccess.DataQueue) offers a collection that
automatically fills up with new data as it receives notifications. See that
module for more information.
2. Depending on the type of data you want, use either getGridDataUpdates() or
getGeometryDataUpdates() in this module. Either one will give you back an
object that will retrieve new data for you and will call a function you specify
each time new data is received.
Example code follows. This example prints temperature as observed from KOMA
each time a METAR is received from there.
from ufpy.dataaccess import DataAccessLayer as DAL
from ufpy.dataaccess import DataNotificationLayer as DNL
def process_obs(list_of_data):
for item in list_of_data:
print(item.getNumber('temperature'))
request = DAL.newDataRequest('obs')
request.setParameters('temperature')
request.setLocationNames('KOMA')
notifier = DNL.getGeometryDataUpdates(request)
notifier.subscribe(process_obs)
# process_obs will called with a list of data each time new data comes in
"""
import sys
import subprocess
from ufpy.dataaccess.PyGeometryNotification import PyGeometryNotification
from ufpy.dataaccess.PyGridNotification import PyGridNotification
THRIFT_HOST = subprocess.check_output(
"source /awips2/fxa/bin/setup.env; echo $DEFAULT_HOST",
shell=True).decode().strip()
USING_NATIVE_THRIFT = False
if 'jep' in sys.modules:
# intentionally do not catch if this fails to import, we want it to
# be obvious that something is configured wrong when running from within
# Java instead of allowing false confidence and fallback behavior
import JepRouter
router = JepRouter
else:
from ufpy.dataaccess import ThriftClientRouter
router = ThriftClientRouter.ThriftClientRouter(THRIFT_HOST)
USING_NATIVE_THRIFT = True
def _getJmsConnectionInfo(notifFilterResponse):
serverString = notifFilterResponse.getJmsConnectionInfo()
try:
host, port = serverString.split(':')
except Exception as e:
raise ValueError('Got bad JMS connection info from server: "' + serverString + '"') from e
return {'host': host, 'port': port}
def getGridDataUpdates(request):
"""
Get a notification object that receives updates to grid data.
Args:
request: the IDataRequest specifying the data you want to receive
Returns:
an update request object that you can listen for updates to by
calling its subscribe() method
"""
response = router.getNotificationFilter(request)
filter = response.getNotificationFilter()
jmsInfo = _getJmsConnectionInfo(response)
notifier = PyGridNotification(request, filter, requestHost=THRIFT_HOST, program="daf-gridDataUpdates", **jmsInfo)
return notifier
def getGeometryDataUpdates(request):
"""
Get a notification object that receives updates to geometry data.
Args:
request: the IDataRequest specifying the data you want to receive
Returns:
an update request object that you can listen for updates to by
calling its subscribe() method
"""
response = router.getNotificationFilter(request)
filter = response.getNotificationFilter()
jmsInfo = _getJmsConnectionInfo(response)
notifier = PyGeometryNotification(request, filter, requestHost=THRIFT_HOST, program="daf-geometryDataUpdates", **jmsInfo)
return notifier
def changeEDEXHost(newHostName):
"""
Changes the EDEX host the Data Access Framework is communicating with. Only
works if using the native Python client implementation, otherwise, this
method will throw a TypeError.
Args:
newHostHame: the EDEX host to connect to
"""
if USING_NATIVE_THRIFT:
global THRIFT_HOST
THRIFT_HOST = newHostName
global router
router = ThriftClientRouter.ThriftClientRouter(THRIFT_HOST)
else:
raise TypeError("Cannot call changeEDEXHost when using JepRouter.")

209
pythonPackages/ufpy/dataaccess/DataQueue.py
View file

@ -1,209 +0,0 @@
# #
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
# #
#
# Convenience class for using the DAF's notifications feature. This is a
# collection that, once connected to EDEX by calling start(), fills with
# data as notifications come in. Runs on a separate thread to allow
# non-blocking data retrieval.
#
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 07/29/16 2416 tgurney Initial creation
#
from ufpy.dataaccess import DataNotificationLayer as DNL
import time
from threading import Thread
from queue import Queue, Empty
"""Used to indicate a DataQueue that will produce geometry data."""
GEOMETRY = object()
"""Used to indicate a DataQueue that will produce grid data."""
GRID = object()
"""Default maximum queue size."""
_DEFAULT_MAXSIZE = 100
class Closed(Exception):
"""Raised when attempting to get data from a closed queue."""
pass
class DataQueue(object):
"""
Convenience class for using the DAF's notifications feature. This is a
collection that, once connected to EDEX by calling start(), fills with
data as notifications come in.
Example for getting obs data:
from DataQueue import DataQueue, GEOMETRY
request = DataAccessLayer.newDataRequest('obs')
request.setParameters('temperature')
request.setLocationNames('KOMA')
q = DataQueue(GEOMETRY, request)
q.start()
for item in q:
print(item.getNumber('temperature'))
"""
def __init__(self, dtype, request, maxsize=_DEFAULT_MAXSIZE):
"""
Create a new DataQueue.
Args:
dtype: Either GRID or GEOMETRY; must match the type of data
requested.
request: IDataRequest describing the data you want. It must at
least have datatype set. All data produced will satisfy the
constraints you specify.
maxsize: Maximum number of data objects the queue can hold at
one time. If the limit is reached, any data coming in after
that will not appear until one or more items are removed using
DataQueue.get().
"""
assert maxsize > 0
assert dtype in (GEOMETRY, GRID)
self._maxsize = maxsize
self._queue = Queue(maxsize=maxsize)
self._thread = None
if dtype is GEOMETRY:
self._notifier = DNL.getGeometryDataUpdates(request)
elif dtype is GRID:
self._notifier = DNL.getGridDataUpdates(request)
def start(self):
"""Start listening for notifications and requesting data."""
if self._thread is not None:
# Already started
return
kwargs = {'callback': self._data_received}
self._thread = Thread(target=self._notifier.subscribe, kwargs=kwargs)
self._thread.daemon = True
self._thread.start()
timer = 0
while not self._notifier.subscribed:
time.sleep(0.1)
timer += 1
if timer >= 100: # ten seconds
raise RuntimeError('timed out when attempting to subscribe')
def _data_received(self, data):
for d in data:
if not isinstance(d, list):
d = [d]
for item in d:
self._queue.put(item)
def get(self, block=True, timeout=None):
"""
Get and return the next available data object. By default, if there is
no data yet available, this method will not return until data becomes
available.
Args:
block: Specifies behavior when the queue is empty. If True, wait
until an item is available before returning (the default). If
False, return None immediately if the queue is empty.
timeout: If block is True, wait this many seconds, and return None
if data is not received in that time.
Returns:
IData
"""
if self.closed:
raise Closed
try:
return self._queue.get(block, timeout)
except Empty:
return None
def get_all(self):
"""
Get all data waiting for processing, in a single list. Always returns
immediately. Returns an empty list if no data has arrived yet.
Returns:
List of IData
"""
data = []
for _ in range(self._maxsize):
next_item = self.get(False)
if next_item is None:
break
data.append(next_item)
return data
def close(self):
"""Close the queue. May not be re-opened after closing."""
if not self.closed:
self._notifier.close()
self._thread.join()
def qsize(self):
"""Return number of items in the queue."""
return self._queue.qsize()
def empty(self):
"""Return True if the queue is empty."""
return self._queue.empty()
def full(self):
"""Return True if the queue is full."""
return self._queue.full()
@property
def closed(self):
"""True if the queue has been closed."""
return not self._notifier.subscribed
@property
def maxsize(self):
"""
Maximum number of data objects the queue can hold at one time.
If this limit is reached, any data coming in after that will not appear
until one or more items are removed using get().
"""
return self._maxsize
def __iter__(self):
if self._thread is not None:
while not self.closed:
yield self.get()
def __enter__(self):
self.start()
return self
def __exit__(self, *unused):
self.close()

57
pythonPackages/ufpy/dataaccess/PyData.py
View file

@ -1,57 +0,0 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Implements IData for use by native Python clients to the Data Access
# Framework.
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 06/03/13 dgilling Initial Creation.
#
#
from ufpy.dataaccess import IData
class PyData(IData):
def __init__(self, dataRecord):
self.__time = dataRecord.getTime()
self.__level = dataRecord.getLevel()
self.__locationName = dataRecord.getLocationName()
self.__attributes = dataRecord.getAttributes()
def getAttribute(self, key):
return self.__attributes[key]
def getAttributes(self):
return list(self.__attributes.keys())
def getDataTime(self):
return self.__time
def getLevel(self):
return self.__level
def getLocationName(self):
return self.__locationName

76
pythonPackages/ufpy/dataaccess/PyGeometryData.py
View file

@ -1,76 +0,0 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Implements IGeometryData for use by native Python clients to the Data Access
# Framework.
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 06/03/13 dgilling Initial Creation.
# 01/06/14 2537 bsteffen Share geometry WKT.
# 03/19/14 2882 dgilling Raise an exception when getNumber()
# is called for data that is not a
# numeric Type.
# 06/09/16 5574 mapeters Handle 'SHORT' type in getNumber().
#
#
from ufpy.dataaccess import IGeometryData
from ufpy.dataaccess import PyData
class PyGeometryData(IGeometryData, PyData.PyData):
def __init__(self, geoDataRecord, geometry):
PyData.PyData.__init__(self, geoDataRecord)
self.__geometry = geometry
self.__dataMap = {}
tempDataMap = geoDataRecord.getDataMap()
for key, value in tempDataMap.items():
self.__dataMap[key] = (value[0], value[1], value[2])
def getGeometry(self):
return self.__geometry
def getParameters(self):
return list(self.__dataMap.keys())
def getString(self, param):
value = self.__dataMap[param][0]
return str(value)
def getNumber(self, param):
value = self.__dataMap[param][0]
t = self.getType(param)
if t in ('INT', 'SHORT', 'LONG'):
return int(value)
elif t in ('DOUBLE', 'FLOAT'):
return float(value)
else:
raise TypeError("Data for parameter " + param + " is not a numeric type.")
def getUnit(self, param):
return self.__dataMap[param][2]
def getType(self, param):
return self.__dataMap[param][1]

55
pythonPackages/ufpy/dataaccess/PyGeometryNotification.py
View file

@ -1,55 +0,0 @@
# #
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
# #
#
# Notification object that produces geometry data
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 07/22/2016 2416 tgurney Initial creation
# 09/07/2017 6175 tgurney Override messageReceived
# 11/05/2019 7884 tgurney Add missing import
#
import dynamicserialize
import traceback
from ufpy.dataaccess.PyNotification import PyNotification
class PyGeometryNotification(PyNotification):
def messageReceived(self, msg):
dataUriMsg = dynamicserialize.deserialize(msg)
dataUris = dataUriMsg.getDataURIs()
dataTimes = set()
for dataUri in dataUris:
if self.notificationFilter.accept(dataUri):
dataTimes.add(self.getDataTime(dataUri))
if dataTimes:
try:
data = self.getData(self.request, list(dataTimes))
self.callback(data)
except Exception as e:
traceback.print_exc()
def getData(self, request, dataTimes):
return self.DAL.getGeometryData(request, dataTimes)

81
pythonPackages/ufpy/dataaccess/PyGridData.py
View file

@ -1,81 +0,0 @@
# #
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
# #
#
# Implements IGridData for use by native Python clients to the Data Access
# Framework.
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 06/03/13 #2023 dgilling Initial Creation.
# 10/13/16 #5916 bsteffen Correct grid shape, allow lat/lon
# 11/10/16 #5900 bsteffen Correct grid shape
# to be requested by a delegate
#
#
import numpy
import warnings
from ufpy.dataaccess import IGridData
from ufpy.dataaccess import PyData
NO_UNIT_CONVERT_WARNING = """
The ability to unit convert grid data is not currently available in this version of the Data Access Framework.
"""
class PyGridData(IGridData, PyData.PyData):
def __init__(self, gridDataRecord, nx, ny, latLonGrid = None, latLonDelegate = None):
PyData.PyData.__init__(self, gridDataRecord)
nx = nx
ny = ny
self.__parameter = gridDataRecord.getParameter()
self.__unit = gridDataRecord.getUnit()
self.__gridData = numpy.reshape(numpy.array(gridDataRecord.getGridData()), (ny, nx))
self.__latLonGrid = latLonGrid
self.__latLonDelegate = latLonDelegate
def getParameter(self):
return self.__parameter
def getUnit(self):
return self.__unit
def getRawData(self, unit=None):
# TODO: Find a proper python library that deals will with numpy and
# javax.measure style unit strings and hook it in to this method to
# allow end-users to perform unit conversion for grid data.
if unit is not None:
warnings.warn(NO_UNIT_CONVERT_WARNING, stacklevel=2)
return self.__gridData
def getLatLonCoords(self):
if self.__latLonGrid is not None:
return self.__latLonGrid
elif self.__latLonDelegate is not None:
return self.__latLonDelegate()
return self.__latLonGrid

60
pythonPackages/ufpy/dataaccess/PyGridNotification.py
View file

@ -1,60 +0,0 @@
# #
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
# #
#
# Notification object that produces grid data
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 06/03/2016 2416 rjpeter Initial Creation.
# 09/06/2017 6175 tgurney Override messageReceived
# 11/05/2019 7884 tgurney Add missing import
#
import dynamicserialize
import traceback
from ufpy.dataaccess.PyNotification import PyNotification
class PyGridNotification(PyNotification):
def messageReceived(self, msg):
dataUriMsg = dynamicserialize.deserialize(msg)
dataUris = dataUriMsg.getDataURIs()
for dataUri in dataUris:
if not self.notificationFilter.accept(dataUri):
continue
try:
# This improves performance over requesting by datatime since it requests only the
# parameter that the notification was received for (instead of this and all previous
# parameters for the same forecast hour)
# TODO: This utterly fails for derived requests
newReq = self.DAL.newDataRequest(self.request.getDatatype())
newReq.addIdentifier("dataURI", dataUri)
newReq.setParameters(self.request.getParameters())
data = self.getData(newReq, [])
self.callback(data)
except Exception as e:
traceback.print_exc()
def getData(self, request, dataTimes):
return self.DAL.getGridData(request, dataTimes)

110
pythonPackages/ufpy/dataaccess/PyNotification.py
View file

@ -1,110 +0,0 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Implements IData for use by native Python clients to the Data Access
# Framework.
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------- -------- ------------ --------------------------------------------
# Jun 22, 2016 2416 rjpeter Initial creation
# Jul 22, 2016 2416 tgurney Finish implementation
# Sep 07, 2017 6175 tgurney Override messageReceived in subclasses
# Nov 05, 2019 7884 tgurney Fix in subscribed()
# Jun 24, 2020 8187 randerso Added program for qpid connection_id
#
import abc
from ufpy.dataaccess import DataAccessLayer
from ufpy.dataaccess import INotificationSubscriber
from ufpy.QpidSubscriber import QpidSubscriber
from dynamicserialize.dstypes.com.raytheon.uf.common.time import DataTime
class PyNotification(INotificationSubscriber, metaclass=abc.ABCMeta):
"""
Receives notifications for new data and retrieves the data that meets
specified filtering criteria.
"""
def __init__(self, request, filter, host='localhost', port=5672, requestHost='localhost', program="PyNotification"):
self.DAL = DataAccessLayer
self.DAL.changeEDEXHost(requestHost)
self.request = request
self.notificationFilter = filter
self.host = host
self.port = port
self.program=program
self.__topicName = "edex.alerts"
self.callback = None
def subscribe(self, callback):
"""
Start listening for notifications.
Args:
callback: Function to call with a list of received data objects.
Will be called once for each request made for data.
"""
assert hasattr(callback, '__call__'), 'callback arg must be callable'
self.callback = callback
self.qs = QpidSubscriber(host=self.host, port=self.port, decompress=True, program=self.program)
self.qs.topicSubscribe(self.__topicName, self.messageReceived)
# Blocks here
def close(self):
self.qs.close()
def getDataTime(self, dataURI):
dataTimeStr = dataURI.split('/')[2]
return DataTime(dataTimeStr)
@abc.abstractmethod
def messageReceived(self, msg):
"""Called when a message is received from QpidSubscriber.
This method must call self.callback once for each request made for data
"""
pass
@abc.abstractmethod
def getData(self, request, dataTimes):
"""
Retrieve and return data
Args:
request: IDataRequest to send to the server
dataTimes: list of data times
Returns:
list of IData
"""
pass
@property
def subscribed(self):
"""True if currently subscribed to notifications."""
try:
return self.qs.queueStarted
except AttributeError:
return False

283
pythonPackages/ufpy/dataaccess/SoundingsSupport.py
View file

@ -1,283 +0,0 @@
# #
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
# #
#
# Classes for retrieving soundings based on gridded data from the Data Access
# Framework
#
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 06/24/15 #4480 dgilling Initial Creation.
#
from collections import defaultdict
from shapely.geometry import Point
from ufpy import DateTimeConverter
from ufpy.dataaccess import DataAccessLayer
from dynamicserialize.dstypes.com.raytheon.uf.common.time import DataTime
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.level import Level
def getSounding(modelName, weatherElements, levels, samplePoint, refTime=None, timeRange=None):
""""
Performs a series of Data Access Framework requests to retrieve a sounding object
based on the specified request parameters.
Args:
modelName: the grid model datasetid to use as the basis of the sounding.
weatherElements: a list of parameters to return in the sounding.
levels: a list of levels to sample the given weather elements at
samplePoint: a lat/lon pair to perform the sampling of data at.
refTime: (optional) the grid model reference time to use for the sounding.
If not specified, the latest ref time in the system will be used.
timeRange: (optional) a TimeRange to specify which forecast hours to use.
If not specified, will default to all forecast hours.
Returns:
A _SoundingCube instance, which acts a 3-tiered dictionary, keyed
by DataTime, then by level and finally by weather element. If no
data is available for the given request parameters, None is returned.
"""
(locationNames, parameters, levels, envelope, refTime, timeRange) = \
__sanitizeInputs(modelName, weatherElements, levels, samplePoint, refTime, timeRange)
requestArgs = { 'datatype' : 'grid',
'locationNames' : locationNames,
'parameters' : parameters,
'levels' : levels,
'envelope' : envelope,
}
req = DataAccessLayer.newDataRequest(**requestArgs)
forecastHours = __determineForecastHours(req, refTime, timeRange)
if not forecastHours:
return None
response = DataAccessLayer.getGeometryData(req, forecastHours)
soundingObject = _SoundingCube(response)
return soundingObject
def setEDEXHost(host):
"""
Changes the EDEX host the Data Access Framework is communicating with.
Args:
host: the EDEX host to connect to
"""
if host:
DataAccessLayer.changeEDEXHost(str(host))
def __sanitizeInputs(modelName, weatherElements, levels, samplePoint, refTime, timeRange):
locationNames = [str(modelName)]
parameters = __buildStringList(weatherElements)
levels = __buildStringList(levels)
envelope = Point(samplePoint)
if refTime is not None:
refTime = DataTime(refTime=DateTimeConverter.convertToDateTime(refTime))
if timeRange is not None:
timeRange = DateTimeConverter.constructTimeRange(*timeRange)
return (locationNames, parameters, levels, envelope, refTime, timeRange)
def __determineForecastHours(request, refTime, timeRange):
dataTimes = DataAccessLayer.getAvailableTimes(request, False)
timesGen = [(DataTime(refTime=dataTime.getRefTime()), dataTime) for dataTime in dataTimes]
dataTimesMap = defaultdict(list)
for baseTime, dataTime in timesGen:
dataTimesMap[baseTime].append(dataTime)
if refTime is None:
refTime = max(dataTimesMap.keys())
forecastHours = dataTimesMap[refTime]
if timeRange is None:
return forecastHours
else:
return [forecastHour for forecastHour in forecastHours if timeRange.contains(forecastHour.getValidPeriod())]
def __buildStringList(param):
if __notStringIter(param):
return [str(item) for item in param]
else:
return [str(param)]
def __notStringIter(iterable):
if not isinstance(iterable, str):
try:
iter(iterable)
return True
except TypeError:
return False
class _SoundingCube(object):
"""
The top-level sounding object returned when calling SoundingsSupport.getSounding.
This object acts as a 3-tiered dict which is keyed by time then level
then parameter name. Calling times() will return all valid keys into this
object.
"""
def __init__(self, geometryDataObjects):
self._dataDict = {}
self._sortedTimes = []
if geometryDataObjects:
for geometryData in geometryDataObjects:
dataTime = geometryData.getDataTime()
level = geometryData.getLevel()
for parameter in geometryData.getParameters():
self.__addItem(parameter, dataTime, level, geometryData.getNumber(parameter))
def __addItem(self, parameter, dataTime, level, value):
timeLayer = self._dataDict.get(dataTime, _SoundingTimeLayer(dataTime))
self._dataDict[dataTime] = timeLayer
timeLayer._addItem(parameter, level, value)
if dataTime not in self._sortedTimes:
self._sortedTimes.append(dataTime)
self._sortedTimes.sort()
def __getitem__(self, key):
return self._dataDict[key]
def __len__(self):
return len(self._dataDict)
def times(self):
"""
Returns the valid times for this sounding.
Returns:
A list containing the valid DataTimes for this sounding in order.
"""
return self._sortedTimes
class _SoundingTimeLayer(object):
"""
The second-level sounding object returned when calling SoundingsSupport.getSounding.
This object acts as a 2-tiered dict which is keyed by level then parameter
name. Calling levels() will return all valid keys into this
object. Calling time() will return the DataTime for this particular layer.
"""
def __init__(self, dataTime):
self._dataTime = dataTime
self._dataDict = {}
def _addItem(self, parameter, level, value):
asString = str(level)
levelLayer = self._dataDict.get(asString, _SoundingTimeAndLevelLayer(self._dataTime, asString))
levelLayer._addItem(parameter, value)
self._dataDict[asString] = levelLayer
def __getitem__(self, key):
asString = str(key)
if asString in self._dataDict:
return self._dataDict[asString]
else:
raise KeyError("Level " + str(key) + " is not a valid level for this sounding.")
def __len__(self):
return len(self._dataDict)
def time(self):
"""
Returns the DataTime for this sounding cube layer.
Returns:
The DataTime for this sounding layer.
"""
return self._dataTime
def levels(self):
"""
Returns the valid levels for this sounding.
Returns:
A list containing the valid levels for this sounding in order of
closest to surface to highest from surface.
"""
sortedLevels = [Level(level) for level in list(self._dataDict.keys())]
sortedLevels.sort()
return [str(level) for level in sortedLevels]
class _SoundingTimeAndLevelLayer(object):
"""
The bottom-level sounding object returned when calling SoundingsSupport.getSounding.
This object acts as a dict which is keyed by parameter name. Calling
parameters() will return all valid keys into this object. Calling time()
will return the DataTime for this particular layer. Calling level() will
return the level for this layer.
"""
def __init__(self, time, level):
self._time = time
self._level = level
self._parameters = {}
def _addItem(self, parameter, value):
self._parameters[parameter] = value
def __getitem__(self, key):
return self._parameters[key]
def __len__(self):
return len(self._parameters)
def level(self):
"""
Returns the level for this sounding cube layer.
Returns:
The level for this sounding layer.
"""
return self._level
def parameters(self):
"""
Returns the valid parameters for this sounding.
Returns:
A list containing the valid parameter names.
"""
return list(self._parameters.keys())
def time(self):
"""
Returns the DataTime for this sounding cube layer.
Returns:
The DataTime for this sounding layer.
"""
return self._time

247
pythonPackages/ufpy/dataaccess/ThriftClientRouter.py
View file

@ -1,247 +0,0 @@
# #
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
# #
#
# Routes requests to the Data Access Framework through Python Thrift.
#
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 05/21/13 2023 dgilling Initial Creation.
# 01/06/14 2537 bsteffen Share geometry WKT.
# 03/03/14 2673 bsteffen Add ability to query only ref times.
# 07/22/14 3185 njensen Added optional/default args to newDataRequest
# 07/23/14 3185 njensen Added new methods
# 07/30/14 3185 njensen Renamed valid identifiers to optional
# 06/30/15 4569 nabowle Use hex WKB for geometries.
# 04/13/15 5379 tgurney Add getIdentifierValues()
# 06/01/16 5587 tgurney Add new signatures for
# getRequiredIdentifiers() and
# getOptionalIdentifiers()
# 08/01/16 2416 tgurney Add getNotificationFilter()
# 10/13/16 5916 bsteffen Correct grid shape, allow lazy grid lat/lon
# 10/26/16 5919 njensen Speed up geometry creation in getGeometryData()
#
import numpy
import shapely.wkb
from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.impl import DefaultDataRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableLocationNamesRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableTimesRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetGeometryDataRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetGridDataRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetGridLatLonRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableParametersRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableLevelsRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetRequiredIdentifiersRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetOptionalIdentifiersRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetIdentifierValuesRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetSupportedDatatypesRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetNotificationFilterRequest
from ufpy import ThriftClient
from ufpy.dataaccess import PyGeometryData
from ufpy.dataaccess import PyGridData
class LazyGridLatLon(object):
def __init__(self, client, nx, ny, envelope, crsWkt):
self._latLonGrid = None
self._client = client
self._request = GetGridLatLonRequest()
self._request.setNx(nx)
self._request.setNy(ny)
self._request.setEnvelope(envelope)
self._request.setCrsWkt(crsWkt)
def __call__(self):
# Its important that the data is cached internally so that if multiple
# GridData are sharing the same delegate then they can also share a
# single request for the LatLon information.
if self._latLonGrid is None:
response = self._client.sendRequest(self._request)
nx = response.getNx()
ny = response.getNy()
latData = numpy.reshape(numpy.array(response.getLats()), (ny, nx))
lonData = numpy.reshape(numpy.array(response.getLons()), (ny, nx))
self._latLonGrid = (lonData, latData)
return self._latLonGrid
class ThriftClientRouter(object):
def __init__(self, host='localhost'):
self._client = ThriftClient.ThriftClient(host)
self._lazyLoadGridLatLon = False
def setLazyLoadGridLatLon(self, lazyLoadGridLatLon):
self._lazyLoadGridLatLon = lazyLoadGridLatLon
def getAvailableTimes(self, request, refTimeOnly):
timesRequest = GetAvailableTimesRequest()
timesRequest.setRequestParameters(request)
timesRequest.setRefTimeOnly(refTimeOnly)
response = self._client.sendRequest(timesRequest)
return response
def getGridData(self, request, times):
gridDataRequest = GetGridDataRequest()
gridDataRequest.setIncludeLatLonData(not self._lazyLoadGridLatLon)
gridDataRequest.setRequestParameters(request)
# if we have an iterable times instance, then the user must have asked
# for grid data with the List of DataTime objects
# else, we assume it was a single TimeRange that was meant for the
# request
try:
iter(times)
gridDataRequest.setRequestedTimes(times)
except TypeError:
gridDataRequest.setRequestedPeriod(times)
response = self._client.sendRequest(gridDataRequest)
locSpecificData = {}
locNames = list(response.getSiteNxValues().keys())
for location in locNames:
nx = response.getSiteNxValues()[location]
ny = response.getSiteNyValues()[location]
if self._lazyLoadGridLatLon:
envelope = response.getSiteEnvelopes()[location]
crsWkt = response.getSiteCrsWkt()[location]
delegate = LazyGridLatLon(
self._client, nx, ny, envelope, crsWkt)
locSpecificData[location] = (nx, ny, delegate)
else:
latData = numpy.reshape(numpy.array(
response.getSiteLatGrids()[location]), (ny, nx))
lonData = numpy.reshape(numpy.array(
response.getSiteLonGrids()[location]), (ny, nx))
locSpecificData[location] = (nx, ny, (lonData, latData))
retVal = []
for gridDataRecord in response.getGridData():
locationName = gridDataRecord.getLocationName()
locData = locSpecificData[locationName]
if self._lazyLoadGridLatLon:
retVal.append(PyGridData.PyGridData(gridDataRecord, locData[
0], locData[1], latLonDelegate=locData[2]))
else:
retVal.append(PyGridData.PyGridData(
gridDataRecord, locData[0], locData[1], locData[2]))
return retVal
def getGeometryData(self, request, times):
geoDataRequest = GetGeometryDataRequest()
geoDataRequest.setRequestParameters(request)
# if we have an iterable times instance, then the user must have asked
# for geometry data with the List of DataTime objects
# else, we assume it was a single TimeRange that was meant for the
# request
try:
iter(times)
geoDataRequest.setRequestedTimes(times)
except TypeError:
geoDataRequest.setRequestedPeriod(times)
response = self._client.sendRequest(geoDataRequest)
geometries = []
for wkb in response.getGeometryWKBs():
# the wkb is a numpy.ndarray of dtype int8
# convert the bytearray to a byte string and load it
geometries.append(shapely.wkb.loads(wkb.tostring()))
retVal = []
for geoDataRecord in response.getGeoData():
geom = geometries[geoDataRecord.getGeometryWKBindex()]
retVal.append(PyGeometryData.PyGeometryData(geoDataRecord, geom))
return retVal
def getAvailableLocationNames(self, request):
locNamesRequest = GetAvailableLocationNamesRequest()
locNamesRequest.setRequestParameters(request)
response = self._client.sendRequest(locNamesRequest)
return response
def getAvailableParameters(self, request):
paramReq = GetAvailableParametersRequest()
paramReq.setRequestParameters(request)
response = self._client.sendRequest(paramReq)
return response
def getAvailableLevels(self, request):
levelReq = GetAvailableLevelsRequest()
levelReq.setRequestParameters(request)
response = self._client.sendRequest(levelReq)
return response
def getRequiredIdentifiers(self, request):
if str(request) == request:
# Handle old version getRequiredIdentifiers(str)
request = self.newDataRequest(request)
idReq = GetRequiredIdentifiersRequest()
idReq.setRequest(request)
response = self._client.sendRequest(idReq)
return response
def getOptionalIdentifiers(self, request):
if str(request) == request:
# Handle old version getOptionalIdentifiers(str)
request = self.newDataRequest(request)
idReq = GetOptionalIdentifiersRequest()
idReq.setRequest(request)
response = self._client.sendRequest(idReq)
return response
def getIdentifierValues(self, request, identifierKey):
idValReq = GetIdentifierValuesRequest()
idValReq.setIdentifierKey(identifierKey)
idValReq.setRequestParameters(request)
response = self._client.sendRequest(idValReq)
return response
def newDataRequest(self, datatype, parameters=[], levels=[], locationNames=[], envelope=None, **kwargs):
req = DefaultDataRequest()
if datatype:
req.setDatatype(datatype)
if parameters:
req.setParameters(*parameters)
if levels:
req.setLevels(*levels)
if locationNames:
req.setLocationNames(*locationNames)
if envelope:
req.setEnvelope(envelope)
if kwargs:
# any args leftover are assumed to be identifiers
req.identifiers = kwargs
return req
def getSupportedDatatypes(self):
response = self._client.sendRequest(GetSupportedDatatypesRequest())
return response
def getNotificationFilter(self, request):
notifReq = GetNotificationFilterRequest()
notifReq.setRequestParameters(request)
response = self._client.sendRequest(notifReq)
return response

385
pythonPackages/ufpy/dataaccess/__init__.py
View file

@ -1,385 +0,0 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# __init__.py for ufpy.dataaccess package
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 12/10/12 njensen Initial Creation.
# Feb 14, 2013 1614 bsteffen refactor data access framework
# to use single request.
# Apr 09, 2013 1871 njensen Add doc strings
# Jun 03, 2013 2023 dgilling Add getAttributes to IData, add
# getLatLonGrids() to IGridData.
# Aug 01, 2016 2416 tgurney Add INotificationSubscriber
# and INotificationFilter
#
#
__all__ = [
]
import abc
class IDataRequest(object, metaclass=abc.ABCMeta):
"""
An IDataRequest to be submitted to the DataAccessLayer to retrieve data.
"""
@abc.abstractmethod
def setDatatype(self, datatype):
"""
Sets the datatype of the request.
Args:
datatype: A string of the datatype, such as "grid", "radar", "gfe", "obs"
"""
return
@abc.abstractmethod
def addIdentifier(self, key, value):
"""
Adds an identifier to the request. Identifiers are specific to the
datatype being requested.
Args:
key: the string key of the identifier
value: the value of the identifier
"""
return
@abc.abstractmethod
def setParameters(self, params):
"""
Sets the parameters of data to request.
Args:
params: a list of strings of parameters to request
"""
return
@abc.abstractmethod
def setLevels(self, levels):
"""
Sets the levels of data to request. Not all datatypes support levels.
Args:
levels: a list of strings of level abbreviations to request
"""
return
@abc.abstractmethod
def setEnvelope(self, env):
"""
Sets the envelope of the request. If supported by the datatype factory,
the data returned for the request will be constrained to only the data
within the envelope.
Args:
env: a shapely geometry
"""
return
@abc.abstractmethod
def setLocationNames(self, locationNames):
"""
Sets the location names of the request.
Args:
locationNames: a list of strings of location names to request
"""
return
@abc.abstractmethod
def getDatatype(self):
"""
Gets the datatype of the request
Returns:
the datatype set on the request
"""
return
@abc.abstractmethod
def getIdentifiers(self):
"""
Gets the identifiers on the request
Returns:
a dictionary of the identifiers
"""
return
@abc.abstractmethod
def getLevels(self):
"""
Gets the levels on the request
Returns:
a list of strings of the levels
"""
return
@abc.abstractmethod
def getLocationNames(self):
"""
Gets the location names on the request
Returns:
a list of strings of the location names
"""
return
@abc.abstractmethod
def getEnvelope(self):
"""
Gets the envelope on the request
Returns:
a rectangular shapely geometry
"""
return
class IData(object, metaclass=abc.ABCMeta):
"""
An IData representing data returned from the DataAccessLayer.
"""
@abc.abstractmethod
def getAttribute(self, key):
"""
Gets an attribute of the data.
Args:
key: the key of the attribute
Returns:
the value of the attribute
"""
return
@abc.abstractmethod
def getAttributes(self):
"""
Gets the valid attributes for the data.
Returns:
a list of strings of the attribute names
"""
return
@abc.abstractmethod
def getDataTime(self):
"""
Gets the data time of the data.
Returns:
the data time of the data, or None if no time is associated
"""
return
@abc.abstractmethod
def getLevel(self):
"""
Gets the level of the data.
Returns:
the level of the data, or None if no level is associated
"""
return
@abc.abstractmethod
def getLocationName(self, param):
"""
Gets the location name of the data.
Returns:
the location name of the data, or None if no location name is
associated
"""
return
class IGridData(IData):
"""
An IData representing grid data that is returned by the DataAccessLayer.
"""
@abc.abstractmethod
def getParameter(self):
"""
Gets the parameter of the data.
Returns:
the parameter of the data
"""
return
@abc.abstractmethod
def getUnit(self):
"""
Gets the unit of the data.
Returns:
the string abbreviation of the unit, or None if no unit is associated
"""
return
@abc.abstractmethod
def getRawData(self):
"""
Gets the grid data as a numpy array.
Returns:
a numpy array of the data
"""
return
@abc.abstractmethod
def getLatLonCoords(self):
"""
Gets the lat/lon coordinates of the grid data.
Returns:
a tuple where the first element is a numpy array of lons, and the
second element is a numpy array of lats
"""
return
class IGeometryData(IData):
"""
An IData representing geometry data that is returned by the DataAccessLayer.
"""
@abc.abstractmethod
def getGeometry(self):
"""
Gets the geometry of the data.
Returns:
a shapely geometry
"""
return
@abc.abstractmethod
def getParameters(self):
"""Gets the parameters of the data.
Returns:
a list of strings of the parameter names
"""
return
@abc.abstractmethod
def getString(self, param):
"""
Gets the string value of the specified param.
Args:
param: the string name of the param
Returns:
the string value of the param
"""
return
@abc.abstractmethod
def getNumber(self, param):
"""
Gets the number value of the specified param.
Args:
param: the string name of the param
Returns:
the number value of the param
"""
return
@abc.abstractmethod
def getUnit(self, param):
"""
Gets the unit of the specified param.
Args:
param: the string name of the param
Returns:
the string abbreviation of the unit of the param
"""
return
@abc.abstractmethod
def getType(self, param):
"""
Gets the type of the param.
Args:
param: the string name of the param
Returns:
a string of the type of the parameter, such as
"STRING", "INT", "LONG", "FLOAT", or "DOUBLE"
"""
return
class INotificationSubscriber(object, metaclass=abc.ABCMeta):
"""
An INotificationSubscriber representing a notification filter returned from
the DataNotificationLayer.
"""
@abc.abstractmethod
def subscribe(self, callback):
"""
Subscribes to the requested data. Method will not return until close is
called in a separate thread.
Args:
callback: the method to call with the IGridData/IGeometryData
"""
pass
@abc.abstractmethod
def close(self):
"""Closes the notification subscriber"""
pass
class INotificationFilter(object, metaclass=abc.ABCMeta):
"""
Represents data required to filter a set of URIs and
return a corresponding list of IDataRequest to retrieve data for.
"""
@abc.abstractmethod
def accept(dataUri):
pass

173
pythonPackages/ufpy/gfe/IFPClient.py
View file

@ -1,173 +0,0 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
from ufpy import ThriftClient
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import DatabaseID
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import ParmID
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import CommitGridsRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import GetGridInventoryRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import GetParmListRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import GetSelectTimeRangeRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.server.request import CommitGridRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.message import WsId
from dynamicserialize.dstypes.com.raytheon.uf.common.site.requests import GetActiveSitesRequest
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.server.message import ServerResponse
#
# Provides a Python-based interface for executing GFE requests.
#
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 07/26/12 dgilling Initial Creation.
#
#
#
class IFPClient(object):
def __init__(self, host, port, user, site=None, progName=None):
self.__thrift = ThriftClient.ThriftClient(host, port)
self.__wsId = WsId(userName=user, progName=progName)
# retrieve default site
if site is None:
sr = self.getSiteID()
if len(sr.getPayload()) > 0:
site = sr.getPayload()[0]
self.__siteId = site
def commitGrid(self, request):
if type(request) is CommitGridRequest:
return self.__commitGrid([request])
elif self.__isHomogenousIterable(request, CommitGridRequest):
return self.__commitGrid([cgr for cgr in request])
raise TypeError("Invalid type: " + str(type(request)) + " specified to commitGrid(). Only accepts CommitGridRequest or lists of CommitGridRequest.")
def __commitGrid(self, requests):
ssr = ServerResponse()
request = CommitGridsRequest()
request.setCommits(requests)
sr = self.__makeRequest(request)
ssr.setMessages(sr.getMessages())
return ssr
def getParmList(self, id):
argType = type(id)
if argType is DatabaseID:
return self.__getParmList([id])
elif self.__isHomogenousIterable(id, DatabaseID):
return self.__getParmList([dbid for dbid in id])
raise TypeError("Invalid type: " + str(argType) + " specified to getParmList(). Only accepts DatabaseID or lists of DatabaseID.")
def __getParmList(self, ids):
ssr = ServerResponse()
request = GetParmListRequest()
request.setDbIds(ids)
sr = self.__makeRequest(request)
ssr.setMessages(sr.getMessages())
list = sr.getPayload() if sr.getPayload() is not None else []
ssr.setPayload(list)
return ssr
def __isHomogenousIterable(self, iterable, classType):
try:
iterator = iter(iterable)
for item in iterator:
if not isinstance(item, classType):
return False
except TypeError:
return False
return True
def getGridInventory(self, parmID):
if type(parmID) is ParmID:
sr = self.__getGridInventory([parmID])
list = []
try:
list = sr.getPayload()[parmID]
except KeyError:
# no-op, we've already default the TimeRange list to empty
pass
sr.setPayload(list)
return sr
elif self.__isHomogenousIterable(parmID, ParmID):
return self.__getGridInventory([id for id in parmID])
raise TypeError("Invalid type: " + str(type(parmID)) + " specified to getGridInventory(). Only accepts ParmID or lists of ParmID.")
def __getGridInventory(self, parmIDs):
ssr = ServerResponse()
request = GetGridInventoryRequest()
request.setParmIds(parmIDs)
sr = self.__makeRequest(request)
ssr.setMessages(sr.getMessages())
trs = sr.getPayload() if sr.getPayload() is not None else {}
ssr.setPayload(trs)
return ssr
def getSelectTR(self, name):
request = GetSelectTimeRangeRequest()
request.setName(name)
sr = self.__makeRequest(request)
ssr = ServerResponse()
ssr.setMessages(sr.getMessages())
ssr.setPayload(sr.getPayload())
return ssr
def getSiteID(self):
ssr = ServerResponse()
request = GetActiveSitesRequest()
sr = self.__makeRequest(request)
ssr.setMessages(sr.getMessages())
ids = sr.getPayload() if sr.getPayload() is not None else []
sr.setPayload(ids)
return sr
def __makeRequest(self, request):
try:
request.setSiteID(self.__siteId)
except AttributeError:
pass
try:
request.setWorkstationID(self.__wsId)
except AttributeError:
pass
sr = ServerResponse()
response = None
try:
response = self.__thrift.sendRequest(request)
except ThriftClient.ThriftRequestException as e:
sr.setMessages([str(e)])
try:
sr.setPayload(response.getPayload())
except AttributeError:
sr.setPayload(response)
try:
sr.setMessages(response.getMessages())
except AttributeError:
# not a server response, nothing else to do
pass
return sr

37
pythonPackages/ufpy/gfe/__init__.py
View file

@ -1,37 +0,0 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# __init__.py for ufpy.gfe package
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 07/26/12 dgilling Initial Creation.
#
#
#
__all__ = [
]

64
pythonPackages/ufpy/ignite_password.py
View file

@ -1,64 +0,0 @@
##############################################################################
# Encryption/decryption for ignite passwords
#
# TODO RODO #8677: The ignite password encryption/decryption code in this and
# associated files is based off similar JMS password code that exists in a
# later version, so the similar code should be consolidated later on.
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------- -------- -------------- -------------------------------------------
# Oct 12, 2021 8667 mapeters Initial version
# Mar 03, 2022 8762 mapeters Handle cache VM jars being moved under
# /awips2/ignite
#
##############################################################################
import os
import subprocess
class IgniteConfigurationException(Exception):
"""Exception subclass for ignite password errors."""
pass
def updateIgnitePasswords(keystore_password, truststore_password, password_props_path):
igniteJar = __findPluginJar("com.raytheon.uf.common.datastore.ignite")
cryptoJar = __findPluginJar("com.raytheon.uf.common.security")
statusJar = __findPluginJar("com.raytheon.uf.common.status")
apacheJar = __findFossJar("org.apache.commons.codec")
classPath = ":".join([igniteJar, cryptoJar, statusJar, apacheJar])
passwords_dict = {'a2.ignite.keystore.password': keystore_password, 'a2.ignite.truststore.password': truststore_password}
for password_key, password in passwords_dict.items():
# need full java path since this is run as root, which doesn't have appropriate path vars set
process = subprocess.run(["/awips2/java/bin/java", "-cp", classPath, "com.raytheon.uf.common.datastore.ignite.IgnitePasswordUtils", "--update", password, password_key, password_props_path],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
process.check_returncode()
except subprocess.CalledProcessError:
raise IgniteConfigurationException(f"Failed to update {password_key}: {process.stderr.decode()}")
def __findPluginJar(pluginName):
# for cache VMs
ignitePluginJar = f"/awips2/ignite/lib/plugins/{pluginName}.jar"
if os.path.isfile(ignitePluginJar):
return ignitePluginJar
# for edex VMs
edexPluginJar = f"/awips2/edex/lib/plugins/{pluginName}.jar"
if os.path.isfile(edexPluginJar):
return edexPluginJar
raise RuntimeError(f"Could not locate plugin {pluginName}.jar")
def __findFossJar(libraryName):
# for cache VMs
igniteFossDir = f"/awips2/ignite/lib/dependencies/{libraryName}"
if os.path.isdir(igniteFossDir):
return f"{igniteFossDir}/*"
# for edex VMs
edexFossDir = f"/awips2/edex/lib/dependencies/{libraryName}"
if os.path.isdir(edexFossDir):
return f"{edexFossDir}/*"
raise RuntimeError(f"Could not locate plugin {libraryName}")

477
pythonPackages/ufpy/localization/LocalizationFileManager.py
View file

@ -1,477 +0,0 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Library for accessing localization files from python.
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# --------- -------- --------- --------------------------
# 08/09/17 5731 bsteffen Initial Creation.
# 04/23/19 7756 mapeters Make user context name determination work with IdM
# 04/22/20 7883 tgurney Python 3 string/bytes fixes
# 01/10/22 8735 mapeters Set Content-Type for PUTs
import urllib.request, urllib.error, urllib.parse
from json import load as loadjson
from xml.etree.ElementTree import parse as parseXml
from base64 import b64encode
from io import BytesIO
import dateutil.parser
import contextlib
import os
from urllib.parse import urlunparse, urljoin
from . import LocalizationUtil
NON_EXISTENT_CHECKSUM = 'NON_EXISTENT_CHECKSUM'
DIRECTORY_CHECKSUM = 'DIRECTORY_CHECKSUM'
class LocalizationFileVersionConflictException(Exception):
pass
class LocalizationFileDoesNotExistException(Exception):
pass
class LocalizationFileIsNotDirectoryException(Exception):
pass
class LocalizationContext(object):
"""A localization context defines the scope of a localization file.
For example the base localization context includes all the default files
installed with EDEX, while a particular user context has custom files for
that user.
A localization context consists of a level and name. The level defines what
kind of entity this context is valid for, such as 'base', 'site', or 'user'.
The name identifies the specific entity, for example the name of a 'user'
level context is usually the username. The 'base' level does not have a name
because there cannot be only one 'base' context.
Attributes:
level: the localization level
name: the context name
"""
def __init__(self, level="base", name=None, localizationType="common_static"):
if level != "base":
assert name is not None
self.level = level
self.name = name
self.type = localizationType
def isBase(self):
return self.level == "base"
def _getUrlComponent(self):
if self.isBase():
return self.type + '/' + "base/"
else:
return self.type + '/' + self.level + '/' + self.name + '/'
def __str__(self):
if self.isBase():
return self.type + ".base"
else:
return self.type + "." + self.level + "." + self.name
def __eq__(self, other):
return self.level == other.level and \
self.name == other.name and \
self.type == other.type
def __hash__(self):
return hash((self.level, self.name, self.type))
class _LocalizationOutput(BytesIO):
"""A file-like object for writing a localization file.
The contents being written are stored in memory and written to a
localization server only when the writing is finished.
This object should be used as a context manager, a save operation will be
executed if the context exits with no errors. If errors occur the partial
contents are abandoned and the server is unchanged.
It is also possible to save the contents to the server with the save()
method.
"""
def __init__(self, manager, lFile):
super().__init__()
self._manager = manager
self._file = lFile
def save(self):
"""Send the currently written contents to the server."""
request = self._manager._buildRequest(self._file.context, self._file.path, method="PUT")
request.data = self.getvalue()
request.add_header("If-Match", self._file.checksum)
# An empty file is created if Content-Type isn't specified (defaults to
# "application/x-www-form-urlencoded"). We aren't encoding the
# request.data bytes in any special way for either text files (e.g. .py
# or .xml) or binary files (e.g. .png), so this works for both.
request.add_header("Content-Type", "application/octet-stream")
try:
urllib.request.urlopen(request)
except urllib.error.HTTPError as e:
if e.code == 409:
raise LocalizationFileVersionConflictException(e.read())
else:
raise e
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
if exc_type is None:
self.save()
def __str__(self):
return '<' + self.__class__.__name__ + " for " + str(self._file) + '>'
class LocalizationFile(object):
"""A specific file stored in localization.
A localization file is uniquely defined by the context and path. There can
only be one valid file for that path and localization at a time. To access
the contents of the file use the open method.
Attributes:
context: A LocalizationContext
path: A path to this file
checksum: A string representation of a checksum generated from the file contents.
timnestamp: A datetime.datetime object indicating when the file was last modified.
"""
def __init__(self, manager, context, path, checksum, timestamp):
"""Initialize a LocalizationFile with the given manager and attributes.
Args:
manager: A LocalizationFileManager to assist with server communication
context: A LocalizationContext
path: A path to this file
checksum: A string representation of a checksum generated from the file contents.
timnestamp: A datetime.datetime object indicating when the file was last modified.
"""
self._manager = manager
self.context = context
self.path = path
self.checksum = checksum
self.timestamp = timestamp
def open(self, mode='r'):
"""Open the file.
This should always be called as as part of a with statement. When
writing the content is not saved on the server until leaving the with
statement normally, if an error occurs the server is left unchanged.
Example:
with locFile.open('w') as output:
output.write('some content')
Args:
mode: 'r' for reading the file, 'w' for writing
Returns:
A file like object that can be used for reads or writes.
"""
if mode == 'r':
request = self._manager._buildRequest(self.context, self.path)
response = urllib.request.urlopen(request)
# Not the recommended way of reading directories.
if not(self.isDirectory()):
checksum = response.headers["Content-MD5"]
if self.checksum != checksum:
raise RuntimeError("Localization checksum mismatch " + self.checksum + " " + checksum)
return contextlib.closing(response)
elif mode == 'w':
return _LocalizationOutput(self._manager, self)
else:
raise ValueError("mode string must be 'r' or 'w' not " + str(r))
def delete(self):
"""Delete this file from the server"""
request = self._manager._buildRequest(self.context, self.path, method='DELETE')
request.add_header("If-Match", self.checksum)
try:
urllib.request.urlopen(request)
except urllib.error.HTTPError as e:
if e.code == 409:
raise LocalizationFileVersionConflictException(e.read())
else:
raise e
def exists(self):
"""Check if this file actually exists.
Returns:
boolean indicating existence of this file
"""
return self.checksum != NON_EXISTENT_CHECKSUM
def isDirectory(self):
"""Check if this file is a directory.
A file must exist to be considered a directory.
Returns:
boolean indicating directorocity of this file
"""
return self.checksum == DIRECTORY_CHECKSUM
def getCheckSum(self):
return self.checksum
def getContext(self):
return self.context
def getPath(self):
return self.path
def getTimeStamp(self):
return self.timestamp
def __str__(self):
return str(self.context) + "/" + self.path
def __eq__(self, other):
return self.context == other.context and \
self.path == other.path and \
self.checksum == other.checksum \
and self.timestamp == other.timestamp
def __hash__(self):
return hash((self.context, self.path, self.checksum, self.timestamp))
def _getHost():
import subprocess
host = subprocess.check_output(
"source /awips2/fxa/bin/setup.env; echo $DEFAULT_HOST",
shell=True).strip().decode()
if host:
return host
return 'localhost'
def _getSiteFromServer(host):
try:
from ufpy import ThriftClient
from dynamicserialize.dstypes.com.raytheon.uf.common.site.requests import GetPrimarySiteRequest
client = ThriftClient.ThriftClient(host)
return client.sendRequest(GetPrimarySiteRequest())
except:
# Servers that don't have GFE installed will not return a site
pass
def _getSiteFromEnv():
site = os.environ.get('FXA_LOCAL_SITE')
if site is None:
site = os.environ.get('SITE_IDENTIFIER')
return site
def _getSite(host):
site = _getSiteFromEnv()
if not(site):
site = _getSiteFromServer(host)
return site
def _parseJsonList(manager, response, context, path):
fileList = []
jsonResponse = loadjson(response)
for name, jsonData in jsonResponse.items():
checksum = jsonData["checksum"]
timestampString = jsonData["timestamp"]
timestamp = dateutil.parser.parse(timestampString)
newpath = urljoin(path, name)
fileList.append(LocalizationFile(manager, context, newpath, checksum, timestamp))
return fileList
def _parseXmlList(manager, response, context, path):
fileList = []
for xmlData in parseXml(response).getroot().findall('file'):
name = xmlData.get("name")
checksum = xmlData.get("checksum")
timestampString = xmlData.get("timestamp")
timestamp = dateutil.parser.parse(timestampString)
newpath = urljoin(path, name)
fileList.append(LocalizationFile(manager, context, newpath, checksum, timestamp))
return fileList
class LocalizationFileManager(object):
"""Connects to a server and retrieves LocalizationFiles."""
def __init__(self, host=None, port=9581, path="/services/localization/", contexts=None, site=None, localizationType="common_static"):
"""Initializes a LocalizationFileManager with connection parameters and context information
All arguments are optional and will use defaults or attempt to figure out appropriate values form the environment.
Args:
host: A hostname of the localization server, such as 'ev'.
port: A port to use to connect to the localization server, usually 9581.
path: A path to reach the localization file service on the server.
contexts: A list of contexts to check for files, the order of the contexts will be used
for the order of incremental results and the priority of absolute results.
site: A site identifier to use for site specific contexts. This is only used if the contexts arg is None.
localizationType: A localization type for contexts. This is only used if the contexts arg is None.
"""
if host is None:
host = _getHost()
if contexts is None:
if site is None:
site = _getSite(host)
contexts = [LocalizationContext("base", None, localizationType)]
if site:
contexts.append(LocalizationContext("configured", site, localizationType))
contexts.append(LocalizationContext("site", site, localizationType))
contexts.append(LocalizationContext("user", LocalizationUtil.getUser(), localizationType))
netloc = host + ':' + str(port)
self._baseUrl = urlunparse(('http', netloc, path, None, None, None))
self._contexts = contexts
def _buildRequest(self, context, path, method='GET'):
url = urljoin(self._baseUrl, context._getUrlComponent())
url = urljoin(url, path)
request = urllib.request.Request(url, method=method)
username = LocalizationUtil.getUser()
# Currently password is ignored in the server
# this is the defacto standard for not providing one to this service.
password = username
base64string = b64encode(b'%s:%s' % (username.encode(), password.encode()))
request.add_header("Authorization", "Basic %s" % base64string.decode())
return request
def _normalizePath(self, path):
if path == '' or path == '/':
path = '.'
if path[0] == '/':
path = path[1:]
return path
def _list(self, path):
path = self._normalizePath(path)
if path[-1] != '/':
path += '/'
fileList = []
exists = False
for context in self._contexts:
try:
request = self._buildRequest(context, path)
request.add_header("Accept", "application/json, application/xml")
response = urllib.request.urlopen(request)
exists = True
if not(response.geturl().endswith("/")):
# For ordinary files the server sends a redirect to remove the slash.
raise LocalizationFileIsNotDirectoryException("Not a directory: " + path)
elif response.headers["Content-Type"] == "application/xml":
fileList += _parseXmlList(self, response, context, path)
else:
fileList += _parseJsonList(self, response, context, path)
except urllib.error.HTTPError as e:
if e.code != 404:
raise e
if not(exists):
raise LocalizationFileDoesNotExistException("No such file or directory: " + path)
return fileList
def _get(self, context, path):
path = self._normalizePath(path)
try:
request = self._buildRequest(context, path, method='HEAD')
resp = urllib.request.urlopen(request)
if (resp.geturl().endswith("/")):
checksum = DIRECTORY_CHECKSUM
else:
if "Content-MD5" not in resp.headers:
raise RuntimeError("Missing Content-MD5 header in response from " + resp.geturl())
checksum = resp.headers["Content-MD5"]
if "Last-Modified" not in resp.headers:
raise RuntimeError("Missing Last-Modified header in response from " + resp.geturl())
timestamp = dateutil.parser.parse(resp.headers["Last-Modified"])
return LocalizationFile(self, context, path, checksum, timestamp)
except urllib.error.HTTPError as e:
if e.code != 404:
raise e
else:
return LocalizationFile(self, context, path, NON_EXISTENT_CHECKSUM, None)
def listAbsolute(self, path):
"""List the files in a localization directory, only a single file is returned for each unique path.
If a file exists in more than one context then the highest level(furthest from base) is used.
Args:
path: A path to a directory that should be the root of the listing
Returns:
A list of LocalizationFiles
"""
merged = dict()
for lFile in self._list(path):
merged[lFile.path] = lFile
return sorted(merged.values(), key=lambda lFile: lFile.path)
def listIncremental(self, path):
"""List the files in a localization directory, this includes all files for all contexts.
Args:
path: A path to a directory that should be the root of the listing
Returns:
A list of tuples, each tuple will contain one or more files for the
same paths but different contexts. Each tuple will be ordered the
same as the contexts in this manager, generally with 'base' first
and 'user' last.
"""
merged = dict()
for lFile in self._list(path):
if lFile.path in merged:
merged[lFile.path] += (lFile,)
else:
merged[lFile.path] = (lFile,)
return sorted(merged.values(), key=lambda t: t[0].path)
def getAbsolute(self, path):
"""Get a single localization file from the highest level context where it exists.
Args:
path: A path to a localization file
Returns:
A Localization File with the specified path or None if the file does not exist in any context.
"""
for context in reversed(self._contexts):
f = self._get(context, path)
if f.exists():
return f
def getIncremental(self, path):
"""Get all the localization files that exist in any context for the provided path.
Args:
path: A path to a localization file
Returns:
A tuple containing all the files that exist for this path in any context. The tuple
will be ordered the same as the contexts in this manager, generally with 'base' first
and 'user' last.
"""
result = ()
for context in self._contexts:
f = self._get(context, path)
if f.exists():
result += (f,)
return result
def getSpecific(self, level, path):
"""Get a specific localization file at a given level, the file may not exist.
The file is returned for whichever context is valid for the provided level in this manager.
For writing new files this is the only way to get access to a file that
does not exist in order to create it.
Args:
level: the name of a localization level, such as "base", "site", "user"
path: A path to a localization file
Returns:
A Localization File with the specified path and a context for the specified level.
"""
for context in self._contexts:
if context.level == level:
return self._get(context, path)
raise ValueError("No context defined for level " + level)
def __str__(self):
contextsStr = '[' + ' '.join((str(c) for c in self._contexts)) + ']'
return '<' + self.__class__.__name__ + " for " + self._baseUrl + ' ' + contextsStr + '>'

45
pythonPackages/ufpy/localization/LocalizationUtil.py
View file

@ -1,45 +0,0 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Utilities for localization.
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# --------- -------- --------- --------------------------
# 04/23/19 7756 mapeters Initial creation
import getpass
def getUser():
'''
Get the user context name.
'''
try:
# Match Java's way of determining the user if we have Jep access
from java.lang import System
user = System.getProperty('user.name')
except:
# Otherwise use built-in getpass module. With IdM, this can return
# user.name@REALM, so strip the @REALM portion if it exists.
user = getpass.getuser()
user = user.split('@')[0]
return user

32
pythonPackages/ufpy/localization/__init__.py
View file

@ -1,32 +0,0 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# __init__.py for ufpy.localization package
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# --------- -------- --------- --------------------------
# 08/10/17 5731 bsteffen Initial Creation.
__all__ = [
]

142
pythonPackages/ufpy/qpidingest.py
View file

@ -1,142 +0,0 @@
#===============================================================================
# qpidingest.py
#
# @author: Aaron Anderson
# @organization: NOAA/WDTB OU/CIMMS
# @version: 1.0 02/19/2010
# @requires: awips2-python and awips2-qpid-proton-python RPMs
#
# ***EDEX and QPID must be running for this module to work***
#
# DESCRIPTION:
# This module is used to connect to QPID and send messages to the external.dropbox queue
# which tells EDEX to ingest a data file from a specified path. This avoids having to copy
# a data file into an endpoint. Each message also contains a header which is used to determine
# which plugin should be used to decode the file. Each plugin has an xml file located in
# $EDEX_HOME/data/utility/edex_static/base/distribution that contains regular expressions
# that the header is compared to. When the header matches one of these regular expressions
# the file is decoded with that plugin. If you make changes to one of these xml files you
# must restart EDEX for the changes to take effect.
#
# NOTE: If the message is being sent but you do not see it being ingested in the EDEX log
# check the xml files to make sure the header you are passing matches one of the regular
# expressions. Beware of spaces, some regular expressions require spaces while others use
# a wildcard character so a space is optional. It seems you are better off having the space
# as this will be matched to both patterns. For the file in the example below,
# 20100218_185755_SAUS46KLOX.metar, I use SAUS46 KLOX as the header to make sure it matches.
#
#
# EXAMPLE:
# Simple example program:
#
#------------------------------------------------------------------------------
# import qpidingest
# #Tell EDEX to ingest a metar file from data_store. The filepath is
# #/data_store/20100218/metar/00/standard/20100218_005920_SAUS46KSEW.metar
#
# conn=qpidingest.IngestViaQPID() #defaults to localhost port 5672
#
# #If EDEX is not on the local machine you can make the connection as follows
# #conn=qpidingest.IngestViaQPID(host='<MACHINE NAME>',port=<PORT NUMBER>)
#
# conn.sendmessage('/data_store/20100218/metar/18/standard/20100218_185755_SAUS46KLOX.metar','SAUS46 KLOX')
# conn.close()
#-------------------------------------------------------------------------------
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------- -------- ------------ ------------------------------------------
# Jun 13, 2013 16242 D. Friedman Add Qpid authentication info
# Mar 06, 2014 17907 D. Friedman Workaround for issue QPID-5569
# Feb 16, 2017 6084 bsteffen Support ssl connections
# Jun 14, 2019 7870 mrichardson MHS env workaround
# Jul 23, 2019 7724 mrichardson Upgrade Qpid to Qpid Proton
# Nov 06, 2019 7724 tgurney Remove the unnecessary
# QpidQueueManager
# Dec 12, 2019 7995 dgilling Revert interface changes from #7724.
# Jul 07, 2020 8187 randerso Added qpid connection_id
#
#===============================================================================
import logging
import os
import pwd
import os.path
import socket
import proton
import proton.utils
import proton.reactor
log = logging.getLogger("qpidingest")
class QpidIngestException(Exception):
"""Exception subclass for broker communication exceptions."""
pass
class IngestViaQPID:
def __init__(self, host="localhost", port=5672, program="qpidingest"):
'''
Connect to QPID and make bindings to route message to external.dropbox queue
@param host: string hostname of computer running EDEX and QPID (default localhost)
@param port: integer port used to connect to QPID (default 5672)
'''
pwuid = pwd.getpwuid(os.getuid())
certdb = os.getenv("QPID_SSL_CERT_DB", os.path.join(pwuid.pw_dir, ".qpid"))
certname = os.getenv("QPID_SSL_CERT_NAME", "guest")
cert_password = os.getenv("QPID_SSL_CERT_PASSWORD", "password")
certfile = os.path.join(certdb, f"{certname}.crt")
keyfile = os.path.join(certdb, f"{certname}.key")
url = f"amqps://{host}:{port}"
ADDRESS = "external.dropbox"
ssl_domain = proton.SSLDomain(mode=proton.SSLDomain.MODE_CLIENT)
ssl_domain.set_credentials(certfile, keyfile, cert_password)
clientID = ":".join([
socket.gethostname(),
pwuid.pw_name,
program,
str(os.getpid()),
])
try:
container = proton.reactor.Container()
container.container_id = clientID
self._conn = proton.utils.BlockingConnection(url, ssl_domain=ssl_domain)
self._sender = self._conn.create_sender(ADDRESS)
log.debug("Connected to broker [%s], endpoint [%s].", url, ADDRESS)
except proton.ProtonException as e:
log.exception("Failed to connect to broker [%s].", url)
raise QpidIngestException("Failed to connect to broker [{}].".format(url)) from e
def sendmessage(self, filepath, header):
'''
This function sends a message to the external.dropbox queue providing the path
to the file to be ingested and a header to determine the plugin to be used to
decode the file.
@param filepath: string full path to file to be ingested
@param header: string header used to determine plugin decoder to use
'''
try:
self._sender.send(proton.Message(body=filepath, subject=header))
except proton.ProtonException as e:
log.exception("Failed to send file [%s] to broker.", filepath)
raise QpidIngestException("Failed to send file [{}] to broker.".format(filepath)) from e
def close(self):
'''
After all messages are sent call this function to close connection and make sure
there are no threads left open
'''
try:
self._sender.close()
self._conn.close()
log.debug("Disconnected from broker.")
except proton.ProtonException as e:
log.warning("Failed to disconnect from broker.", exc_info=True)
raise QpidIngestException("Failed to disconnect from broker.") from e

48
pythonPackages/ufpy/test/Record.py
View file

@ -1,48 +0,0 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# Pure python logging mechanism for logging to AlertViz from
# pure python (ie not JEP). DO NOT USE IN PYTHON CALLED
# FROM JAVA.
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# 11/03/10 5849 cjeanbap Initial Creation.
#
#
#
import os
import sys
class Record():
def __init__(self, level=0, msg='Test Message'):
self.levelno=level
self.message=msg
self.exc_info=sys.exc_info()
self.exc_text="TEST"
def getMessage(self):
return self.message

Some files were not shown because too many files have changed in this diff Show more