commit 895976c5372b6fe68529d24592278c85f45e539b Author: mjames-upc Date: Fri Jun 12 11:57:06 2015 -0600 initial commit diff --git a/acars/a2acars.csh b/acars/a2acars.csh new file mode 100755 index 0000000..4c146db --- /dev/null +++ b/acars/a2acars.csh @@ -0,0 +1,128 @@ +#!/bin/csh +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + + +# +# A script wrapper around a UEngine call that is meant to get all available +# acars data in the A-II database over a specified range of times. +# The data is output to stdout as ASCII. Each line is one time/platform +# combination. The individual data items are comma delimited. +# This version can adapt to use a python stub that calls the +# data access framework. +# +# Usage: +# +# a2acars.csh {p} yyyy-mm-dd hh:mm yyyy-mm-dd hh:mm +# +# The literal p flag is optional. The p flag means preserve +# the final version of the python submitted to the UEngine instead of +# cleaning it up. The path to the finalized python is /tmp/a2acarsNNNNN.py +# where NNNNN is a unix process id. This flag does nothing when using the +# DataAccessFramework. +# +# This outputs the following set of variables for each line: +# +# tailNumber,asciitime,latitude,longitude,acarsReceivingStation, +# pressure,flightPhase,rollQuality,temperature,windDir,windSpeed, +# relHumidity,mixRatio +# +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 07/23/2014 3392 nanbowle Initial modification. Passes parameters straight to DAF script instead of using sed. +# +# +# + + +set rmpy = yes +if ( "$1" == "p" ) then + set rmpy = no + shift +endif +# +# Identify directory this script is in, will be one of the directories we +# search for other files in. +# +set mydir = `dirname $0` +set d1 = `echo $mydir | cut -c1` +if ( "$mydir" == '.' ) then + set mydir = $PWD +else if ( "$d1" != "/" ) then + set mydir = $PWD/$mydir +endif +set mydir = `(cd $mydir ; pwd)` +if ( ! $?FXA_HOME ) set FXA_HOME = xxxx +# +# Locate python stub that we will modify to create the final python logic. +# +if ( -e ./a2acarsStub.py ) then + set stubpy = ./a2acarsStub.py +else if ( -e $mydir/a2acarsStub.py ) then + set stubpy = $mydir/a2acarsStub.py +else if ( -e $FXA_HOME/src/dm/bufr/acars/a2acarsStub.py ) then + set stubpy = $FXA_HOME/src/dm/bufr/acars/a2acarsStub.py +else if ( -e $FXA_HOME/bin/a2acarsStub.py ) then + set stubpy = $FXA_HOME/bin/a2acarsStub.py +else + bash -c "echo could not find a2acarsStub.py 1>&2" + exit +endif +# +# Determine if we are using the data access framework or the uEngine. +# +grep DataAccessLayer $stubpy >& /dev/null +if ( $status == 0 ) then + /awips2/python/bin/python $stubpy -b "$1 $2" -e "$3 $4" +else + # + # Set up the environment we need to run the UEngine. + # + set method = "uengine" + if ( -e ./UEngine.cshsrc ) then + set ueenv = ./UEngine.cshsrc + else if ( -e $mydir/UEngine.cshsrc ) then + set ueenv = $mydir/UEngine.cshsrc + else if ( -e $FXA_HOME/src/dm/point/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/src/dm/point/UEngine.cshsrc + else if ( -e $FXA_HOME/bin/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/bin/UEngine.cshsrc + else + bash -c "echo could not find UEngine.cshsrc 1>&2" + exit + endif + source $ueenv + + set specpy = /tmp/a2acars${$}.py + rm -rf $specpy >& /dev/null + touch $specpy + chmod 775 $specpy + cat $stubpy | sed "s/BBBBB/$1 $2/g" | sed "s/EEEEE/$3 $4/g" > $specpy + + cd $UE_BIN_PATH + ( uengine -r python < $specpy ) | grep -v '<' | grep -v Response + + if ( "$rmpy" == "yes" ) rm -rf $specpy >& /dev/null +endif + diff --git a/acars/a2acarsStub.py b/acars/a2acarsStub.py new file mode 100644 index 0000000..2976b81 --- /dev/null +++ b/acars/a2acarsStub.py @@ -0,0 +1,184 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + + +# +# Gets all available acars data in the A-II database over a specified range of +# times. The data is output to stdout as ASCII. Each line is one time/platform +# combination. The individual data items are comma delimited. +# +# +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 07/23/2014 3392 nabowle Initial modification. Replaces UEngine with DAF. +# 07/28/2014 3392 nabowle Strip tail and receiver to match original formatting. +# 09/04/2014 3405 nabowle NO_DATA and date to string extracted to a2dafcommon +# +# + + +import a2dafcommon +import argparse +import sys + +from datetime import datetime +from ufpy.dataaccess import DataAccessLayer +from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange + +def get_args(): + parser = argparse.ArgumentParser(conflict_handler="resolve") + parser.add_argument("-h", action="store", dest="host", + help="EDEX server hostname (optional)", + metavar="hostname") + parser.add_argument("-b", action="store", dest="start", + help="The start of the time range in YYYY-MM-DD HH:MM", + metavar="start") + parser.add_argument("-e", action="store", dest="end", + help="The end of the time range in YYYY-MM-DD HH:MM", + metavar="end") + return parser.parse_args() + +def main(): + user_args = get_args() + + # Set the host in the DataAcessLayer if supplied + if user_args.host: + DataAccessLayer.changeEDEXHost(user_args.host) + + start = user_args.start + end = user_args.end + + if not start or not end: + print >> sys.stderr, "Start or End date not provided" + return + + + req = DataAccessLayer.newDataRequest("acars") + req.setParameters("tailNumber", "receiver", "pressure", "flightPhase", + "rollAngleQuality", "temp", "windDirection", "windSpeed", + "humidity", "mixingRatio", "icing") + + + + beginRange = datetime.strptime( start + ":00.0", "%Y-%m-%d %H:%M:%S.%f") + endRange = datetime.strptime( end + ":59.9", "%Y-%m-%d %H:%M:%S.%f") + timerange = TimeRange(beginRange, endRange) + + geometries = DataAccessLayer.getGeometryData(req, timerange) + + if len(geometries) == 0: +# print("No data available.") + return + + for geoData in geometries: + mytail = geoData.getString("tailNumber") + if a2dafcommon.is_no_data(mytail): + mytail = "" + else: + mytail = mytail.strip() + + mytime = geoData.getDataTime() + if mytime == None: + continue + #2014-07-16 00:00:00 (0) => 2014-07-16_00:00:00 + mytime = a2dafcommon.datatime_to_string(mytime) + + geo = geoData.getGeometry() + if geo == None: + continue + mylon = geo.x + mylat = geo.y + if a2dafcommon.is_no_data(mylat) or a2dafcommon.is_no_data(mylon): + continue + mylat = "%.4f"%float(mylat) + mylon = "%.4f"%float(mylon) + + myrec = geoData.getString("receiver") + if a2dafcommon.is_no_data(myrec): + myrec = "" + else: + myrec = myrec.strip() + + mypres = geoData.getNumber("pressure") + if a2dafcommon.is_no_data(mypres): + mypres = "1e37" + else : + mypres = "%.0f"%mypres + + myphs = geoData.getString("flightPhase") + if a2dafcommon.is_no_data(myphs): + myphs = "7" + else : + myphs = "%d"%int(myphs) + + myrol = geoData.getString("rollAngleQuality") + if a2dafcommon.is_no_data(myrol): + myrol = "3" + else : + myrol = "%d"%int(myrol) + + mytemp = geoData.getNumber("temp") + if a2dafcommon.is_no_data(mytemp): + mytemp = "1e37" + else : + mytemp = "%.1f"%mytemp + + mydir = geoData.getString("windDirection") + if a2dafcommon.is_no_data(mydir): + mydir = "1e37" + else : + mydir = "%d"%int(mydir) + + myspd = geoData.getNumber("windSpeed") + if a2dafcommon.is_no_data(myspd): + myspd = "1e37" + else : + myspd = "%.1f"%myspd + + myhum = geoData.getNumber("humidity") + if a2dafcommon.is_no_data(myhum): + myhum = "1e37" + else : + myhum = "%.0f"%myhum + + mymix = geoData.getNumber("mixingRatio") + if a2dafcommon.is_no_data(mymix): + mymix = "1e37" + else : + mymix = "%.2f"%mymix + +# Icing was commented out of the uengine version +# myicg = geoData.getString("icing") +# if a2dafcommon.is_no_data(myicg): +# myicg = "1e37" +# else : +# myicg = "%d"%int(myicg) + + msg = mytail + "," + mytime + "," + mylat + "," + mylon + "," + \ + myrec + "," + mypres + "," + myphs + "," + myrol + "," + \ + mytemp + "," + mydir + "," + myspd + "," + myhum + "," + mymix + print msg + +if __name__ == '__main__': + main() diff --git a/aireppirep/a2aircraft.csh b/aireppirep/a2aircraft.csh new file mode 100644 index 0000000..53fcb36 --- /dev/null +++ b/aireppirep/a2aircraft.csh @@ -0,0 +1,228 @@ +#!/bin/csh +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# +# A script wrapper around a UEngine call that is meant to get all available +# maritime data in the A-II database over a specified range of times. +# The data is output to stdout as ASCII. Each line is one time/platform +# combination. The individual data items are comma delimited. +# This version can adapt to use a python stub that calls the +# data access framework. +# +# Usage: +# +# a2aircraft.csh {p} {t} yyyy-mm-dd hh:mm yyyy-mm-dd hh:mm +# +# The literal p flag is optional. The p flag means preserve +# the final version of the python submitted to the UEngine instead of +# cleaning it up. The path to the finalized python is /tmp/a2aircraftNNNNN.py +# where NNNNN is a unix process id. This flag does nothing when using the DAF +# version. +# +# The optional literal t mean change some of the ascii phenomena descriptions +# for pireps into codes suitable for D-2D format aircraft data files. +# +# This outputs the following set of variables for each line: +# +# lat|lon,asciitime,flightLevel,reportType,aircraftType, +# temperature,windDir,windSpeed,visibility, +# FlightWeather*,FlightHazard*,FlightConditions*,WeatherGroup*, +# numCloudLayers,cloudBaseHeight,cloudTopHeight,cloudAmount, +# numIcingLayers,heightBaseIcing,heightTopIcing,typeIcing,intensityOfIcing, +# numTurbulenceLayers,heightBaseTurb,heightTopTurb, +# typeTurbulence,intensityOfTurbulence +# +# Asterisk (*) means variable that does not directly correspond to a +# variable in the D-2D format aircraft data files. +# +# +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 09/02/2014 3405 nabowle Initial modification to better handle daf. +# +# +set rmpy = yes +set typcod = qwertyuiop +if ( "$1" == "t" ) then + set typcod = "typecode = 'no'" + shift +endif +if ( "$1" == "p" ) then + set rmpy = no + shift +endif +set repType = "" +set rt = `echo $1 | tr 'a-z' 'A-Z' | grep '^.IREP$'` +if ( "$rt" != "" ) then + set repType = $rt + shift +endif +if ( "$1" == "p" ) then + set rmpy = no + shift +endif +if ( "$1" == "t" ) then + set typcod = "typecode = 'no'" + shift +endif +# +# Identify directory this script is in, will be one of the directories we +# search for other files in. +# +set mydir = `dirname $0` +set d1 = `echo $mydir | cut -c1` +if ( "$mydir" == '.' ) then + set mydir = $PWD +else if ( "$d1" != "/" ) then + set mydir = $PWD/$mydir +endif +set mydir = `(cd $mydir ; pwd)` +if ( ! $?FXA_HOME ) set FXA_HOME = xxxx +# +# Locate python stub that we will modify to create the final python logic. +# +if ( -e ./a2airepStub.py ) then + set stubpy = ./a2airepStub.py +else if ( -e $mydir/a2airepStub.py ) then + set stubpy = $mydir/a2airepStub.py +else if ( -e $FXA_HOME/src/dm/bufr/acars/a2airepStub.py ) then + set stubpy = $FXA_HOME/src/dm/bufr/acars/a2airepStub.py +else if ( -e $FXA_HOME/bin/a2airepStub.py ) then + set stubpy = $FXA_HOME/bin/a2airepStub.py +else + bash -c "echo could not find a2airepStub.py 1>&2" + exit +endif +# +# Determine if we are using the data access framework or the uEngine. +# +grep DataAccessLayer $stubpy >& /dev/null +if ( $status == 0 ) then + set method = "daf" + if ( "$repType" != "PIREP" ) then + /awips2/python/bin/python $stubpy -b "$1 $2" -e "$3 $4" --match-legacy +# /awips2/python/bin/python $stubpy -b "$1 $2" -e "$3 $4" + endif +else + # + # Set up the environment we need to run the UEngine. + # + set method = "uengine" + if ( -e ./UEngine.cshsrc ) then + set ueenv = ./UEngine.cshsrc + else if ( -e $mydir/UEngine.cshsrc ) then + set ueenv = $mydir/UEngine.cshsrc + else if ( -e $FXA_HOME/src/dm/point/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/src/dm/point/UEngine.cshsrc + else if ( -e $FXA_HOME/bin/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/bin/UEngine.cshsrc + else + bash -c "echo could not find UEngine.cshsrc 1>&2" + exit + endif + source $ueenv + + # + set specpy = /tmp/a2airep${$}.py + rm -rf $specpy >& /dev/null + touch $specpy + chmod 775 $specpy + cat $stubpy | sed "s/BBBBB/$1 $2/g" | sed "s/EEEEE/$3 $4/g" > $specpy + if ( "$repType" != "PIREP" ) then + set here = `pwd` + cd $UE_BIN_PATH + ( uengine -r python < $specpy ) | grep -v '<' | grep -v Response + cd $here + endif + if ( "$rmpy" == "yes" ) rm -rf $specpy >& /dev/null +endif + +# +# Locate python stub that we will modify to create the final python logic. +# +if ( -e ./a2pirepStub.py ) then + set stubpy = ./a2pirepStub.py +else if ( -e $mydir/a2pirepStub.py ) then + set stubpy = $mydir/a2pirepStub.py +else if ( -e $FXA_HOME/src/dm/bufr/acars/a2pirepStub.py ) then + set stubpy = $FXA_HOME/src/dm/bufr/acars/a2pirepStub.py +else if ( -e $FXA_HOME/bin/a2pirepStub.py ) then + set stubpy = $FXA_HOME/bin/a2pirepStub.py +else + bash -c "echo could not find a2pirepStub.py 1>&2" + exit +endif +# +# Determine if we are using the data access framework or the uEngine. +# +grep DataAccessLayer $stubpy >& /dev/null +if ( $status == 0 ) then + set method = "daf" + if ( "$repType" != "AIREP" ) then + # By default, match the legacy output. + set pyargs = "--match-legacy" +# set pyargs = "" + if ( "$typcod" != "qwertyuiop" ) then + set pyargs = "$pyargs -t" + endif + + /awips2/python/bin/python $stubpy -b "$1 $2" -e "$3 $4" $pyargs + endif +else + if ( "$method" == "daf" ) then + # + # Set up the environment we need to run the UEngine. + # + set method = "uengine" + if ( -e ./UEngine.cshsrc ) then + set ueenv = ./UEngine.cshsrc + else if ( -e $mydir/UEngine.cshsrc ) then + set ueenv = $mydir/UEngine.cshsrc + else if ( -e $FXA_HOME/src/dm/point/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/src/dm/point/UEngine.cshsrc + else if ( -e $FXA_HOME/bin/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/bin/UEngine.cshsrc + else + bash -c "echo could not find UEngine.cshsrc 1>&2" + exit + endif + source $ueenv + endif + + # + set specpy = /tmp/a2pirep${$}.py + rm -rf $specpy >& /dev/null + touch $specpy + chmod 775 $specpy + cat $stubpy | sed "s/BBBBB/$1 $2/g" | sed "s/EEEEE/$3 $4/g" | \ + grep -v "$typcod" > $specpy + cd $UE_BIN_PATH + if ( "$repType" != "AIREP" ) then + cd $UE_BIN_PATH + ( uengine -r python < $specpy ) | grep -v '<' | grep -v Response + endif + if ( "$rmpy" == "yes" ) rm -rf $specpy >& /dev/null +endif + diff --git a/aireppirep/a2airepStub.py b/aireppirep/a2airepStub.py new file mode 100644 index 0000000..ad219be --- /dev/null +++ b/aireppirep/a2airepStub.py @@ -0,0 +1,168 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + + +# +# Gets all available pirep data in the A-II database over a specified range of +# times. The data is output to stdout as ASCII. Each line is one record. +# The individual data items are comma delimited. +# +# +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 08/25/2014 3405 nabowle Initial modification. Replaces UEngine with DAF. +# +# + + +import a2dafcommon +import argparse +import sys + +from datetime import datetime +from ufpy.dataaccess import DataAccessLayer +from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange + + +def get_args(): + parser = argparse.ArgumentParser(conflict_handler="resolve") + parser.add_argument("-h", action="store", dest="host", + help="EDEX server hostname (optional)", + metavar="hostname") + parser.add_argument("-b", action="store", dest="start", + help="The start of the time range in YYYY-MM-DD HH:MM", + metavar="start") + parser.add_argument("-e", action="store", dest="end", + help="The end of the time range in YYYY-MM-DD HH:MM", + metavar="end") + parser.add_argument("--match-legacy", action="store_true", dest="match", + help="If set, the legacy script output will be matched.", + default=False) + return parser.parse_args() + + +def main(): + user_args = get_args() + + if user_args.host: + DataAccessLayer.changeEDEXHost(user_args.host) + + start = user_args.start + end = user_args.end + + if not start or not end: + print >> sys.stderr, "Start or End date not provided" + return + + beginRange = datetime.strptime( start + ":00.0", "%Y-%m-%d %H:%M:%S.%f") + endRange = datetime.strptime( end + ":59.9", "%Y-%m-%d %H:%M:%S.%f") + timerange = TimeRange(beginRange, endRange) + + req = DataAccessLayer.newDataRequest("airep") + req.setParameters("id", "flightLevel", "temp", "windDirection", "windSpeed", + "flightWeather", "flightHazard", "flightConditions") + geometries = DataAccessLayer.getGeometryData(req, timerange) + + if not geometries : +# print "No data available." + return + + msg = ""; + + geometries.sort(key=lambda geo: str(geo.getDataTime())) + for geoData in geometries : + mytime = geoData.getDataTime() + if not mytime: + continue + mytime = a2dafcommon.datatime_to_string(mytime) + + geo = geoData.getGeometry() + if not geo: + continue + + mylon = geo.x + mylat = geo.y + if a2dafcommon.is_no_data(mylat) or a2dafcommon.is_no_data(mylon): + continue + mylat = "%.4f"%mylat + mylon = "%.4f"%mylon + + myflvl = geoData.getNumber("flightLevel") + if a2dafcommon.is_no_data(myflvl) : + myflvl = "1e37" + else : + myflvl = "%d"%myflvl + +# Temp is not stored. +# mytemp = geoData.getString("temp") +# if a2dafcommon.is_no_data(mytemp) : +# mytemp = "1e37" +# else : +# mytemp = "%.1f"%float(mytemp) + mytemp = "1e37" + +# Wind Direction is not stored. +# mydir = geoData.getString("windDirection") +# if a2dafcommon.is_no_data(mydir) : +# mydir = "1e37" +# else : +# mydir = "%d"%int(mydir) + mydir = "1e37" + + myspd = geoData.getNumber("windSpeed") + if a2dafcommon.is_no_data(myspd) : + myspd = "1e37" + else : + myspd = "%.1f"%myspd + + myfwx = geoData.getNumber("flightWeather") + if myfwx : + myfwx = "-1" + else : + myfwx = "%d"%myfwx + + myhaz = geoData.getNumber("flightHazard") + if a2dafcommon.is_no_data(myhaz) : + myhaz = "-1" + else : + myhaz = "%d"%myhaz + + mycond = geoData.getNumber("flightConditions") + if a2dafcommon.is_no_data(mycond) : + mycond = "-1" + else : + mycond = "%d"%mycond + + if user_args.match: +# Wind Speed not returned to legacy script. + myspd = "1e37" + + msg += mylat + "|" + mylon + "," + mytime + "," + myflvl + ",AIREP,," + \ + mytemp + "," + mydir + "," + myspd + ",1e37," + \ + myfwx + "," + myhaz + "," + mycond + ",,0,,,,0,,,,,0,,,,\n" + + print msg.strip() + +if __name__ == '__main__': + main() diff --git a/aireppirep/a2pirepStub.py b/aireppirep/a2pirepStub.py new file mode 100644 index 0000000..77f3f6e --- /dev/null +++ b/aireppirep/a2pirepStub.py @@ -0,0 +1,385 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + + +# +# Gets all available pirep data in the A-II database over a specified time +# range. The data is output to stdout as ASCII. Each line is one record. The +# individual data items are comma delimited. +# +# The legacy script does not retrieve any values not stored in the postgres db. +# To compensate for this in side-by-side comparison, a --match-legacy flag is +# provided that will ignore these fields. +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 08/25/2014 3405 nabowle Initial modification. Replaces UEngine with DAF. +# +# + + +import a2dafcommon +import argparse +import sys + +from datetime import datetime +from ufpy.dataaccess import DataAccessLayer +from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange + + + +def get_args(): + parser = argparse.ArgumentParser(conflict_handler="resolve") + parser.add_argument("-h", action="store", dest="host", + help="EDEX server hostname (optional)", + metavar="hostname") + parser.add_argument("-b", action="store", dest="start", + help="The start of the time range in YYYY-MM-DD HH:MM", + metavar="start") + parser.add_argument("-e", action="store", dest="end", + help="The end of the time range in YYYY-MM-DD HH:MM", + metavar="end") + parser.add_argument("--match-legacy", action="store_true", dest="match", + help="If set, the legacy script output will be matched.", + default=False) + parser.add_argument("-t", action="store_true", dest="typecode", + help="If set, more type information is displayed.", + default=False) + return parser.parse_args() + + +def main(): + MULTI_DIM_PARAMS = set(["hazardType", + "turbType", "turbBaseHeight", "turbTopHeight", + "iceType", "iceBaseHeight", "iceTopHeight", + "skyCover1", "skyCover2", "skyBaseHeight", "skyTopHeight" + ]) + + user_args = get_args() + + if user_args.host: + DataAccessLayer.changeEDEXHost(user_args.host) + + start = user_args.start + end = user_args.end + + if not start or not end: + print >> sys.stderr, "Start or End date not provided" + return + + beginRange = datetime.strptime( start + ":00.0", "%Y-%m-%d %H:%M:%S.%f") + endRange = datetime.strptime( end + ":59.9", "%Y-%m-%d %H:%M:%S.%f") + timerange = TimeRange(beginRange, endRange) + + req = DataAccessLayer.newDataRequest("pirep") + req.setParameters('id', 'flightLevel', 'temp', 'windDirection', 'windSpeed', + 'horzVisibility', 'aircraftType', 'weatherGroup') + req.getParameters().extend(MULTI_DIM_PARAMS) + geometries = DataAccessLayer.getGeometryData(req, timerange) + + if not geometries : +# print "No data available." + return + + typecode = user_args.typecode + match = user_args.match + msg = "" + layerData = [] + combinedGeos = [] + for geoData in geometries : +# The DAF returns multi-dimensional parameters as separate results before +# the base result that contain the single-dimensional parameters. +# Because of the separation of parameters and known ordering of result +# types, we can easily figure out what each result is and correlate the +# ancillary data with the base data. + if set(geoData.getParameters()) & MULTI_DIM_PARAMS : + layerData.append(geoData) + continue + + combinedGeos.append({"base":geoData, "layers":layerData}) + layerData = [] + + combinedGeos.sort(key=lambda geoMap: str(geoMap['base'].getDataTime())) + + for geoMap in combinedGeos : + geoData = geoMap['base'] + layerData = geoMap['layers'] + + mytime = geoData.getDataTime() + if not mytime: + continue + mytime = mytime = a2dafcommon.datatime_to_string(mytime) + + geo = geoData.getGeometry() + if not geo: + continue + + mylon = geo.x + mylat = geo.y + if a2dafcommon.is_no_data(mylat) or a2dafcommon.is_no_data(mylon) : + continue + mylat = "%.4f"%mylat + mylon = "%.4f"%mylon + + myflvl = geoData.getNumber("flightLevel") + if a2dafcommon.is_no_data(myflvl) : + myflvl = "1e37" + else : + myflvl = "%d"%myflvl + +# Temp is not stored. +# mytemp = geoData.getNumber("temp") +# if a2dafcommon.is_no_data(mytemp) : +# mytemp = "1e37" +# else : +# mytemp = "%.1f"%float(mytemp) + mytemp = "1e37" + +# Wind Direction is not stored. +# mydir = geoData.getString("windDirection") +# if a2dafcommon.is_no_data(mydir) : +# mydir = "1e37" +# else : +# mydir = "%d"%int(mydir) + mydir = "1e37" + + myspd = geoData.getNumber("windSpeed") + if a2dafcommon.is_no_data(myspd) : + myspd = "1e37" + else : + myspd = "%.1f"%myspd + + myvis = geoData.getNumber("horzVisibility") + if a2dafcommon.is_no_data(myvis) : + myvis = "1e37" + else : + myvis = "%.1f"%myvis + + mycraft = geoData.getString("aircraftType") + if a2dafcommon.is_no_data(mycraft) : + mycraft = "" + + mywx = geoData.getString("weatherGroup") + if a2dafcommon.is_no_data(mywx) : + mywx = "" + + + cc = 0 + cldBas = "" + cldTop = "" + cldVal = "" + ii = 0 + icgBas = "" + icgTop = "" + icgTyp = "" + icgVal = "" + tt = 0 + trbBas = "" + trbTop = "" + trbTyp = "" + trbVal = "" + + if match : +# Speed, Horizontal Visibility, aircraft type, and weather group are +# not returned to the legacy script. Flightlevel of -9999 is output +# as such instead of being replaced by "1e37" + myspd = "1e37" + myvis = "-9999998.0" + mycraft = "" + mywx = "" + if geoData.getNumber("flightLevel") == -9999: + myflvl = "-9999" + else : + for pld in layerData: + sep = "" + ltyp = pld.getString("hazardType") + if a2dafcommon.is_no_data(ltyp) : + continue + + fval = pld.getString("skyCover1") + if fval == "None" : + fval = "" + + sval = pld.getString("skyCover2") + if sval == "None" : + sval = "" + if ltyp == "CLOUD" : + if fval == "TOP" : + fval = "" + if sval == "TOP" : + sval = "" + if sval != "" : + fval += "-"+sval + if typecode : + if fval == "CLR" : + fval = "0" + elif fval == "OVC" : + fval = "8" + elif fval == "SCT" : + fval = "11" + elif fval == "BKN" : + fval = "12" + elif fval == "FEW" : + fval = "13" + else : + continue + if cldBas != "" : + sep = "|" + + base = pld.getNumber("skyBaseHeight") + if a2dafcommon.is_no_data(base) or base == 99999 : + base = "1e37" + else : + base = "%.0f"%float(base) + + top = pld.getNumber("skyTopHeight") + if a2dafcommon.is_no_data(top) or top == 99999 : + top = "1e37" + else : + top = "%.0f"%float(top) + + cldBas += sep+base + cldTop += sep+top + cldVal += sep+fval + cc += 1 + elif ltyp == "ICING" : + dtyp = pld.getString("iceType") + if a2dafcommon.is_no_data(dtyp) : + dtyp = "" + + if sval != "" : + fval += "-"+sval + if icgBas != "" : + sep = "|" + if typecode : + if dtyp == "RIME" : + dtyp = "1" + elif dtyp == "CLR" : + dtyp = "2" + elif dtyp == "MXD" : + dtyp = "3" + else : + dtyp = "-9999" + if fval == "NEG" : + fval = "0"; + elif fval == "TRACE" : + fval = "1" + elif fval == "TRACE-LGT" : + fval = "2" + elif fval == "LGT" : + fval = "3" + elif fval == "LGT-MOD" : + fval = "4" + elif fval == "MOD" : + fval = "5" + elif fval == "MOD-SEV" : + fval = "7" + elif fval == "SEV" : + fval = "8" + else : + fval = "-9999" + if fval == "-9999" and dtyp == "-9999" : + continue + + base = pld.getNumber("iceBaseHeight") + if a2dafcommon.is_no_data(base) or base == 99999 : + base = "1e37" + else : + base = "%.0f"%float(base) + + top = pld.getNumber("iceTopHeight") + if a2dafcommon.is_no_data(top) or top == 99999 : + top = "1e37" + else : + top = "%.0f"%float(top) + + icgBas += sep+base + icgTop += sep+top + icgTyp += sep+dtyp + icgVal += sep+fval + ii += 1 + elif ltyp == "TURBC" : + dtyp = pld.getString("turbType") + if a2dafcommon.is_no_data(dtyp) : + dtyp = "" + + if sval != "" : + fval += "-"+sval + if typecode : + if dtyp == "CAT" : + dtyp = "1" + elif dtyp == "CHOP" : + dtyp = "2" + else : + dtyp = "-9999" + if fval == "NEG" : + fval = "0"; + elif fval == "LGT" : + fval = "2" + elif fval == "LGT-MOD" : + fval = "3" + elif fval == "MOD" : + fval = "4" + elif fval == "MOD-SEV" : + fval = "5" + elif fval == "SEV" : + fval = "6" + elif fval == "EXTRM" : + fval = "8" + else : + fval = "-9999" + if fval == "-9999" and dtyp == "-9999" : + continue + if trbBas != "" : + sep = "|" + + base = pld.getNumber("turbBaseHeight") + if a2dafcommon.is_no_data(base) or base == 99999 : + base = "1e37" + else : + base = "%.0f"%float(base) + + top = pld.getNumber("turbTopHeight") + if a2dafcommon.is_no_data(top) or top == 99999 : + top = "1e37" + else : + top = "%.0f"%float(top) + + trbBas += sep+base + trbTop += sep+top + trbTyp += sep+dtyp + trbVal += sep+fval + tt += 1 + + msg += mylat + "|" + mylon + "," + mytime + "," + myflvl + ",PIREP," + \ + mycraft + "," + mytemp + "," + mydir + "," + myspd + "," + \ + myvis + ",-1,-1,-1," + mywx + "," + \ + str(cc) + "," + cldBas + "," + cldTop + "," + cldVal + "," + \ + str(ii) + "," + icgBas + "," + icgTop + "," + \ + icgTyp + "," + icgVal + "," + \ + str(tt) + "," + trbBas + "," + trbTop + "," + \ + trbTyp + "," + trbVal + "\n" + + print msg.strip() + +if __name__ == '__main__': + main() diff --git a/common/UEngine.cshsrc b/common/UEngine.cshsrc new file mode 100755 index 0000000..59c98da --- /dev/null +++ b/common/UEngine.cshsrc @@ -0,0 +1,28 @@ +# +# Set up the environment we need to run the UEngine. +# +#set release = "" +#if ( -e /awips2/edex/conf/banner.txt ) then +# set release = `grep Version /awips2/edex/conf/banner.txt | tr '*' ' ' | \ +# sed 's/ *$//g' | sed 's/^.* //g'` +#endif +# +while (1) + if ( $?INSTALL_BASE ) then + if ( -d $INSTALL_BASE/python ) break + endif + setenv INSTALL_BASE /awips2 + break +end +if ( -e $INSTALL_BASE/bin/uengine ) then + setenv UE_BIN_PATH $INSTALL_BASE/bin +else + setenv UE_BIN_PATH $INSTALL_BASE/fxa/bin +endif +if ( ! $?PRE_ADE_PATH ) then + setenv PRE_ADE_PATH ${PATH} +endif +setenv JAVA_HOME $INSTALL_BASE/java +setenv LD_LIBRARY_PATH /usr/lib +setenv PYTHONPATH .:$INSTALL_BASE/bin/src +setenv PATH ${JAVA_HOME}/bin:${UE_BIN_PATH}:${PRE_ADE_PATH} diff --git a/common/a2dafcommon.py b/common/a2dafcommon.py new file mode 100644 index 0000000..ee3a7b0 --- /dev/null +++ b/common/a2dafcommon.py @@ -0,0 +1,43 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + + +# +# Common functionality for MSAS/LAPS scripts using the Data Acess Framework. +# +# +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 09/02/2014 3405 nabowle Initial creation. +# +# + +NO_DATA = [None, "None", -9999, -9999.0, "-9999", "-9999.0", ""] + +def is_no_data(val): + return val in NO_DATA + + +def datatime_to_string(datatime): + return str(datatime)[0:19].replace(" ","_") diff --git a/common/gtasUtil b/common/gtasUtil new file mode 100644 index 0000000..aa73c56 Binary files /dev/null and b/common/gtasUtil differ diff --git a/common/msgCodeSeds.txt b/common/msgCodeSeds.txt new file mode 100644 index 0000000..cac25bf --- /dev/null +++ b/common/msgCodeSeds.txt @@ -0,0 +1,108 @@ +# 35 36 37 38 41 50 51 57 65 66 67 81 85 86 90 +s/^0 *$/0|STImotion/g +s/^2 *$/2|GSM/g +s/^3 *$/3|PRR/g +s/^6 *$/6|AAP/g +s/^8 *$/8|PTL/g +s/^9 *$/9|AM/g +s/^16 *$/16|Z 1km 3bit/g +s/^17 *$/17|Z 2km 3bit/g +s/^18 *$/18|Z 4km 3bit/g +s/^19 *$/19|Z 1km 4bit/g +s/^20 *$/20|Z 2km 4bit/g +s/^21 *$/21|Z 4km 4bit/g +s/^22 *$/22|V 0.25km 3bit/g +s/^23 *$/23|V 0.5km 3bit/g +s/^24 *$/24|V 1km 3bit/g +s/^25 *$/25|V 0.25km 4bit/g +s/^26 *$/26|V 0.5km 4bit/g +s/^27 *$/27|V 1km 4bit/g +s/^28 *$/28|SW 0.25km 3bit/g +s/^29 *$/29|SW 0.5km 3bit/g +s/^30 *$/30|SW 1km 3bit/g +s/^31 *$/31|USP 2km 4bit/g +s/^32 *$/32|DHR 1km 8bit/g +s/^33 *$/33|HSR 1km 4bit/g +s/^34 *$/34|CFC 1km 3bit/g +s/^35 *$/35|CZ 1km 3bit/g +s/^36 *$/36|CZ 4km 3bit/g +s/^37 *$/37|CZ 1km 4bit/g +s/^38 *$/38|CZ 4km 4bit/g +s/^41 *$/41|ET 4km 4bit/g +s/^48 *$/48|VWP/g +s/^50 *$/50|RCS 1km 4bit/g +s/^51 *$/51|VCS 0.5km 4bit/g +s/^55 *$/55|SRR 0.5km 4bit/g +s/^56 *$/56|SRM 1km 4bit/g +s/^57 *$/57|VIL 4km 4bit/g +s/^58 *$/58|STI/g +s/^59 *$/59|HI/g +s/^60 *$/60|M/g +s/^61 *$/61|TVS/g +s/^62 *$/62|SS/g +s/^65 *$/65|LRM 4km 3bit/g +s/^66 *$/66|LRM 4km 3bit/g +s/^67 *$/67|APR 4km 3bit/g +s/^73 *$/73|UAM/g +s/^74 *$/74|RCM/g +s/^75 *$/75|FTM/g +s/^78 *$/78|OHP 2km 4bit/g +s/^79 *$/79|THP 2km 4bit/g +s/^80 *$/80|STP 2km 4bit/g +s/^81 *$/81|DPA 4km 8bit/g +s/^82 *$/82|SPD 40km 3bit/g +s/^84 *$/84|VAD/g +s/^85 *$/85|RCS 1km 3bit/g +s/^86 *$/86|VCS 0.5km 3bit/g +s/^90 *$/90|LRM 4km 3bit/g +s/^93 *$/93|DBV 1km 8bit/g +s/^94 *$/94|Z 1km 8bit/g +s/^99 *$/99|V 0.25km 8bit/g +s/^100 *$/100|VSDT/g +s/^101 *$/101|STIT/g +s/^102 *$/102|HIT/g +s/^103 *$/103|MT/g +s/^104 *$/104|TVST/g +s/^107 *$/107|OHPT/g +s/^108 *$/108|THPT/g +s/^109 *$/109|STPT/g +s/^132 *$/132|CLR 1km 4bit/g +s/^133 *$/133|CLD 1km 4bit/g +s/^134 *$/134|DVL 1km 8bit/g +s/^135 *$/135|EET 1km 8bit/g +s/^136 *$/136|SO/g +s/^137 *$/137|ULR 1km 4bit/g +s/^138 *$/138|STP 2km 8bit/g +s/^139 *$/139|MRU/g +s/^140 *$/140|GFM/g +s/^141 *$/141|MD/g +s/^143 *$/143|TRU/g +s/^144 *$/144|OSW 2km 4bit/g +s/^145 *$/145|OSD 2km 4bit/g +s/^146 *$/146|SSW 2km 4bit/g +s/^147 *$/147|SSD 2km 4bit/g +s/^149 *$/149|DMD/g +s/^150 *$/150|USW 2km 4bit/g +s/^151 *$/151|USD 2km 4bit/g +s/^152 *$/152|RSS/g +s/^153 *$/153|Z 0.25km 0.5Deg 8bit/g +s/^154 *$/154|V 0.25km 0.5Deg 8bit/g +s/^155 *$/155|SW 0.25km 0.5Deg 8bit/g +s/^158 *$/158|ZDR 1km 4bit/g +s/^159 *$/159|ZDR 0.25km 8bit/g +s/^160 *$/160|CC 1km 4bit/g +s/^161 *$/161|CC 0.25km 8bit/g +s/^162 *$/162|KDP 1km 4bit/g +s/^163 *$/163|KDP 0.25km 8bit/g +s/^164 *$/164|HC 1km 4bit/g +s/^165 *$/165|HC 0.25km 8bit/g +s/^166 *$/166|ML/g +s/^169 *$/169|OHA 2km 4bit/g +s/^170 *$/170|DAA 0.25km 8bit/g +s/^171 *$/171|STA 2km 4bit/g +s/^172 *$/172|STA 0.25km 8bit/g +s/^173 *$/173|DUA 0.25km 8bit/g +s/^174 *$/174|DOD 0.25km 8bit/g +s/^175 *$/175|DSD 0.25km 8bit/g +s/^176 *$/176|DPR 0.25km gen/g +s/^177 *$/177|HHC 0.25km 8bit/g diff --git a/common/wmoToNameRaob.txt b/common/wmoToNameRaob.txt new file mode 100755 index 0000000..425a31b --- /dev/null +++ b/common/wmoToNameRaob.txt @@ -0,0 +1,290 @@ +s/^04202,/BGTL,/g +s/^08594,/GVAC,/g +s/^21824,/RUTS,/g +s/^25563,/UHMA,/g +s/^25594,/UHMD,/g +s/^25913,/UHMM,/g +s/^31329,/UHPB,/g +s/^31735,/RUHB,/g +s/^31960,/UHWW,/g +s/^32150,/RUSH,/g +s/^32540,/UHPP,/g +s/^46692,/RCSS,/g +s/^46697,/RCGM,/g +s/^46747,/RCMJ,/g +s/^47058,/ZKKK,/g +s/^47110,/RKSS,/g +s/^47122,/RKSO,/g +s/^47138,/RKTH,/g +s/^47158,/RKJJ,/g +s/^47187,/RKPM,/g +s/^47401,/RJCW,/g +s/^47412,/RJCO,/g +s/^47580,/RJSM,/g +s/^47582,/RJSK,/g +s/^47590,/RJSS,/g +s/^47678,/RJTH,/g +s/^47681,/RJNH,/g +s/^47808,/RJFF,/g +s/^47827,/RJFK,/g +s/^47881,/RJOS,/g +s/^47918,/ROIG,/g +s/^47931,/RODN,/g +s/^47936,/ROAH,/g +s/^47945,/ROMD,/g +s/^47971,/RJAO,/g +s/^47981,/RJAW,/g +s/^47991,/RJAM,/g +s/^50745,/ZYQQ,/g +s/^50953,/ZYHB,/g +s/^54161,/ZYCC,/g +s/^54342,/ZYYY,/g +s/^54662,/ZYTL,/g +s/^54857,/ZSQD,/g +s/^58457,/ZSHC,/g +s/^61641,/GOOY,/g +s/^61902,/FHAW,/g +s/^61967,/DJDG,/g +s/^70026,/PABR,/g +s/^70086,/PABA,/g +s/^70133,/PAOT,/g +s/^70200,/PAOM,/g +s/^70219,/PABE,/g +s/^70231,/PAMC,/g +s/^70261,/PAFA,/g +s/^70273,/PAFC,/g +s/^70308,/PASN,/g +s/^70316,/PACB,/g +s/^70326,/PAKN,/g +s/^70350,/PADQ,/g +s/^70361,/PAYA,/g +s/^70398,/PANT,/g +s/^70414,/PASY,/g +s/^71043,/CYVQ,/g +s/^71051,/CWSY,/g +s/^71081,/CYUX,/g +s/^71082,/CYLT,/g +s/^71109,/CYZT,/g +s/^71119,/CWEG,/g +s/^71120,/CYOD,/g +s/^71121,/CYED,/g +s/^71123,/CYEG,/g +s/^71203,/CYLW,/g +s/^71396,/CYHZ,/g +s/^71399,/CWOS,/g +s/^71600,/CWSA,/g +s/^71603,/CYQI,/g +s/^71624,/CYYZ,/g +s/^71627,/CYUL,/g +s/^71701,/CYCX,/g +s/^71722,/CWMW,/g +s/^71799,/CYYJ,/g +s/^71801,/CYYT,/g +s/^71807,/CWAR,/g +s/^71811,/CYZV,/g +s/^71815,/CYJT,/g +s/^71816,/CYYR,/g +s/^71823,/CYAH,/g +s/^71836,/CYMO,/g +s/^71845,/CWPL,/g +s/^71853,/CWLO,/g +s/^71867,/CYQD,/g +s/^71879,/CYXD,/g +s/^71896,/CYXS,/g +s/^71906,/CYVP,/g +s/^71907,/CYPH,/g +s/^71909,/CYFB,/g +s/^71913,/CYYQ,/g +s/^71915,/CYZS,/g +s/^71917,/CWEU,/g +s/^71924,/CYRB,/g +s/^71925,/CYCB,/g +s/^71926,/CYBK,/g +s/^71934,/CYSM,/g +s/^71945,/CYYE,/g +s/^71957,/CYEV,/g +s/^71964,/CYXY,/g +s/^72201,/KKEY,/g +s/^72202,/KMFL,/g +s/^72206,/KJAX,/g +s/^72208,/KCHS,/g +s/^72210,/KTBW,/g +s/^72214,/KTAE,/g +s/^72215,/KFFC,/g +s/^72221,/KVPS,/g +s/^72230,/KBMX,/g +s/^72233,/KLIX,/g +s/^72235,/KJAN,/g +s/^72240,/KLCH,/g +s/^72248,/KSHV,/g +s/^72249,/KFWD,/g +s/^72250,/KBRO,/g +s/^72251,/KCRP,/g +s/^72257,/KHLR,/g +s/^72261,/KDRT,/g +s/^72265,/KMAF,/g +s/^72269,/KWSD,/g +s/^72274,/KTWC,/g +s/^72280,/KYUM,/g +s/^72291,/KNSI,/g +s/^72293,/KNKX,/g +s/^72305,/KMHX,/g +s/^72317,/KGSO,/g +s/^72318,/KRNK,/g +s/^72327,/KOHX,/g +s/^72340,/KLZK,/g +s/^72357,/KOUN,/g +s/^72363,/KAMA,/g +s/^72364,/KEPZ,/g +s/^72365,/KABQ,/g +s/^72376,/KFGZ,/g +s/^72381,/KEDW,/g +s/^72386,/KLAS,/g +s/^72387,/KDRA,/g +s/^72391,/KNTD,/g +s/^72393,/KVBG,/g +s/^72402,/KWAL,/g +s/^72403,/KLWX,/g +s/^72426,/KILN,/g +s/^72440,/KSGF,/g +s/^72451,/KDDC,/g +s/^72456,/KTOP,/g +s/^72469,/KDNR,/g +s/^72476,/KGJT,/g +s/^72489,/KREV,/g +s/^72493,/KOAK,/g +s/^72501,/KOKX,/g +s/^72518,/KALY,/g +s/^72520,/KPBZ,/g +s/^72528,/KBUF,/g +s/^72558,/KOAX,/g +s/^72562,/KLBF,/g +s/^72572,/KSLC,/g +s/^72582,/KLKN,/g +s/^72597,/KMFR,/g +s/^72632,/KDTX,/g +s/^72634,/KAPX,/g +s/^72645,/KGRB,/g +s/^72649,/KMPX,/g +s/^72659,/KABR,/g +s/^72662,/KUNR,/g +s/^72672,/KRIW,/g +s/^72681,/KBOI,/g +s/^72694,/KSLE,/g +s/^72712,/KCAR,/g +s/^72747,/KINL,/g +s/^72764,/KBIS,/g +s/^72768,/KGGW,/g +s/^72776,/KTFX,/g +s/^72786,/KOTX,/g +s/^72797,/KUIL,/g +s/^74002,/KAPG,/g +s/^74004,/K1Y7,/g +s/^74005,/K1Y8,/g +s/^74006,/K1Y9,/g +s/^74389,/KGYX,/g +s/^74455,/KDVN,/g +s/^74494,/KCHH,/g +s/^74560,/KILX,/g +s/^74626,/KPSR,/g +s/^74794,/KXMR,/g +s/^76151,/MILS,/g +s/^76225,/MMCU,/g +s/^76256,/MMGM,/g +s/^76342,/MMOV,/g +s/^76361,/MLMM,/g +s/^76382,/MTRC,/g +s/^76394,/MMMY,/g +s/^76405,/MLAP,/g +s/^76458,/MMMZ,/g +s/^76491,/MCVM,/g +s/^76499,/MSOT,/g +s/^76548,/MTAM,/g +s/^76612,/MMGL,/g +s/^76644,/MMMD,/g +s/^76649,/MCZM,/g +s/^76654,/MMZO,/g +s/^76665,/MMOR,/g +s/^76679,/MMMX,/g +s/^76680,/MMDF,/g +s/^76692,/MMVR,/g +s/^76723,/MISC,/g +s/^76749,/MCME,/g +s/^76805,/MACA,/g +s/^76830,/MIZT,/g +s/^76840,/MARR,/g +s/^76904,/MTAP,/g +s/^78016,/TXKF,/g +s/^78073,/MYNN,/g +s/^78367,/MUGM,/g +s/^78384,/MKCG,/g +s/^78397,/MKJP,/g +s/^78486,/MDSD,/g +s/^78526,/TJSJ,/g +s/^78583,/MZBZ,/g +s/^78806,/MPHO,/g +s/^78808,/MABK,/g +s/^78866,/MACM,/g +s/^78897,/TFFR,/g +s/^78954,/MKPB,/g +s/^78970,/MKPP,/g +s/^78988,/MACC,/g +s/^80001,/MCSP,/g +s/^80035,/SKRH,/g +s/^81405,/SOCA,/g +s/^91066,/PMDY,/g +s/^91165,/PHLI,/g +s/^91212,/PGUM,/g +s/^91217,/PGAC,/g +s/^91218,/PGUA,/g +s/^91245,/PWAK,/g +s/^91250,/PKMA,/g +s/^91275,/PJON,/g +s/^91285,/PHTO,/g +s/^91334,/PTKK,/g +s/^91348,/PTPN,/g +s/^91366,/PKWA,/g +s/^91376,/PMKJ,/g +s/^91408,/PTRO,/g +s/^91413,/PTYA,/g +s/^91517,/AGGH,/g +s/^91558,/NHHH,/g +s/^91592,/NWWN,/g +s/^91610,/NGTA,/g +s/^91643,/NGFU,/g +s/^91680,/NFFN,/g +s/^91700,/PCIS,/g +s/^91765,/NSTU,/g +s/^91843,/NCRG,/g +s/^91925,/NTMN,/g +s/^91938,/NTTT,/g +s/^91944,/NTTO,/g +s/^93997,/NZRN,/g +s/^94027,/AYLA,/g +s/^94035,/AYPY,/g +s/^94120,/YPDN,/g +s/^94203,/YPBR,/g +s/^94212,/AHHC,/g +s/^94238,/YDTC,/g +s/^94287,/YBCS,/g +s/^94294,/YBTL,/g +s/^94326,/YBAS,/g +s/^94332,/YBMA,/g +s/^94346,/YBLR,/g +s/^94367,/YBMK,/g +s/^94380,/YBGL,/g +s/^94510,/YBCV,/g +s/^94578,/YBBN,/g +s/^94637,/YPKG,/g +s/^94646,/YPFT,/g +s/^94653,/YPCD,/g +s/^94659,/YPWR,/g +s/^94672,/YPAD,/g +s/^94693,/YMMI,/g +s/^94750,/YSNW,/g +s/^94767,/YSSY,/g +s/^94776,/YSWM,/g +s/^94791,/YSCH,/g +s/^94995,/ASLH,/g +s/^94996,/YSNF,/g +s/^98327,/RPMK,/g diff --git a/grid/a2invmdlStub.py b/grid/a2invmdlStub.py new file mode 100644 index 0000000..b56d679 --- /dev/null +++ b/grid/a2invmdlStub.py @@ -0,0 +1,124 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# Gets inventories of gridded data from the A-II database. The data is output +# to stdout as ASCII. +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 2014-10-22 3599 nabowle Initial modification. Convert to DAF. +# 2014-11-17 3599 nabowle Fix call to get_args(). +# + +import argparse +import numpy +import sys + +from ufpy.dataaccess import DataAccessLayer + +def get_args(): + parser = argparse.ArgumentParser(conflict_handler="resolve") + parser.add_argument("-h", action="store", dest="host", + help="EDEX server hostname (optional)", metavar="hostname") + parser.add_argument("--srcId", action="store", dest="srcId", + help="Unique alphanumeric name for gridded data source", + metavar="srcId") + parser.add_argument("--varAbrev", action="store", dest="varAbrev", + help="Variable abreviation", metavar="varAbrev") + parser.add_argument("--lvlOne", action="store", dest="lvlOne", + help="Level One value", metavar="lvlOne", type=float) + parser.add_argument("--lvlTwo", action="store", dest="lvlTwo", + help="Level Two value", metavar="lvlTwo", type=float) + parser.add_argument("--lvlName", action="store", dest="lvlName", + help="Master level name", metavar="lvlName") + parser.add_argument("--mode", action="store", dest="mode", default="time", + help="Mode - time, plane, field, or fieldplane") + return parser.parse_args() + +def main(): + user_args = get_args() + + if user_args.host: + DataAccessLayer.changeEDEXHost(user_args.host) + + req = DataAccessLayer.newDataRequest("grid") + + if not user_args.srcId: + print >> sys.stderr, "srcId not provided" + return + req.addIdentifier("info.datasetId", user_args.srcId) + + if user_args.varAbrev: + req.setParameters(user_args.varAbrev) + if user_args.lvlName is not None: + req.addIdentifier("info.level.masterLevel.name", user_args.lvlName) + if user_args.lvlOne is not None: + req.addIdentifier("info.level.levelonevalue", numpy.float64(user_args.lvlOne)) + if user_args.lvlTwo is not None: + req.addIdentifier("info.level.leveltwovalue", numpy.float64(user_args.lvlTwo)) + + mode = user_args.mode + if mode not in ["time", "plane", "field", "fieldplane"]: + print >> sys.stderr, "mode must be one of time, plane, field, or fieldplane." + return + + msg = ""; + if mode == "time": + times = DataAccessLayer.getAvailableTimes(req) + for time in times: + timeStr = str(time) + if "--" in timeStr: + timeStr = timeStr[0:-22] + ".0" + timeStr[-22:-1] + ".0" + timeStr[-1] + msg += timeStr[0:19] + ".0" + timeStr[19:] + "\n" + elif mode == "plane": + levels = DataAccessLayer.getAvailableLevels(req) + for level in levels: + msg += level_to_string(level) + "\n" + elif mode == "field": + params = DataAccessLayer.getAvailableParameters(req) + msg = "\n".join(params) + else: #fieldplane + params = DataAccessLayer.getAvailableParameters(req) + for param in params: + msg += param + ":\n" + req.setParameters(param) + levels = DataAccessLayer.getAvailableLevels(req) + if levels: + levelStr = [] + for level in levels: + levelStr.append(level_to_string(level)) + msg += " ".join(levelStr) + " \n" + + print msg.strip("\n") + + +def level_to_string(level): + name = level.getMasterLevel().getName() + lvlOne = str(level.getLevelonevalue()) + lvlTwo = str(level.getLeveltwovalue()) + msg = name + " " + lvlOne + if lvlTwo not in ["None", "-999999.0"]: + msg += " " + lvlTwo + return msg + +if __name__ == '__main__': + main() diff --git a/grid/a2rdmdlCommon.py b/grid/a2rdmdlCommon.py new file mode 100644 index 0000000..2566db9 --- /dev/null +++ b/grid/a2rdmdlCommon.py @@ -0,0 +1,142 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# Gets all available raob data in the A-II database over a specified range of +# times. The data is output to stdout as ASCII. +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 2014-10-15 3598 nabowle Initial creation. Extracted common code from a2rdmdl*.py +# 2014-12-15 3598 nabowle Fix retrieval when fcst is 0. +# + +import argparse +import numpy + +from datetime import datetime +from ufpy.dataaccess import DataAccessLayer +from dynamicserialize.dstypes.com.raytheon.uf.common.time import DataTime +from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange + +def get_parser(): + parser = argparse.ArgumentParser(conflict_handler="resolve") + parser.add_argument("-h", action="store", dest="host", + help="EDEX server hostname (optional)", metavar="hostname") + parser.add_argument("--date", action="store", dest="date", + help="The date in YYYY-MM-DD", metavar="date") + parser.add_argument("--hour", action="store", dest="hour", + help="The hour in HH", metavar="hour") + parser.add_argument("--fcst", action="store", dest="fcst", + help="The forecast time in hours", metavar="fcst") + parser.add_argument("--srcId", action="store", dest="srcId", + help="Unique alphanumeric name for gridded data source", + metavar="srcId") + parser.add_argument("--varAbrev", action="store", dest="varAbrev", + help="Variable abreviation", metavar="varAbrev") + parser.add_argument("--lvlOne", action="store", dest="lvlOne", + help="Level One value", metavar="lvlOne", type=float) + parser.add_argument("--lvlTwo", action="store", dest="lvlTwo", + help="Level Two value", metavar="lvlTwo", type=float) + parser.add_argument("--lvlName", action="store", dest="lvlName", + help="Master level name", metavar="lvlName") + return parser + +def do_request(user_args): + if user_args.host: + DataAccessLayer.changeEDEXHost(user_args.host) + + srcId = user_args.srcId + varAbrev = user_args.varAbrev + + if not srcId or not varAbrev: + raise Exception("srcId or varAbrev not provided") + return + + date = user_args.date + hour = user_args.hour + fcst = user_args.fcst + + if not date or not hour or not fcst: + raise Exception("date, hour, or fcst not provided") + return + + dt = datetime.strptime( str(date) + " " + str(hour) + ":00:00.0", "%Y-%m-%d %H:%M:%S.%f") + +# check for and build date range if necessary + daterange = None + if varAbrev.endswith("hr"): + import re + matches = re.findall(r'\d+', varAbrev) + if matches: + from datetime import timedelta + hourRange = int(matches[-1]) + endDate = dt + timedelta(hours=int(fcst)) + beginDate = endDate - timedelta(hours=hourRange) + daterange = TimeRange(beginDate, endDate) + +# convert hours to seconds because DataTime does the reverse + time = DataTime(dt, int(fcst)*3600, daterange) + + + req = DataAccessLayer.newDataRequest("grid") + req.setParameters(varAbrev) + req.addIdentifier("info.datasetId", srcId) + +# To handle partial level matches, use identifiers instead of a Level. + if user_args.lvlName is not None: + req.addIdentifier("info.level.masterLevel.name", user_args.lvlName) + if user_args.lvlOne is not None: + req.addIdentifier("info.level.levelonevalue", numpy.float64(user_args.lvlOne)) + if user_args.lvlTwo is not None: + req.addIdentifier("info.level.leveltwovalue", numpy.float64(user_args.lvlTwo)) + + times = [time] + +# If fcst is 0, also query for times with FCST_USED flag + if fcst == '0': + time = DataTime(dt, int(fcst)*3600, daterange) + time.utilityFlags.add("FCST_USED") + times.append(time) + + grids = DataAccessLayer.getGridData(req, times) + + if not grids: +# print "Data not available" + raise Exception("") + + grid = grids[0] + rawData = grid.getRawData() + + yLen = len(rawData[0]) + xLen = len(rawData) + + return grid, xLen, yLen + +def get_indices(j, rowLen): +# the lengths are reversed from how getRawData() returns the data and forces +# the need to calculate the dataIdx and arrIdx instead of +# for row in reversed(rawData): for k in row: ... +# it's important to check that arrIdx < len(arr) when arrIdx is incremented + dataIdx = int(j / rowLen) # index into rawData + arrIdx = j % rowLen # index into rawData[dataIdx] + return dataIdx, arrIdx + diff --git a/grid/a2rdmdlStub.py b/grid/a2rdmdlStub.py new file mode 100644 index 0000000..59b2c9d --- /dev/null +++ b/grid/a2rdmdlStub.py @@ -0,0 +1,167 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# Gets all available raob data in the A-II database over a specified range of +# times. The data is output to stdout as ASCII. +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 2014-10-14 3598 nabowle Initial modification. Changed to use DataAccessLayer. +# + +import a2rdmdlCommon +import argparse +import numpy +import sys + +def get_args(): + parser = a2rdmdlCommon.get_parser() + parser.add_argument("--dimLine", action="store_true", dest="dimLine", + help="Output dimensions", default=False) + return parser.parse_args() + +def main(): + user_args = get_args() + + try: + grid, xLen, yLen = a2rdmdlCommon.do_request(user_args) + except Exception as e: + print >> sys.stderr, str(e) + return + + rawData = grid.getRawData() + + msg = "" + if user_args.dimLine: + msg += str(xLen) + " " + str(yLen) + "\n" + + nxy = yLen*xLen + j = nxy-xLen + while j>=0 : + dataIdx, arrIdx = a2rdmdlCommon.get_indices(j, yLen) + arr = rawData[dataIdx] + i = 0 + while i= yLen: + arrIdx = 0 + dataIdx += 1 + arr = rawData[dataIdx] + + k = arr[arrIdx] + if numpy.isnan(k) : + k = 0 + if k<0 : + a = -k + else : + a = k + if a>=999998 : + msg += "1e37 " + elif a<0.00005 : + msg += "%g"%k + " " + elif a<0.0009 : + if round(k,8) == round(k,4) : + msg += "%.4f"%k + " " + elif round(k,8) == round(k,5) : + msg += "%.5f"%k + " " + elif round(k,8) == round(k,6) : + msg += "%.6f"%k + " " + elif round(k,8) == round(k,7) : + msg += "%.7f"%k + " " + else : + msg += "%.8f"%k + " " + elif a<0.009 : + if round(k,7) == round(k,3) : + msg += "%.3f"%k + " " + elif round(k,7) == round(k,4) : + msg += "%.4f"%k + " " + elif round(k,7) == round(k,5) : + msg += "%.5f"%k + " " + elif round(k,7) == round(k,6) : + msg += "%.6f"%k + " " + else : + msg += "%.7f"%k + " " + elif a<0.09 : + if round(k,6) == round(k,2) : + msg += "%.2f"%k + " " + elif round(k,6) == round(k,3) : + msg += "%.3f"%k + " " + elif round(k,6) == round(k,4) : + msg += "%.4f"%k + " " + elif round(k,6) == round(k,5) : + msg += "%.5f"%k + " " + else : + msg += "%.6f"%k + " " + elif a<0.9 : + if round(k,5) == round(k,1) : + msg += "%.1f"%k + " " + elif round(k,5) == round(k,2) : + msg += "%.2f"%k + " " + elif round(k,5) == round(k,3) : + msg += "%.3f"%k + " " + elif round(k,5) == round(k,4) : + msg += "%.4f"%k + " " + else : + msg += "%.5f"%k + " " + elif a<9 : + if round(k,4) == round(k,0) : + msg += "%.0f"%k + " " + elif round(k,4) == round(k,1) : + msg += "%.1f"%k + " " + elif round(k,4) == round(k,2) : + msg += "%.2f"%k + " " + elif round(k,4) == round(k,3) : + msg += "%.3f"%k + " " + else : + msg += "%.4f"%k + " " + elif a<99 : + if round(k,3) == round(k,0) : + msg += "%.0f"%k + " " + elif round(k,3) == round(k,1) : + msg += "%.1f"%k + " " + elif round(k,3) == round(k,2) : + msg += "%.2f"%k + " " + else : + msg += "%.3f"%k + " " + elif a<999 : + if round(k,2) == round(k,0) : + msg += "%.0f"%k + " " + elif round(k,2) == round(k,1) : + msg += "%.1f"%k + " " + else : + msg += "%.2f"%k + " " + elif a<9999 : + if round(k,1) == round(k,0) : + msg += "%.0f"%k + " " + else : + msg += "%.1f"%k + " " + else : + msg += "%.0f"%k + " " + i += 1 + arrIdx += 1 + + msg += "\n" + j -= xLen + + print msg.strip() + " " + +if __name__ == '__main__': + main() diff --git a/grid/a2rdmdlXdr.py b/grid/a2rdmdlXdr.py new file mode 100644 index 0000000..eb2ac35 --- /dev/null +++ b/grid/a2rdmdlXdr.py @@ -0,0 +1,91 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# Gets all available raob data in the A-II database over a specified range of +# times. The data is output to stdout as ASCII. +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 2014-10-15 3598 nabowle Initial modification. Changed to use DataAccessLayer. +# + +import a2rdmdlCommon +import argparse +import numpy +import xdrlib +import bz2 +import sys + +def get_args(): + return a2rdmdlCommon.get_parser().parse_args() + +def main(): + user_args = get_args() + + try: + grid, xLen, yLen = a2rdmdlCommon.do_request(user_args) + except Exception as e: + print >> sys.stderr, str(e) + return + + rawData = grid.getRawData() + + msg = str(xLen) + " " + str(yLen) + "\n" + + nxy = yLen*xLen + j = nxy-xLen + mypacker = xdrlib.Packer() + mypacker.reset() + while j>=0 : + dataIdx, arrIdx = a2rdmdlCommon.get_indices(j, yLen) + arr = rawData[dataIdx] + i = 0 + while i= yLen: + arrIdx = 0 + dataIdx += 1 + arr = rawData[dataIdx] + + k = arr[arrIdx] + if numpy.isnan(k) : + k = 0 + + mypacker.pack_float(float(k)) + i += 1 + arrIdx += 1 + + j -= xLen + + packLen = len(mypacker.get_buffer()) + xdrbuf = bz2.compress(mypacker.get_buffer()) + cmpLen = len(xdrbuf) + msg += str(packLen)+" "+str(cmpLen*2)+"\t\n" + i = 0 + while i tag in the associated +# entry, the ggid must be supplied as a quoted empty string. +# +# With no arguments after the grid number, returns a list of variables for +# the data source specified by the process and grid id. With only a variable, +# returns information for the list of planes for that variable. With more +# arguments, returns a list of times for that variable and plane. +# +# Level value arguments are meaningless without the level type argument, +# but it is meaningful to provide only a level type. +# +# If the only argument after the process and grid is a literal at sign ('@') +# then what is returned is a list of all times for which there is data +# available for the given process/grid combination. +# +# If the only argument after the process and grid is a literal plus sign (+), +# then what will be returned will be a level inventory for all variables. +# +# The literal p option means preserve the final version of the python +# submitted to the UEngine instead of cleaning it up. The path to the +# finalized python is /tmp/a2rdmdlNNNNN.py where NNNNN is a unix process id. +# +# Gets all available raob data in the A-II database over a specified range of +# times. The data is output to stdout as ASCII. +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 2014-10-22 3599 nabowle Initial modification. Changed to properly call DAF version. +# +set rmpy = yes +if ( "$1" == "p" ) then + set rmpy = no + shift +endif +# +# Identify directory this script is in, will be one of the directories we +# search for other files in. +# +set mydir = `dirname $0` +set d1 = `echo $mydir | cut -c1` +if ( "$mydir" == '.' ) then + set mydir = $PWD +else if ( "$d1" != "/" ) then + set mydir = $PWD/$mydir +endif +set mydir = `(cd $mydir ; pwd)` +if ( -x $mydir/$0 ) then + set me = $mydir/$0 +else + set me = $0 +endif +if ( ! $?FXA_HOME ) set FXA_HOME = xxxx +# +# Primarily base stuff on source name, but try to use the old interface. +# +set sss = "$1" +shift +set ids = `echo $sss | tr ',' ' '` +echo "$ids[1]" | grep '^[0-9][0-9]*$' >& /dev/null +if ( $status == 0 || $#ids > 1 ) then + set mroot = /awips2/edex/data/utility/edex_static/base/grib/models + set ids = `echo $ids | tr ' ' '\n' | grep -v '^ *$' | \ + sed 's#^##g' | sed 's#$#<|#g'` + set ids = `echo ${ids}quertyuiop | sed 's/ *//g'` + set ggg = "$1" + shift + if ( "$ggg" == "" ) then + set mmm = `find $mroot -name '*xml' ! -name '*ECMWF*' \ + -exec cat '{}' \; | sed 's|-->|~|g' | \ + tr '\t' ' ' | sed 's/ *//g' | sed 's||~|g' | \ + tr '\n' ' ' | tr '~' '\n' | grep -E "$ids" | \ + grep -v "" | sed 's/^.*//g' | \ + cut '-d<' -f1 | sort -u` + else + set mmm = `find $mroot -name '*xml' -exec cat '{}' \; | \ + sed 's|-->|~|g' | \ + tr '\t' ' ' | sed 's/ *//g' | sed 's||~|g' | \ + tr '\n' ' ' | tr '~' '\n' | grep -E "$ids" | \ + grep "$ggg<" | sed 's/^.*//g' | \ + cut '-d<' -f1 | sort -u` + endif + if ( $#mmm != 1 ) then + echo "$mmm" + echo "Could not look up model name based on $sss '$ggg'" + exit 1 + endif + set sss = $mmm +endif + +# +# Locate python stub that we will modify to create the final python logic. +# +if ( -e ./a2invmdlStub.py ) then + set stubpy = ./a2invmdlStub.py +else if ( -e $mydir/a2invmdlStub.py ) then + set stubpy = $mydir/a2invmdlStub.py +else if ( -e $FXA_HOME/src/dm/grid/a2invmdlStub.py ) then + set stubpy = $FXA_HOME/src/dm/grid/a2invmdlStub.py +else if ( -e $FXA_HOME/bin/a2invmdlStub.py ) then + set stubpy = $FXA_HOME/bin/a2invmdlStub.py +else + bash -c "echo could not find a2invmdlStub.py 1>&2" + exit +endif +# +# Determine if we are using the data access framework or the uEngine. +# +grep DataAccessLayer $stubpy >& /dev/null +if ( $status == 0 ) then + if ( "$*" == "+" ) then + python $stubpy --mode fieldplane --srcId $sss + else if ( "$1" == "" ) then + python $stubpy --mode field --srcId $sss + else if ( "$1" == "@" ) then + python $stubpy --mode time --srcId $sss + else if ( "$2" == "" ) then + python $stubpy --mode plane --srcId $sss --varAbrev $1 + else if ( "$3" == "" ) then + python $stubpy --mode time --srcId $sss --lvlName $1 --varAbrev $2 + else if ( "$4" == "" ) then + python $stubpy --mode time --srcId $sss --lvlName $1 --lvlOne $2 --varAbrev $3 + else + python $stubpy --mode time --srcId $sss --lvlName $1 --lvlOne $2 --lvlTwo $3 --varAbrev $4 + endif +else + # + if ( "$*" == "+" ) then + set varList = `$me $sss` + foreach onevar ( $varList ) + echo ${onevar}: + $me $sss $onevar | tr '\n' ' ' + echo + end + exit + endif + + # + # Set up the environment we need to run the UEngine. + # + if ( -e ./UEngine.cshsrc ) then + set ueenv = ./UEngine.cshsrc + else if ( -e $mydir/UEngine.cshsrc ) then + set ueenv = $mydir/UEngine.cshsrc + else if ( -e $FXA_HOME/src/dm/point/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/src/dm/point/UEngine.cshsrc + else if ( -e $FXA_HOME/bin/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/bin/UEngine.cshsrc + else + bash -c "echo could not find UEngine.cshsrc 1>&2" + exit + endif + source $ueenv + + # + # Modify the text of special tags in stub to create finalized script. + # + set specpy = /tmp/a2invmdl${$}.py + rm -rf $specpy >& /dev/null + touch $specpy + chmod 775 $specpy + set plane = no + if ( "$1" == "" ) then + cat $stubpy | sed "s/MMMMM/field/g" | sed "s/SSSSS/$sss/g" | \ + sed 's/^.*TTTTT.*$//g' | sed 's/^.*LLLLL.*$//g' | \ + sed 's/^.*22222.*$//g' | sed 's/^.*VVVVV.*$//g' >> $specpy + else if ( "$1" == "@" ) then + cat $stubpy | sed "s/MMMMM/time/g" | sed "s/SSSSS/$sss/g" | \ + sed 's/^.*TTTTT.*$//g' | sed 's/^.*LLLLL.*$//g' | \ + sed 's/^.*22222.*$//g' | sed 's/^.*VVVVV.*$//g' >> $specpy + else if ( "$2" == "" ) then + set plane = yes + cat $stubpy | sed "s/MMMMM/plane/g" | sed "s/SSSSS/$sss/g" | \ + sed 's/^.*TTTTT.*$//g' | sed 's/^.*LLLLL.*$//g' | \ + sed 's/^.*22222.*$//g' | sed "s/VVVVV/$1/g" >> $specpy + else if ( "$3" == "" ) then + cat $stubpy | sed "s/MMMMM/time/g" | sed "s/SSSSS/$sss/g" | \ + sed "s/TTTTT/$1/g" | sed 's/^.*LLLLL.*$//g' | \ + sed 's/^.*22222.*$//g' | sed "s/VVVVV/$2/g" >> $specpy + else if ( "$4" == "" ) then + cat $stubpy | sed "s/MMMMM/time/g" | sed "s/SSSSS/$sss/g" | \ + sed "s/TTTTT/$1/g" | sed "s/LLLLL/$2/g"| \ + sed 's/^.*22222.*$//g' | sed "s/VVVVV/$3/g" >> $specpy + else + cat $stubpy | sed "s/MMMMM/time/g" | sed "s/SSSSS/$sss/g" | \ + sed "s/TTTTT/$1/g" | sed "s/LLLLL/$2/g" | \ + sed "s/22222/$3/g" | sed "s/VVVVV/$4/g" >> $specpy + endif + if ( "$plane" == "no" ) then + cd $UE_BIN_PATH + ( uengine -r python < $specpy ) |& grep attributes | cut '-d"' -f4 + else + cd $UE_BIN_PATH + ( uengine -r python < $specpy ) |& sed 's|.*.*|@|g' | \ + grep -E 'attributes|@' | cut '-d"' -f4 | tr '\n' ' ' | tr '@' '\n' | \ + sed 's/ -999999.0//g' | sed 's/^ *//g' | sed 's/ *$//g' + endif + #if ( "$rmpy" == "yes" ) rm -rf $specpy >& /dev/null + # +endif + diff --git a/grid/gridRead.csh b/grid/gridRead.csh new file mode 100755 index 0000000..34b79a1 --- /dev/null +++ b/grid/gridRead.csh @@ -0,0 +1,305 @@ +#!/bin/csh -f +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# Gets all available raob data in the A-II database over a specified range of +# times. The data is output to stdout as ASCII. +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 2014-10-16 3598 nabowle Initial modification. Changed to handle DataAccessLayer. +# +# +# A script wrapper that is meant to get a single slab of gridded data +# from the A-II database. The data is output to stdout as ASCII. +# This version can adapt to use a python stub that calls the +# data access framework. +# +# Usage: +# +# a2rdmdl.csh p n x srcid ctyp lval1 lval2 varAbrev date hour fcst +# +# p - A literal p. (optional) +# n - A literal n. (optional) +# x - A literal x. (optional) +# srcid - Unique alphanumeric name for gridded data source. +# ctyp - Level type id (optional) +# lval1 - First level value (optional) +# lval2 - Second level value (optional) +# varAbrev - Variable abreviation. +# date - Date of model run time as yyyy-mm-dd +# hour - Hour of model run time as hh +# fcst - Forecast time in hours +# +# Legacy usage, not supported in all cases: +# +# a2rdmdl.csh p n x gproc ggid ctyp lval1 lval2 varAbrev date hour fcst +# +# p - A literal p. (optional) +# n - A literal n. (optional) +# x - A literal x. (optional) +# gproc - GRIB process number (can be multiple comma delimited) +# ggid - GRIB grid number +# ctyp - Level type id (optional) +# lval1 - First level value (optional) +# lval2 - Second level value (optional) +# varAbrev - Variable abreviation. +# date - Date of model run time as yyyy-mm-dd +# hour - Hour of model run time as hh +# fcst - Forecast time in hours +# +# With the new unified GRIB decoder, instead of gproc ggid, it is best +# to supply the srcid, which is like ETA or GFS254; e.g. the directory +# under /awips2/edex/data/hdf5/grid where the data is stored. +# +# Note that now for sources with no tag in the associated +# entry, the ggid must be supplied as a quoted empty string. +# +# The literal p option means preserve the final version of the python +# submitted to the UEngine instead of cleaning it up. The path to the +# finalized python is /tmp/a2rdmdlNNNNN.py where NNNNN is a unix process id. +# The literal n option means the first line of output is the dimension of +# the grid returned. The literal x option means return the data in xdr +# format; in this case the xdr data begins immediately after a tab-linefeed. +# +# Process any one character options. +# +set rmpy = yes +set dimStr = dimStr +set specpyName = a2rdmdlStub +while (1) + if ( "$1" == "p" ) then + set rmpy = no + else if ( "$1" == "n" ) then + set dimStr = qwertyuiop + else if ( "$1" == "x" ) then + set specpyName = a2rdmdlXdr + set dimStr = qwertyuiopx + else + break + endif + shift +end +# +# Identify directory this script is in, will be one of the directories we +# search for other files we need. +# +set mydir = `dirname $0` +set d1 = `echo $mydir | cut -c1` +if ( "$mydir" == '.' ) then + set mydir = $PWD +else if ( "$d1" != "/" ) then + set mydir = $PWD/$mydir +endif +set mydir = `(cd $mydir ; pwd)` +if ( ! $?FXA_HOME ) set FXA_HOME = xxxx +# +# Special case of topo, we will hard code it. +# +echo " $* " | grep " Topo " >& /dev/null +if ( $status == 0 ) then + set outfile = `find . -maxdepth 1 \ + -name "$1.Topo" -o -name "$1.$2.Topo" -o -name "*,$1.$2.Topo" -o \ + -name "$1,*.$2.Topo" -o -name "*,$1,*.$2.Topo"` >& /dev/null + if ( ( $#outfile != 1 ) && ( $mydir != $PWD ) ) then + set outfile = `find $mydir -maxdepth 1 \ + -name "$1.Topo" -o -name "$1.$2.Topo" -o -name "*,$1.$2.Topo" -o \ + -name "$1,*.$2.Topo" -o -name "*,$1,*.$2.Topo"` >& /dev/null + endif + if ( ( $#outfile != 1 ) && ( -d $FXA_HOME/data ) ) then + set outfile = `find $FXA_HOME/data -maxdepth 1 \ + -name "$1.Topo" -o -name "$1.$2.Topo" -o -name "*,$1.$2.Topo" -o \ + -name "$1,*.$2.Topo" -o -name "*,$1,*.$2.Topo"` >& /dev/null + endif + if ( $#outfile != 1 ) then + bash -c "echo No flat file available with topo for $1 $2 1>&2" + exit + endif + if ( "$dimStr" == "qwertyuiop" ) then + set nnn = `cat $outfile | wc` + set ny = $nnn[1] + @ nx = $nnn[2] / $ny + echo $nx $ny + endif + cat $outfile + exit +endif +# +# Primarily base stuff on source name, but try to use the old interface. +# +set sss = "$1" +shift +set ids = `echo $sss | tr ',' ' '` +echo "$ids[1]" | grep '^[0-9][0-9]*$' >& /dev/null +if ( $status == 0 || $#ids > 1 ) then + set mroot = /awips2/edex/data/utility/edex_static/base/grib/models + set ids = `echo $ids | tr ' ' '\n' | grep -v '^ *$' | \ + sed 's#^##g' | sed 's#$#<|#g'` + set ids = `echo ${ids}quertyuiop | sed 's/ *//g'` + set ggg = "$1" + shift + if ( "$ggg" == "" ) then + set mmm = `find $mroot -name '*xml' ! -name '*ECMWF*' \ + -exec cat '{}' \; | sed 's|-->|~|g' | \ + tr '\t' ' ' | sed 's/ *//g' | sed 's||~|g' | \ + tr '\n' ' ' | tr '~' '\n' | grep -E "$ids" | \ + grep -v "" | sed 's/^.*//g' | \ + cut '-d<' -f1 | sort -u` + else + set mmm = `find $mroot -name '*xml' -exec cat '{}' \; | \ + sed 's|-->|~|g' | \ + tr '\t' ' ' | sed 's/ *//g' | sed 's||~|g' | \ + tr '\n' ' ' | tr '~' '\n' | grep -E "$ids" | \ + grep "$ggg<" | sed 's/^.*//g' | \ + cut '-d<' -f1 | sort -u` + endif + if ( $#mmm != 1 ) then + echo "$mmm" + echo "Could not look up model name based on $sss '$ggg'" + exit 1 + endif + set sss = $mmm +endif + +@ i = $#argv - 3 +set vvvvv = $argv[$i] + +# +# Locate python stub that we will modify to create the final python logic. +# +if ( -e ./${specpyName}.py ) then + set stubpy = ./${specpyName}.py +else if ( -e $mydir/${specpyName}.py ) then + set stubpy = $mydir/${specpyName}.py +else if ( -e $FXA_HOME/src/dm/grid/${specpyName}.py ) then + set stubpy = $FXA_HOME/src/dm/grid/${specpyName}.py +else if ( -e $FXA_HOME/bin/${specpyName}.py ) then + set stubpy = $FXA_HOME/bin/${specpyName}.py +else + bash -c "echo could not find ${specpyName}.py 1>&2" + exit +endif +# +# Determine if we are using the data access framework or the uEngine. +# +grep DataAccessLayer $stubpy >& /dev/null +if ( $status == 0 ) then + set userArgs = "--srcId $sss --varAbrev $vvvvv" + if ( ( "$dimStr" != "dimStr" ) && ( "$specpyName" != "a2rdmdlXdr" ) ) then + set userArgs = "$userArgs --dimLine" + endif + + if ( "$5" == "" ) then + set userArgs = "$userArgs --date $2 --hour $3 --fcst $4" + else if ( "$6" == "" ) then + set userArgs = "$userArgs --lvlName $1 --date $3 --hour $4 --fcst $5" + else if ( "$7" == "" ) then + set userArgs = "$userArgs --lvlName $1 --lvlOne $2 --date $4 --hour $5 --fcst $6" + else + set userArgs = "$userArgs --lvlName $1 --lvlOne $2 --lvlTwo $3 --date $5 --hour $6 --fcst $7" + endif + python $stubpy $userArgs +else + # + # Set up the environment we need to run the UEngine. + # + if ( -e ./UEngine.cshsrc ) then + set ueenv = ./UEngine.cshsrc + else if ( -e $mydir/UEngine.cshsrc ) then + set ueenv = $mydir/UEngine.cshsrc + else if ( -e $FXA_HOME/src/dm/point/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/src/dm/point/UEngine.cshsrc + else if ( -e $FXA_HOME/bin/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/bin/UEngine.cshsrc + else + bash -c "echo could not find UEngine.cshsrc 1>&2" + exit + endif + source $ueenv + + set rrrrr = "" + set aaa = `echo $vvvvv | grep -E '^CP|^TP|^LgSP' | tr 'A-z' ' '` + set aaa = `echo $aaa` + # + # Special case of formatting the times for accumulated precip + # + if ( "$aaa" != "" ) then + if ( -x ./gtasUtil ) then + set gtasUtil = ./gtasUtil + else if ( -x $mydir/gtasUtil ) then + set gtasUtil = $mydir/gtasUtil + else if ( -x $FXA_HOME/src/dm/point/gtasUtil ) then + set gtasUtil = $FXA_HOME/src/dm/point/gtasUtil + else if ( -x $FXA_HOME/bin/gtasUtil ) then + set gtasUtil = $FXA_HOME/bin/gtasUtil + else + bash -c "echo could not find gtasUtil executable 1>&2" + exit + endif + @ i++ + set t = `echo $* | cut '-d ' -f${i}-$#argv` + @ fff = $t[3] * 3600 + set vt = `$gtasUtil = $t[1] ${t[2]}:00:00.0 $fff` + @ aaa = $aaa * 3600 + set bt = `$gtasUtil = $vt -$aaa` + set rrrrr = "[$bt--$vt]" + endif + + # + # Modify the text of special tags in stub to create finalized script. + # + set specpy = /tmp/a2rdmdl${$}.py + rm -rf $specpy >& /dev/null + touch $specpy + chmod 775 $specpy + if ( "$5" == "" ) then + cat $stubpy | grep -v $dimStr | sed "s/SSSSS/$sss/g" | \ + sed 's/^.*TTTTT.*$//g' | sed 's/^.*LLLLL.*$//g' | \ + sed 's/^.*22222.*$//g' | sed "s/VVVVV/$1/g" | sed "s/DDDDD/$2/g" | \ + sed "s/HHHHH/$3/g" | sed "s/FFFFF/$4/g" | sed "s/RRRRR/$rrrrr/g" >> \ + $specpy + else if ( "$6" == "" ) then + cat $stubpy | grep -v $dimStr | sed "s/SSSSS/$sss/g" | \ + sed "s/TTTTT/$1/g" | sed 's/^.*LLLLL.*$//g' | sed 's/^.*22222.*$//g' | \ + sed "s/VVVVV/$2/g" | sed "s/DDDDD/$3/g" | \ + sed "s/HHHHH/$4/g" | sed "s/FFFFF/$5/g" | sed "s/RRRRR/$rrrrr/g" >> \ + $specpy + else if ( "$7" == "" ) then + cat $stubpy | grep -v $dimStr | sed "s/SSSSS/$sss/g" | \ + sed "s/TTTTT/$1/g" | sed "s/LLLLL/$2/g" | sed 's/^.*22222.*$//g' | \ + sed "s/VVVVV/$3/g" | sed "s/DDDDD/$4/g" | \ + sed "s/HHHHH/$5/g" | sed "s/FFFFF/$6/g" | sed "s/RRRRR/$rrrrr/g" >> \ + $specpy + else + cat $stubpy | grep -v $dimStr | sed "s/SSSSS/$sss/g" | \ + sed "s/TTTTT/$1/g" | sed "s/LLLLL/$2/g" | sed "s/22222/$3/g" | \ + sed "s/VVVVV/$4/g" | sed "s/DDDDD/$5/g" | \ + sed "s/HHHHH/$6/g" | sed "s/FFFFF/$7/g" | sed "s/RRRRR/$rrrrr/g" >> \ + $specpy + endif + # + # Submit the temporary python script stripping any xml stuff, then remove it + # + cd $UE_BIN_PATH + ( uengine -r python < $specpy ) | grep -v '<' | grep -v Response + if ( "$rmpy" == "yes" ) rm -rf $specpy >& /dev/null +endif diff --git a/maritime/a2cvboyStub.py b/maritime/a2cvboyStub.py new file mode 100644 index 0000000..8f42369 --- /dev/null +++ b/maritime/a2cvboyStub.py @@ -0,0 +1,160 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# Gets all available Laps maritime data in the A-II database over a specified +# range of times. The data is output to stdout as ASCII. Each line is one +# time/platform combination. The individual data items are comma delimited. +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 09/18/2014 3591 nabowle Initial modification. Replace UEngine with DAF. +# +# + +import argparse +import sys + +from datetime import datetime +from ufpy.dataaccess import DataAccessLayer +from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange + +def get_args(): + parser = argparse.ArgumentParser(conflict_handler="resolve") + parser.add_argument("-h", action="store", dest="host", + help="EDEX server hostname (optional)", + metavar="hostname") + parser.add_argument("-b", action="store", dest="start", + help="The start of the time range in YYYY-MM-DD HH:MM", + metavar="start") + parser.add_argument("-e", action="store", dest="end", + help="The end of the time range in YYYY-MM-DD HH:MM", + metavar="end") + return parser.parse_args() + + +def main(): + user_args = get_args() + + if user_args.host: + DataAccessLayer.changeEDEXHost(user_args.host) + + start = user_args.start + end = user_args.end + + if not start or not end: + print >> sys.stderr, "Start or End date not provided" + return + + beginRange = datetime.strptime( start + ":00.0", "%Y-%m-%d %H:%M:%S.%f") + endRange = datetime.strptime( end + ":59.9", "%Y-%m-%d %H:%M:%S.%f") + timerange = TimeRange(beginRange, endRange) + + req = DataAccessLayer.newDataRequest("sfcobs") + req.setParameters("stationId","timeObs","elevation","reportType", + "wx_present","visibility","seaLevelPress","stationPress", + "pressChange3Hour","pressChangeChar","temperature", + "dewpoint","seaSurfaceTemp","wetBulb","windDir", + "windSpeed","equivWindSpeed10m","windGust","precip1Hour", + "precip6Hour","precip24Hour" ) + geometries = DataAccessLayer.getGeometryData(req, timerange) + + if not geometries : +# print "No data available." + return + + # Initialize conversion array for wx. + wxstr = [ " ", " ", " ", " ", "FU", "HZ", "DU", "BLSA", "PO", "VCSS", \ + "BR", "BCFG", "MIFG", "VCTS", "VCSH", "VCSH", "VCSH", " ", "SQ", "+FC", \ + "DZ", "RA", "SN", "RA SN", "FZRA", "SHRA", "SHRA SHSN", "SHGR", "FG FZFG", "TS", \ + "SS", "SS", "SS", "+SS", "+SS", "+SS", "DRSN", " ", "BLSN", "+BLSN", \ + "VCFG", "BCFG", "FG FZFG", "FG FZFG", "FG FZFG", "FG FZFG", "FG FZFG", "FG FZFG", "FZFG", "FZFG", \ + "-DZ", "-DZ", "DZ", "DZ", "+DZ", "+DZ", "-FZDZ", "FZDZ", "-DZ -RA", "DZ RA", \ + "-RA", "-RA", "RA", "RA", "+RA", "+RA", "-FZRA", "FZRA", "-RA -SN", "RA SN", \ + "-SN", "-SN", "SN", "SN", "+SN", "+SN", "IC", "SG", "IC", "PE", \ + "-SHRA", "SHRA", "+SHRA", "-SHSN -SHRA", "SHSN SHRA", "-SNSN", "SHSN", "-SHPE", "SHPE", " ", \ + "SHGR", "-RA", "+RA", "-RA -SN -GR", "+RA +SN +GR", "TSRA", "TSPE", "+TSRA", " ", "+TSPE" ] + + msg = "" + for geo in geometries : + lon = geo.getGeometry().x + lat = geo.getGeometry().y + + sName = geo.getString("stationId") + tobs = geo.getNumber("timeObs") + elev = geo.getNumber("elevation") + typ = geo.getNumber("reportType") + wx = geo.getNumber("wx_present") + vis = geo.getNumber("visibility") + msl = geo.getNumber("seaLevelPress") + p = geo.getNumber("stationPress") + pchg = geo.getNumber("pressChange3Hour") + pchr = geo.getNumber("pressChangeChar") + temp = geo.getNumber("temperature") + dpt = geo.getNumber("dewpoint") + th2o = geo.getNumber("seaSurfaceTemp") + tw = geo.getNumber("wetBulb") + dir = geo.getNumber("windDir") + spd = geo.getNumber("windSpeed") + s10 = geo.getNumber("equivWindSpeed10m") + gust = geo.getNumber("windGust") + pr1 = geo.getNumber("precip1Hour") + pr6 = geo.getNumber("precip6Hour") + pr24 = geo.getNumber("precip24Hour") + + msg += sName + "," + msg += str(tobs/1000) + "," + msg += "%.4f"%lat + "," + msg += "%.4f"%lon + "," + msg += "%.0f"%elev + "," + if typ < 1001 or typ > 1007 : + msg += "-32767," + elif typ == 1001 or typ == 1004 or typ == 1005 : + msg += "0," + else : + msg += "1," + if wx < 0 or wx > 99 : + msg += " ," + else : + msg += wxstr[wx] + "," + msg += str(vis) + "," + msg += "%.2f"%msl + "," + msg += "%.2f"%p + "," + msg += "%.0f"%pchg + "," + if pchr <= -9999 : + pchr = -32767 + msg += str(pchr) + " ," + msg += "%.1f"%temp + "," + msg += "%.1f"%dpt + "," + msg += "%.1f"%th2o + "," + msg += "%.1f"%tw + "," + msg += "%.0f"%dir + "," + msg += "%.1f"%spd + "," + msg += "%.1f"%s10 + "," + msg += "%.1f"%gust + "," + msg += "%.2f"%pr1 + "," + msg += "%.2f"%pr6 + "," + msg += "%.2f"%pr24 + "\n" + + print msg.strip() + +if __name__ == '__main__': + main() diff --git a/maritime/a2gtboy.csh b/maritime/a2gtboy.csh new file mode 100644 index 0000000..0d7e7d9 --- /dev/null +++ b/maritime/a2gtboy.csh @@ -0,0 +1,134 @@ +#!/bin/csh +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## +# +# A script wrapper around a UEngine call that is meant to get all available +# maritime data in the A-II database over a specified range of times. +# The data is output to stdout as ASCII. Each line is one time/platform +# combination. The individual data items are comma delimited. +# This version can adapt to use a python stub that calls the +# data access framework. +# +# Usage: +# +# a2gtboy.csh {p} {c} yyyy-mm-dd hh:mm yyyy-mm-dd hh:mm +# +# The literal p and c flags are optional. The p flag means preserve +# the final version of the python submitted to the UEngine instead of +# cleaning it up. The path to the finalized python is /tmp/a2gtboyNNNNN.py +# where NNNNN is a unix process id. The c flag means to retreive the +# Laps set of variables, instead of the default MSAS set. +# +# +# Not using the 'c' format, the MSAS set of variables, outputs the following +# variables for each line: +# +# stationId,timeObs,latitude,longitude,elevation,seaLevelPress, +# stationPress,temperature,dewpoint,windDir,windSpeed,pressChange3Hour +# +# Using the 'c' format, the Laps set of variables, outputs the following +# variables for each line: +# +# stationId,timeObs,latitude,longitude,elevation,reportType,wx_present, +# visibility,seaLevelPress,stationPress,pressChange3Hour,pressChangeChar, +# temperature,dewpoint,seaSurfaceTemp,wetBulb,windDir,windSpeed, +# equivWindSpeed10m,windGust,precip1Hour,precip6Hour,precip24Hour +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 09/22/2014 3591 nabowle Initial modification. Properly call DAF script. +# +# +set rmpy = yes +if ( "$1" == "p" ) then + set rmpy = no + shift +endif +# +# Identify directory this script is in, will be one of the directories we +# search for other files in. +# +set mydir = `dirname $0` +set d1 = `echo $mydir | cut -c1` +if ( "$mydir" == '.' ) then + set mydir = $PWD +else if ( "$d1" != "/" ) then + set mydir = $PWD/$mydir +endif +set mydir = `(cd $mydir ; pwd)` +if ( ! $?FXA_HOME ) set FXA_HOME = xxxx +# +set stubbase = a2gtboyStub.py +if ( "$1" == "c" ) then + shift + set stubbase = a2cvboyStub.py +endif +# +# Locate python stub that we will modify to create the final python logic. +# +if ( -e ./$stubbase ) then + set stubpy = ./$stubbase +else if ( -e $mydir/$stubbase ) then + set stubpy = $mydir/$stubbase +else if ( -e $FXA_HOME/src/dm/maritime/$stubbase ) then + set stubpy = $FXA_HOME/src/dm/maritime/$stubbase +else if ( -e $FXA_HOME/bin/$stubbase ) then + set stubpy = $FXA_HOME/bin/$stubbase +else + bash -c "echo could not find $stubbase 1>&2" + exit +endif +# +# Determine if we are using the data access framework or the uEngine. +# +grep DataAccessLayer $stubpy >& /dev/null +if ( $status == 0 ) then + /awips2/python/bin/python $stubpy -b "$1 $2" -e "$3 $4" +else + # + # Set up the environment we need to run the UEngine. + # + if ( -e ./UEngine.cshsrc ) then + set ueenv = ./UEngine.cshsrc + else if ( -e $mydir/UEngine.cshsrc ) then + set ueenv = $mydir/UEngine.cshsrc + else if ( -e $FXA_HOME/src/dm/point/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/src/dm/point/UEngine.cshsrc + else if ( -e $FXA_HOME/bin/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/bin/UEngine.cshsrc + else + bash -c "echo could not find UEngine.cshsrc 1>&2" + exit + endif + source $ueenv + + set specpy = /tmp/a2gtboy${$}.py + rm -rf $specpy >& /dev/null + touch $specpy + chmod 775 $specpy + cat $stubpy | sed "s/BBBBB/$1 $2/g" | sed "s/EEEEE/$3 $4/g" > $specpy + + cd $UE_BIN_PATH + ( uengine -r python < $specpy ) | grep -v '<' | sed -n '2,$p' + + if ( "$rmpy" == "yes" ) rm -rf $specpy >& /dev/null +endif diff --git a/maritime/a2gtboyStub.py b/maritime/a2gtboyStub.py new file mode 100644 index 0000000..64e9d47 --- /dev/null +++ b/maritime/a2gtboyStub.py @@ -0,0 +1,114 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# Gets all available MSAS maritime data in the A-II database over a specified +# range of times. The data is output to stdout as ASCII. Each line is one +# time/platform combination. The individual data items are comma delimited. +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 09/18/2014 3591 nabowle Initial modification. Replace UEngine with DAF. +# +# + +import argparse +import sys + +from datetime import datetime +from ufpy.dataaccess import DataAccessLayer +from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange + +def get_args(): + parser = argparse.ArgumentParser(conflict_handler="resolve") + parser.add_argument("-h", action="store", dest="host", + help="EDEX server hostname (optional)", + metavar="hostname") + parser.add_argument("-b", action="store", dest="start", + help="The start of the time range in YYYY-MM-DD HH:MM", + metavar="start") + parser.add_argument("-e", action="store", dest="end", + help="The end of the time range in YYYY-MM-DD HH:MM", + metavar="end") + return parser.parse_args() + + +def main(): + user_args = get_args() + + if user_args.host: + DataAccessLayer.changeEDEXHost(user_args.host) + + start = user_args.start + end = user_args.end + + if not start or not end: + print >> sys.stderr, "Start or End date not provided" + return + + beginRange = datetime.strptime( start + ":00.0", "%Y-%m-%d %H:%M:%S.%f") + endRange = datetime.strptime( end + ":59.9", "%Y-%m-%d %H:%M:%S.%f") + timerange = TimeRange(beginRange, endRange) + + req = DataAccessLayer.newDataRequest("sfcobs") + req.setParameters("stationId","timeObs","elevation","seaLevelPress", + "stationPress","temperature","dewpoint","windDir", + "windSpeed","pressChange3Hour" ) + geometries = DataAccessLayer.getGeometryData(req, timerange) + + if not geometries : +# print "No data available." + return + + msg = "" + for geo in geometries : + lon = geo.getGeometry().x + lat = geo.getGeometry().y + + sName = geo.getString("stationId") + tobs = geo.getNumber("timeObs") + elev = geo.getNumber("elevation") + msl = geo.getNumber("seaLevelPress") + p = geo.getNumber("stationPress") + temp = geo.getNumber("temperature") + dpt = geo.getNumber("dewpoint") + dir = geo.getNumber("windDir") + spd = geo.getNumber("windSpeed") + pchg = geo.getNumber("pressChange3Hour") + + + msg += sName + "," + msg += str(tobs/1000) + "," + msg += "%.4f"%lat + "," + msg += "%.4f"%lon + "," + msg += "%.0f"%elev + "," + msg += "%.2f"%msl + "," + msg += "%.2f"%p + "," + msg += "%.1f"%temp + "," + msg += "%.1f"%dpt + "," + msg += "%.0f"%dir + "," + msg += "%.1f"%spd + "," + msg += "%.0f"%pchg + "\n" + + print msg.strip() + +if __name__ == '__main__': + main() diff --git a/metar/a2cvmtrStub.py b/metar/a2cvmtrStub.py new file mode 100644 index 0000000..43bfcc5 --- /dev/null +++ b/metar/a2cvmtrStub.py @@ -0,0 +1,210 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# Gets all available Laps metar data in the A-II database over a specified range +# of times within a specifed area. The data is output to stdout as ASCII. +# Each line is one time/station combination. The individual data items are comma +# delimited. +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 09/15/2014 3593 nabowle Initial modification. Fix index issues on 2D parameters. +# 09/15/2014 3593 nabowle Replace UEngine with DAF. +# +# + +import argparse +import sys + +from datetime import datetime +from ufpy.dataaccess import DataAccessLayer +from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange + +def get_args(): + parser = argparse.ArgumentParser(conflict_handler="resolve") + parser.add_argument("-h", action="store", dest="host", + help="EDEX server hostname (optional)", + metavar="hostname") + parser.add_argument("-b", action="store", dest="start", + help="The start of the time range in YYYY-MM-DD HH:MM", + metavar="start") + parser.add_argument("-e", action="store", dest="end", + help="The end of the time range in YYYY-MM-DD HH:MM", + metavar="end") + parser.add_argument("--lat-min", action="store", dest="latMin", type=float, + help="Minimum latitude", default=0.0, metavar="lat") + parser.add_argument("--lat-max", action="store", dest="latMax", type=float, + help="Maximum latitude", default=90.0, metavar="lat") + parser.add_argument("--lon-min", action="store", dest="lonMin", type=float, + help="Minimum longitude", default=-180.0, metavar="lon") + parser.add_argument("--lon-max", action="store", dest="lonMax", type=float, + help="Maximum longitude", default=180.0, metavar="lon") + return parser.parse_args() + + +def main(): + # The multi-dimensional parameters. + PRES_PARAMS = set(["presWeather"]) + SKY_PARAMS = set(["skyCover", "skyLayerBase"]) + + user_args = get_args() + + if user_args.host: + DataAccessLayer.changeEDEXHost(user_args.host) + + start = user_args.start + end = user_args.end + + if not start or not end: + print >> sys.stderr, "Start or End date not provided" + return + + latMin = user_args.latMin + latMax = user_args.latMax + lonMin = user_args.lonMin + lonMax = user_args.lonMax + + beginRange = datetime.strptime( start + ":00.0", "%Y-%m-%d %H:%M:%S.%f") + endRange = datetime.strptime( end + ":59.9", "%Y-%m-%d %H:%M:%S.%f") + timerange = TimeRange(beginRange, endRange) + + req = DataAccessLayer.newDataRequest("obs") + req.setParameters("stationName","timeObs","wmoId","autoStationType", + "elevation","reportType","presWeather","visibility","skyCover", + "skyLayerBase","altimeter","seaLevelPress","pressChange3Hour", + "pressChangeChar","temperature","tempFromTenths","dewpoint", + "dpFromTenths","windDir","windSpeed","windGust","maxTemp24Hour", + "minTemp24Hour""","precip1Hour","precip3Hour","precip6Hour", + "precip24Hour") + geometries = DataAccessLayer.getGeometryData(req, timerange) + + if not geometries : +# print "No data available." + return + + msg = "" + wx = [] + cvr = [] + bas = [] + for geoData in geometries: + if set(geoData.getParameters()) & PRES_PARAMS : + wx.append(geoData.getString("presWeather")) + continue + if set(geoData.getParameters()) & SKY_PARAMS : + cvr.append(geoData.getString("skyCover")) + bas.append(geoData.getNumber("skyLayerBase")) + continue + + lon = geoData.getGeometry().x + lat = geoData.getGeometry().y + if lon < lonMin or lon > lonMax or lat < latMin or lat > latMax: + wx = [] + cvr = [] + bas = [] + continue + + sName = geoData.getString("stationName") + tobs = geoData.getNumber("timeObs") + elev = geoData.getNumber("elevation") + ista = geoData.getNumber("wmoId") + atype = geoData.getString("autoStationType") + repTyp = geoData.getString("reportType") + vis = geoData.getNumber("visibility") + alt = geoData.getNumber("altimeter") + msl = geoData.getNumber("seaLevelPress") + pchg = geoData.getNumber("pressChange3Hour") + pchr = geoData.getString("pressChangeChar") + temp = geoData.getNumber("temperature") + t10 = geoData.getNumber("tempFromTenths") + dpt = geoData.getNumber("dewpoint") + td10 = geoData.getNumber("dpFromTenths") + dir = geoData.getNumber("windDir") + spd = geoData.getNumber("windSpeed") + gust = geoData.getNumber("windGust") + tmx = geoData.getNumber("maxTemp24Hour") + tmn = geoData.getNumber("minTemp24Hour") + pr1 = geoData.getNumber("precip1Hour") + pr3 = geoData.getNumber("precip3Hour") + pr6 = geoData.getNumber("precip6Hour") + pr24 = geoData.getNumber("precip24Hour") + + + msg += sName + "," + msg += str(tobs/1000) + "," + msg += "%.4f"%lat + "," + msg += "%.4f"%lon + "," + msg += "%.0f"%elev + "," + if ista < 0 : + msg += "-99," + else : + msg += str(ista) + "," + msg += atype + " ," + msg += repTyp + " ," + msg += wx[0] + " ," + msg += "%.3f"%vis + "," + + + msg += cvr[0]; + kk = 5 + while kk > 0 and cvr[0+kk] == "" : + kk -= 1 + k = 1 + while k <= kk : + msg += "|" + cvr[0+k]; + k += 1 + msg += " ," + msg += "%.1f"%bas[0]; + kk = 5 + while kk > 0 and bas[0+kk] < -9998 : + kk -= 1 + k = 1 + while k <= kk : + msg += "|" + "%.1f"%bas[0+k]; + k += 1 + msg += "," + + msg += "%.2f"%alt + "," + msg += "%.2f"%msl + "," + msg += "%.0f"%pchg + "," + msg += pchr + " ," + msg += "%.1f"%temp + "," + msg += "%.1f"%t10 + "," + msg += "%.1f"%dpt + "," + msg += "%.1f"%td10 + "," + msg += "%.0f"%dir + "," + msg += "%.1f"%spd + "," + msg += "%.1f"%gust + "," + msg += "%.1f"%tmx + "," + msg += "%.1f"%tmn + "," + msg += "%.2f"%pr1 + "," + msg += "%.2f"%pr3 + "," + msg += "%.2f"%pr6 + "," + msg += "%.2f"%pr24 + "\n" + + wx = [] + cvr = [] + bas = [] + + print msg.strip() + +if __name__ == '__main__': + main() diff --git a/metar/a2gtmtr.csh b/metar/a2gtmtr.csh new file mode 100755 index 0000000..4da9251 --- /dev/null +++ b/metar/a2gtmtr.csh @@ -0,0 +1,181 @@ +#!/bin/csh +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# +# A script wrapper around a UEngine call that is meant to get all available +# metar data in the A-II database over a specified range of times. The data +# is output to stdout as ASCII. Each line is one time/station combination. +# The individual data items are comma delimited. This version can adapt to +# use apython stub that calls the data access framework. +# +# Usage: +# +# a2gtmtr.csh {p} {c} yyyy-mm-dd hh:mm yyyy-mm-dd hh:mm +# +# The literal p and c flags are optional. The p flag means preserve +# the final version of the python submitted to the UEngine instead of +# cleaning it up. The path to the finalized python is /tmp/a2gtmtrNNNNN.py +# where NNNNN is a unix process id. The c flag means to retreive the +# Laps set of variables, instead of the default MSAS set. +# +# Not using the 'c' format, the MSAS set of variables, outputs the following +# variables for each line: +# +# stationName,timeObs,latitude,longitude,elevation,wmoId,autoStationType +# seaLevelPress,temperature,dewpoint,windDir,windSpeed,altimeter +# +# Using the 'c' format, the Laps set of variables, outputs the following +# variables for each line: +# +# stationName,timeObs,latitude,longitude,elevation,wmoId,autoStationType +# reportType,presWeather,visibility,skyCover,skyLayerBase,altimeter +# seaLevelPress,pressChange3Hour,pressChangeChar,temperature,tempFromTenths +# dewpoint,dpFromTenths,windDir,windSpeed,windGust,maxTemp24Hour, +# minTemp24Hour,precip1Hour,precip3Hour,precip6Hour,precip24Hour +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 09/15/2014 3593 nabowle Initial modification to properly use DAF version of the script. +# +# + +set rmpy = yes +if ( "$1" == "p" ) then + set rmpy = no + shift +endif +# +# Identify directory this script is in, will be one of the directories we +# search for other files in. +# +set mydir = `dirname $0` +set d1 = `echo $mydir | cut -c1` +if ( "$mydir" == '.' ) then + set mydir = $PWD +else if ( "$d1" != "/" ) then + set mydir = $PWD/$mydir +endif +set mydir = `(cd $mydir ; pwd)` +if ( ! $?FXA_HOME ) set FXA_HOME = xxxx +if ( ! $?FXA_LOCAL_SITE ) set FXA_LOCAL_SITE = xxxx +if ( ! $?FXA_INGEST_SITE ) set FXA_INGEST_SITE = $FXA_LOCAL_SITE +# +set stubbase = a2gtmtrStub.py +if ( "$1" == "c" ) then + shift + set stubbase = a2cvmtrStub.py +endif +# +# Locate python stub that we will modify to create the final python logic. +# +if ( -e ./$stubbase ) then + set stubpy = ./$stubbase +else if ( -e $mydir/$stubbase ) then + set stubpy = $mydir/$stubbase +else if ( -e $FXA_HOME/src/dm/metar/$stubbase ) then + set stubpy = $FXA_HOME/src/dm/metar/$stubbase +else if ( -e $FXA_HOME/bin/$stubbase ) then + set stubpy = $FXA_HOME/bin/$stubbase +else + bash -c "echo could not find $stubbase 1>&2" + exit +endif + +# +# Attempt to use current D-2D localization to determine lat/lon bounds. +# +set ltmn = 0 +set ltmx = 90 +set lnmn = -180 +set lnmx = 180 +set locDir = $FXA_HOME/data/localizationDataSets/$FXA_INGEST_SITE +while ( -e $locDir/CenterPoint.dat ) + grep conusScale $locDir/scaleInfo.txt >& /dev/null + if ( $status == 0 ) then + set ltmn = 15 + set ltmx = 60 + set lnmn = -145 + set lnmx = -55 + break + endif + set cenLoc = `cat $locDir/CenterPoint.dat` + if ( $#cenLoc != 2 ) break + set cenlat = `echo $cenLoc[1] | cut '-d.' -f1` + set cenlat = `( @ x = $cenlat + 0 >& /dev/null ; echo $x )` + if ( "$cenlat" == "" ) break + set cenlon = `echo $cenLoc[2] | cut '-d.' -f1` + set cenlon = `( @ x = $cenlon + 0 >& /dev/null ; echo $x )` + if ( "$cenlon" == "" ) break + if ( $cenlat > 75 ) then + set ltmn = 55 + break + endif + if ( $cenlat > 50 ) then + @ ltmn = $cenlat - 20 + break + endif + @ ltmn = $cenlat - 20 + @ ltmx = $cenlat + 20 + @ lnmn = $cenlon - 20 + @ lnmx = $cenlon + 20 + break +end +# +# +# Determine if we are using the data access framework or the uEngine. +# +grep DataAccessLayer $stubpy >& /dev/null +if ( $status == 0 ) then + /awips2/python/bin/python $stubpy -b "$1 $2" -e "$3 $4" --lat-min $ltmn --lat-max $ltmx --lon-min $lnmn --lon-max $lnmx +else + # + # Set up the environment we need to run the UEngine. + # + set method = "uengine" + if ( -e ./UEngine.cshsrc ) then + set ueenv = ./UEngine.cshsrc + else if ( -e $mydir/UEngine.cshsrc ) then + set ueenv = $mydir/UEngine.cshsrc + else if ( -e $FXA_HOME/src/dm/point/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/src/dm/point/UEngine.cshsrc + else if ( -e $FXA_HOME/bin/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/bin/UEngine.cshsrc + else + bash -c "echo could not find UEngine.cshsrc 1>&2" + exit + endif + source $ueenv + + set specpy = /tmp/a2gtmtr${$}.py + rm -rf $specpy >& /dev/null + touch $specpy + chmod 775 $specpy + cat $stubpy | sed "s/LTMN/$ltmn/g" | sed "s/LTMX/ /g" | \ + sed "s/LNMN/$lnmn/g" | sed "s/LNMX/$lnmx/g" | \ + sed "s/BBBBB/$1 $2/g" | sed "s/EEEEE/$3 $4/g" > $specpy + cd $UE_BIN_PATH + ( uengine -r python < $specpy ) | grep -v '<' | sed -n '3,$p' + if ( "$rmpy" == "yes" ) rm -rf $specpy >& /dev/null +endif +# diff --git a/metar/a2gtmtrStub.py b/metar/a2gtmtrStub.py new file mode 100644 index 0000000..55faf83 --- /dev/null +++ b/metar/a2gtmtrStub.py @@ -0,0 +1,132 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# Gets all available metar data in the A-II database over a specified range of +# times within a specifed area. The data is output to stdout as ASCII. +# Each line is one time/station combination. The individual data items are comma +# delimited. +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 09/15/2014 3593 nabowle Initial modification. Fix losing first record. +# 09/15/2014 3593 nabowle Replace UEngine with DAF. +# +# + +import argparse +import sys + +from datetime import datetime +from ufpy.dataaccess import DataAccessLayer +from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange + +def get_args(): + parser = argparse.ArgumentParser(conflict_handler="resolve") + parser.add_argument("-h", action="store", dest="host", + help="EDEX server hostname (optional)", + metavar="hostname") + parser.add_argument("-b", action="store", dest="start", + help="The start of the time range in YYYY-MM-DD HH:MM", + metavar="start") + parser.add_argument("-e", action="store", dest="end", + help="The end of the time range in YYYY-MM-DD HH:MM", + metavar="end") + parser.add_argument("--lat-min", action="store", dest="latMin", type=float, + help="Minimum latitude", default=0.0, metavar="lat") + parser.add_argument("--lat-max", action="store", dest="latMax", type=float, + help="Maximum latitude", default=90.0, metavar="lat") + parser.add_argument("--lon-min", action="store", dest="lonMin", type=float, + help="Minimum longitude", default=-180.0, metavar="lon") + parser.add_argument("--lon-max", action="store", dest="lonMax", type=float, + help="Maximum longitude", default=180.0, metavar="lon") + return parser.parse_args() + + +def main(): + user_args = get_args() + + if user_args.host: + DataAccessLayer.changeEDEXHost(user_args.host) + + start = user_args.start + end = user_args.end + + if not start or not end: + print >> sys.stderr, "Start or End date not provided" + return + + latMin = user_args.latMin + latMax = user_args.latMax + lonMin = user_args.lonMin + lonMax = user_args.lonMax + + beginRange = datetime.strptime( start + ":00.0", "%Y-%m-%d %H:%M:%S.%f") + endRange = datetime.strptime( end + ":59.9", "%Y-%m-%d %H:%M:%S.%f") + timerange = TimeRange(beginRange, endRange) + + req = DataAccessLayer.newDataRequest("obs") + req.setParameters("stationName","timeObs","wmoId","autoStationType", + "elevation","seaLevelPress","temperature","dewpoint", + "windDir","windSpeed","altimeter" ) + geometries = DataAccessLayer.getGeometryData(req, timerange) + + if not geometries : +# print "No data available." + return + + msg = "" + for geo in geometries : + lon = geo.getGeometry().x + lat = geo.getGeometry().y + if lon < lonMin or lon > lonMax or lat < latMin or lat > latMax: + continue + + sName = geo.getString("stationName") + tobs = geo.getNumber("timeObs") + elev = geo.getNumber("elevation") + ista = geo.getString("wmoId") + atype = geo.getString("autoStationType") + msl = geo.getNumber("seaLevelPress") + temp = geo.getNumber("temperature") + dpt = geo.getNumber("dewpoint") + dir = geo.getNumber("windDir") + spd = geo.getNumber("windSpeed") + alt = geo.getNumber("altimeter") + + msg += sName + "," + msg += str(tobs/1000) + "," + msg += "%.4f"%lat + "," + msg += "%.4f"%lon + "," + msg += "%.0f"%elev + "," + msg += str(ista) + "," + msg += atype + " ," + msg += "%.2f"%msl + "," + msg += "%.1f"%temp + "," + msg += "%.1f"%dpt + "," + msg += "%.0f"%dir + "," + msg += "%.1f"%spd + "," + msg += "%.2f"%alt + "\n" + + print msg.strip() + +if __name__ == '__main__': + main() diff --git a/profiler/a2gtprof.csh b/profiler/a2gtprof.csh new file mode 100644 index 0000000..855eb64 --- /dev/null +++ b/profiler/a2gtprof.csh @@ -0,0 +1,129 @@ +#!/bin/csh +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## +# +# A script wrapper around a UEngine call that is meant to get all available +# profiler data in the A-II database over a specified range of times. The +# data is output to stdout as ASCII. Each line is one time/station combination. +# The individual data variables are comma delimited, and when what is returned +# for a data item is a profile, each item in the profile is vertical bar +# delimited. This version can adapt to use a python stub that calls the +# data access framework. +# +# Usage: +# +# a2gtprof.csh {p} yyyy-mm-dd hh:mm yyyy-mm-dd hh:mm +# +# The literal p flag is optional. The p flag means preserve +# the final version of the python submitted to the UEngine instead of +# cleaning it up. The path to the finalized python is /tmp/a2gtprofNNNNN.py +# where NNNNN is a unix process id. +# +# The following data variables are output for each line: +# +# profilerId,validTime,latitude,longitude,elevation,pressure, +# temperature,relHumidity,windDirSfc,windSpeedSfc,rainRate,submode, +# height,levelMode,uComponent,vComponent,wComponent, +# peakPower,HorizSpStdDev,VertSpStdDev,uvQualityCode,consensusNum +# +# Everything from height onward are profiles. +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# Oct 6, 2014 3594 nabowle Initial modification. Handle DAF version. +# +# +set rmpy = yes +if ( "$1" == "p" ) then + set rmpy = no + shift +endif +# +# Identify directory this script is in, will be one of the directories we +# search for other files in. +# +set mydir = `dirname $0` +set d1 = `echo $mydir | cut -c1` +if ( "$mydir" == '.' ) then + set mydir = $PWD +else if ( "$d1" != "/" ) then + set mydir = $PWD/$mydir +endif +set mydir = `(cd $mydir ; pwd)` +if ( ! $?FXA_HOME ) set FXA_HOME = xxxx +if ( ! $?FXA_LOCAL_SITE ) set FXA_LOCAL_SITE = xxxx +if ( ! $?FXA_INGEST_SITE ) set FXA_INGEST_SITE = $FXA_LOCAL_SITE +# +# Locate python stub that we will modify to create the final python logic. +# +set stubbase = a2gtprofStub.py +if ( -e ./$stubbase ) then + set stubpy = ./$stubbase +else if ( -e $mydir/$stubbase ) then + set stubpy = $mydir/$stubbase +else if ( -e $FXA_HOME/src/dm/profiler/$stubbase ) then + set stubpy = $FXA_HOME/src/dm/profiler/$stubbase +else if ( -e $FXA_HOME/bin/$stubbase ) then + set stubpy = $FXA_HOME/bin/$stubbase +else + bash -c "echo could not find $stubbase 1>&2" + exit +endif +# +# Determine if we are using the data access framework or the uEngine. +# +grep DataAccessLayer $stubpy >& /dev/null +if ( $status == 0 ) then + /awips2/python/bin/python $stubpy -b "$1 $2" -e "$3 $4" +else + # + # Set up the environment we need to run the UEngine. + # + set method = "uengine" + if ( -e ./UEngine.cshsrc ) then + set ueenv = ./UEngine.cshsrc + else if ( -e $mydir/UEngine.cshsrc ) then + set ueenv = $mydir/UEngine.cshsrc + else if ( -e $FXA_HOME/src/dm/point/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/src/dm/point/UEngine.cshsrc + else if ( -e $FXA_HOME/bin/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/bin/UEngine.cshsrc + else + bash -c "echo could not find UEngine.cshsrc 1>&2" + exit + endif + source $ueenv + + set specpy = /tmp/a2gtprof${$}.py + rm -rf $specpy >& /dev/null + touch $specpy + chmod 775 $specpy + cat $stubpy | sed "s/BBBBB/$1 $2/g" | sed "s/EEEEE/$3 $4/g" > $specpy + + cd $UE_BIN_PATH + #uengine -r python < $specpy + ( uengine -r python < $specpy ) | grep -v '<' | sed -n '2,$p' + + if ( "$rmpy" == "yes" ) rm -rf $specpy >& /dev/null +endif + diff --git a/profiler/a2gtprofStub.py b/profiler/a2gtprofStub.py new file mode 100644 index 0000000..560cff5 --- /dev/null +++ b/profiler/a2gtprofStub.py @@ -0,0 +1,184 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# Gets all available profiler data in the A-II database over a specified range +# of times. The data is output to stdout as ASCII. Each line is one +# time/station combination. The individual data variables are comma delimited, +# and when what is returned for a data item is a profile, each item in the +# profile is vertical bar delimited +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# Oct 6, 2014 3594 nabowle Initial modification. Replace UEngine with DAF. +# +# + +import argparse +import sys + +from datetime import datetime +from ufpy.dataaccess import DataAccessLayer +from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange + +def get_args(): + parser = argparse.ArgumentParser(conflict_handler="resolve") + parser.add_argument("-h", action="store", dest="host", + help="EDEX server hostname (optional)", + metavar="hostname") + parser.add_argument("-b", action="store", dest="start", + help="The start of the time range in YYYY-MM-DD HH:MM", + metavar="start") + parser.add_argument("-e", action="store", dest="end", + help="The end of the time range in YYYY-MM-DD HH:MM", + metavar="end") + return parser.parse_args() + +def main(): + # The multi-dimensional parameters. + MULTI_DIM_PARAMS = set(['vComponent', 'uComponent', 'peakPower', + 'levelMode', 'uvQualityCode', 'consensusNum', + 'HorizSpStdDev', 'wComponent', 'height', + 'VertSpStdDev']) + + user_args = get_args() + + if user_args.host: + DataAccessLayer.changeEDEXHost(user_args.host) + + start = user_args.start + end = user_args.end + + if not start or not end: + print >> sys.stderr, "Start or End date not provided" + return + + beginRange = datetime.strptime( start + ":00.0", "%Y-%m-%d %H:%M:%S.%f") + endRange = datetime.strptime( end + ":59.9", "%Y-%m-%d %H:%M:%S.%f") + timerange = TimeRange(beginRange, endRange) + + req = DataAccessLayer.newDataRequest("profiler") + req.setParameters('numProfLvls', 'elevation', 'windDirSfc', 'validTime', + 'windSpeedSfc', 'pressure', 'submode', 'relHumidity', + 'profilerId', 'rainRate', 'temperature') + req.getParameters().extend(MULTI_DIM_PARAMS) + + geometries = DataAccessLayer.getGeometryData(req, timerange) + + if not geometries : +# print "couldn't get data" + return + + + subgeos = [] + msg = "" + for geoData in geometries : + if set(geoData.getParameters()) & MULTI_DIM_PARAMS : + subgeos.append(geoData) + continue + + elev = geoData.getNumber("elevation") + msg += geoData.getString("profilerId") + "," + msg += str(geoData.getNumber("validTime")/1000) + "," + msg += "%.4f"%geoData.getGeometry().y + "," + msg += "%.4f"%geoData.getGeometry().x + "," + msg += "%.0f"%elev + "," + msg += "%.1f"%geoData.getNumber("pressure") + "," + msg += "%.1f"%geoData.getNumber("temperature") + "," + msg += "%.1f"%geoData.getNumber("relHumidity") + "," + msg += "%.0f"%geoData.getNumber("windDirSfc") + "," + msg += "%.1f"%geoData.getNumber("windSpeedSfc") + "," + msg += "%.1f"%geoData.getNumber("rainRate") + "," + msg += str(geoData.getNumber("submode")) + "," + + kk = len(subgeos) +# msg += str(kk) + "," + + subgeos[0].getString("consensusNum") + if kk>0 : msg += "%.0f"%(subgeos[0].getNumber("height")-elev) + k = 1 + while k < kk : + msg += "|" + "%.0f"%(subgeos[k].getNumber("height")-elev) + k += 1 + msg += "," + if kk>0 : msg += str(subgeos[0].getNumber("levelMode")) + k = 1 + while k < kk : + msg += "|" + str(subgeos[k].getNumber("levelMode")) + k += 1 + msg += "," + + if kk>0 : msg += "%.1f"%subgeos[0].getNumber("uComponent") + k = 1 + while k < kk : + msg += "|" + "%.1f"%subgeos[k].getNumber("uComponent") + k += 1 + msg += "," + if kk>0 : msg += "%.1f"%subgeos[0].getNumber("vComponent") + k = 1 + while k < kk : + msg += "|" + "%.1f"%subgeos[k].getNumber("vComponent") + k += 1 + msg += "," + if kk>0 : msg += "%.2f"%subgeos[0].getNumber("wComponent") + k = 1 + while k < kk : + msg += "|" + "%.2f"%subgeos[k].getNumber("wComponent") + k += 1 + msg += "," + + if kk>0 : msg += "%.1f"%subgeos[0].getNumber("peakPower") + k = 1 + while k < kk : + msg += "|" + "%.1f"%subgeos[k].getNumber("peakPower") + k += 1 + msg += "," + if kk>0 : msg += "%.1f"%subgeos[0].getNumber("HorizSpStdDev") + k = 1 + while k < kk : + msg += "|" + "%.1f"%subgeos[k].getNumber("HorizSpStdDev") + k += 1 + msg += "," + if kk>0 : msg += "%.1f"%subgeos[0].getNumber("VertSpStdDev") + k = 1 + while k < kk : + msg += "|" + "%.1f"%subgeos[k].getNumber("VertSpStdDev") + k += 1 + msg += "," + + if kk>0 : msg += subgeos[0].getString("uvQualityCode") + k = 1 + while k < kk : + msg += "|" + subgeos[k].getString("uvQualityCode") + k += 1 + msg += "," + if kk>0 : msg += subgeos[0].getString("consensusNum") + k = 1 + while k < kk : + msg += "|" + subgeos[k].getString("consensusNum") + k += 1 + msg += "\n" + subgeos = [] + + print msg.strip() + +if __name__ == '__main__': + main() diff --git a/radar/a2advrad.csh b/radar/a2advrad.csh new file mode 100644 index 0000000..f9f0bfc --- /dev/null +++ b/radar/a2advrad.csh @@ -0,0 +1,232 @@ +#!/bin/csh +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# +# A script wrapper that is meant to get data for a single radar product +# from the A-II database. The result is output to stdout as ASCII. +# The first line has the dimensions of the data, the volume scan time, +# 'radial' or 'raster', elevation number, a product description, the tilt +# angle or layer, and the VCP. The second line contains the level labels, +# and the third line has a partial list of the product dependent parameters. +# If the product is radial, the fourth line contains a list of the +# radial angles. Otherwise each line after that has data for one +# radial/row, as a sequence of space delimited plain decimal integers. +# +# The product description includes the mnemonic, the resolution, and +# the bit depth. If the azimuthal resolution is finer than 1 degree, +# the product description will additionally include a note of that. +# The product dependent parameters as defined in an 88D RPG product +# are 28 signed two byte integers. The list here includes those +# with indices from 17 to 26 (C indexing). +# +# Note that for now, this is only designed to return data for image +# products. +# +# Usage: +# +# a2advrad.csh {p} {x} radar msgcode {elev} date time {slop} +# +# p - A literal p. (optional) +# x - A literal x. (optional) Expanded set of header information. +# radar - four character radar id +# msgcode - RPG message code +# elev - tilt angle/layer value. defaults to 0. +# date - yyyy-mm-dd +# time - hh:mm +# slop - seconds of slop either side, defaults to 60 +# +# The tilt angles specified are primary tilt angles for a range of tilts. +# +# The literal p option means preserve the final version of the python +# submitted to the UEngine instead of cleaning it up. The path to the +# finalized python is /tmp/a2advradNNNNN.py where NNNNN is a unix process id. +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 08/07/2014 3393 nabowle Initial modification. Passes parameters straight to +# non-uengine script instead of sed. Remove use of gtasUtil +# if not using the uengine stub. +# +# +set rmpy = yes +set fff = "" +if ( "$1" == "p" ) then + set rmpy = no + shift +endif +if ( "$1" == "x" ) then + set fff = "x" + shift +endif +if ( "$1" == "p" ) then + set rmpy = no + shift +endif +# +# Identify directory this script is in, will be one of the directories we +# search for other files in. +# +set mydir = `dirname $0` +set d1 = `echo $mydir | cut -c1` +if ( "$mydir" == '.' ) then + set mydir = $PWD +else if ( "$d1" != "/" ) then + set mydir = $PWD/$mydir +endif +set mydir = `(cd $mydir ; pwd)` +if ( ! $?FXA_HOME ) set FXA_HOME = xxxx +set fxa_home = $FXA_HOME +if ( $?STAGING ) then + set fxa_home = $STAGING/D-2D +endif + +# +# Locate python stub that we will modify to create the final UEngine script +# +if ( -e ./a2advradStub.py ) then + set stubpy = ./a2advradStub.py +else if ( -e $mydir/a2advradStub.py ) then + set stubpy = $mydir/a2advradStub.py +else if ( -e $fxa_home/src/dm/radar/a2advradStub.py ) then + set stubpy = $fxa_home/src/dm/radar/a2advradStub.py +else if ( -e $FXA_HOME/bin/a2advradStub.py ) then + set stubpy = $FXA_HOME/bin/a2advradStub.py +else + bash -c "echo could not find a2advradStub.py 1>&2" + exit +endif + +# +# Determine if we are using the data access framework or the uEngine. +# +grep DataAccessLayer $stubpy >& /dev/null +if ( $status == 0 ) then + set method = "daf" +else + # + # Set up the environment we need to run the UEngine. + # + set method = "uengine" + if ( -e ./UEngine.cshsrc ) then + set ueenv = ./UEngine.cshsrc + else if ( -e $mydir/UEngine.cshsrc ) then + set ueenv = $mydir/UEngine.cshsrc + else if ( -e $FXA_HOME/src/dm/point/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/src/dm/point/UEngine.cshsrc + else if ( -e $FXA_HOME/bin/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/bin/UEngine.cshsrc + else + bash -c "echo could not find UEngine.cshsrc 1>&2" + exit + endif + source $ueenv +endif + +# +# Locate file that lets us provide a description of the data set. +# +if ( -e ./msgCodeSeds.txt ) then + set mctrans = $PWD/msgCodeSeds.txt +else if ( -e $mydir/msgCodeSeds.txt ) then + set mctrans = $mydir/msgCodeSeds.txt +else if ( -e $fxa_home/src/dm/radar/msgCodeSeds.txt ) then + set mctrans = $fxa_home/src/dm/radar/msgCodeSeds.txt +else if ( -e $FXA_HOME/data/msgCodeSeds.txt ) then + set mctrans = $FXA_HOME/data/msgCodeSeds.txt +else + bash -c "echo could not find msgCodeSeds.txt 1>&2" + exit +endif +# +# +set rrr = $1 +set mmm = $2 +shift +shift +set ddd = `echo $mmm | sed -f $mctrans | cut '-d|' -f2 ` + +set eee = `echo $1 | grep -v '.*-'` +if ( "$eee" != "" ) shift +set slop = `echo $3 | grep '[0-9]'` +if ( "$slop" == "" ) set slop = 60 + + +if ( "$method" == "daf" ) then + set datetime = $1' '$2 + set opts = "" + + if ( "$eee" != "" ) then + set opts = "$opts --angle $eee" + endif + + if ( "$fff" == "x" ) then + set opts = "$opts --extended" + endif + + /awips2/python/bin/python $stubpy --radar $rrr --code $mmm --datetime="${datetime}" --slop $slop --description="${ddd}" $opts +else + # + # Get program that can do math with ascii time string, then use this to + # properly encode range of times for which we look for data. + # + if ( -x ./gtasUtil ) then + set gtasUtil = ./gtasUtil + else if ( -x $mydir/gtasUtil ) then + set gtasUtil = $mydir/gtasUtil + else if ( -x $fxa_home/src/dm/point/gtasUtil ) then + set gtasUtil = $fxa_home/src/dm/point/gtasUtil + else if ( -x $FXA_HOME/bin/gtasUtil ) then + set gtasUtil = $FXA_HOME/bin/gtasUtil + else + bash -c "echo could not find gtasUtil executable 1>&2" + exit + endif + + set aaa = `$gtasUtil = $1 $2 -$slop` + set bbb = `$gtasUtil = $1 $2 $slop` + + # + # Modify the text of special tags in stub to create finalized script. + # + set specpy = /tmp/a2advrad${$}.py + rm -rf $specpy >& /dev/null + touch $specpy + chmod 775 $specpy + if ( "$eee" == "" ) then + cat $stubpy | sed "s/KKKK/$rrr/g" | sed "s/MMMM/$mmm/g" | \ + sed "s/AAAAA/$aaa/g" | sed "s/BBBBB/$bbb/g" | sed "s/FFF/$fff/g" | \ + sed "s/DDDDD/$ddd/g" | sed 's/^.*EEEE.*$//g' >> $specpy + else + cat $stubpy | sed "s/KKKK/$rrr/g" | sed "s/MMMM/$mmm/g" | \ + sed "s/AAAAA/$aaa/g" | sed "s/BBBBB/$bbb/g" | sed "s/FFF/$fff/g" | \ + sed "s/DDDDD/$ddd/g" | sed "s/EEEE/$eee/g" >> $specpy + endif + # + # Submit the temporary python script stripping xml stuff, then remove it + # + cd $UE_BIN_PATH + ( uengine -r python < $specpy ) | grep -v '<' | sed 's/>/>/g' | \ + sed 's/</& /dev/null +endif +# diff --git a/radar/a2advradStub.py b/radar/a2advradStub.py new file mode 100644 index 0000000..06dda8f --- /dev/null +++ b/radar/a2advradStub.py @@ -0,0 +1,128 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# +# Gets data for a single radar product from the A-II database. The result is +# output to stdout as ASCII. This uses a data-specific Request/Response instead +# of the DataAccessLayer in order to preserve data-genericness of the interface. +# +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 08/13/2014 3393 nabowle Initial modification. Replaces UEngine +# with a custom Request/Response. +# +# + +import argparse +import a2radcommon + +def get_args(): + parser = a2radcommon.get_args_parser() + parser.add_argument("--extended", action='store_true', default=False, + dest="extended", help="Output the extended header.") + return parser.parse_args() + + +def main(): + user_args = get_args() + + records = a2radcommon.send_request(user_args) + + if not records: + # print "Data not available" + return + + description = user_args.description + if not description: + print >> sys.stderr, "Description not provided" + return + + format = user_args.extended + + print_record(records[0], format, description) + + +def print_record(record, format, description): + idra = record.getHdf5Data() + + rdat,azdat,depVals,threshVals = a2radcommon.get_hdf5_data(idra) + + if not rdat: + # Graphic, XY + # print "Unsupported radar format" + return + + dim = rdat.getDimension() + if dim != 2: + # print "Data not available" + return + + yLen = rdat.getSizes()[0] + xLen = rdat.getSizes()[1] + + # byte[] -- the raw data + array = rdat.getByteData() + arraySize = len(array) + if xLen * yLen != arraySize: + # print "Data not available" + return + + # get data for azimuth angles if we have them. + if azdat : + azVals = azdat.getFloatData() + azValsLen = len(azVals) + if yLen != azValsLen: + # print "Data not available" + return + + msg = a2radcommon.get_header(record, format, xLen, yLen, azdat, description) + + msg += a2radcommon.encode_thresh_vals(threshVals) + msg += a2radcommon.encode_dep_vals(depVals) + if azdat : + msg += a2radcommon.encode_radial(azVals) + msg += encode_data(yLen, xLen, array) + + print msg + + +def encode_data(yLen, xLen, array): + nxy = yLen*xLen + j = 0 + msg = "" + while j&2" + exit +endif +# +# Determine if we are using the data access framework or the uEngine. +# +grep DataAccessLayer $stubpy >& /dev/null +if ( $status == 0 ) then + set method = "daf" +else + # + # Set up the environment we need to run the UEngine. + # + set method = "uengine" + if ( -e ./UEngine.cshsrc ) then + set ueenv = ./UEngine.cshsrc + else if ( -e $mydir/UEngine.cshsrc ) then + set ueenv = $mydir/UEngine.cshsrc + else if ( -e $FXA_HOME/src/dm/point/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/src/dm/point/UEngine.cshsrc + else if ( -e $FXA_HOME/bin/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/bin/UEngine.cshsrc + else + bash -c "echo could not find UEngine.cshsrc 1>&2" + exit + endif + source $ueenv +endif +# +# Locate file that lets us provide a description of the data set. +# +if ( -e ./msgCodeSeds.txt ) then + set mctrans = $PWD/msgCodeSeds.txt +else if ( -e $mydir/msgCodeSeds.txt ) then + set mctrans = $mydir/msgCodeSeds.txt +else if ( -e $fxa_home/src/dm/radar/msgCodeSeds.txt ) then + set mctrans = $fxa_home/src/dm/radar/msgCodeSeds.txt +else if ( -e $FXA_HOME/data/msgCodeSeds.txt ) then + set mctrans = $FXA_HOME/data/msgCodeSeds.txt +else + bash -c "echo could not find msgCodeSeds.txt 1>&2" + exit +endif +# +# +set rrr = $1 +set mmm = $2 +shift +shift +set ddd = `echo $mmm | sed -f $mctrans | cut '-d|' -f2 ` + +set eee = `echo $1 | grep -v '.*-'` +if ( "$eee" != "" ) shift +set slop = `echo $3 | grep '[0-9]'` +if ( "$slop" == "" ) set slop = 60 + +if ( "$method" == "daf" ) then + set datetime = $1' '$2 + set opts = "" + + if ( "$eee" != "" ) then + set opts = "$opts --angle $eee" + endif + + if ( "$fff" == "x" ) then + set opts = "$opts --extended" + endif + if ( "$encoding" == "1" ) then + set opts = "$opts --hex" + else if ( "$encoding" == "0" ) then + set opts = "$opts --int" + endif + + /awips2/python/bin/python $stubpy --radar $rrr --code $mmm --datetime="${datetime}" --slop $slop --description="${ddd}" $opts +else + # + # Get program that can do math with ascii time string, then use this to + # properly encode range of times for which we look for data. + # + if ( -x ./gtasUtil ) then + set gtasUtil = ./gtasUtil + else if ( -x $mydir/gtasUtil ) then + set gtasUtil = $mydir/gtasUtil + else if ( -x $fxa_home/src/dm/point/gtasUtil ) then + set gtasUtil = $fxa_home/src/dm/point/gtasUtil + else if ( -x $FXA_HOME/bin/gtasUtil ) then + set gtasUtil = $FXA_HOME/bin/gtasUtil + else + bash -c "echo could not find gtasUtil executable 1>&2" + exit + endif + + set aaa = `$gtasUtil = $1 $2 -$slop` + set bbb = `$gtasUtil = $1 $2 $slop` + # + # Modify the text of special tags in stub to create finalized script. + # + set specpy = /tmp/a2gtrad${$}.py + rm -rf $specpy >& /dev/null + touch $specpy + chmod 775 $specpy + if ( "$eee" == "" ) then + cat $stubpy | sed "s/KKKK/$rrr/g" | sed "s/MMMM/$mmm/g" | \ + sed "s/AAAAA/$aaa/g" | sed "s/BBBBB/$bbb/g" | sed "s/FFF/$fff/g" | \ + sed "s/DDDDD/$ddd/g" | sed 's/^.*EEEE.*$//g' | \ + sed "s/XXXXX/$encoding/g" >> $specpy + else + cat $stubpy | sed "s/KKKK/$rrr/g" | sed "s/MMMM/$mmm/g" | \ + sed "s/AAAAA/$aaa/g" | sed "s/BBBBB/$bbb/g" | sed "s/FFF/$fff/g" | \ + sed "s/DDDDD/$ddd/g" | sed "s/EEEE/$eee/g" | \ + sed "s/XXXXX/$encoding/g" >> $specpy + endif + # + # Submit the temporary python script stripping xml stuff, then remove it + # + + cd $UE_BIN_PATH + ( uengine -r python < $specpy ) | grep -v '<' | sed 's/>/>/g' | \ + sed 's/</& /dev/null +endif + +# diff --git a/radar/a2gtradStub.py b/radar/a2gtradStub.py new file mode 100644 index 0000000..c5af7ce --- /dev/null +++ b/radar/a2gtradStub.py @@ -0,0 +1,167 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# +# Gets data for a single radar product from the A-II database. The result is +# output to stdout as ASCII. This uses a data-specific Request/Response instead +# of the DataAccessLayer in order to preserve data-genericness of the interface. +# +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 08/11/2014 3393 nabowle Initial modification. Replaces UEngine +# with a custom Request/Response. +# 12/16/2014 3393 nabowle Fix negative k values. +# +# + +import argparse +import a2radcommon + +def get_args(): + parser = a2radcommon.get_args_parser() + parser.add_argument("--hex", action='store_const', dest="encoding", + const=1, help="Hex encoding.", metavar="encoding") + parser.add_argument("--int", action='store_const', dest="encoding", + const=0, help="Delimited integer encoding.", + metavar="encoding") + parser.add_argument("--extended", action='store_true', default=False, + dest="extended", help="Output the extended header.") + return parser.parse_args() + + +def main(): + user_args = get_args() + + records = a2radcommon.send_request(user_args) + + if not records: + # print "Data not available" + return + + description = user_args.description + if not description: + print >> sys.stderr, "Description not provided" + return + + format = user_args.extended + encoding = user_args.encoding + + print_record(records[0], format, description, encoding) + + +def print_record(record, format, description, encoding): + + idra = record.getHdf5Data() + + rdat,azdat,depVals,threshVals = a2radcommon.get_hdf5_data(idra) + + if not rdat: + # Graphic, XY + # print "Unsupported radar format" + return + + dim = rdat.getDimension() + if dim != 2: + # print "Data not available" + return + + yLen = rdat.getSizes()[0] + xLen = rdat.getSizes()[1] + + # byte[] -- the raw data + array = rdat.getByteData() + arraySize = len(array) + if xLen * yLen != arraySize: + # print "Data not available" + return + + # get data for azimuth angles if we have them. + if azdat : + azVals = azdat.getFloatData() + azValsLen = len(azVals) + if yLen != azValsLen: + # print "Data not available" + return + + msg = a2radcommon.get_header(record, format, xLen, yLen, azdat, description) + + msg += a2radcommon.encode_thresh_vals(threshVals) + msg += a2radcommon.encode_dep_vals(depVals) + if azdat : + msg += a2radcommon.encode_radial(azVals) + msg += encode_data(yLen, xLen, array, encoding) + + print msg.strip() + + +def encode_data(yLen, xLen, array, encoding): + plus = " ghijklmnopqrstuvwxyz" + minus = " GHIJKLMNOPQRSTUVWXYZ" + nxy = yLen*xLen + j = 0 + msg = "" + while jkk+20 or kkk : + msg += plus[k-kk] + else : + msg += minus[kk-k] + kk = k + i += 1 + msg += "\n" + j += xLen + return msg + + +if __name__ == '__main__': + main() + diff --git a/radar/a2gtradStubAll.py b/radar/a2gtradStubAll.py new file mode 100644 index 0000000..34dc4a5 --- /dev/null +++ b/radar/a2gtradStubAll.py @@ -0,0 +1,267 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# +# Gets data for a single radar product from the A-II database. The result is +# output to stdout as ASCII. This uses a data-specific Request/Response instead +# of the DataAccessLayer in order to preserve data-genericness of the interface. +# +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 08/11/2014 3393 nabowle Initial modification. Replaces UEngine +# with a custom Request/Response. +# +# + +import argparse +import a2radcommon + +def get_args(): + parser = a2radcommon.get_args_parser() + return parser.parse_args() + + +def main(): + user_args = get_args() + + records = a2radcommon.send_request(user_args) + + if not records: + # print "Data not available" + return + + description = user_args.description + if not description: + print >> sys.stderr, "Description not provided" + return + + print_record(records[0], description) + + +def print_record(record, description): + + idra = record.getHdf5Data() + + rdat,azdat,depVals,prodVals,recVals,stormVals,symVals,symData,threshVals = get_hdf5_data(idra) + + if not rdat: + # Graphic, XY + # print "Unsupported radar format" + return + + dim = rdat.getDimension() + if dim != 2: + # print "Data not available" + return + + yLen = rdat.getSizes()[0] + xLen = rdat.getSizes()[1] + + # byte[] -- the raw data + array = rdat.getByteData() + arraySize = len(array) + if xLen * yLen != arraySize: + # print "Data not available" + return + + # get data for azimuth angles if we have them. + if azdat : + azVals = azdat.getFloatData() + azValsLen = len(azVals) + if yLen != azValsLen: + # print "Data not available" + return + + msg = get_header(record, xLen, yLen, azdat, description) + msg += encode_dep_vals(depVals) + msg += encode_prod_vals(prodVals) + msg += encode_rec_vals(recVals) + msg += encode_storm_vals(stormVals) + msg += encode_sym_vals(symVals) + msg += encode_sym_data(symData) + msg += encode_thresh_vals(threshVals) + + if azdat : + msg += a2radcommon.encode_radial(azVals) + + msg += encode_data(yLen, xLen, array) + + print msg + + +def get_hdf5_data(idra): + rdat = [] + azdat = [] + depVals = [] + prodVals = [] + recVals = [] + stormVals = [] + symVals = [] + symData = [] + threshVals = [] + if len(idra) > 0: + for ii in range(len(idra)): + if idra[ii].getName() == "Data": + rdat = idra[ii] + elif idra[ii].getName() == "Angles": + azdat = idra[ii] + dattyp = "radial" + elif idra[ii].getName() == "DependentValues": + depVals = idra[ii].getShortData() + elif idra[ii].getName() == "ProductVals": + prodVals = idra[ii].getByteData() + elif idra[ii].getName() == "RecordVals": + recVals = idra[ii].getByteData() + elif idra[ii].getName() == "StormIds": + stormVals = idra[ii].getByteData() + elif idra[ii].getName() == "Symbology": + symVals = idra[ii].getByteData() + elif idra[ii].getName() == "SymbologyData": + symData = idra[ii].getByteData() + elif idra[ii].getName() == "Thresholds": + threshVals = idra[ii].getShortData() + + return rdat,azdat,depVals,prodVals,recVals,stormVals,symVals,symData,threshVals + + +def get_header(record, xLen, yLen, azdat, description): + # Encode dimensions, time, mapping, description, tilt, and VCP + mytime = a2radcommon.get_datetime_str(record) + dattyp = a2radcommon.get_data_type(azdat) + + msg = str(xLen) + " " + str(yLen) + " " + mytime + " " + dattyp + \ + " " + description + "\n" + \ + str(record.getTrueElevationAngle()) + " " + \ + str(record.getVolumeCoveragePattern()) + "\n" + return msg + + +def encode_dep_vals(depVals): + nnn = len(depVals) + msg = str(nnn) + j = 0 + while j&2" + exit +endif + +set rrr = `echo $1 | grep '[a-z][a-z][a-z][a-z]'` +if ( "$rrr" != "" ) shift +set lastcmd = cat + +# +# Determine if we are using the data access framework or the uEngine. +# +grep DataAccessLayer $stubpy >& /dev/null +if ( $status == 0 ) then + set mydate = `echo "$1" | grep '.*-.*-'` + set mytime = `echo "$2" | grep ':'` + if ( "$mydate" != "" && "$mytime" != "" ) then + shift + shift + set userargs = "--date ${mydate} --time ${mytime}" + else + set userargs = "" + endif + + if ( "$rrr" == "" ) then + #done + else if ( "$1" == "" ) then + set userargs = "$userargs --icao $rrr" + else if ( "$1" == "+" ) then + set userargs = "$userargs --icao $rrr" + if ( -e ./msgCodeSeds.txt ) then + set mctrans = $PWD/msgCodeSeds.txt + else if ( -e $mydir/msgCodeSeds.txt ) then + set mctrans = $mydir/msgCodeSeds.txt + else if ( -e $fxa_home/src/dm/radar/msgCodeSeds.txt ) then + set mctrans = $fxa_home/src/dm/radar/msgCodeSeds.txt + else if ( -e $FXA_HOME/data/msgCodeSeds.txt ) then + set mctrans = $FXA_HOME/data/msgCodeSeds.txt + else + bash -c "echo could not find msgCodeSeds.txt 1>&2" + exit + endif + set lastcmd = "sed -f $mctrans" + else if ( "$2" == "" ) then + set userargs = "$userargs --icao $rrr --productCode $1 --angle 0.0 --outputDate" + else if ( "$2" == "+" ) then + set userargs = "$userargs --icao $rrr --productCode $1 --outputPrimary" + else + set userargs = "$userargs --icao $rrr --productCode $1 --angle $2 --outputTrue --outputDate" + endif + /awips2/python/bin/python $stubpy ${userargs} | $lastcmd +else + # + # Set up the environment we need to run the UEngine. + # + set method = "uengine" + if ( -e ./UEngine.cshsrc ) then + set ueenv = ./UEngine.cshsrc + else if ( -e $mydir/UEngine.cshsrc ) then + set ueenv = $mydir/UEngine.cshsrc + else if ( -e $FXA_HOME/src/dm/point/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/src/dm/point/UEngine.cshsrc + else if ( -e $FXA_HOME/bin/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/bin/UEngine.cshsrc + else + bash -c "echo could not find UEngine.cshsrc 1>&2" + exit + endif + source $ueenv + + # + # Set range of time we will request this information over, will default to + # essentially forever. + # + set aaa = "1970-01-01 00:00:00.0" + set bbb = "2038-01-01 00:00:00.0" + set mydate = `echo "$1" | grep '.*-.*-'` + set mytime = `echo "$2" | grep ':'` + if ( "$mydate" != "" && "$mytime" != "" ) then + shift + shift + if ( -x ./gtasUtil ) then + set gtasUtil = ./gtasUtil + else if ( -x $mydir/gtasUtil ) then + set gtasUtil = $mydir/gtasUtil + else if ( -x $fxa_home/src/dm/point/gtasUtil ) then + set gtasUtil = $fxa_home/src/dm/point/gtasUtil + else if ( -x $FXA_HOME/bin/gtasUtil ) then + set gtasUtil = $FXA_HOME/bin/gtasUtil + else + bash -c "echo could not find gtasUtil executable 1>&2" + exit + endif + set aaa = `$gtasUtil = $mydate $mytime -60` + set bbb = `$gtasUtil = $mydate $mytime 60` + endif + + + # + # Modify the text of special tags in stub to create finalized script. + # + set binary = no + set specpy = /tmp/a2invrad${$}.py + rm -rf $specpy >& /dev/null + touch $specpy + chmod 775 $specpy + if ( "$rrr" == "" ) then + cat $stubpy | grep -v "KKKK" | grep -v "MMMM" | grep -v "EEEE" | \ + sed "s/AAAAA/$aaa/g" | sed "s/BBBBB/$bbb/g" | \ + grep -v 'Code")' | grep -v 'Time")' | grep -v 'Angle")' \ + >> $specpy + else if ( "$1" == "" ) then + cat $stubpy | sed "s/KKKK/$rrr/g" | grep -v "MMMM" | \ + sed "s/AAAAA/$aaa/g" | sed "s/BBBBB/$bbb/g" | grep -v 'icao")' | \ + sed 's/^.*EEEE.*$//g' | grep -v 'Time")' | grep -v 'Angle")' \ + >> $specpy + else if ( "$1" == "+" ) then + cat $stubpy | sed "s/KKKK/$rrr/g" | grep -v "MMMM" | \ + sed "s/AAAAA/$aaa/g" | sed "s/BBBBB/$bbb/g" | grep -v 'icao")' | \ + sed 's/^.*EEEE.*$//g' | grep -v 'Time")' | grep -v 'Angle")' \ + >> $specpy + if ( -e ./msgCodeSeds.txt ) then + set mctrans = $PWD/msgCodeSeds.txt + else if ( -e $mydir/msgCodeSeds.txt ) then + set mctrans = $mydir/msgCodeSeds.txt + else if ( -e $fxa_home/src/dm/radar/msgCodeSeds.txt ) then + set mctrans = $fxa_home/src/dm/radar/msgCodeSeds.txt + else if ( -e $FXA_HOME/data/msgCodeSeds.txt ) then + set mctrans = $FXA_HOME/data/msgCodeSeds.txt + else + bash -c "echo could not find msgCodeSeds.txt 1>&2" + exit + endif + set lastcmd = "sed -f $mctrans" + else if ( "$2" == "" ) then + cat $stubpy | sed "s/KKKK/$rrr/g" | sed "s/MMMM/$1/g" | \ + sed "s/AAAAA/$aaa/g" | sed "s/BBBBB/$bbb/g" | grep -v 'icao")' | \ + sed 's/EEEE/0.0/g' | grep -v 'Angle")' | grep -v 'Code")' \ + >> $specpy + else if ( "$2" == "+" ) then + cat $stubpy | sed "s/KKKK/$rrr/g" | sed "s/MMMM/$1/g" | \ + sed "s/AAAAA/$aaa/g" | sed "s/BBBBB/$bbb/g" | grep -v 'icao")' | \ + sed 's/^.*EEEE.*$//g' | grep -v 'Time")' | grep -v 'Code")' | \ + sed 's/true/primary/g' >> $specpy + else + cat $stubpy | sed "s/KKKK/$rrr/g" | sed "s/MMMM/$1/g" | \ + sed "s/AAAAA/$aaa/g" | sed "s/BBBBB/$bbb/g" | grep -v 'icao")' | \ + sed "s/EEEE/$2/g" | grep -v 'Code")' >> $specpy + set binary = yes + endif + # + # Submit the temporary python script stripping xml stuff, then remove it + # + if ( "$binary" == "yes" ) then + cd $UE_BIN_PATH + ( uengine -r python < $specpy ) |& sed 's|.*.*|@|g' | \ + grep -E 'attributes|@' | cut '-d"' -f4 | tr '\n' ' ' | tr '@' '\n' | \ + sed 's/00000.*$//g' | sed 's/^ *//g' | sed 's/ *$//g' + else + cd $UE_BIN_PATH + ( uengine -r python < $specpy ) |& grep attributes | cut '-d"' -f4 | \ + $lastcmd + endif + + if ( "$rmpy" == "yes" ) rm -rf $specpy >& /dev/null +endif + diff --git a/radar/a2invradStub.py b/radar/a2invradStub.py new file mode 100644 index 0000000..15738b1 --- /dev/null +++ b/radar/a2invradStub.py @@ -0,0 +1,195 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# Gets inventories of radar grid data from the A-II database. The data is +# output to stdout as ASCII. Inventories are limited to Radial and Raster +# formats. +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 2014-10-27 3600 nabowle Initial modification. Convert to DAF. +# 2014-12-18 3600 nabowle Use new getAvailableLevels() to speed up retrieval. +# + +import argparse +import numpy +import sys + +from datetime import datetime +from datetime import timedelta + +from ufpy.dataaccess import DataAccessLayer +from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange +from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.level import Level + + +def get_args(): + parser = argparse.ArgumentParser(conflict_handler="resolve") + parser.add_argument("-h", action="store", dest="host", + help="EDEX server hostname (optional)", metavar="hostname") + + parser.add_argument("--icao", action="store", dest="icao", + help="The ICAO (optional)", metavar="icao") + + parser.add_argument("--productCode", action="store", dest="productCode", + help="Product Code (optional)", metavar="productCode") + + parser.add_argument("--angle", action="store", dest="angle", type=float, + help="The Primary Elevation Angle (optional)", + metavar="angle") + + parser.add_argument("--date", action="store", dest="date", + help="A date to find data within a minute of. (optional, --time required if provided)", + metavar="YYYY-MM-DD") + + parser.add_argument("--time", action="store", dest="time", + help="A time to find data within a minute of. (optional, --date required if provided)", + metavar="HH:MM") + + parser.add_argument("--outputDate", action="store_true", + dest="outputDate", help="Output the datetime (optional)") + + parser.add_argument("--outputTrue", action="store_const", + dest="outputAngle", const="true", + help="Output true elevation angle, if relevant. (optional," + + " exclusive with --outputPrimary)") + + parser.add_argument("--outputPrimary", action="store_const", + dest="outputAngle", const="primary", + help="Output primary elevation angle, if relevant. " + + "(optional, exclusive with --outputTrue)") + return parser.parse_args() + +def main(): + user_args = get_args() + + if user_args.host: + DataAccessLayer.changeEDEXHost(user_args.host) + + if (user_args.date and not user_args.time) or (user_args.time and not user_args.date): + print >> sys.stderr, "date and time must be provided together" + return + + # If a time range is provided, results will be filtered based on available times + timeRange = None + if user_args.date: + midRange = datetime.strptime( user_args.date + " " + user_args.time, "%Y-%m-%d %H:%M") + beginRange = midRange - timedelta(0, 60) + endRange = midRange + timedelta(0, 60) + timeRange = TimeRange(beginRange, endRange) + + req = create_request(user_args) + if user_args.icao: + if user_args.productCode: # retrieve available times and/or true or primary elevation angles + if timeRange: + tr = timeRange + else: + tr = None + lines = set() + + if user_args.outputAngle: + levels = DataAccessLayer.getAvailableLevels(req) + for level in levels: + line = "" + req.setLevels(level) + if user_args.outputDate: + times = DataAccessLayer.getAvailableTimes(req) + for time in times: + if not tr or tr.contains(time.getValidPeriod()): + line = str(time) + ".0" + line += " " + if user_args.outputAngle == "true": + line += "%.1f"%level.getLeveltwovalue() + else: + line += "%.1f"%level.getLevelonevalue() + lines.add(line) + else: + if not tr or data_in_time_range(req, tr): + if user_args.outputAngle == "true": + line = "%.1f"%level.getLeveltwovalue() + else: + line = "%.1f"%level.getLevelonevalue() + lines.add(line) + else : # just output time + times = DataAccessLayer.getAvailableTimes(req) + for time in times: + if not tr or tr.contains(time.getValidPeriod()): + lines.add(str(time) + ".0") + msg = "\n".join(lines) + else: #retrieve available product codes + unfiltered = DataAccessLayer.getAvailableParameters(req) + productCodes = [] + for parameter in unfiltered: #filter to just productCodes + if parameter.isdigit(): + productCodes.append(parameter) + if timeRange: + unfiltered = productCodes + productCodes = [] + for productCode in unfiltered: + req = create_request(user_args) + req.setParameters(productCode) + if data_in_time_range(req, timeRange): + productCodes.append(productCode) + msg = "\n".join(productCodes) + + else: # retrieve available icaos + icaos = DataAccessLayer.getAvailableLocationNames(req) + if timeRange: + unfiltered = icaos + icaos = [] + for icao in unfiltered: + req = create_request(user_args) + req.addIdentifier("icao", icao) + if data_in_time_range(req, timeRange): + icaos.append(icao) + + msg = "\n".join(icaos) + + print msg.strip() + +def create_request(user_args): + req = DataAccessLayer.newDataRequest("radar") + if user_args.icao: + req.addIdentifier("icao", user_args.icao) + if user_args.productCode: + req.setParameters(user_args.productCode) + if user_args.angle is not None: + level = Level() + level.setLevelonevalue(user_args.angle) + req.setLevels(level) + # Indicate that when providing or requesting levels, the Levelonevalue + # is the primaryElevationAngle and the Leveltwovalue value is the + # trueElevationAngle + req.addIdentifier("level.one.field", "primaryElevationAngle") + req.addIdentifier("level.two.field", "trueElevationAngle") + + return req + +def data_in_time_range(req, timeRange): + times = DataAccessLayer.getAvailableTimes(req) + for time in times: + if timeRange.contains(time.getValidPeriod()): + return True + return False + +if __name__ == '__main__': + main() diff --git a/radar/a2radcommon.py b/radar/a2radcommon.py new file mode 100644 index 0000000..1200261 --- /dev/null +++ b/radar/a2radcommon.py @@ -0,0 +1,251 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# +# Common methods for the a2gtrad and a2advrad scripts. +# +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 08/13/2014 3393 nabowle Initial creation to contain common +# code for a2*radStub scripts. +# +# + +import argparse +import sys + +from datetime import datetime +from datetime import timedelta +from ufpy import ThriftClient + +from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange +from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.level import Level +from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.radar.request import GetRadarDataRecordRequest + +def get_default_host(): + from ufpy.dataaccess import DataAccessLayer + return DataAccessLayer.THRIFT_HOST + + +def get_args_parser(): + parser = argparse.ArgumentParser(conflict_handler="resolve") + parser.add_argument("--host", action="store", dest="host", + help="EDEX server hostname", + metavar="hostname") + parser.add_argument("--datetime", action="store", dest="datetime", + help="The start of the time range in YYYY-MM-DD HH:MM", + metavar="datetime") + parser.add_argument("--radar", action="store", dest="radar", + help="The ICAO code for the radar", + metavar="radar") + parser.add_argument("--code", action="store", dest="code", + help="The product code.", type=int, + metavar="code") + parser.add_argument("--angle", action="store", dest="angle", default=0, + help="The Elevation Angle", metavar="angle") + parser.add_argument("--description", action="store", dest="description", + help="The description.", + metavar="desc") + parser.add_argument("--slop", action="store", dest="slop", default=60, + help="The amount of slop, in seconds, to allow around the datetime.", + metavar="slop", type=int) + return parser + + +def send_request(user_args): + slop = user_args.slop + + dateTimeStr = user_args.datetime + if not dateTimeStr: + print >> sys.stderr, "DateTime not provided" + return + dateTime = datetime.strptime(dateTimeStr, "%Y-%m-%d %H:%M") + beginRange = dateTime - timedelta(0, slop) + endRange = dateTime + timedelta(0, slop) + + timerange = TimeRange(beginRange, endRange) + + radar = user_args.radar + if not radar: + print >> sys.stderr, "Radar code not provided" + return + + code = user_args.code + if not code: + print >> sys.stderr, "Product code not provided" + return + + angle = user_args.angle + + slop = int(user_args.slop) + + host = user_args.host + if not host: + host = get_default_host() + + client = ThriftClient.ThriftClient(host) + + # Perform a GetRadarHDF5Request + req = GetRadarDataRecordRequest() + req.setRadarId(radar) + req.setPrimaryElevationAngle(float(angle)) + req.setTimeRange(timerange) + req.setProductCode(int(code)) + + response = client.sendRequest(req) + + if response is None: + # print "Data not available" + return + + records = response.getData() + return records + + +def get_datetime_str(record): + #2014-07-16 00:00:00 (0) => 2014-07-16_00:03:00.0 + return str(record.getDataTime())[0:19].replace(" ","_") + ".0" + + +def get_data_type(azdat): + if azdat: + dattyp = "radial" + else : + dattyp = "raster" + return dattyp + + +def get_hdf5_data(idra): + rdat = [] + azdat = [] + depVals = [] + threshVals = [] + if len(idra) > 0: + for ii in range(len(idra)): + if idra[ii].getName() == "Data": + rdat = idra[ii] + elif idra[ii].getName() == "Angles": + azdat = idra[ii] + dattyp = "radial" + elif idra[ii].getName() == "DependentValues": + depVals = idra[ii].getShortData() +## Commented out from the original. May not be available. +# elif idra[ii].getName() == "ProductVals": +# prodVals = idra[ii].getByteData() +# elif idra[ii].getName() == "RecordVals": +# recVals = idra[ii].getByteData() +# elif idra[ii].getName() == "StormIds": +# stormVals = idra[ii].getByteData() +# elif idra[ii].getName() == "Symbology": +# symVals = idra[ii].getByteData() +# elif idra[ii].getName() == "SymbologyData": +# symData = idra[ii].getByteData() +## + elif idra[ii].getName() == "Thresholds": + threshVals = idra[ii].getShortData() + + return rdat,azdat,depVals,threshVals + + +def get_header(record, format, xLen, yLen, azdat, description): + # Encode dimensions, time, mapping, description, tilt, and VCP + mytime = get_datetime_str(record) + dattyp = get_data_type(azdat) + + if format : + msg = str(xLen) + " " + str(yLen) + " " + mytime + " " + \ + dattyp + " " + str(record.getLatitude()) + " " + \ + str(record.getLongitude()) + " " + \ + str(record.getElevation()) + " " + \ + str(record.getElevationNumber()) + " " + \ + description + " " + str(record.getTrueElevationAngle()) + " " + \ + str(record.getVolumeCoveragePattern()) + "\n" +#"%.1f"% + else : + msg = str(xLen) + " " + str(yLen) + " " + mytime + " " + \ + dattyp + " " + description + " " + \ + str(record.getTrueElevationAngle()) + " " + \ + str(record.getVolumeCoveragePattern()) + "\n" + + return msg + + +def encode_thresh_vals(threshVals): + spec = [".", "TH", "ND", "RF", "BI", "GC", "IC", "GR", "WS", "DS", + "RA", "HR", "BD", "HA", "UK"] + nnn = len(threshVals) + j = 0 + msg = "" + while j 14 : + msg += "." + else : + msg += spec[lo] + continue + if hi % 16 >= 8 : + msg += ">" + elif hi % 8 >= 4 : + msg += "<" + if hi % 4 >= 2 : + msg += "+" + elif hi % 2 >= 1 : + msg += "-" + if hi >= 64 : + msg += "%.2f"%(lo*0.01) + elif hi % 64 >= 32 : + msg += "%.2f"%(lo*0.05) + elif hi % 32 >= 16 : + msg += "%.1f"%(lo*0.1) + else : + msg += str(lo) + msg += "\n" + return msg + + +def encode_dep_vals(depVals): + nnn = len(depVals) + j = 0 + msg = "" + while j& /dev/null +if ( $status == 0 ) then +# Replace first field with station names substitutions in wmoToNameRaob.txt or +# with what's stored in the database. They are similar but do not match 100%. + /awips2/python/bin/python $stubpy -b "$1 $2" -e "$3 $4" | sed -f $staInf +# /awips2/python/bin/python $stubpy -b "$1 $2" -e "$3 $4" --use-station-name +else + # + # Set up the environment we need to run the UEngine. + # + set method = "uengine" + if ( -e ./UEngine.cshsrc ) then + set ueenv = ./UEngine.cshsrc + else if ( -e $mydir/UEngine.cshsrc ) then + set ueenv = $mydir/UEngine.cshsrc + else if ( -e $FXA_HOME/src/dm/point/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/src/dm/point/UEngine.cshsrc + else if ( -e $FXA_HOME/bin/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/bin/UEngine.cshsrc + else + bash -c "echo could not find UEngine.cshsrc 1>&2" + exit + endif + source $ueenv + + set specpy = /tmp/a2gtraob${$}.py + rm -rf $specpy >& /dev/null + touch $specpy + chmod 775 $specpy + cat $stubpy | sed "s/BBBBB/$1 $2/g" | sed "s/EEEEE/$3 $4/g" > $specpy + + cd $UE_BIN_PATH + ( uengine -r python < $specpy ) | grep -v '<' | sed -n '2,$p' | \ + sed -f $staInf + + if ( "$rmpy" == "yes" ) rm -rf $specpy >& /dev/null +endif + diff --git a/raobs/a2gtraobStub.py b/raobs/a2gtraobStub.py new file mode 100644 index 0000000..ff5bb2e --- /dev/null +++ b/raobs/a2gtraobStub.py @@ -0,0 +1,338 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# Gets all available raob data in the A-II database over a specified range of +# times. The data is output to stdout as ASCII. +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# Oct 10, 2014 3595 nabowle Initial modification. Fix Man and SigW indices. +# Oct 10, 2014 3595 nabowle Replace UEngine with DAF. +# +# + +import argparse +import sys + +from datetime import datetime +from ufpy.dataaccess import DataAccessLayer +from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange + +def get_args(): + parser = argparse.ArgumentParser(conflict_handler="resolve") + parser.add_argument("-h", action="store", dest="host", + help="EDEX server hostname (optional)", + metavar="hostname") + parser.add_argument("-b", action="store", dest="start", + help="The start of the time range in YYYY-MM-DD HH:MM", + metavar="start") + parser.add_argument("-e", action="store", dest="end", + help="The end of the time range in YYYY-MM-DD HH:MM", + metavar="end") + parser.add_argument("--use-station-name", action='store_true', default=False, + dest="stationname", help="Output the station name instead of station id for the first output field.") + return parser.parse_args() + +def main(): + # The multi-dimensional parameters. + MAN_PARAMS = set(['prMan', 'htMan', 'tpMan', 'tdMan', 'wdMan', 'wsMan']) + TROP_PARAMS = set(['prTrop', 'tpTrop', 'tdTrop', 'wdTrop', 'wsTrop']) + MAXW_PARAMS = set(['prMaxW', 'wdMaxW', 'wsMaxW']) + SIGT_PARAMS = set(['prSigT', 'tpSigT', 'tdSigT']) + SIGW_PARAMS = set(['htSigW', 'wdSigW', 'wsSigW']) + + user_args = get_args() + + if user_args.host: + DataAccessLayer.changeEDEXHost(user_args.host) + + start = user_args.start + end = user_args.end + + if not start or not end: + print >> sys.stderr, "Start or End date not provided" + return + + beginRange = datetime.strptime( start + ":00.0", "%Y-%m-%d %H:%M:%S.%f") + endRange = datetime.strptime( end + ":59.9", "%Y-%m-%d %H:%M:%S.%f") + timerange = TimeRange(beginRange, endRange) + + req = DataAccessLayer.newDataRequest("bufrua") + req.setParameters("wmoStaNum", "validTime", "rptType", "staElev", "numMand", + "numSigT", "numSigW", "numTrop", "numMwnd", "staName") + req.getParameters().extend(MAN_PARAMS) + req.getParameters().extend(TROP_PARAMS) + req.getParameters().extend(MAXW_PARAMS) + req.getParameters().extend(SIGT_PARAMS) + req.getParameters().extend(SIGW_PARAMS) + + geometries = DataAccessLayer.getGeometryData(req, timerange) + + if not geometries : +# print "couldn't get data" + return + + + manGeos = [] + tropGeos = [] + maxwGeos = [] + sigtGeos = [] + sigwGeos = [] + # D-2D format files expect depression + tdman = [] + tdsigt = [] + tdtrop = [] + msg = "" + for geoData in geometries : + if set(geoData.getParameters()) & MAN_PARAMS : + manGeos.append(geoData) + td = geoData.getNumber("tdMan") + tp = geoData.getNumber("tpMan") + if td >150 and td<=tp : + tdman.append(tp-td) + else : + tdman.append(-9999.0) + continue + if set(geoData.getParameters()) & TROP_PARAMS : + tropGeos.append(geoData) + td = geoData.getNumber("tdTrop") + tp = geoData.getNumber("tpTrop") + if td>150 and td<=tp : + tdtrop.append(tp-td) + else : + tdtrop.append(-9999.0) + continue + if set(geoData.getParameters()) & MAXW_PARAMS : + maxwGeos.append(geoData) + continue + if set(geoData.getParameters()) & SIGT_PARAMS : + sigtGeos.append(geoData) + td = geoData.getNumber("tdSigT") + tp = geoData.getNumber("tpSigT") + if td>150 and td<=tp : + tdsigt.append(tp-td) + else : + tdsigt.append(-9999.0) + continue + if set(geoData.getParameters()) & SIGW_PARAMS : + sigwGeos.append(geoData) + continue + + if len(manGeos) == 0 and len(sigtGeos) == 0 and len(sigwGeos) == 0 or geoData.getNumber("rptType") > 2022 : + manGeos = [] + tropGeos = [] + maxwGeos = [] + sigtGeos = [] + sigwGeos = [] + tdman = [] + tdsigt = [] + tdtrop = [] + continue + + if user_args.stationname and geoData.getString("staName") : + msg += geoData.getString("staName") + "," + else : + msg += geoData.getString("wmoStaNum") + "," + msg += str(geoData.getNumber("validTime")/1000) + "," + msg += "%.4f"%geoData.getGeometry().y + "," + msg += "%.4f"%geoData.getGeometry().x + "," + msg += "%.0f"%geoData.getNumber("staElev") + "," + msg += geoData.getString("wmoStaNum") + "," + + kk = len(manGeos) + msg += str(kk) + "," + if kk>0 : + msg += "%.1f"%manGeos[0].getNumber("prMan") + k = 1 + while k < kk : + msg += "|" + "%.1f"%manGeos[k].getNumber("prMan") + k += 1 + msg += "," + + msg += "%.1f"%geoData.getNumber("staElev") + k = 1 + while k < kk : + msg += "|" + "%.1f"%manGeos[k].getNumber("htMan") + k += 1 + msg += "," + + msg += "%.1f"%manGeos[0].getNumber("tpMan") + k = 1 + while k < kk : + msg += "|" + "%.1f"%manGeos[k].getNumber("tpMan") + k += 1 + msg += "," + + msg += "%.1f"%tdman[0] + k = 1 + while k < kk : + msg += "|" + "%.1f"%tdman[k]; + k += 1 + msg += "," + + msg += "%.1f"%manGeos[0].getNumber("wsMan") + k = 1 + while k < kk : + msg += "|" + "%.1f"%manGeos[k].getNumber("wsMan") + k += 1 + msg += "," + + msg += "%.1f"%manGeos[0].getNumber("wdMan") + k = 1 + while k < kk : + msg += "|" + "%.1f"%manGeos[k].getNumber("wdMan") + k += 1 + msg += "," + else : + msg += ",,,,,," + + kk = len(sigtGeos) + msg += str(kk) + "," + if kk>0 : + msg += "%.1f"%sigtGeos[0].getNumber("prSigT") + k = 1 + while k < kk : + msg += "|" + "%.1f"%sigtGeos[k].getNumber("prSigT") + k += 1 + msg += "," + + msg += "%.1f"%sigtGeos[0].getNumber("tpSigT") + k = 1 + while k < kk : + msg += "|" + "%.1f"%sigtGeos[k].getNumber("tpSigT") + k += 1 + msg += "," + + msg += "%.1f"%tdsigt[0] + k = 1 + while k < kk : + msg += "|" + "%.1f"%tdsigt[k] + k += 1 + msg += "," + else : + msg += ",,," + + kk = len(sigwGeos) + msg += str(kk) + "," + if kk>0 : + msg += "%.1f"%sigwGeos[0].getNumber("htSigW") + k = 1 + while k < kk : + msg += "|" + "%.1f"%sigwGeos[k].getNumber("htSigW") + k += 1 + msg += "," + + msg += "%.1f"%sigwGeos[0].getNumber("wsSigW") + k = 1 + while k < kk : + msg += "|" + "%.1f"%sigwGeos[k].getNumber("wsSigW") + k += 1 + msg += "," + + msg += "%.1f"%sigwGeos[0].getNumber("wdSigW") + k = 1 + while k < kk : + msg += "|" + "%.1f"%sigwGeos[k].getNumber("wdSigW") + k += 1 + msg += "," + else : + msg += ",,," + + kk = len(tropGeos) + msg += str(kk) + "," + if kk>0 : + msg += "%.1f"%tropGeos[0].getNumber("prTrop") + k = 1 + while k < kk : + msg += "|" + "%.1f"%tropGeos[k].getNumber("prTrop") + k += 1 + msg += "," + + msg += "%.1f"%tropGeos[0].getNumber("tpTrop") + k = 1 + while k < kk : + msg += "|" + "%.1f"%tropGeos[k].getNumber("tpTrop") + k += 1 + msg += "," + + msg += "%.1f"%tdtrop[0] + k = 1 + while k < kk : + msg += "|" + "%.1f"%tdtrop[k] + k += 1 + msg += "," + + msg += "%.1f"%tropGeos[0].getNumber("wsTrop") + k = 1 + while k < kk : + msg += "|" + "%.1f"%tropGeos[k].getNumber("wsTrop") + k += 1 + msg += "," + + msg += "%.1f"%tropGeos[0].getNumber("wdTrop") + k = 1 + while k < kk : + msg += "|" + "%.1f"%tropGeos[k].getNumber("wdTrop") + k += 1 + msg += "," + else : + msg += ",,,,," + + kk = len(maxwGeos) + msg += str(kk) + "," + if kk>0 : + msg += "%.1f"%maxwGeos[0].getNumber("prMaxW") + k = 1 + while k < kk : + msg += "|" + "%.1f"%maxwGeos[k].getNumber("prMaxW") + k += 1 + msg += "," + + msg += "%.1f"%maxwGeos[0].getNumber("wsMaxW") + k = 1 + while k < kk : + msg += "|" + "%.1f"%maxwGeos[k].getNumber("wsMaxW") + k += 1 + msg += "," + + msg += "%.1f"%maxwGeos[0].getNumber("wdMaxW") + k = 1 + while k < kk : + msg += "|" + "%.1f"%maxwGeos[k].getNumber("wdMaxW") + k += 1 + else : + msg += ",," + msg += "\n" + + manGeos = [] + tropGeos = [] + maxwGeos = [] + sigtGeos = [] + sigwGeos = [] + tdman = [] + tdsigt = [] + tdtrop = [] + + print msg.strip() + +if __name__ == '__main__': + main() diff --git a/satellite/a2invsat.csh b/satellite/a2invsat.csh new file mode 100644 index 0000000..e3c47f8 --- /dev/null +++ b/satellite/a2invsat.csh @@ -0,0 +1,174 @@ +#!/bin/csh +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## +# +# A script wrapper that is meant to get inventories of satellite data +# from the A-II database. The data is output to stdout as ASCII. +# This version can adapt to use a python stub that calls the +# data access framework. +# +# Usage: +# +# a2invsat.csh {p} sector channel {satid} +# +# p - A literal p. (optional) +# sector - sector id +# channel - channel id +# satid - (optional) satellite id +# +# Returns a list of times with data for the specified sector/channel. +# +# The ids can be either D-2D integer ids, or AWIPS-II ascii ids, in which +# case they need to be quoted on the command line. +# +# Integer ids can be looked up in a2satInfo.txt, channel id corresponds to +# the physicalElement, and satid corresponds to the creatingEntity. +# +# The literal p option means preserve the final version of the python +# submitted to the UEngine instead of cleaning it up. The path to the +# finalized python is /tmp/a2invsatNNNNN.py where NNNNN is a unix process id. +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 2014-10-23 3601 nabowle Initial modification. Properly calls DAF. Make a2satInfo.txt optional. +# + +set rmpy = yes +if ( "$1" == "p" ) then + set rmpy = no + shift +endif +# +# Identify directory this script is in, will be one of the directories we +# search for other files in. +# +set mydir = `dirname $0` +set d1 = `echo $mydir | cut -c1` +if ( "$mydir" == '.' ) then + set mydir = $PWD +else if ( "$d1" != "/" ) then + set mydir = $PWD/$mydir +endif +set mydir = `(cd $mydir ; pwd)` +if ( ! $?FXA_HOME ) set FXA_HOME = xxxx +# +# Locate python stub that we will modify to create the final python logic. +# +if ( -e ./a2invsatStub.py ) then + set stubpy = ./a2invsatStub.py +else if ( -e $mydir/a2invsatStub.py ) then + set stubpy = $mydir/a2invsatStub.py +else if ( -e $FXA_HOME/src/dm/sat/a2invsatStub.py ) then + set stubpy = $FXA_HOME/src/dm/sat/a2invsatStub.py +else if ( -e $FXA_HOME/bin/a2invsatStub.py ) then + set stubpy = $FXA_HOME/bin/a2invsatStub.py +else + bash -c "echo could not find a2invsatStub.py 1>&2" + exit +endif + +# +# Locate file containing mapping between D-2D interger ids and AWIPS-II ascii +# ids for sectors, channels, and satellites. +# +if ( -e ./a2satInfo.txt ) then + set satInf = ./a2satInfo.txt +else if ( -e $mydir/a2satInfo.txt ) then + set satInf = $mydir/a2satInfo.txt +else if ( -e $FXA_HOME/src/dm/sat/a2satInfo.txt ) then + set satInf = $FXA_HOME/src/dm/sat/a2satInfo.txt +else if ( -e $FXA_HOME/data/a2satInfo.txt ) then + set satInf = $FXA_HOME/data/a2satInfo.txt +else + set satInf = "" + set sss = "$1" + set ccc = "$2" + if ( "$3" == "" ) then + set eee = "" + else + set eee = "$3" + endif +endif +# +# +if ( $satInf != "" ) then + set sss = `grep "^ *$1|.*sectorID" $satInf | cut '-d|' -f3` + if ( "$sss" == "" ) set sss = "$1" + set ccc = `grep "^ *$2|.*physicalElement" $satInf | cut '-d|' -f3` + if ( "$ccc" == "" ) set ccc = "$2" + if ( "$3" == "" ) then + set eee = "" + else + set eee = `grep "^ *$3|.*creatingEntity" $satInf | cut '-d|' -f3` + if ( "$eee" == "" ) set eee = "$3" + endif +endif + +# +# Determine if we are using the data access framework or the uEngine. +# +grep DataAccessLayer $stubpy >& /dev/null +if ( $status == 0 ) then + if ( "$eee" == "" ) then + /awips2/python/bin/python $stubpy --sectorID "$sss" --physicalElement "$ccc" + else + /awips2/python/bin/python $stubpy --sectorID "$sss" --physicalElement "$ccc" --creatingEntity "$eee" + endif +else + # + # Set up the environment we need to run the UEngine. + # + if ( -e ./UEngine.cshsrc ) then + set ueenv = ./UEngine.cshsrc + else if ( -e $mydir/UEngine.cshsrc ) then + set ueenv = $mydir/UEngine.cshsrc + else if ( -e $FXA_HOME/src/dm/point/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/src/dm/point/UEngine.cshsrc + else if ( -e $FXA_HOME/bin/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/bin/UEngine.cshsrc + else + bash -c "echo could not find UEngine.cshsrc 1>&2" + exit + endif + source $ueenv + # + # Modify the text of special tags in stub to create finalized script. + # + set specpy = /tmp/a2invsat${$}.py + rm -rf $specpy >& /dev/null + touch $specpy + chmod 775 $specpy + if ( "$eee" == "" ) then + cat $stubpy | sed "s/SSSSS/$sss/g" | sed "s/CCCCC/$ccc/g" | \ + sed 's/^.*EEEEE.*$//g' >> $specpy + else + cat $stubpy | sed "s/SSSSS/$sss/g" | sed "s/CCCCC/$ccc/g" | \ + sed "s/EEEEE/$eee/g" >> $specpy + endif + + cd $UE_BIN_PATH + ( uengine -r python < $specpy ) |& grep attributes | cut '-d"' -f4 + + if ( "$rmpy" == "yes" ) rm -rf $specpy >& /dev/null +endif + diff --git a/satellite/a2invsatStub.py b/satellite/a2invsatStub.py new file mode 100644 index 0000000..fcd4fc8 --- /dev/null +++ b/satellite/a2invsatStub.py @@ -0,0 +1,76 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# Gets inventories of satellite data from the A-II database. The data is output +# to stdout as ASCII. +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 2014-10-23 3601 nabowle Initial modification. Convert to DAF. +# + +import argparse +import numpy +import sys + +from ufpy.dataaccess import DataAccessLayer + +def get_args(): + parser = argparse.ArgumentParser(conflict_handler="resolve") + parser.add_argument("-h", action="store", dest="host", + help="EDEX server hostname (optional)", metavar="hostname") + parser.add_argument("--sectorID", action="store", dest="sectorID", + help="The sector ID.", metavar="sectorID") + parser.add_argument("--physicalElement", action="store", dest="physicalElement", + help="The physical element.", metavar="physicalElement") + parser.add_argument("--creatingEntity", action="store", dest="creatingEntity", + help="(optional) The creating entity", metavar="creatingEntity") + + return parser.parse_args() + +def main(): + user_args = get_args() + + if user_args.host: + DataAccessLayer.changeEDEXHost(user_args.host) + + req = DataAccessLayer.newDataRequest("satellite") + + if not user_args.sectorID or not user_args.physicalElement: + print >> sys.stderr, "sectorID or physicalElement not provided" + return + req.setParameters(user_args.physicalElement) + req.addIdentifier("sectorID", user_args.sectorID) + + if user_args.creatingEntity: + req.addIdentifier("creatingEntity", user_args.creatingEntity) + + msg = ""; + times = DataAccessLayer.getAvailableTimes(req) + for time in times: + timeStr = str(time) + msg += timeStr[0:19] + ".0" + timeStr[19:] + "\n" + + print msg.strip() + +if __name__ == '__main__': + main() diff --git a/satellite/a2rdsat.csh b/satellite/a2rdsat.csh new file mode 100644 index 0000000..96bab07 --- /dev/null +++ b/satellite/a2rdsat.csh @@ -0,0 +1,243 @@ +#!/bin/csh +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## +# +# A script wrapper that is meant to get data for a single satellite sector +# from the A-II database. The result is output to stdout as ASCII. +# The first line returned has the dimensions of the image, the time, and the +# source satellite of the data set returned. The rest is one line per row +# of satellite data. The data for each row undergoes second order compression +# Each pixel value of 0 or 255 is encoded as @ or #, respectively. Otherwise +# the first pixel on the row and any pixel that is more than 20 counts +# different than the previous one is encoded as two hex digits. Pixels the +# same as the previous are encoded as a period, pixels from 1 to 20 counts less +# than the previous are encoded as G through Z, and pixels from 1 to 20 counts +# more than the previous are encoded as g through z. There are no delimeters +# between the encoding for each pixel. +# +# This version can adapt to use a python stub that calls the +# data access framework. +# +# Usage: +# +# a2rdsat.csh {p} {h|i} sector channel {satid} date time {slop} {partition} +# +# p - (optional) A literal p. +# h|i - (optional) A literal h or literal i. +# Output pure undelimited hex or delimited integer values. +# sector - sector id +# channel - channel id +# satid - (optional) satellite id +# date - yyyy-mm-dd +# time - hh:mm +# slop - seconds of slop either side, defaults to 180 +# partition - (optional) upper case letter indicating partition to get. For +# very large images data may need to be returned in pieces. +# Allowable partitions are A through D. +# +# The ids can be either D-2D integer ids, or AWIPS-II ascii ids, in which +# case they may need to be quoted on the command line. +# +# Integer ids can be looked up in a2satInfo.txt, channel id corresponds to +# the physicalElement, and satid corresponds to the creatingEntity. +# +# The literal p option means preserve the final version of the python +# submitted to the UEngine instead of cleaning it up. The path to the +# finalized python is /tmp/a2rdsatNNNNN.py where NNNNN is a unix process id. +# The literal n option means the first line of output is the dimension of +# the grid returned. +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# Sep 29, 2014 3596 nabowle Initial modification. Handle daf conversion. +# +# +# +set rmpy = yes +if ( "$1" == "p" ) then + set rmpy = no + shift +endif +set encoding = 2 +if ( "$1" == "h" ) then + set encoding = 1 + shift +endif +if ( "$1" == "i" ) then + set encoding = 0 + shift +endif +if ( "$1" == "p" ) then + set rmpy = no + shift +endif +# +# Identify directory this script is in, will be one of the directories we +# search for other files in. +# +set mydir = `dirname $0` +set d1 = `echo $mydir | cut -c1` +if ( "$mydir" == '.' ) then + set mydir = $PWD +else if ( "$d1" != "/" ) then + set mydir = $PWD/$mydir +endif +set mydir = `(cd $mydir ; pwd)` +if ( ! $?FXA_HOME ) set FXA_HOME = xxxx +# +# Locate python stub that we will modify to create the final python logic. +# +if ( -e ./a2rdsatStub.py ) then + set stubpy = ./a2rdsatStub.py +else if ( -e $mydir/a2rdsatStub.py ) then + set stubpy = $mydir/a2rdsatStub.py +else if ( -e $FXA_HOME/src/dm/sat/a2rdsatStub.py ) then + set stubpy = $FXA_HOME/src/dm/sat/a2rdsatStub.py +else if ( -e $FXA_HOME/bin/a2rdsatStub.py ) then + set stubpy = $FXA_HOME/bin/a2rdsatStub.py +else + bash -c "echo could not find a2rdsatStub.py 1>&2" + exit +endif + + +# +# Locate file containing mapping between D-2D interger ids and AWIPS-II ascii +# ids for sectors, channels, and satellites. +# +if ( -e ./a2satInfo.txt ) then + set satInf = ./a2satInfo.txt +else if ( -e $mydir/a2satInfo.txt ) then + set satInf = $mydir/a2satInfo.txt +else if ( -e $FXA_HOME/src/dm/sat/a2satInfo.txt ) then + set satInf = $FXA_HOME/src/dm/sat/a2satInfo.txt +else if ( -e $FXA_HOME/data/a2satInfo.txt ) then + set satInf = $FXA_HOME/data/a2satInfo.txt +else + set satInf = "" + set sss = "$1" + set ccc = "$2" + set eee = `echo $3 | grep -v '.*-'` +endif +# +# +if ( $satInf != "" ) then + set sss = `grep "^ *$1|.*sectorID" $satInf | cut '-d|' -f3` + if ( "$sss" == "" ) set sss = "$1" + set ccc = `grep "^ *$2|.*physicalElement" $satInf | cut '-d|' -f3` + if ( "$ccc" == "" ) set ccc = "$2" + set eee = `echo $3 | grep -v '.*-'` + if ( "$eee" != "" ) then + set eee = `grep "^ *$eee|.*creatingEntity" $satInf | cut '-d|' -f3` + if ( "$eee" == "" ) set eee = "$3" + endif +endif + +shift +shift + +if ( "$eee" != "" ) shift +set slop = `echo $3 | grep '[0-9]'` +if ( "$slop" == "" ) set slop = 180 + +set ppp = `echo $argv[$#argv] | grep '^[A-Z]$'` +if ( "$ppp" == "" ) set ppp = 0 + +# +# Determine if we are using the data access framework or the uEngine. +# +grep DataAccessLayer $stubpy >& /dev/null +if ( $status == 0 ) then + set opts = "" + if ( "$eee" != "" ) then + set opts = "--entity ${eee}" + endif + + if ( "$encoding" == "1" ) then + set opts = "$opts --hex" + else if ( "$encoding" == "0" ) then + set opts = "$opts --int" + endif + + /awips2/python/bin/python $stubpy --sector "${sss}" --physical "${ccc}" --datetime "$1 $2" --part $ppp --slop $slop $opts +else + # + # Get program that can do math with ascii time string, then use this to + # properly encode range of times for which we look for data. + # + if ( -x ./gtasUtil ) then + set gtasUtil = ./gtasUtil + else if ( -x $mydir/gtasUtil ) then + set gtasUtil = $mydir/gtasUtil + else if ( -x $FXA_HOME/src/dm/point/gtasUtil ) then + set gtasUtil = $FXA_HOME/src/dm/point/gtasUtil + else if ( -x $FXA_HOME/bin/gtasUtil ) then + set gtasUtil = $FXA_HOME/bin/gtasUtil + else + bash -c "echo could not find gtasUtil executable 1>&2" + exit + endif + + # + # Set up the environment we need to run the UEngine. + # + set method = "uengine" + if ( -e ./UEngine.cshsrc ) then + set ueenv = ./UEngine.cshsrc + else if ( -e $mydir/UEngine.cshsrc ) then + set ueenv = $mydir/UEngine.cshsrc + else if ( -e $FXA_HOME/src/dm/point/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/src/dm/point/UEngine.cshsrc + else if ( -e $FXA_HOME/bin/UEngine.cshsrc ) then + set ueenv = $FXA_HOME/bin/UEngine.cshsrc + else + bash -c "echo could not find UEngine.cshsrc 1>&2" + exit + endif + source $ueenv + + set aaa = `$gtasUtil = $1 $2 -$slop` + set bbb = `$gtasUtil = $1 $2 $slop` + + set specpy = /tmp/a2rdsat${$}.py + rm -rf $specpy >& /dev/null + touch $specpy + chmod 775 $specpy + if ( "$eee" == "" ) then + cat $stubpy | sed "s/SSSSS/$sss/g" | sed "s/CCCCC/$ccc/g" | \ + sed "s/AAAAA/$aaa/g" | sed "s/BBBBB/$bbb/g" | \ + sed 's/^.*EEEEE.*$//g'| sed "s/PPPPP/$ppp/g" | \ + sed "s/XXXXX/$encoding/g" >> $specpy + else + cat $stubpy | sed "s/SSSSS/$sss/g" | sed "s/CCCCC/$ccc/g" | \ + sed "s/AAAAA/$aaa/g" | sed "s/BBBBB/$bbb/g" | \ + sed "s/EEEEE/$eee/g" | sed "s/PPPPP/$ppp/g" | \ + sed "s/XXXXX/$encoding/g" >> $specpy + endif + + cd $UE_BIN_PATH + ( uengine -r python < $specpy ) | grep -v '<' | grep -v Response + + if ( "$rmpy" == "yes" ) rm -rf $specpy >& /dev/null + # +endif diff --git a/satellite/a2rdsatStub.py b/satellite/a2rdsatStub.py new file mode 100644 index 0000000..4ee8b1a --- /dev/null +++ b/satellite/a2rdsatStub.py @@ -0,0 +1,227 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# Gets data for a single satellite sector from the A-II database. The result is +# output to stdout as ASCII. The first line returned has the dimensions of the +# image, the time, and the source satellite of the data set returned. The rest +# is one line per row of satellite data. The data for each row undergoes second +# order compression Each pixel value of 0 or 255 is encoded as @ or #, +# respectively. Otherwise the first pixel on the row and any pixel that is more +# than 20 counts different than the previous one is encoded as two hex digits. +# Pixels the same as the previous are encoded as a period, pixels from 1 to 20 +# counts less than the previous are encoded as G through Z, and pixels from 1 to +# 20 counts more than the previous are encoded as g through z. There are no +# delimeters between the encoding for each pixel. +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# Sep 29, 2014 3596 nabowle Initial modification. Replace UEngine with DAF. +# +# + +import a2dafcommon +import argparse +import sys + +from datetime import datetime +from datetime import timedelta +from ufpy.dataaccess import DataAccessLayer +from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange + +def get_args(): + parser = argparse.ArgumentParser(conflict_handler="resolve") + parser.add_argument("-h", action="store", dest="host", + help="EDEX server hostname (optional)", + metavar="hostname") + parser.add_argument("--datetime", action="store", dest="datetime", + help="The start of the time range in YYYY-MM-DD HH:MM", + metavar="datetime") + parser.add_argument("--slop", action="store", dest="slop", default=180, + help="The amount of slop, in seconds, to allow around the datetime.", + metavar="slop", type=int) + parser.add_argument("--sector", action="store", dest="sector", + help="The sector ID.", metavar="sectorID") + parser.add_argument("--physical", action="store", dest="physical", + help="The physical element.", metavar="physicalElement") + parser.add_argument("--entity", action="store", dest="entity", + help="The creating entity (optional)", + metavar="creatingEntity") + parser.add_argument("--partition", action="store", dest="partition", + help="Upper case letter indicating partition to get.", + metavar="partition", default="0") + parser.add_argument("--hex", action='store_const', dest="encoding", + const=1, help="Hex encoding.", metavar="encoding") + parser.add_argument("--int", action='store_const', dest="encoding", + const=0, help="Delimited integer encoding.", + metavar="encoding") + return parser.parse_args() + + +def main(): + user_args = get_args() + + if user_args.host: + DataAccessLayer.changeEDEXHost(user_args.host) + + slop = user_args.slop + + dateTimeStr = user_args.datetime + if not dateTimeStr: + print >> sys.stderr, "DateTime not provided" + return + + physicalElement = user_args.physical + if not physicalElement: + print >> sys.stderr, "PhysicalElement not provided" + return + + sectorID = user_args.sector + if not sectorID: + print >> sys.stderr, "SectorID not provided" + return + + creatingEntity = user_args.entity + part = user_args.partition + encoding = user_args.encoding + + dateTime = datetime.strptime(dateTimeStr, "%Y-%m-%d %H:%M") + beginRange = dateTime - timedelta(0, slop) + endRange = dateTime + timedelta(0, slop) + + timerange = TimeRange(beginRange, endRange) + + req = DataAccessLayer.newDataRequest("satellite") + req.setParameters(physicalElement) + req.setLocationNames(sectorID) + + if creatingEntity: + req.addIdentifier("creatingEntity", creatingEntity) + + grids = DataAccessLayer.getGridData(req, timerange) + + if not grids: +# print "Data not available" + return + + grid = grids[0] + data = grid.getRawData() + myent = grid.getAttribute("creatingEntity") + mytime = a2dafcommon.datatime_to_string(grid.getDataTime()) + ".0" + + if data is None or len(data) == 0: +# print "No data." + return + + yLen = len(data[0]) + xLen = len(data) + + plus = " ghijklmnopqrstuvwxyz" + minus = " GHIJKLMNOPQRSTUVWXYZ" + limit = 10000000 + if encoding == 1 : + limit = limit/2 + elif encoding == 0 : + limit = limit/8 + + k = xLen * ( yLen / 4 ) + j = 0 + nxy = yLen*xLen + if part=="D" : + j = k+k+k + elif part=="C" : + j = k+k + nxy = j+k + elif part=="B" : + j = k + nxy = j+k + elif part=="A" or nxy>limit : + nxy = k + + msg = "" + if part<="A" : + msg += str(xLen) + " " + str(yLen) + " " + msg += mytime + " " + myent + "\n" + + dataIdx = int(j / yLen) + arrIdx = j % yLen + arr = data[dataIdx] + i = 0 + kk = None + while j= yLen : + arrIdx = 0 + dataIdx += 1 + if dataIdx >= xLen: + break + arr = data[dataIdx] + while ikk+20 or kkk : + msg += plus[k-kk] + else : + msg += minus[kk-k] + kk = k + + arrIdx+=1 + i+=1 + if arrIdx >= yLen : + arrIdx = 0 + dataIdx += 1 + if dataIdx >= xLen: + break + arr = data[dataIdx] + + msg += "\n" + j+= xLen + + print msg.strip() + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..946020a --- /dev/null +++ b/setup.py @@ -0,0 +1,8 @@ +from distutils.core import setup + +setup( + name='ufpy', + version='', + packages=['ufpy','ufpy.dataaccess',], + license='Creative Commons Attribution-Noncommercial-Share Alike license', +) diff --git a/ufpy/AlertVizHandler.py b/ufpy/AlertVizHandler.py new file mode 100644 index 0000000..aeb2ab0 --- /dev/null +++ b/ufpy/AlertVizHandler.py @@ -0,0 +1,70 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + + +# +# Pure python logging mechanism for logging to AlertViz from +# pure python (ie not JEP). DO NOT USE IN PYTHON CALLED +# FROM JAVA. +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 08/18/10 njensen Initial Creation. +# +# +# + +import logging +import NotificationMessage + +class AlertVizHandler(logging.Handler): + + def __init__(self, host='localhost', port=61999, category='LOCAL', source='ANNOUNCER', level=logging.NOTSET): + logging.Handler.__init__(self, level) + self._category = category + self._host = host + self._port = port + self._source = source + + + def emit(self, record): + "Implements logging.Handler's interface. Record argument is a logging.LogRecord." + priority = None + if record.levelno >= 50: + priority = 'CRITICAL' + elif record.levelno >= 40: + priority = 'SIGNIFICANT' + elif record.levelno >= 30: + priority = 'PROBLEM' + elif record.levelno >= 20: + priority = 'EVENTA' + elif record.levelno >= 10: + priority = 'EVENTB' + else: + priority = 'VERBOSE' + + msg = self.format(record) + + notify = NotificationMessage.NotificationMessage(self._host, self._port, msg, priority, self._category, self._source) + notify.send() + \ No newline at end of file diff --git a/ufpy/ConfigFileUtil.py b/ufpy/ConfigFileUtil.py new file mode 100644 index 0000000..58f8d67 --- /dev/null +++ b/ufpy/ConfigFileUtil.py @@ -0,0 +1,56 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# +# A set of utility functions for dealing with configuration files. +# +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 09/27/10 dgilling Initial Creation. +# +# +# + + +def parseKeyValueFile(fileName): + propDict= dict() + + try: + propFile= open(fileName, "rU") + for propLine in propFile: + propDef= propLine.strip() + if len(propDef) == 0: + continue + if propDef[0] in ( '#' ): + continue + punctuation= [ propDef.find(c) for c in ':= ' ] + [ len(propDef) ] + found= min( [ pos for pos in punctuation if pos != -1 ] ) + name= propDef[:found].rstrip() + value= propDef[found:].lstrip(":= ").rstrip() + propDict[name]= value + propFile.close() + except: + pass + + return propDict \ No newline at end of file diff --git a/ufpy/NotificationMessage.py b/ufpy/NotificationMessage.py new file mode 100644 index 0000000..6ec2174 --- /dev/null +++ b/ufpy/NotificationMessage.py @@ -0,0 +1,178 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +from string import Template + +import ctypes +import stomp +import socket +import sys +import time +import threading +import xml.etree.ElementTree as ET + +import ThriftClient +from dynamicserialize.dstypes.com.raytheon.uf.common.alertviz import AlertVizRequest +from dynamicserialize import DynamicSerializationManager + +# +# Provides a capability of constructing notification messages and sending +# them to a STOMP data source. +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 09/30/08 chammack Initial Creation. +# 11/03/10 5849 cjeanbap Moved to ufpy package from +# com.raytheon.uf.tools.cli +# 01/07/11 5645 cjeanbap Added audio file to Status Message. +# 05/27/11 3050 cjeanbap Added if-statement to check Priority +# value +# +class NotificationMessage: + + priorityMap = { + 0: 'CRITICAL', + 1: 'SIGNIFICANT', + 2: 'PROBLEM', + 3: 'EVENTA', + 4: 'EVENTB', + 5: 'VERBOSE'} + + def __init__(self, host='localhost', port=61999, message='', priority='PROBLEM', category="LOCAL", source="ANNOUNCER", audioFile="NONE"): + self.host = host + self.port = port + self.message = message + self.audioFile = audioFile + self.source = source + self.category = category + + priorityInt = None + + try: + priorityInt = int(priority) + except: + pass + + if priorityInt is None: + #UFStatus.java contains mapping of Priority to Logging level mapping + if priority == 'CRITICAL' or priority == 'FATAL': + priorityInt = int(0) + elif priority == 'SIGNIFICANT' or priority == 'ERROR': + priorityInt = int(1) + elif priority == 'PROBLEM' or priority == 'WARN': + priorityInt = int(2) + elif priority == 'EVENTA' or priority == 'INFO': + priorityInt = int(3) + elif priority == 'EVENTB': + priorityInt = int(4) + elif priority == 'VERBOSE' or priority == 'DEBUG': + priorityInt = int(5) + + if (priorityInt < 0 or priorityInt > 5): + print "Error occurred, supplied an invalid Priority value: " + str(priorityInt) + print "Priority values are 0, 1, 2, 3, 4 and 5." + sys.exit(1) + + if priorityInt is not None: + self.priority = self.priorityMap[priorityInt] + else: + self.priority = priority + + def connection_timeout(self, connection): + if (connection is not None and not connection.is_connected()): + print "Connection Retry Timeout" + for tid, tobj in threading._active.items(): + if tobj.name is "MainThread": + res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(SystemExit)) + if res != 0 and res != 1: + # problem, reset state + ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, 0) + + def send(self): + # depending on the value of the port number indicates the distribution + # of the message to AlertViz + # 9581 is global distribution thru ThriftClient to Edex + # 61999 is local distribution + if (self.port == 61999): + # use stomp.py + conn = stomp.Connection(host_and_ports=[(self.host, self.port)]) + timeout = threading.Timer(5.0, self.connection_timeout, [conn]) + + try: + timeout.start(); + conn.start() + finally: + timeout.cancel() + + conn.connect() + + sm = ET.Element("statusMessage") + sm.set("machine", socket.gethostname()) + sm.set("priority", self.priority) + sm.set("category", self.category) + sm.set("sourceKey", self.source) + sm.set("audioFile", self.audioFile) + msg = ET.SubElement(sm, "message") + msg.text = self.message + details = ET.SubElement(sm, "details") + msg = ET.tostring(sm, "UTF-8") + + try : + conn.send(msg, destination='/queue/messages') + time.sleep(2) + finally: + conn.stop() + else: + # use ThriftClient + alertVizRequest = createRequest(self.message, self.priority, self.source, self.category, self.audioFile) + thriftClient = ThriftClient.ThriftClient(self.host, self.port, "/services") + + serverResponse = None + try: + serverResponse = thriftClient.sendRequest(alertVizRequest) + except Exception, ex: + print "Caught exception submitting AlertVizRequest: ", str(ex) + + if (serverResponse != "None"): + print "Error occurred submitting Notification Message to AlertViz receiver: ", serverResponse + sys.exit(1) + else: + print "Response: " + str(serverResponse) + +def createRequest(message, priority, source, category, audioFile): + obj = AlertVizRequest() + + obj.setMachine(socket.gethostname()) + obj.setPriority(priority) + obj.setCategory(category) + obj.setSourceKey(source) + obj.setMessage(message) + if (audioFile is not None): + obj.setAudioFile(audioFile) + else: + obj.setAudioFile('\0') + + return obj + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/ufpy/QpidSubscriber.py b/ufpy/QpidSubscriber.py new file mode 100644 index 0000000..567e5c8 --- /dev/null +++ b/ufpy/QpidSubscriber.py @@ -0,0 +1,100 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# +# Provides a Python-based interface for subscribing to qpid queues and topics. +# +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 11/17/10 njensen Initial Creation. +# 08/15/13 2169 bkowal Optionally gzip decompress any data that is read. +# +# + +import qpid +import zlib + +from Queue import Empty +from qpid.exceptions import Closed + +class QpidSubscriber: + + def __init__(self, host='127.0.0.1', port=5672, decompress=False): + self.host = host + self.port = port + self.decompress = decompress; + socket = qpid.util.connect(host, port) + self.__connection = qpid.connection.Connection(sock=socket, username='guest', password='guest') + self.__connection.start() + self.__session = self.__connection.session(str(qpid.datatypes.uuid4())) + self.subscribed = True + + def topicSubscribe(self, topicName, callback): + # if the queue is edex.alerts, set decompress to true always for now to + # maintain compatibility with existing python scripts. + if (topicName == 'edex.alerts'): + self.decompress = True + + print "Establishing connection to broker on", self.host + queueName = topicName + self.__session.name + self.__session.queue_declare(queue=queueName, exclusive=True, auto_delete=True, arguments={'qpid.max_count':100, 'qpid.policy_type':'ring'}) + self.__session.exchange_bind(exchange='amq.topic', queue=queueName, binding_key=topicName) + self.__innerSubscribe(queueName, callback) + + def __innerSubscribe(self, serverQueueName, callback): + local_queue_name = 'local_queue_' + serverQueueName + queue = self.__session.incoming(local_queue_name) + self.__session.message_subscribe(serverQueueName, destination=local_queue_name) + queue.start() + print "Connection complete to broker on", self.host + + while self.subscribed: + try: + message = queue.get(timeout=10) + content = message.body + self.__session.message_accept(qpid.datatypes.RangedSet(message.id)) + if (self.decompress): + print "Decompressing received content" + try: + # http://stackoverflow.com/questions/2423866/python-decompressing-gzip-chunk-by-chunk + d = zlib.decompressobj(16+zlib.MAX_WBITS) + content = d.decompress(content) + except: + # decompression failed, return the original content + pass + callback(content) + except Empty: + pass + except Closed: + self.close() + + def close(self): + self.subscribed = False + try: + self.__session.close(timeout=10) + except: + pass + + + \ No newline at end of file diff --git a/ufpy/ThriftClient.py b/ufpy/ThriftClient.py new file mode 100644 index 0000000..2bd5ee7 --- /dev/null +++ b/ufpy/ThriftClient.py @@ -0,0 +1,102 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +import httplib +from dynamicserialize import DynamicSerializationManager +from dynamicserialize.dstypes.com.raytheon.uf.common.serialization.comm.response import ServerErrorResponse +from dynamicserialize.dstypes.com.raytheon.uf.common.serialization import SerializableExceptionWrapper + +# +# Provides a Python-based interface for executing Thrift requests. +# +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 09/20/10 dgilling Initial Creation. +# +# +# + + +class ThriftClient: + + # How to call this constructor: + # 1. Pass in all arguments separately (e.g., + # ThriftClient.ThriftClient("localhost", 9581, "/services")) + # will return a Thrift client pointed at http://localhost:9581/services. + # 2. Pass in all arguments through the host string (e.g., + # ThriftClient.ThriftClient("localhost:9581/services")) + # will return a Thrift client pointed at http://localhost:9581/services. + # 3. Pass in host/port arguments through the host string (e.g., + # ThriftClient.ThriftClient("localhost:9581", "/services")) + # will return a Thrift client pointed at http://localhost:9581/services. + def __init__(self, host, port=9581, uri="/services"): + hostParts = host.split("/", 1) + if (len(hostParts) > 1): + hostString = hostParts[0] + self.__uri = "/" + hostParts[1] + self.__httpConn = httplib.HTTPConnection(hostString) + else: + if (port is None): + self.__httpConn = httplib.HTTPConnection(host) + else: + self.__httpConn = httplib.HTTPConnection(host, port) + + self.__uri = uri + + self.__dsm = DynamicSerializationManager.DynamicSerializationManager() + + def sendRequest(self, request, uri="/thrift"): + message = self.__dsm.serializeObject(request) + + self.__httpConn.connect() + self.__httpConn.request("POST", self.__uri + uri, message) + + response = self.__httpConn.getresponse() + if (response.status != 200): + raise ThriftRequestException("Unable to post request to server") + + rval = self.__dsm.deserializeBytes(response.read()) + self.__httpConn.close() + + # let's verify we have an instance of ServerErrorResponse + # IF we do, through an exception up to the caller along + # with the original Java stack trace + # ELSE: we have a valid response and pass it back + try: + forceError = rval.getException() + raise ThriftRequestException(forceError) + except AttributeError: + pass + + return rval + + +class ThriftRequestException(Exception): + def __init__(self, value): + self.parameter = value + + def __str__(self): + return repr(self.parameter) + + \ No newline at end of file diff --git a/ufpy/TimeUtil.py b/ufpy/TimeUtil.py new file mode 100644 index 0000000..e4dc631 --- /dev/null +++ b/ufpy/TimeUtil.py @@ -0,0 +1,108 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## +# ---------------------------------------------------------------------------- +# This software is in the public domain, furnished "as is", without technical +# support, and with no warranty, express or implied, as to its usefulness for +# any purpose. +# +# offsetTime.py +# Handles Displaced Real Time for various applications +# +# Author: hansen/romberg +# ---------------------------------------------------------------------------- + +import string +import time + +# Given the timeStr, return the offset (in seconds) +# from the current time. +# Also return the launchStr i.e. Programs launched from this +# offset application will use the launchStr as the -z argument. +# The offset will be positive for time in the future, +# negative for time in the past. +# +# May still want it to be normalized to the most recent midnight. +# +# NOTES about synchronizing: +# --With synchronizing on, the "current time" for all processes started +# within a given hour will be the same. +# This guarantees that GFE's have the same current time and ISC grid +# time stamps are syncrhonized and can be exchanged. +# Formatters launched from the GFE in this mode will be synchronized as +# well by setting the launchStr to use the time difference format +# (YYYYMMDD_HHMM,YYYYMMDD_HHMM). +# --This does not solve the problem in the general case. +# For example, if someone starts the GFE at 12:59 and someone +# else starts it at 1:01, they will have different offsets and +# current times. +# --With synchronizing off, when the process starts, the current time +# matches the drtTime in the command line. However, with synchronizing +# on, the current time will be offset by the fraction of the hour at +# which the process was started. Examples: +# Actual Starting time: 20040617_1230 +# drtTime 20040616_0000 +# Synchronizing off: +# GFE Spatial Editor at StartUp: 20040616_0000 +# Synchronizing on: +# GFE Spatial Editor at StartUp: 20040616_0030 +# +def determineDrtOffset(timeStr): + launchStr = timeStr + # Check for time difference + if timeStr.find(",") >=0: + times = timeStr.split(",") + t1 = makeTime(times[0]) + t2 = makeTime(times[1]) + #print "time offset", t1-t2, (t1-t2)/3600 + return t1-t2, launchStr + # Check for synchronized mode + synch = 0 + if timeStr[0] == "S": + timeStr = timeStr[1:] + synch = 1 + drt_t = makeTime(timeStr) + #print "input", year, month, day, hour, minute + gm = time.gmtime() + cur_t = time.mktime(gm) + + # Synchronize to most recent hour + # i.e. "truncate" cur_t to most recent hour. + #print "gmtime", gm + if synch: + cur_t = time.mktime((gm[0], gm[1], gm[2], gm[3], 0, 0, 0, 0, 0)) + curStr = '%4s%2s%2s_%2s00\n' % (`gm[0]`,`gm[1]`,`gm[2]`,`gm[3]`) + curStr = curStr.replace(' ','0') + launchStr = timeStr + "," + curStr + + #print "drt, cur", drt_t, cur_t + offset = drt_t - cur_t + #print "offset", offset, offset/3600, launchStr + return int(offset), launchStr + +def makeTime(timeStr): + year = string.atoi(timeStr[0:4]) + month = string.atoi(timeStr[4:6]) + day = string.atoi(timeStr[6:8]) + hour = string.atoi(timeStr[9:11]) + minute = string.atoi(timeStr[11:13]) + # Do not use daylight savings because gmtime is not in daylight + # savings time. + return time.mktime((year, month, day, hour, minute, 0, 0, 0, 0)) + diff --git a/ufpy/UsageArgumentParser.py b/ufpy/UsageArgumentParser.py new file mode 100644 index 0000000..3d51564 --- /dev/null +++ b/ufpy/UsageArgumentParser.py @@ -0,0 +1,58 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +import argparse +import sys + +from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import DatabaseID +from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import ParmID + + +class UsageArgumentParser(argparse.ArgumentParser): + """ + A subclass of ArgumentParser that overrides error() to print the + whole help text, rather than just the usage string. + """ + def error(self, message): + sys.stderr.write('%s: error: %s\n' % (self.prog, message)) + self.print_help() + sys.exit(2) + +## Custom actions for ArgumentParser objects ## +class StoreDatabaseIDAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + did = DatabaseID(values) + if did.isValid(): + setattr(namespace, self.dest, did) + else: + parser.error("DatabaseID [" + values + "] not a valid identifier") + +class AppendParmNameAndLevelAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + tx = ParmID.parmNameAndLevel(values) + comp = tx[0] + '_' + tx[1] + if (hasattr(namespace, self.dest)) and \ + (getattr(namespace, self.dest) is not None): + currentValues = getattr(namespace, self.dest) + currentValues.append(comp) + setattr(namespace, self.dest, currentValues) + else: + setattr(namespace, self.dest, [comp]) + diff --git a/ufpy/UsageOptionParser.py b/ufpy/UsageOptionParser.py new file mode 100644 index 0000000..0f2938f --- /dev/null +++ b/ufpy/UsageOptionParser.py @@ -0,0 +1,38 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +import sys +from optparse import OptionParser + +class UsageOptionParser(OptionParser): + """ + A subclass of OptionParser that prints that overrides error() to print the + whole help text, rather than just the usage string. + """ + def error(self, msg): + """ + Print the help text and exit. + """ + self.print_help(sys.stderr) + sys.stderr.write("\n") + sys.stderr.write(msg) + sys.stderr.write("\n") + sys.exit(2) + diff --git a/ufpy/__init__.py b/ufpy/__init__.py new file mode 100644 index 0000000..6ed9ac6 --- /dev/null +++ b/ufpy/__init__.py @@ -0,0 +1,37 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + + +# +# __init__.py for ufpy package +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 09/21/10 dgilling Initial Creation. +# +# +# + + +__all__ = [ + ] diff --git a/ufpy/dataaccess/DataAccessLayer.py b/ufpy/dataaccess/DataAccessLayer.py new file mode 100644 index 0000000..1bda6fd --- /dev/null +++ b/ufpy/dataaccess/DataAccessLayer.py @@ -0,0 +1,216 @@ +# # +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +# # + + +# +# Published interface for ufpy.dataaccess package +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 12/10/12 njensen Initial Creation. +# Feb 14, 2013 1614 bsteffen refactor data access framework +# to use single request. +# 04/10/13 1871 mnash move getLatLonCoords to JGridData and add default args +# 05/29/13 2023 dgilling Hook up ThriftClientRouter. +# 03/03/14 2673 bsteffen Add ability to query only ref times. +# 07/22/14 3185 njensen Added optional/default args to newDataRequest +# 07/30/14 3185 njensen Renamed valid identifiers to optional +# +# +# + + +import sys +import subprocess + +THRIFT_HOST = subprocess.check_output( + "source /awips2/fxa/bin/setup.env; echo $DEFAULT_HOST", + shell=True).strip() +USING_NATIVE_THRIFT = False + +if sys.modules.has_key('JavaImporter'): + # intentionally do not catch if this fails to import, we want it to + # be obvious that something is configured wrong when running from within + # Java instead of allowing false confidence and fallback behavior + import JepRouter + router = JepRouter +else: + from ufpy.dataaccess import ThriftClientRouter + router = ThriftClientRouter.ThriftClientRouter(THRIFT_HOST) + USING_NATIVE_THRIFT = True + + +def getAvailableTimes(request, refTimeOnly=False): + """ + Get the times of available data to the request. + + Args: + request: the IDataRequest to get data for + refTimeOnly: optional, use True if only unique refTimes should be + returned (without a forecastHr) + + Returns: + a list of DataTimes + """ + return router.getAvailableTimes(request, refTimeOnly) + +def getGridData(request, times=[]): + """ + Gets the grid data that matches the request at the specified times. Each + combination of parameter, level, and dataTime will be returned as a + separate IGridData. + + Args: + request: the IDataRequest to get data for + times: a list of DataTimes, a TimeRange, or None if the data is time + agnostic + + Returns: + a list of IGridData + """ + return router.getGridData(request, times) + +def getGeometryData(request, times=[]): + """ + Gets the geometry data that matches the request at the specified times. + Each combination of geometry, level, and dataTime will be returned as a + separate IGeometryData. + + Args: + request: the IDataRequest to get data for + times: a list of DataTimes, a TimeRange, or None if the data is time + agnostic + + Returns: + a list of IGeometryData + """ + return router.getGeometryData(request, times) + +def getAvailableLocationNames(request): + """ + Gets the available location names that match the request without actually + requesting the data. + + Args: + request: the request to find matching location names for + + Returns: + a list of strings of available location names. + """ + return router.getAvailableLocationNames(request) + +def getAvailableParameters(request): + """ + Gets the available parameters names that match the request without actually + requesting the data. + + Args: + request: the request to find matching parameter names for + + Returns: + a list of strings of available parameter names. + """ + return router.getAvailableParameters(request) + +def getAvailableLevels(request): + """ + Gets the available levels that match the request without actually + requesting the data. + + Args: + request: the request to find matching levels for + + Returns: + a list of strings of available levels. + """ + return router.getAvailableLevels(request) + +def getRequiredIdentifiers(datatype): + """ + Gets the required identifiers for this datatype. These identifiers + must be set on a request for the request of this datatype to succeed. + + Args: + datatype: the datatype to find required identifiers for + + Returns: + a list of strings of required identifiers + """ + return router.getRequiredIdentifiers(datatype) + +def getOptionalIdentifiers(datatype): + """ + Gets the optional identifiers for this datatype. + + Args: + datatype: the datatype to find optional identifiers for + + Returns: + a list of strings of optional identifiers + """ + return router.getOptionalIdentifiers(datatype) + +def newDataRequest(datatype=None, **kwargs): + """" + Creates a new instance of IDataRequest suitable for the runtime environment. + All args are optional and exist solely for convenience. + + Args: + datatype: the datatype to create a request for + parameters: a list of parameters to set on the request + levels: a list of levels to set on the request + locationNames: a list of locationNames to set on the request + envelope: an envelope to limit the request + **kwargs: any leftover kwargs will be set as identifiers + + Returns: + a new IDataRequest + """ + return router.newDataRequest(datatype, **kwargs) + +def getSupportedDatatypes(): + """ + Gets the datatypes that are supported by the framework + + Returns: + a list of strings of supported datatypes + """ + return router.getSupportedDatatypes() + + +def changeEDEXHost(newHostName): + """ + Changes the EDEX host the Data Access Framework is communicating with. Only + works if using the native Python client implementation, otherwise, this + method will throw a TypeError. + + Args: + newHostHame: the EDEX host to connect to + """ + if USING_NATIVE_THRIFT: + global THRIFT_HOST + THRIFT_HOST = newHostName + global router + router = ThriftClientRouter.ThriftClientRouter(THRIFT_HOST) + else: + raise TypeError("Cannot call changeEDEXHost when using JepRouter.") diff --git a/ufpy/dataaccess/PyData.py b/ufpy/dataaccess/PyData.py new file mode 100644 index 0000000..0d1a1ef --- /dev/null +++ b/ufpy/dataaccess/PyData.py @@ -0,0 +1,57 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# +# Implements IData for use by native Python clients to the Data Access +# Framework. +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 06/03/13 dgilling Initial Creation. +# +# + +from ufpy.dataaccess import IData + +class PyData(IData): + + def __init__(self, dataRecord): + self.__time = dataRecord.getTime() + self.__level = dataRecord.getLevel() + self.__locationName = dataRecord.getLocationName() + self.__attributes = dataRecord.getAttributes() + + def getAttribute(self, key): + return self.__attributes[key] + + def getAttributes(self): + return self.__attributes.keys() + + def getDataTime(self): + return self.__time + + def getLevel(self): + return self.__level + + def getLocationName(self): + return self.__locationName diff --git a/ufpy/dataaccess/PyGeometryData.py b/ufpy/dataaccess/PyGeometryData.py new file mode 100644 index 0000000..0bf300d --- /dev/null +++ b/ufpy/dataaccess/PyGeometryData.py @@ -0,0 +1,79 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# +# Implements IGeometryData for use by native Python clients to the Data Access +# Framework. +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 06/03/13 dgilling Initial Creation. +# 01/06/14 #2537 bsteffen Share geometry WKT. +# 03/19/14 #2882 dgilling Raise an exception when getNumber() +# is called for data that is not a +# numeric Type. +# +# + +from ufpy.dataaccess import IGeometryData +from ufpy.dataaccess import PyData + +class PyGeometryData(IGeometryData, PyData.PyData): + + def __init__(self, geoDataRecord, geometry): + PyData.PyData.__init__(self, geoDataRecord) + self.__geometry = geometry + self.__dataMap = {} + tempDataMap = geoDataRecord.getDataMap() + for key, value in tempDataMap.items(): + self.__dataMap[key] = (value[0], value[1], value[2]) + + def getGeometry(self): + return self.__geometry + + def getParameters(self): + return self.__dataMap.keys() + + def getString(self, param): + value = self.__dataMap[param][0] + return str(value) + + def getNumber(self, param): + value = self.__dataMap[param][0] + t = self.getType(param) + if t == 'INT': + return int(value) + elif t == 'LONG': + return long(value) + elif t == 'FLOAT': + return float(value) + elif t == 'DOUBLE': + return float(value) + else: + raise TypeError("Data for parameter " + param + " is not a numeric type.") + + def getUnit(self, param): + return self.__dataMap[param][2] + + def getType(self, param): + return self.__dataMap[param][1] diff --git a/ufpy/dataaccess/PyGridData.py b/ufpy/dataaccess/PyGridData.py new file mode 100644 index 0000000..eff1898 --- /dev/null +++ b/ufpy/dataaccess/PyGridData.py @@ -0,0 +1,72 @@ +# # +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +# # + +# +# Implements IGridData for use by native Python clients to the Data Access +# Framework. +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 06/03/13 #2023 dgilling Initial Creation. +# +# + + +import numpy +import warnings + +from ufpy.dataaccess import IGridData +from ufpy.dataaccess import PyData + +NO_UNIT_CONVERT_WARNING = """ +The ability to unit convert grid data is not currently available in this version of the Data Access Framework. +""" + + +class PyGridData(IGridData, PyData.PyData): + + def __init__(self, gridDataRecord, nx, ny, latLonGrid): + PyData.PyData.__init__(self, gridDataRecord) + nx = nx + ny = ny + self.__parameter = gridDataRecord.getParameter() + self.__unit = gridDataRecord.getUnit() + self.__gridData = numpy.reshape(numpy.array(gridDataRecord.getGridData()), (nx, ny)) + self.__latLonGrid = latLonGrid + + def getParameter(self): + return self.__parameter + + def getUnit(self): + return self.__unit + + def getRawData(self, unit=None): + # TODO: Find a proper python library that deals will with numpy and + # javax.measure style unit strings and hook it in to this method to + # allow end-users to perform unit conversion for grid data. + if unit is not None: + warnings.warn(NO_UNIT_CONVERT_WARNING, stacklevel=2) + return self.__gridData + + def getLatLonCoords(self): + return self.__latLonGrid diff --git a/ufpy/dataaccess/ThriftClientRouter.py b/ufpy/dataaccess/ThriftClientRouter.py new file mode 100644 index 0000000..6c96301 --- /dev/null +++ b/ufpy/dataaccess/ThriftClientRouter.py @@ -0,0 +1,172 @@ +# # +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +# # + +# +# Routes requests to the Data Access Framework through Python Thrift. +# +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 05/21/13 #2023 dgilling Initial Creation. +# 01/06/14 #2537 bsteffen Share geometry WKT. +# 03/03/14 #2673 bsteffen Add ability to query only ref times. +# 07/22/14 #3185 njensen Added optional/default args to newDataRequest +# 07/23/14 #3185 njensen Added new methods +# 07/30/14 #3185 njensen Renamed valid identifiers to optional +# + + +import numpy +import shapely.wkt + +from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.impl import DefaultDataRequest +from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableLocationNamesRequest +from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableTimesRequest +from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetGeometryDataRequest +from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetGridDataRequest +from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableParametersRequest +from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableLevelsRequest +from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetRequiredIdentifiersRequest +from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetOptionalIdentifiersRequest +from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetSupportedDatatypesRequest + +from ufpy import ThriftClient +from ufpy.dataaccess import PyGeometryData +from ufpy.dataaccess import PyGridData + + +class ThriftClientRouter(object): + + def __init__(self, host='localhost'): + self._client = ThriftClient.ThriftClient(host) + + def getAvailableTimes(self, request, refTimeOnly): + timesRequest = GetAvailableTimesRequest() + timesRequest.setRequestParameters(request) + timesRequest.setRefTimeOnly(refTimeOnly) + response = self._client.sendRequest(timesRequest) + return response + + def getGridData(self, request, times): + gridDataRequest = GetGridDataRequest() + gridDataRequest.setRequestParameters(request) + # if we have an iterable times instance, then the user must have asked + # for grid data with the List of DataTime objects + # else, we assume it was a single TimeRange that was meant for the + # request + try: + iter(times) + gridDataRequest.setRequestedTimes(times) + except TypeError: + gridDataRequest.setRequestedPeriod(times) + response = self._client.sendRequest(gridDataRequest) + + locSpecificData = {} + locNames = response.getSiteNxValues().keys() + for location in locNames: + nx = response.getSiteNxValues()[location] + ny = response.getSiteNyValues()[location] + latData = numpy.reshape(numpy.array(response.getSiteLatGrids()[location]), (nx, ny)) + lonData = numpy.reshape(numpy.array(response.getSiteLonGrids()[location]), (nx, ny)) + locSpecificData[location] = (nx, ny, (lonData, latData)) + + retVal = [] + for gridDataRecord in response.getGridData(): + locationName = gridDataRecord.getLocationName() + locData = locSpecificData[locationName] + retVal.append(PyGridData.PyGridData(gridDataRecord, locData[0], locData[1], locData[2])) + return retVal + + def getGeometryData(self, request, times): + geoDataRequest = GetGeometryDataRequest() + geoDataRequest.setRequestParameters(request) + # if we have an iterable times instance, then the user must have asked + # for geometry data with the List of DataTime objects + # else, we assume it was a single TimeRange that was meant for the + # request + try: + iter(times) + geoDataRequest.setRequestedTimes(times) + except TypeError: + geoDataRequest.setRequestedPeriod(times) + response = self._client.sendRequest(geoDataRequest) + geometries = [] + for wkt in response.getGeometryWKTs(): + geometries.append(shapely.wkt.loads(wkt)) + + retVal = [] + for geoDataRecord in response.getGeoData(): + geom = geometries[geoDataRecord.getGeometryWKTindex()] + retVal.append(PyGeometryData.PyGeometryData(geoDataRecord, geom)) + return retVal + + def getAvailableLocationNames(self, request): + locNamesRequest = GetAvailableLocationNamesRequest() + locNamesRequest.setRequestParameters(request) + response = self._client.sendRequest(locNamesRequest) + return response + + def getAvailableParameters(self, request): + paramReq = GetAvailableParametersRequest() + paramReq.setRequestParameters(request) + response = self._client.sendRequest(paramReq) + return response + + def getAvailableLevels(self, request): + levelReq = GetAvailableLevelsRequest() + levelReq.setRequestParameters(request) + response = self._client.sendRequest(levelReq) + return response + + def getRequiredIdentifiers(self, datatype): + idReq = GetRequiredIdentifiersRequest() + idReq.setDatatype(datatype) + response = self._client.sendRequest(idReq) + return response + + def getOptionalIdentifiers(self, datatype): + idReq = GetOptionalIdentifiersRequest() + idReq.setDatatype(datatype) + response = self._client.sendRequest(idReq) + return response + + def newDataRequest(self, datatype, parameters=[], levels=[], locationNames = [], envelope=None, **kwargs): + req = DefaultDataRequest() + if datatype: + req.setDatatype(datatype) + if parameters: + req.setParameters(*parameters) + if levels: + req.setLevels(*levels) + if locationNames: + req.setLocationNames(*locationNames) + if envelope: + req.setEnvelope(envelope) + if kwargs: + # any args leftover are assumed to be identifiers + req.identifiers = kwargs + return req + + def getSupportedDatatypes(self): + response = self._client.sendRequest(GetSupportedDatatypesRequest()) + return response diff --git a/ufpy/dataaccess/__init__.py b/ufpy/dataaccess/__init__.py new file mode 100644 index 0000000..e581bfe --- /dev/null +++ b/ufpy/dataaccess/__init__.py @@ -0,0 +1,353 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + + +# +# __init__.py for ufpy.dataaccess package +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 12/10/12 njensen Initial Creation. +# Feb 14, 2013 1614 bsteffen refactor data access framework +# to use single request. +# Apr 09, 2013 1871 njensen Add doc strings +# Jun 03, 2013 2023 dgilling Add getAttributes to IData, add +# getLatLonGrids() to IGridData. +# +# + +__all__ = [ + + ] + +import abc + +class IDataRequest(object): + """ + An IDataRequest to be submitted to the DataAccessLayer to retrieve data. + """ + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def setDatatype(self, datatype): + """ + Sets the datatype of the request. + + Args: + datatype: A string of the datatype, such as "grid", "radar", "gfe", "obs" + """ + return + + @abc.abstractmethod + def addIdentifier(self, key, value): + """ + Adds an identifier to the request. Identifiers are specific to the + datatype being requested. + + Args: + key: the string key of the identifier + value: the value of the identifier + """ + return + + @abc.abstractmethod + def setParameters(self, params): + """ + Sets the parameters of data to request. + + Args: + params: a list of strings of parameters to request + """ + return + + @abc.abstractmethod + def setLevels(self, levels): + """ + Sets the levels of data to request. Not all datatypes support levels. + + Args: + levels: a list of strings of level abbreviations to request + """ + return + + @abc.abstractmethod + def setEnvelope(self, env): + """ + Sets the envelope of the request. If supported by the datatype factory, + the data returned for the request will be constrained to only the data + within the envelope. + + Args: + env: a shapely geometry + """ + return + + @abc.abstractmethod + def setLocationNames(self, locationNames): + """ + Sets the location names of the request. + + Args: + locationNames: a list of strings of location names to request + """ + return + + @abc.abstractmethod + def getDatatype(self): + """ + Gets the datatype of the request + + Returns: + the datatype set on the request + """ + return + + @abc.abstractmethod + def getIdentifiers(self): + """ + Gets the identifiers on the request + + Returns: + a dictionary of the identifiers + """ + return + + @abc.abstractmethod + def getLevels(self): + """ + Gets the levels on the request + + Returns: + a list of strings of the levels + """ + return + + @abc.abstractmethod + def getLocationNames(self): + """ + Gets the location names on the request + + Returns: + a list of strings of the location names + """ + return + + @abc.abstractmethod + def getEnvelope(self): + """ + Gets the envelope on the request + + Returns: + a rectangular shapely geometry + """ + return + + + +class IData(object): + """ + An IData representing data returned from the DataAccessLayer. + """ + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def getAttribute(self, key): + """ + Gets an attribute of the data. + + Args: + key: the key of the attribute + + Returns: + the value of the attribute + """ + return + + @abc.abstractmethod + def getAttributes(self): + """ + Gets the valid attributes for the data. + + Returns: + a list of strings of the attribute names + """ + return + + @abc.abstractmethod + def getDataTime(self): + """ + Gets the data time of the data. + + Returns: + the data time of the data, or None if no time is associated + """ + return + + @abc.abstractmethod + def getLevel(self): + """ + Gets the level of the data. + + Returns: + the level of the data, or None if no level is associated + """ + return + + @abc.abstractmethod + def getLocationName(self, param): + """ + Gets the location name of the data. + + Returns: + the location name of the data, or None if no location name is + associated + """ + return + + + +class IGridData(IData): + """ + An IData representing grid data that is returned by the DataAccessLayer. + """ + + @abc.abstractmethod + def getParameter(self): + """ + Gets the parameter of the data. + + Returns: + the parameter of the data + """ + return + + @abc.abstractmethod + def getUnit(self): + """ + Gets the unit of the data. + + Returns: + the string abbreviation of the unit, or None if no unit is associated + """ + return + + @abc.abstractmethod + def getRawData(self): + """ + Gets the grid data as a numpy array. + + Returns: + a numpy array of the data + """ + return + + @abc.abstractmethod + def getLatLonCoords(self): + """ + Gets the lat/lon coordinates of the grid data. + + Returns: + a tuple where the first element is a numpy array of lons, and the + second element is a numpy array of lats + """ + return + + + +class IGeometryData(IData): + """ + An IData representing geometry data that is returned by the DataAccessLayer. + """ + + @abc.abstractmethod + def getGeometry(self): + """ + Gets the geometry of the data. + + Returns: + a shapely geometry + """ + return + + @abc.abstractmethod + def getParameters(self): + """Gets the parameters of the data. + + Returns: + a list of strings of the parameter names + """ + return + + @abc.abstractmethod + def getString(self, param): + """ + Gets the string value of the specified param. + + Args: + param: the string name of the param + + Returns: + the string value of the param + """ + return + + @abc.abstractmethod + def getNumber(self, param): + """ + Gets the number value of the specified param. + + Args: + param: the string name of the param + + Returns: + the number value of the param + """ + return + + @abc.abstractmethod + def getUnit(self, param): + """ + Gets the unit of the specified param. + + Args: + param: the string name of the param + + Returns: + the string abbreviation of the unit of the param + """ + return + + @abc.abstractmethod + def getType(self, param): + """ + Gets the type of the param. + + Args: + param: the string name of the param + + Returns: + a string of the type of the parameter, such as + "STRING", "INT", "LONG", "FLOAT", or "DOUBLE" + """ + return + diff --git a/ufpy/gfe/IFPClient.py b/ufpy/gfe/IFPClient.py new file mode 100644 index 0000000..1288094 --- /dev/null +++ b/ufpy/gfe/IFPClient.py @@ -0,0 +1,173 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +from ufpy import ThriftClient + +from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import DatabaseID +from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import ParmID +from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import CommitGridsRequest +from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import GetGridInventoryRequest +from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import GetParmListRequest +from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import GetSelectTimeRangeRequest +from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.server.request import CommitGridRequest +from dynamicserialize.dstypes.com.raytheon.uf.common.message import WsId +from dynamicserialize.dstypes.com.raytheon.uf.common.site.requests import GetActiveSitesRequest +from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.server.message import ServerResponse + + +# +# Provides a Python-based interface for executing GFE requests. +# +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 07/26/12 dgilling Initial Creation. +# +# +# + + +class IFPClient(object): + def __init__(self, host, port, user, site=None, progName=None): + self.__thrift = ThriftClient.ThriftClient(host, port) + self.__wsId = WsId(userName=user, progName=progName) + # retrieve default site + if site is None: + sr = self.getSiteID() + if len(sr.getPayload()) > 0: + site = sr.getPayload()[0] + self.__siteId = site + + def commitGrid(self, request): + if type(request) is CommitGridRequest: + return self.__commitGrid([request]) + elif self.__isHomogenousIterable(request, CommitGridRequest): + return self.__commitGrid([cgr for cgr in request]) + raise TypeError("Invalid type: " + str(type(request)) + " specified to commitGrid(). Only accepts CommitGridRequest or lists of CommitGridRequest.") + + def __commitGrid(self, requests): + ssr = ServerResponse() + request = CommitGridsRequest() + request.setCommits(requests) + sr = self.__makeRequest(request) + ssr.setMessages(sr.getMessages()) + return ssr + + def getParmList(self, id): + argType = type(id) + if argType is DatabaseID: + return self.__getParmList([id]) + elif self.__isHomogenousIterable(id, DatabaseID): + return self.__getParmList([dbid for dbid in id]) + raise TypeError("Invalid type: " + str(argType) + " specified to getParmList(). Only accepts DatabaseID or lists of DatabaseID.") + + def __getParmList(self, ids): + ssr = ServerResponse() + request = GetParmListRequest() + request.setDbIds(ids) + sr = self.__makeRequest(request) + ssr.setMessages(sr.getMessages()) + list = sr.getPayload() if sr.getPayload() is not None else [] + ssr.setPayload(list) + return ssr + + def __isHomogenousIterable(self, iterable, classType): + try: + iterator = iter(iterable) + for item in iterator: + if not isinstance(item, classType): + return False + except TypeError: + return False + return True + + def getGridInventory(self, parmID): + if type(parmID) is ParmID: + sr = self.__getGridInventory([parmID]) + list = [] + try: + list = sr.getPayload()[parmID] + except KeyError: + # no-op, we've already default the TimeRange list to empty + pass + sr.setPayload(list) + return sr + elif self.__isHomogenousIterable(parmID, ParmID): + return self.__getGridInventory([id for id in parmID]) + raise TypeError("Invalid type: " + str(type(parmID)) + " specified to getGridInventory(). Only accepts ParmID or lists of ParmID.") + + def __getGridInventory(self, parmIDs): + ssr = ServerResponse() + request = GetGridInventoryRequest() + request.setParmIds(parmIDs) + sr = self.__makeRequest(request) + ssr.setMessages(sr.getMessages()) + trs = sr.getPayload() if sr.getPayload() is not None else {} + ssr.setPayload(trs) + return ssr + + def getSelectTR(self, name): + request = GetSelectTimeRangeRequest() + request.setName(name) + sr = self.__makeRequest(request) + ssr = ServerResponse() + ssr.setMessages(sr.getMessages()) + ssr.setPayload(sr.getPayload()) + return ssr + + def getSiteID(self): + ssr = ServerResponse() + request = GetActiveSitesRequest() + sr = self.__makeRequest(request) + ssr.setMessages(sr.getMessages()) + ids = sr.getPayload() if sr.getPayload() is not None else [] + sr.setPayload(ids) + return sr + + def __makeRequest(self, request): + try: + request.setSiteID(self.__siteId) + except AttributeError: + pass + try: + request.setWorkstationID(self.__wsId) + except AttributeError: + pass + + sr = ServerResponse() + response = None + try: + response = self.__thrift.sendRequest(request) + except ThriftClient.ThriftRequestException as e: + sr.setMessages([str(e)]) + try: + sr.setPayload(response.getPayload()) + except AttributeError: + sr.setPayload(response) + try: + sr.setMessages(response.getMessages()) + except AttributeError: + # not a server response, nothing else to do + pass + + return sr diff --git a/ufpy/gfe/__init__.py b/ufpy/gfe/__init__.py new file mode 100644 index 0000000..f2a16d5 --- /dev/null +++ b/ufpy/gfe/__init__.py @@ -0,0 +1,37 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + + +# +# __init__.py for ufpy.gfe package +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 07/26/12 dgilling Initial Creation. +# +# +# + + +__all__ = [ + ] diff --git a/ufpy/qpidingest.py b/ufpy/qpidingest.py new file mode 100644 index 0000000..7a920a5 --- /dev/null +++ b/ufpy/qpidingest.py @@ -0,0 +1,111 @@ +#=============================================================================== +# qpidingest.py +# +# @author: Aaron Anderson +# @organization: NOAA/WDTB OU/CIMMS +# @version: 1.0 02/19/2010 +# @requires: QPID Python Client available from http://qpid.apache.org/download.html +# The Python Client is located under Single Component Package/Client +# +# From the README.txt Installation Instructions +# = INSTALLATION = +# Extract the release archive into a directory of your choice and set +# your PYTHONPATH accordingly: +# +# tar -xzf qpid-python-.tar.gz -C +# export PYTHONPATH=/qpid-/python +# +# ***EDEX and QPID must be running for this module to work*** +# +# DESCRIPTION: +# This module is used to connect to QPID and send messages to the external.dropbox queue +# which tells EDEX to ingest a data file from a specified path. This avoids having to copy +# a data file into an endpoint. Each message also contains a header which is used to determine +# which plugin should be used to decode the file. Each plugin has an xml file located in +# $EDEX_HOME/data/utility/edex_static/base/distribution that contains regular expressions +# that the header is compared to. When the header matches one of these regular expressions +# the file is decoded with that plugin. If you make changes to one of these xml files you +# must restart EDEX for the changes to take effect. +# +# NOTE: If the message is being sent but you do not see it being ingested in the EDEX log +# check the xml files to make sure the header you are passing matches one of the regular +# expressions. Beware of spaces, some regular expressions require spaces while others use +# a wildcard character so a space is optional. It seems you are better off having the space +# as this will be matched to both patterns. For the file in the example below, +# 20100218_185755_SAUS46KLOX.metar, I use SAUS46 KLOX as the header to make sure it matches. +# +# +# EXAMPLE: +# Simple example program: +# +#------------------------------------------------------------------------------ +# import qpidingest +# #Tell EDEX to ingest a metar file from data_store. The filepath is +# #/data_store/20100218/metar/00/standard/20100218_005920_SAUS46KSEW.metar +# +# conn=qpidingest.IngestViaQPID() #defaults to localhost port 5672 +# +# #If EDEX is not on the local machine you can make the connection as follows +# #conn=qpidingest.IngestViaQPID(host='',port=) +# +# conn.sendmessage('/data_store/20100218/metar/18/standard/20100218_185755_SAUS46KLOX.metar','SAUS46 KLOX') +# conn.close() +#------------------------------------------------------------------------------- +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# .... +# 06/13/2013 DR 16242 D. Friedman Add Qpid authentication info +# 03/06/2014 DR 17907 D. Friedman Workaround for issue QPID-5569 +# +#=============================================================================== + +import qpid +from qpid.util import connect +from qpid.connection import Connection +from qpid.datatypes import Message, uuid4 + +QPID_USERNAME = 'guest' +QPID_PASSWORD = 'guest' + +class IngestViaQPID: + def __init__(self, host='localhost', port=5672): + ''' + Connect to QPID and make bindings to route message to external.dropbox queue + @param host: string hostname of computer running EDEX and QPID (default localhost) + @param port: integer port used to connect to QPID (default 5672) + ''' + + try: + # + self.socket = connect(host, port) + self.connection = Connection (sock=self.socket, username=QPID_USERNAME, password=QPID_PASSWORD) + self.connection.start() + self.session = self.connection.session(str(uuid4())) + self.session.exchange_bind(exchange='amq.direct', queue='external.dropbox', binding_key='external.dropbox') + print 'Connected to Qpid' + except: + print 'Unable to connect to Qpid' + + def sendmessage(self, filepath, header): + ''' + This function sends a message to the external.dropbox queue providing the path + to the file to be ingested and a header to determine the plugin to be used to + decode the file. + @param filepath: string full path to file to be ingested + @param header: string header used to determine plugin decoder to use + ''' + props = self.session.delivery_properties(routing_key='external.dropbox') + head = self.session.message_properties(application_headers={'header':header}, + user_id=QPID_USERNAME) # For issue QPID-5569. Fixed in Qpid 0.27 + self.session.message_transfer(destination='amq.direct', message=Message(props, head, filepath)) + + def close(self): + ''' + After all messages are sent call this function to close connection and make sure + there are no threads left open + ''' + self.session.close(timeout=10) + print 'Connection to Qpid closed' \ No newline at end of file diff --git a/ufpy/stomp.py b/ufpy/stomp.py new file mode 100644 index 0000000..9356af9 --- /dev/null +++ b/ufpy/stomp.py @@ -0,0 +1,934 @@ +#!/usr/bin/env python +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## +"""Stomp Protocol Connectivity + + This provides basic connectivity to a message broker supporting the 'stomp' protocol. + At the moment ACK, SEND, SUBSCRIBE, UNSUBSCRIBE, BEGIN, ABORT, COMMIT, CONNECT and DISCONNECT operations + are supported. + + This changes the previous version which required a listener per subscription -- now a listener object + just calls the 'addlistener' method and will receive all messages sent in response to all/any subscriptions. + (The reason for the change is that the handling of an 'ack' becomes problematic unless the listener mechanism + is decoupled from subscriptions). + + Note that you must 'start' an instance of Connection to begin receiving messages. For example: + + conn = stomp.Connection([('localhost', 62003)], 'myuser', 'mypass') + conn.start() + + Meta-Data + --------- + Author: Jason R Briggs + License: http://www.apache.org/licenses/LICENSE-2.0 + Start Date: 2005/12/01 + Last Revision Date: $Date: 2008/09/11 00:16 $ + + Notes/Attribution + ----------------- + * uuid method courtesy of Carl Free Jr: + http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/213761 + * patch from Andreas Schobel + * patches from Julian Scheid of Rising Sun Pictures (http://open.rsp.com.au) + * patch from Fernando + * patches from Eugene Strulyov + + Updates + ------- + * 2007/03/31 : (Andreas Schobel) patch to fix newlines problem in ActiveMQ 4.1 + * 2007/09 : (JRB) updated to get stomp.py working in Jython as well as Python + * 2007/09/05 : (Julian Scheid) patch to allow sending custom headers + * 2007/09/18 : (JRB) changed code to use logging instead of just print. added logger for jython to work + * 2007/09/18 : (Julian Scheid) various updates, including: + - change incoming message handling so that callbacks are invoked on the listener not only for MESSAGE, but also for + CONNECTED, RECEIPT and ERROR frames. + - callbacks now get not only the payload but any headers specified by the server + - all outgoing messages now sent via a single method + - only one connection used + - change to use thread instead of threading + - sends performed on the calling thread + - receiver loop now deals with multiple messages in one received chunk of data + - added reconnection attempts and connection fail-over + - changed defaults for "user" and "passcode" to None instead of empty string (fixed transmission of those values) + - added readline support + * 2008/03/26 : (Fernando) added cStringIO for faster performance on large messages + * 2008/09/10 : (Eugene) remove lower() on headers to support case-sensitive header names + * 2008/09/11 : (JRB) fix incompatibilities with RabbitMQ, add wait for socket-connect + * 2008/10/28 : (Eugene) add jms map (from stomp1.1 ideas) + * 2008/11/25 : (Eugene) remove superfluous (incorrect) locking code + * 2009/02/05 : (JRB) remove code to replace underscores with dashes in header names (causes a problem in rabbit-mq) + * 2009/03/29 : (JRB) minor change to add logging config file + (JRB) minor change to add socket timeout, suggested by Israel + * 2009/04/01 : (Gavin) patch to change md5 to hashlib (for 2.6 compatibility) + * 2009/04/02 : (Fernando Ciciliati) fix overflow bug when waiting too long to connect to the broker + +""" + +import hashlib +import math +import random +import re +import socket +import sys +import thread +import threading +import time +import types +import xml.dom.minidom +from cStringIO import StringIO + +# +# stomp.py version number +# +_version = 1.8 + + +def _uuid( *args ): + """ + uuid courtesy of Carl Free Jr: + (http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/213761) + """ + + t = long( time.time() * 1000 ) + r = long( random.random() * 100000000000000000L ) + + try: + a = socket.gethostbyname( socket.gethostname() ) + except: + # if we can't get a network address, just imagine one + a = random.random() * 100000000000000000L + data = str(t) + ' ' + str(r) + ' ' + str(a) + ' ' + str(args) + md5 = hashlib.md5() + md5.update(data) + data = md5.hexdigest() + return data + + +class DevNullLogger(object): + """ + dummy logging class for environments without the logging module + """ + def log(self, msg): + print msg + + def devnull(self, msg): + pass + + debug = devnull + info = devnull + warning = log + error = log + critical = log + exception = log + + def isEnabledFor(self, lvl): + return False + + +# +# add logging if available +# +try: + import logging + import logging.config + logging.config.fileConfig("stomp.log.conf") + log = logging.getLogger('root') +except: + log = DevNullLogger() + + +class ConnectionClosedException(Exception): + """ + Raised in the receiver thread when the connection has been closed + by the server. + """ + pass + + +class NotConnectedException(Exception): + """ + Raised by Connection.__send_frame when there is currently no server + connection. + """ + pass + + +class ConnectionListener(object): + """ + This class should be used as a base class for objects registered + using Connection.add_listener(). + """ + def on_connecting(self, host_and_port): + """ + Called by the STOMP connection once a TCP/IP connection to the + STOMP server has been established or re-established. Note that + at this point, no connection has been established on the STOMP + protocol level. For this, you need to invoke the "connect" + method on the connection. + + \param host_and_port a tuple containing the host name and port + number to which the connection has been established. + """ + pass + + def on_connected(self, headers, body): + """ + Called by the STOMP connection when a CONNECTED frame is + received, that is after a connection has been established or + re-established. + + \param headers a dictionary containing all headers sent by the + server as key/value pairs. + + \param body the frame's payload. This is usually empty for + CONNECTED frames. + """ + pass + + def on_disconnected(self): + """ + Called by the STOMP connection when a TCP/IP connection to the + STOMP server has been lost. No messages should be sent via + the connection until it has been reestablished. + """ + pass + + def on_message(self, headers, body): + """ + Called by the STOMP connection when a MESSAGE frame is + received. + + \param headers a dictionary containing all headers sent by the + server as key/value pairs. + + \param body the frame's payload - the message body. + """ + pass + + def on_receipt(self, headers, body): + """ + Called by the STOMP connection when a RECEIPT frame is + received, sent by the server if requested by the client using + the 'receipt' header. + + \param headers a dictionary containing all headers sent by the + server as key/value pairs. + + \param body the frame's payload. This is usually empty for + RECEIPT frames. + """ + pass + + def on_error(self, headers, body): + """ + Called by the STOMP connection when an ERROR frame is + received. + + \param headers a dictionary containing all headers sent by the + server as key/value pairs. + + \param body the frame's payload - usually a detailed error + description. + """ + pass + + +class Connection(object): + """ + Represents a STOMP client connection. + """ + + def __init__(self, + host_and_ports = [ ('localhost', 61613) ], + user = None, + passcode = None, + prefer_localhost = True, + try_loopback_connect = True, + reconnect_sleep_initial = 0.1, + reconnect_sleep_increase = 0.5, + reconnect_sleep_jitter = 0.1, + reconnect_sleep_max = 60.0): + """ + Initialize and start this connection. + + \param host_and_ports + a list of (host, port) tuples. + + \param prefer_localhost + if True and the local host is mentioned in the (host, + port) tuples, try to connect to this first + + \param try_loopback_connect + if True and the local host is found in the host + tuples, try connecting to it using loopback interface + (127.0.0.1) + + \param reconnect_sleep_initial + + initial delay in seconds to wait before reattempting + to establish a connection if connection to any of the + hosts fails. + + \param reconnect_sleep_increase + + factor by which the sleep delay is increased after + each connection attempt. For example, 0.5 means + to wait 50% longer than before the previous attempt, + 1.0 means wait twice as long, and 0.0 means keep + the delay constant. + + \param reconnect_sleep_max + + maximum delay between connection attempts, regardless + of the reconnect_sleep_increase. + + \param reconnect_sleep_jitter + + random additional time to wait (as a percentage of + the time determined using the previous parameters) + between connection attempts in order to avoid + stampeding. For example, a value of 0.1 means to wait + an extra 0%-10% (randomly determined) of the delay + calculated using the previous three parameters. + """ + + sorted_host_and_ports = [] + sorted_host_and_ports.extend(host_and_ports) + + # If localhost is preferred, make sure all (host, port) tuples + # that refer to the local host come first in the list + if prefer_localhost: + def is_local_host(host): + return host in Connection.__localhost_names + + sorted_host_and_ports.sort(lambda x, y: (int(is_local_host(y[0])) + - int(is_local_host(x[0])))) + + # If the user wishes to attempt connecting to local ports + # using the loopback interface, for each (host, port) tuple + # referring to a local host, add an entry with the host name + # replaced by 127.0.0.1 if it doesn't exist already + loopback_host_and_ports = [] + if try_loopback_connect: + for host_and_port in sorted_host_and_ports: + if is_local_host(host_and_port[0]): + port = host_and_port[1] + if (not ("127.0.0.1", port) in sorted_host_and_ports + and not ("localhost", port) in sorted_host_and_ports): + loopback_host_and_ports.append(("127.0.0.1", port)) + + # Assemble the final, possibly sorted list of (host, port) tuples + self.__host_and_ports = [] + self.__host_and_ports.extend(loopback_host_and_ports) + self.__host_and_ports.extend(sorted_host_and_ports) + + self.__recvbuf = '' + + self.__listeners = [ ] + + self.__reconnect_sleep_initial = reconnect_sleep_initial + self.__reconnect_sleep_increase = reconnect_sleep_increase + self.__reconnect_sleep_jitter = reconnect_sleep_jitter + self.__reconnect_sleep_max = reconnect_sleep_max + + self.__connect_headers = {} + if user is not None and passcode is not None: + self.__connect_headers['login'] = user + self.__connect_headers['passcode'] = passcode + + self.__socket = None + self.__current_host_and_port = None + + self.__receiver_thread_exit_condition = threading.Condition() + self.__receiver_thread_exited = False + + # + # Manage the connection + # + + def start(self): + """ + Start the connection. This should be called after all + listeners have been registered. If this method is not called, + no frames will be received by the connection. + """ + self.__running = True + self.__attempt_connection() + thread.start_new_thread(self.__receiver_loop, ()) + + def stop(self): + """ + Stop the connection. This is equivalent to calling + disconnect() but will do a clean shutdown by waiting for the + receiver thread to exit. + """ + self.disconnect() + + self.__receiver_thread_exit_condition.acquire() + if not self.__receiver_thread_exited: + self.__receiver_thread_exit_condition.wait() + self.__receiver_thread_exit_condition.release() + + def get_host_and_port(self): + """ + Return a (host, port) tuple indicating which STOMP host and + port is currently connected, or None if there is currently no + connection. + """ + return self.__current_host_and_port + + def is_connected(self): + try: + return self.__socket is not None and self.__socket.getsockname()[1] != 0 + except socket.error: + return False + + # + # Manage objects listening to incoming frames + # + + def add_listener(self, listener): + self.__listeners.append(listener) + + def remove_listener(self, listener): + self.__listeners.remove(listener) + + # + # STOMP transmissions + # + + def subscribe(self, headers={}, **keyword_headers): + self.__send_frame_helper('SUBSCRIBE', '', self.__merge_headers([headers, keyword_headers]), [ 'destination' ]) + + def unsubscribe(self, headers={}, **keyword_headers): + self.__send_frame_helper('UNSUBSCRIBE', '', self.__merge_headers([headers, keyword_headers]), [ ('destination', 'id') ]) + + def send(self, message='', headers={}, **keyword_headers): + if '\x00' in message: + content_length_headers = {'content-length': len(message)} + else: + content_length_headers = {} + self.__send_frame_helper('SEND', message, self.__merge_headers([headers, + keyword_headers, + content_length_headers]), [ 'destination' ]) + + def ack(self, headers={}, **keyword_headers): + self.__send_frame_helper('ACK', '', self.__merge_headers([headers, keyword_headers]), [ 'message-id' ]) + + def begin(self, headers={}, **keyword_headers): + use_headers = self.__merge_headers([headers, keyword_headers]) + if not 'transaction' in use_headers.keys(): + use_headers['transaction'] = _uuid() + self.__send_frame_helper('BEGIN', '', use_headers, [ 'transaction' ]) + return use_headers['transaction'] + + def abort(self, headers={}, **keyword_headers): + self.__send_frame_helper('ABORT', '', self.__merge_headers([headers, keyword_headers]), [ 'transaction' ]) + + def commit(self, headers={}, **keyword_headers): + self.__send_frame_helper('COMMIT', '', self.__merge_headers([headers, keyword_headers]), [ 'transaction' ]) + + def connect(self, headers={}, **keyword_headers): + if keyword_headers.has_key('wait') and keyword_headers['wait']: + while not self.is_connected(): time.sleep(0.1) + del keyword_headers['wait'] + self.__send_frame_helper('CONNECT', '', self.__merge_headers([self.__connect_headers, headers, keyword_headers]), [ ]) + + def disconnect(self, headers={}, **keyword_headers): + self.__send_frame_helper('DISCONNECT', '', self.__merge_headers([self.__connect_headers, headers, keyword_headers]), [ ]) + self.__running = False + if hasattr(socket, 'SHUT_RDWR'): + self.__socket.shutdown(socket.SHUT_RDWR) + if self.__socket: + self.__socket.close() + self.__current_host_and_port = None + + # ========= PRIVATE MEMBERS ========= + + + # List of all host names (unqualified, fully-qualified, and IP + # addresses) that refer to the local host (both loopback interface + # and external interfaces). This is used for determining + # preferred targets. + __localhost_names = [ "localhost", + "127.0.0.1", + socket.gethostbyname(socket.gethostname()), + socket.gethostname(), + socket.getfqdn(socket.gethostname()) ] + # + # Used to parse STOMP header lines in the format "key:value", + # + __header_line_re = re.compile('(?P[^:]+)[:](?P.*)') + + # + # Used to parse the STOMP "content-length" header lines, + # + __content_length_re = re.compile('^content-length[:]\\s*(?P[0-9]+)', re.MULTILINE) + + def __merge_headers(self, header_map_list): + """ + Helper function for combining multiple header maps into one. + + Any underscores ('_') in header names (keys) will be replaced by dashes ('-'). + """ + headers = {} + for header_map in header_map_list: + for header_key in header_map.keys(): + headers[header_key] = header_map[header_key] + return headers + + def __convert_dict(self, payload): + """ + Encode python dictionary as ... structure. + """ + + xmlStr = "\n" + for key in payload: + xmlStr += "\n" + xmlStr += "%s" % key + xmlStr += "%s" % payload[key] + xmlStr += "\n" + xmlStr += "" + + return xmlStr + + def __send_frame_helper(self, command, payload, headers, required_header_keys): + """ + Helper function for sending a frame after verifying that a + given set of headers are present. + + \param command the command to send + + \param payload the frame's payload + + \param headers a dictionary containing the frame's headers + + \param required_header_keys a sequence enumerating all + required header keys. If an element in this sequence is itself + a tuple, that tuple is taken as a list of alternatives, one of + which must be present. + + \throws ArgumentError if one of the required header keys is + not present in the header map. + """ + for required_header_key in required_header_keys: + if type(required_header_key) == tuple: + found_alternative = False + for alternative in required_header_key: + if alternative in headers.keys(): + found_alternative = True + if not found_alternative: + raise KeyError("Command %s requires one of the following headers: %s" % (command, str(required_header_key))) + elif not required_header_key in headers.keys(): + raise KeyError("Command %s requires header %r" % (command, required_header_key)) + self.__send_frame(command, headers, payload) + + def __send_frame(self, command, headers={}, payload=''): + """ + Send a STOMP frame. + """ + if type(payload) == dict: + headers["transformation"] = "jms-map-xml" + payload = self.__convert_dict(payload) + + if self.__socket is not None: + frame = '%s\n%s\n%s\x00' % (command, + reduce(lambda accu, key: accu + ('%s:%s\n' % (key, headers[key])), headers.keys(), ''), + payload) + self.__socket.sendall(frame) + log.debug("Sent frame: type=%s, headers=%r, body=%r" % (command, headers, payload)) + else: + raise NotConnectedException() + + def __receiver_loop(self): + """ + Main loop listening for incoming data. + """ + try: + try: + threading.currentThread().setName("StompReceiver") + while self.__running: + log.debug('starting receiver loop') + + if self.__socket is None: + break + + try: + try: + for listener in self.__listeners: + if hasattr(listener, 'on_connecting'): + listener.on_connecting(self.__current_host_and_port) + + while self.__running: + frames = self.__read() + + for frame in frames: + (frame_type, headers, body) = self.__parse_frame(frame) + log.debug("Received frame: result=%r, headers=%r, body=%r" % (frame_type, headers, body)) + frame_type = frame_type.lower() + if frame_type in [ 'connected', + 'message', + 'receipt', + 'error' ]: + for listener in self.__listeners: + if hasattr(listener, 'on_%s' % frame_type): + eval('listener.on_%s(headers, body)' % frame_type) + else: + log.debug('listener %s has no such method on_%s' % (listener, frame_type)) + else: + log.warning('Unknown response frame type: "%s" (frame length was %d)' % (frame_type, len(frame))) + finally: + try: + self.__socket.close() + except: + pass # ignore errors when attempting to close socket + self.__socket = None + self.__current_host_and_port = None + except ConnectionClosedException: + if self.__running: + log.error("Lost connection") + # Notify listeners + for listener in self.__listeners: + if hasattr(listener, 'on_disconnected'): + listener.on_disconnected() + # Clear out any half-received messages after losing connection + self.__recvbuf = '' + continue + else: + break + except: + log.exception("An unhandled exception was encountered in the stomp receiver loop") + + finally: + self.__receiver_thread_exit_condition.acquire() + self.__receiver_thread_exited = True + self.__receiver_thread_exit_condition.notifyAll() + self.__receiver_thread_exit_condition.release() + + def __read(self): + """ + Read the next frame(s) from the socket. + """ + fastbuf = StringIO() + while self.__running: + try: + c = self.__socket.recv(1024) + except: + c = '' + if len(c) == 0: + raise ConnectionClosedException + fastbuf.write(c) + if '\x00' in c: + break + self.__recvbuf += fastbuf.getvalue() + fastbuf.close() + result = [] + + if len(self.__recvbuf) > 0 and self.__running: + while True: + pos = self.__recvbuf.find('\x00') + if pos >= 0: + frame = self.__recvbuf[0:pos] + preamble_end = frame.find('\n\n') + if preamble_end >= 0: + content_length_match = Connection.__content_length_re.search(frame[0:preamble_end]) + if content_length_match: + content_length = int(content_length_match.group('value')) + content_offset = preamble_end + 2 + frame_size = content_offset + content_length + if frame_size > len(frame): + # Frame contains NUL bytes, need to + # read more + if frame_size < len(self.__recvbuf): + pos = frame_size + frame = self.__recvbuf[0:pos] + else: + # Haven't read enough data yet, + # exit loop and wait for more to + # arrive + break + result.append(frame) + self.__recvbuf = self.__recvbuf[pos+1:] + else: + break + return result + + + def __transform(self, body, transType): + """ + Perform body transformation. Currently, the only supported transformation is + 'jms-map-xml', which converts a map into python dictionary. This can be extended + to support other transformation types. + + The body has the following format: + + + name + Dejan + + + city + Belgrade + + + + (see http://docs.codehaus.org/display/STOMP/Stomp+v1.1+Ideas) + """ + + if transType != 'jms-map-xml': + return body + + try: + entries = {} + doc = xml.dom.minidom.parseString(body) + rootElem = doc.documentElement + for entryElem in rootElem.getElementsByTagName("entry"): + pair = [] + for node in entryElem.childNodes: + if not isinstance(node, xml.dom.minidom.Element): continue + pair.append(node.firstChild.nodeValue) + assert len(pair) == 2 + entries[pair[0]] = pair[1] + return entries + except Exception, ex: + # unable to parse message. return original + return body + + + def __parse_frame(self, frame): + """ + Parse a STOMP frame into a (frame_type, headers, body) tuple, + where frame_type is the frame type as a string (e.g. MESSAGE), + headers is a map containing all header key/value pairs, and + body is a string containing the frame's payload. + """ + preamble_end = frame.find('\n\n') + preamble = frame[0:preamble_end] + preamble_lines = preamble.split('\n') + body = frame[preamble_end+2:] + + # Skip any leading newlines + first_line = 0 + while first_line < len(preamble_lines) and len(preamble_lines[first_line]) == 0: + first_line += 1 + + # Extract frame type + frame_type = preamble_lines[first_line] + + # Put headers into a key/value map + headers = {} + for header_line in preamble_lines[first_line+1:]: + header_match = Connection.__header_line_re.match(header_line) + if header_match: + headers[header_match.group('key')] = header_match.group('value') + + if 'transformation' in headers: + body = self.__transform(body, headers['transformation']) + + return (frame_type, headers, body) + + def __attempt_connection(self): + """ + Try connecting to the (host, port) tuples specified at construction time. + """ + + sleep_exp = 1 + while self.__running and self.__socket is None: + for host_and_port in self.__host_and_ports: + try: + log.debug("Attempting connection to host %s, port %s" % host_and_port) + self.__socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.__socket.settimeout(None) + self.__socket.connect(host_and_port) + self.__current_host_and_port = host_and_port + log.info("Established connection to host %s, port %s" % host_and_port) + break + except socket.error: + self.__socket = None + if type(sys.exc_info()[1]) == types.TupleType: + exc = sys.exc_info()[1][1] + else: + exc = sys.exc_info()[1] + log.warning("Could not connect to host %s, port %s: %s" % (host_and_port[0], host_and_port[1], exc)) + + if self.__socket is None: + sleep_duration = (min(self.__reconnect_sleep_max, + ((self.__reconnect_sleep_initial / (1.0 + self.__reconnect_sleep_increase)) + * math.pow(1.0 + self.__reconnect_sleep_increase, sleep_exp))) + * (1.0 + random.random() * self.__reconnect_sleep_jitter)) + sleep_end = time.time() + sleep_duration + log.debug("Sleeping for %.1f seconds before attempting reconnect" % sleep_duration) + while self.__running and time.time() < sleep_end: + time.sleep(0.2) + + if sleep_duration < self.__reconnect_sleep_max: + sleep_exp += 1 + +# +# command line testing +# +if __name__ == '__main__': + + # If the readline module is available, make command input easier + try: + import readline + def stomp_completer(text, state): + commands = [ 'subscribe', 'unsubscribe', + 'send', 'ack', + 'begin', 'abort', 'commit', + 'connect', 'disconnect' + ] + for command in commands[state:]: + if command.startswith(text): + return "%s " % command + return None + + readline.parse_and_bind("tab: complete") + readline.set_completer(stomp_completer) + readline.set_completer_delims("") + except ImportError: + pass # ignore unavailable readline module + + class StompTester(object): + def __init__(self, host='localhost', port=61613, user='', passcode=''): + self.c = Connection([(host, port)], user, passcode) + self.c.add_listener(self) + self.c.start() + + def __print_async(self, frame_type, headers, body): + print "\r \r", + print frame_type + for header_key in headers.keys(): + print '%s: %s' % (header_key, headers[header_key]) + print + print body + print '> ', + sys.stdout.flush() + + def on_connecting(self, host_and_port): + self.c.connect(wait=True) + + def on_disconnected(self): + print "lost connection" + + def on_message(self, headers, body): + self.__print_async("MESSAGE", headers, body) + + def on_error(self, headers, body): + self.__print_async("ERROR", headers, body) + + def on_receipt(self, headers, body): + self.__print_async("RECEIPT", headers, body) + + def on_connected(self, headers, body): + self.__print_async("CONNECTED", headers, body) + + def ack(self, args): + if len(args) < 3: + self.c.ack(message_id=args[1]) + else: + self.c.ack(message_id=args[1], transaction=args[2]) + + def abort(self, args): + self.c.abort(transaction=args[1]) + + def begin(self, args): + print 'transaction id: %s' % self.c.begin() + + def commit(self, args): + if len(args) < 2: + print 'expecting: commit ' + else: + print 'committing %s' % args[1] + self.c.commit(transaction=args[1]) + + def disconnect(self, args): + try: + self.c.disconnect() + except NotConnectedException: + pass # ignore if no longer connected + + def send(self, args): + if len(args) < 3: + print 'expecting: send ' + else: + self.c.send(destination=args[1], message=' '.join(args[2:])) + + def sendtrans(self, args): + if len(args) < 3: + print 'expecting: sendtrans ' + else: + self.c.send(destination=args[1], message="%s\n" % ' '.join(args[3:]), transaction=args[2]) + + def subscribe(self, args): + if len(args) < 2: + print 'expecting: subscribe [ack]' + elif len(args) > 2: + print 'subscribing to "%s" with acknowledge set to "%s"' % (args[1], args[2]) + self.c.subscribe(destination=args[1], ack=args[2]) + else: + print 'subscribing to "%s" with auto acknowledge' % args[1] + self.c.subscribe(destination=args[1], ack='auto') + + def unsubscribe(self, args): + if len(args) < 2: + print 'expecting: unsubscribe ' + else: + print 'unsubscribing from "%s"' % args[1] + self.c.unsubscribe(destination=args[1]) + + if len(sys.argv) > 5: + print 'USAGE: stomp.py [host] [port] [user] [passcode]' + sys.exit(1) + + if len(sys.argv) >= 2: + host = sys.argv[1] + else: + host = "localhost" + if len(sys.argv) >= 3: + port = int(sys.argv[2]) + else: + port = 61613 + + if len(sys.argv) >= 5: + user = sys.argv[3] + passcode = sys.argv[4] + else: + user = None + passcode = None + + st = StompTester(host, port, user, passcode) + try: + while True: + line = raw_input("\r> ") + if not line or line.lstrip().rstrip() == '': + continue + elif 'quit' in line or 'disconnect' in line: + break + split = line.split() + command = split[0] + if not command.startswith("on_") and hasattr(st, command): + getattr(st, command)(split) + else: + print 'unrecognized command' + finally: + st.disconnect(None) + + diff --git a/ufpy/test/Record.py b/ufpy/test/Record.py new file mode 100644 index 0000000..ac69d51 --- /dev/null +++ b/ufpy/test/Record.py @@ -0,0 +1,48 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + + +# +# Pure python logging mechanism for logging to AlertViz from +# pure python (ie not JEP). DO NOT USE IN PYTHON CALLED +# FROM JAVA. +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 11/03/10 5849 cjeanbap Initial Creation. +# +# +# + +import os +import sys + +class Record(): + def __init__(self, level=0, msg='Test Message'): + self.levelno=level + self.message=msg + self.exc_info=sys.exc_info() + self.exc_text="TEST" + + def getMessage(self): + return self.message \ No newline at end of file diff --git a/ufpy/test/Test b/ufpy/test/Test new file mode 100644 index 0000000..4ff1a0b --- /dev/null +++ b/ufpy/test/Test @@ -0,0 +1,48 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + + +# +# Pure python logging mechanism for logging to AlertViz from +# pure python (ie not JEP). DO NOT USE IN PYTHON CALLED +# FROM JAVA. +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 11/03/10 5849 cjeanbap Initial Creation. +# +# +# + +## to execute type python Test + + +import os +import logging +from ufpy import AlertVizHandler +import Record + +avh = AlertVizHandler.AlertVizHandler(host=os.getenv("BROKER_ADDR","localhost"), port=9581, category='LOCAL', source='ANNOUNCER', level=logging.NOTSET) +record = Record.Record(10) +avh.emit(record) + \ No newline at end of file diff --git a/ufpy/test/__init__.py b/ufpy/test/__init__.py new file mode 100644 index 0000000..ae5228c --- /dev/null +++ b/ufpy/test/__init__.py @@ -0,0 +1,34 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + + +# +# __init__.py for ufpy package +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 11/03/10 5489 cjeanbap Initial Creation. +# +# +# + diff --git a/ufpy/test/testQpidTimeToLive.py b/ufpy/test/testQpidTimeToLive.py new file mode 100644 index 0000000..ce3f074 --- /dev/null +++ b/ufpy/test/testQpidTimeToLive.py @@ -0,0 +1,104 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# +# +# +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 03/09/11 njensen Initial Creation. +# 08/15/13 2169 bkowal Decompress data read from the queue +# +# +# + +import time, sys +import threading + +import dynamicserialize + +TIME_TO_SLEEP = 300 + +class ListenThread(threading.Thread): + + def __init__(self, hostname, portNumber, topicName): + self.hostname = hostname + self.portNumber = portNumber + self.topicName = topicName + self.nMessagesReceived = 0 + self.waitSecond = 0 + self.stopped = False + threading.Thread.__init__(self) + + def run(self): + from ufpy import QpidSubscriber + self.qs = QpidSubscriber.QpidSubscriber(self.hostname, self.portNumber, True) + self.qs.topicSubscribe(self.topicName, self.receivedMessage) + + def receivedMessage(self, msg): + print "Received message" + self.nMessagesReceived += 1 + if self.waitSecond == 0: + fmsg = open('/tmp/rawMessage', 'w') + fmsg.write(msg) + fmsg.close() + + while self.waitSecond < TIME_TO_SLEEP and not self.stopped: + if self.waitSecond % 60 == 0: + print time.strftime('%H:%M:%S'), "Sleeping and stuck in not so infinite while loop" + self.waitSecond += 1 + time.sleep(1) + + print time.strftime('%H:%M:%S'), "Received", self.nMessagesReceived, "messages" + + def stop(self): + print "Stopping" + self.stopped = True + self.qs.close() + + + +def main(): + print "Starting up at", time.strftime('%H:%M:%S') + + topic = 'edex.alerts' + host = 'localhost' + port = 5672 + + thread = ListenThread(host, port, topic) + try: + thread.start() + while True: + time.sleep(3) + except KeyboardInterrupt: + pass + finally: + thread.stop() + + +if __name__ == '__main__': + main() + + +