Omaha #3596 Update scripts to use DAF. Always put creatingEntity in returned attributes.

Change-Id: I456ff499ccf9446622d1535323afc0d0d1f9b70d

Former-commit-id: e4779f2e66 [formerly d67d1e941b89357e7d86fa0f810a82f9e01f4c27]
Former-commit-id: 368b6341b0
This commit is contained in:
Nathan Bowler 2014-10-01 14:54:50 -04:00
parent f7eac5f1db
commit 2d5d90c055
3 changed files with 312 additions and 153 deletions

View file

@ -1,19 +1,19 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
@ -57,6 +57,7 @@ import com.raytheon.uf.common.numeric.source.DataSource;
* single request.
* Feb 04, 2014 2672 bsteffen Enable requesting subgrids.
* Jul 30, 2014 3184 njensen Renamed valid identifiers to optional
* Sep 29, 2014 3596 nabowle Always put creatingEntity in attributes.
*
* </pre>
*
@ -66,12 +67,14 @@ import com.raytheon.uf.common.numeric.source.DataSource;
public class SatelliteGridFactory extends AbstractGridDataPluginFactory
implements IDataFactory {
private static final String FIELD_CREATING_ENTITY = "creatingEntity";
private static final String FIELD_PYHSICAL_ELEMENT = "physicalElement";
private static final String FIELD_SECTOR_ID = "sectorID";
private static final String[] OPTIONAL_IDENTIFIERS = { "source",
"creatingEntity", FIELD_SECTOR_ID, FIELD_PYHSICAL_ELEMENT };
FIELD_CREATING_ENTITY, FIELD_SECTOR_ID, FIELD_PYHSICAL_ELEMENT };
public SatelliteGridFactory() {
SatelliteUnits.register();
@ -110,14 +113,21 @@ public class SatelliteGridFactory extends AbstractGridDataPluginFactory
}
}
defaultGridData.setUnit(unit);
defaultGridData.setAttributes(request.getIdentifiers());
Map<String, Object> attributes = new HashMap<>();
attributes.put(FIELD_CREATING_ENTITY,
satelliteRecord.getCreatingEntity());
if (request.getIdentifiers() != null) {
attributes.putAll(request.getIdentifiers());
}
defaultGridData.setAttributes(attributes);
return defaultGridData;
}
/**
* Builds the base constraint map based on the supplied grid request
*
*
* @param request
* the original grid request
* @return the base constraint map

View file

@ -1,4 +1,23 @@
#!/bin/csh
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
#
# A script wrapper that is meant to get data for a single satellite sector
# from the A-II database. The result is output to stdout as ASCII.
@ -38,7 +57,7 @@
#
# Integer ids can be looked up in a2satInfo.txt, channel id corresponds to
# the physicalElement, and satid corresponds to the creatingEntity.
#
#
# The literal p option means preserve the final version of the python
# submitted to the UEngine instead of cleaning it up. The path to the
# finalized python is /tmp/a2rdsatNNNNN.py where NNNNN is a unix process id.
@ -46,6 +65,14 @@
# the grid returned.
#
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# Sep 29, 2014 3596 nabowle Initial modification. Handle daf conversion.
#
#
#
set rmpy = yes
if ( "$1" == "p" ) then
set rmpy = no
@ -92,13 +119,85 @@ else
bash -c "echo could not find a2rdsatStub.py 1>&2"
exit
endif
#
# Locate file containing mapping between D-2D interger ids and AWIPS-II ascii
# ids for sectors, channels, and satellites.
#
if ( -e ./a2satInfo.txt ) then
set satInf = ./a2satInfo.txt
else if ( -e $mydir/a2satInfo.txt ) then
set satInf = $mydir/a2satInfo.txt
else if ( -e $FXA_HOME/src/dm/sat/a2satInfo.txt ) then
set satInf = $FXA_HOME/src/dm/sat/a2satInfo.txt
else if ( -e $FXA_HOME/data/a2satInfo.txt ) then
set satInf = $FXA_HOME/data/a2satInfo.txt
else
set satInf = ""
set sss = "$1"
set ccc = "$2"
set eee = `echo $3 | grep -v '.*-'`
endif
#
#
if ( $satInf != "" ) then
set sss = `grep "^ *$1|.*sectorID" $satInf | cut '-d|' -f3`
if ( "$sss" == "" ) set sss = "$1"
set ccc = `grep "^ *$2|.*physicalElement" $satInf | cut '-d|' -f3`
if ( "$ccc" == "" ) set ccc = "$2"
set eee = `echo $3 | grep -v '.*-'`
if ( "$eee" != "" ) then
set eee = `grep "^ *$eee|.*creatingEntity" $satInf | cut '-d|' -f3`
if ( "$eee" == "" ) set eee = "$3"
endif
endif
shift
shift
if ( "$eee" != "" ) shift
set slop = `echo $3 | grep '[0-9]'`
if ( "$slop" == "" ) set slop = 180
set ppp = `echo $argv[$#argv] | grep '^[A-Z]$'`
if ( "$ppp" == "" ) set ppp = 0
#
# Determine if we are using the data access framework or the uEngine.
#
grep DataAccessLayer $stubpy >& /dev/null
if ( $status == 0 ) then
set method = "daf"
set opts = ""
if ( "$eee" != "" ) then
set opts = "--entity ${eee}"
endif
if ( "$encoding" == "1" ) then
set opts = "$opts --hex"
else if ( "$encoding" == "0" ) then
set opts = "$opts --int"
endif
/awips2/python/bin/python $stubpy --sector "${sss}" --physical "${ccc}" --datetime "$1 $2" --part $ppp --slop $slop $opts
else
#
# Get program that can do math with ascii time string, then use this to
# properly encode range of times for which we look for data.
#
if ( -x ./gtasUtil ) then
set gtasUtil = ./gtasUtil
else if ( -x $mydir/gtasUtil ) then
set gtasUtil = $mydir/gtasUtil
else if ( -x $FXA_HOME/src/dm/point/gtasUtil ) then
set gtasUtil = $FXA_HOME/src/dm/point/gtasUtil
else if ( -x $FXA_HOME/bin/gtasUtil ) then
set gtasUtil = $FXA_HOME/bin/gtasUtil
else
bash -c "echo could not find gtasUtil executable 1>&2"
exit
endif
#
# Set up the environment we need to run the UEngine.
#
@ -116,82 +215,29 @@ else
exit
endif
source $ueenv
endif
#
# Locate file containing mapping between D-2D interger ids and AWIPS-II ascii
# ids for sectors, channels, and satellites.
#
if ( -e ./a2satInfo.txt ) then
set satInf = ./a2satInfo.txt
else if ( -e $mydir/a2satInfo.txt ) then
set satInf = $mydir/a2satInfo.txt
else if ( -e $FXA_HOME/src/dm/sat/a2satInfo.txt ) then
set satInf = $FXA_HOME/src/dm/sat/a2satInfo.txt
else if ( -e $FXA_HOME/data/a2satInfo.txt ) then
set satInf = $FXA_HOME/data/a2satInfo.txt
else
bash -c "echo could not find a2satInfo.txt 1>&2"
exit
endif
#
#
set sss = `grep "^ *$1|.*sectorID" $satInf | cut '-d|' -f3`
if ( "$sss" == "" ) set sss = "$1"
set ccc = `grep "^ *$2|.*physicalElement" $satInf | cut '-d|' -f3`
if ( "$ccc" == "" ) set ccc = "$2"
shift
shift
#
# Get program that can do math with ascii time string, then use this to
# properly encode range of times for which we look for data.
#
if ( -x ./gtasUtil ) then
set gtasUtil = ./gtasUtil
else if ( -x $mydir/gtasUtil ) then
set gtasUtil = $mydir/gtasUtil
else if ( -x $FXA_HOME/src/dm/point/gtasUtil ) then
set gtasUtil = $FXA_HOME/src/dm/point/gtasUtil
else if ( -x $FXA_HOME/bin/gtasUtil ) then
set gtasUtil = $FXA_HOME/bin/gtasUtil
else
bash -c "echo could not find gtasUtil executable 1>&2"
exit
endif
set eee = `echo $1 | grep -v '.*-'`
if ( "$eee" != "" ) shift
set slop = `echo $3 | grep '[0-9]'`
if ( "$slop" == "" ) set slop = 180
set aaa = `$gtasUtil = $1 $2 -$slop`
set bbb = `$gtasUtil = $1 $2 $slop`
set ppp = `echo $argv[$#argv] | grep '^[A-Z]$'`
if ( "$ppp" == "" ) set ppp = 0
#
# Modify the text of special tags in stub to create finalized script.
#
set specpy = /tmp/a2rdsat${$}.py
rm -rf $specpy >& /dev/null
touch $specpy
chmod 775 $specpy
if ( "$eee" == "" ) then
cat $stubpy | sed "s/SSSSS/$sss/g" | sed "s/CCCCC/$ccc/g" | \
sed "s/AAAAA/$aaa/g" | sed "s/BBBBB/$bbb/g" | \
sed 's/^.*EEEEE.*$//g'| sed "s/PPPPP/$ppp/g" | \
sed "s/XXXXX/$encoding/g" >> $specpy
else
set eee = `grep "^ *$eee|.*creatingEntity" $satInf | cut '-d|' -f3`
cat $stubpy | sed "s/SSSSS/$sss/g" | sed "s/CCCCC/$ccc/g" | \
sed "s/AAAAA/$aaa/g" | sed "s/BBBBB/$bbb/g" | \
sed "s/EEEEE/$eee/g" | sed "s/PPPPP/$ppp/g" | \
sed "s/XXXXX/$encoding/g" >> $specpy
endif
#
# Submit the temporary python script stripping xml stuff, then remove it
#
if ( "$method" == "daf" ) then
/awips2/python/bin/python $specpy
else
set aaa = `$gtasUtil = $1 $2 -$slop`
set bbb = `$gtasUtil = $1 $2 $slop`
set specpy = /tmp/a2rdsat${$}.py
rm -rf $specpy >& /dev/null
touch $specpy
chmod 775 $specpy
if ( "$eee" == "" ) then
cat $stubpy | sed "s/SSSSS/$sss/g" | sed "s/CCCCC/$ccc/g" | \
sed "s/AAAAA/$aaa/g" | sed "s/BBBBB/$bbb/g" | \
sed 's/^.*EEEEE.*$//g'| sed "s/PPPPP/$ppp/g" | \
sed "s/XXXXX/$encoding/g" >> $specpy
else
cat $stubpy | sed "s/SSSSS/$sss/g" | sed "s/CCCCC/$ccc/g" | \
sed "s/AAAAA/$aaa/g" | sed "s/BBBBB/$bbb/g" | \
sed "s/EEEEE/$eee/g" | sed "s/PPPPP/$ppp/g" | \
sed "s/XXXXX/$encoding/g" >> $specpy
endif
cd $UE_BIN_PATH
( uengine -r python < $specpy ) | grep -v '<' | grep -v Response
if ( "$rmpy" == "yes" ) rm -rf $specpy >& /dev/null
#
endif
if ( "$rmpy" == "yes" ) rm -rf $specpy >& /dev/null
#

View file

@ -1,59 +1,138 @@
import BaseRequest
from com.raytheon.uf.common.message.response import ResponseMessageGeneric
from com.raytheon.edex.plugin.satellite.dao import SatelliteDao
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
# Perform a satellite request for data of interest
sr = BaseRequest.BaseRequest("satellite")
sr.addParameter("creatingEntity","EEEEE","=")
sr.addParameter("sectorID","SSSSS","=")
sr.addParameter("physicalElement","CCCCC","=")
sr.addParameter("dataTime","AAAAA",">=")
sr.addParameter("dataTime","BBBBB","<=")
# Gets data for a single satellite sector from the A-II database. The result is
# output to stdout as ASCII. The first line returned has the dimensions of the
# image, the time, and the source satellite of the data set returned. The rest
# is one line per row of satellite data. The data for each row undergoes second
# order compression Each pixel value of 0 or 255 is encoded as @ or #,
# respectively. Otherwise the first pixel on the row and any pixel that is more
# than 20 counts different than the previous one is encoded as two hex digits.
# Pixels the same as the previous are encoded as a period, pixels from 1 to 20
# counts less than the previous are encoded as G through Z, and pixels from 1 to
# 20 counts more than the previous are encoded as g through z. There are no
# delimeters between the encoding for each pixel.
#
# SOFTWARE HISTORY
#
# Date Ticket# Engineer Description
# ------------ ---------- ----------- --------------------------
# Sep 29, 2014 3596 nabowle Initial modification. Replace UEngine with DAF.
#
#
result = sr.execute()
size = result.size()
if size == 0:
return ResponseMessageGeneric("Data not available")
import a2dafcommon
import argparse
import sys
# ResponseMessageGeneric. Payload is SatelliteRecord
rmg = result.get(0)
part = "PPPPP"
encoding = XXXXX
from datetime import datetime
from datetime import timedelta
from ufpy.dataaccess import DataAccessLayer
from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange
# SatelliteRecord
srec = rmg.getContents()
mytime = srec.getDataURI().split('/',4)[2]
myent = srec.getCreatingEntity()
def get_args():
parser = argparse.ArgumentParser(conflict_handler="resolve")
parser.add_argument("-h", action="store", dest="host",
help="EDEX server hostname (optional)",
metavar="hostname")
parser.add_argument("--datetime", action="store", dest="datetime",
help="The start of the time range in YYYY-MM-DD HH:MM",
metavar="datetime")
parser.add_argument("--slop", action="store", dest="slop", default=180,
help="The amount of slop, in seconds, to allow around the datetime.",
metavar="slop", type=int)
parser.add_argument("--sector", action="store", dest="sector",
help="The sector ID.", metavar="sectorID")
parser.add_argument("--physical", action="store", dest="physical",
help="The physical element.", metavar="physicalElement")
parser.add_argument("--entity", action="store", dest="entity",
help="The creating entity (optional)",
metavar="creatingEntity")
parser.add_argument("--partition", action="store", dest="partition",
help="Upper case letter indicating partition to get.",
metavar="partition", default="0")
parser.add_argument("--hex", action='store_const', dest="encoding",
const=1, help="Hex encoding.", metavar="encoding")
parser.add_argument("--int", action='store_const', dest="encoding",
const=0, help="Delimited integer encoding.",
metavar="encoding")
return parser.parse_args()
# SatelliteDao. Inherits from PluginDao, which has a getHDF5Data method,
# which takes a PluginDataObject as an arg.
satdao = SatelliteDao("satellite")
# returns IDataRecord[]. IDataRecord is implemented by only one class --
# AbstractStorageRecord. ASR is extended by a few *DataRecord classes; one
# of them is ByteDataRecord
idra = satdao.getHDF5Data(srec,-1)
def main():
user_args = get_args()
msg = "No data."
if len(idra) > 0:
# pick an arbitrary datarecord
idr = idra[0]
if user_args.host:
DataAccessLayer.changeEDEXHost(user_args.host)
# this hints at the IDR's concrete class: ByteDataRecord
#print "true type of IDataRecord:", idr.getDataObject().toString()
slop = user_args.slop
dim = idr.getDimension()
if dim != 2:
return ResponseMessageGeneric(msg)
dateTimeStr = user_args.datetime
if not dateTimeStr:
print >> sys.stderr, "DateTime not provided"
return
xLen = idr.getSizes()[0]
yLen = idr.getSizes()[1]
physicalElement = user_args.physical
if not physicalElement:
print >> sys.stderr, "PhysicalElement not provided"
return
# byte[] -- the raw data
barray = idr.getByteData()
barraySize = len(barray)
if xLen * yLen != barraySize:
return ResponseMessageGeneric(msg)
sectorID = user_args.sector
if not sectorID:
print >> sys.stderr, "SectorID not provided"
return
creatingEntity = user_args.entity
part = user_args.partition
encoding = user_args.encoding
dateTime = datetime.strptime(dateTimeStr, "%Y-%m-%d %H:%M")
beginRange = dateTime - timedelta(0, slop)
endRange = dateTime + timedelta(0, slop)
timerange = TimeRange(beginRange, endRange)
req = DataAccessLayer.newDataRequest("satellite")
req.setParameters(physicalElement)
req.setLocationNames(sectorID)
if creatingEntity:
req.addIdentifier("creatingEntity", creatingEntity)
grids = DataAccessLayer.getGridData(req, timerange)
if not grids:
# print "Data not available"
return
grid = grids[0]
data = grid.getRawData()
myent = grid.getAttribute("creatingEntity")
mytime = a2dafcommon.datatime_to_string(grid.getDataTime()) + ".0"
if data is None or len(data) == 0:
# print "No data."
return
yLen = len(data[0])
xLen = len(data)
plus = " ghijklmnopqrstuvwxyz"
minus = " GHIJKLMNOPQRSTUVWXYZ"
@ -77,48 +156,72 @@ if len(idra) > 0:
elif part=="A" or nxy>limit :
nxy = k
msg = "\n"
msg = ""
if part<="A" :
msg += str(xLen) + " " + str(yLen) + " "
msg += mytime + " " + myent + "\n"
dataIdx = int(j / yLen)
arrIdx = j % yLen
arr = data[dataIdx]
i = 0
kk = None
while j<nxy :
i = 0
kk = barray[i+j]
i=0
kk = int(arr[arrIdx])
if kk<0 : kk += 256
if encoding == 0 :
msg += str(kk)
msg += str(kk)
elif encoding == 1 :
msg += "%2.2x"%kk
msg += "%2.2x"%kk
elif kk == 0 :
msg += "@"
msg += "@"
elif kk == 255 :
msg += "#"
msg += "#"
else :
msg += "%2.2x"%kk
i += 1
msg += "%2.2x"%kk
i+=1
arrIdx+=1
if arrIdx >= yLen :
arrIdx = 0
dataIdx += 1
if dataIdx >= xLen:
break
arr = data[dataIdx]
while i<xLen :
k = barray[i+j]
if k<0 : k += 256
if encoding == 0 :
k = int(arr[arrIdx])
if k<0 : k += 256
if encoding == 0 :
msg += " "+str(k)
elif encoding == 1 :
elif encoding == 1 :
msg += "%2.2x"%k
elif k==0 :
elif k==0 :
msg += "@"
elif k == 255 :
elif k == 255 :
msg += "#"
elif k==kk :
elif k==kk :
msg += "."
elif k>kk+20 or k<kk-20 :
elif k>kk+20 or k<kk-20 :
msg += "%2.2x"%k
elif k>kk :
elif k>kk :
msg += plus[k-kk]
else :
else :
msg += minus[kk-k]
kk = k
i += 1
msg += "\n"
j += xLen
kk = k
return ResponseMessageGeneric(msg)
arrIdx+=1
i+=1
if arrIdx >= yLen :
arrIdx = 0
dataIdx += 1
if dataIdx >= xLen:
break
arr = data[dataIdx]
msg += "\n"
j+= xLen
print msg.strip()
if __name__ == '__main__':
main()