mirror of
https://github.com/Unidata/python-awips.git
synced 2025-02-23 22:57:56 -05:00
add classes from GEMPAK7/scripts/python/
This commit is contained in:
Michael James
parent
98c26ceee1
commit
9e97004e7d
8 changed files with 1054 additions and 0 deletions
128
awips/gempak/GridDataRetriever.py
Normal file
128
awips/gempak/GridDataRetriever.py
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
import os
|
||||
import numpy
|
||||
from datetime import datetime
|
||||
from awips import ThriftClient
|
||||
from dynamicserialize.dstypes.gov.noaa.nws.ncep.common.dataplugin.gempak.request import GetGridDataRequest
|
||||
|
||||
|
||||
class GridDataRetriever:
|
||||
|
||||
def __init__(self, server, pluginName, modelId, cycle, forecast, level1, level2, vcoord, param, nnx, nny):
|
||||
self.pluginName = pluginName
|
||||
self.modelId = modelId
|
||||
self.cycle = cycle
|
||||
self.forecast = forecast
|
||||
self.level1 = level1
|
||||
self.level2 = level2
|
||||
self.vcoord = vcoord
|
||||
self.param = param
|
||||
self.nx = nnx
|
||||
self.ny = nny
|
||||
self.host = os.getenv("DEFAULT_HOST", server)
|
||||
self.port = os.getenv("DEFAULT_PORT", "9581")
|
||||
self.client = ThriftClient.ThriftClient(self.host, self.port)
|
||||
|
||||
def getData(self):
|
||||
""" Sends ThriftClient request and writes out received files."""
|
||||
req = GetGridDataRequest()
|
||||
|
||||
req.setPluginName(self.pluginName)
|
||||
req.setModelId(self.modelId)
|
||||
|
||||
dt = datetime.strptime(self.cycle, '%y%m%d/%H%M')
|
||||
ct = datetime.strftime(dt, '%Y-%m-%d %H:%M:%S')
|
||||
req.setReftime(ct)
|
||||
req.setFcstsec(self.forecast)
|
||||
|
||||
if self.level1 == '-1':
|
||||
f1 = -999999.0
|
||||
else:
|
||||
f1 = float(self.level1)
|
||||
|
||||
if self.level2 == '-1':
|
||||
f2 = -999999.0
|
||||
else:
|
||||
f2 = float(self.level2)
|
||||
|
||||
vcoord = self.vcoord
|
||||
if vcoord == 'SGMA':
|
||||
if f1 >= 0.0:
|
||||
f1 = f1 / 10000
|
||||
if f2 >= 0.0:
|
||||
f2 = f2 / 10000
|
||||
elif vcoord == 'DPTH':
|
||||
if f1 >= 0.0:
|
||||
f1 = f1 / 100.0
|
||||
if f2 >= 0.0:
|
||||
f2 = f2 / 100.0
|
||||
elif vcoord == 'POTV':
|
||||
if f1 >= 0.0:
|
||||
f1 = f1 / 1000.0
|
||||
if f2 >= 0.0:
|
||||
f2 = f2 / 1000.0
|
||||
|
||||
req.setLevel1(str(f1))
|
||||
req.setLevel2(str(f2))
|
||||
req.setVcoord(vcoord)
|
||||
|
||||
req.setParm(self.param)
|
||||
|
||||
resp = self.client.sendRequest(req)
|
||||
|
||||
# Get the dimensions of the grid
|
||||
kx = int(self.nx)
|
||||
ky = int(self.ny)
|
||||
kxky = kx * ky
|
||||
|
||||
# Put the data into a NUMPY array
|
||||
grid = numpy.asarray(resp.getFloatData())
|
||||
|
||||
# All grids need to be flipped from a GEMPAK point of view
|
||||
# Reshape the array into 2D
|
||||
grid = numpy.reshape(grid, (ky, kx))
|
||||
# Flip the array in the up-down direction
|
||||
grid = numpy.flipud(grid)
|
||||
# Reshape the array back into 1D
|
||||
grid = numpy.reshape(grid, kxky)
|
||||
|
||||
return [replacemissing(x) for x in grid]
|
||||
|
||||
|
||||
def getgriddata(server, table, model, cycle, forecast, level1,
|
||||
level2, vcoord, param, nnx, nny):
|
||||
gir = GridDataRetriever(server, table, model, cycle, forecast,
|
||||
level1, level2, vcoord, param, nnx, nny)
|
||||
return gir.getData()
|
||||
|
||||
|
||||
def getheader(server, table, model, cycle, forecast, level1,
|
||||
level2, vcoord, param, nnx, nny):
|
||||
idata = []
|
||||
idata.append(0)
|
||||
idata.append(0)
|
||||
return idata
|
||||
|
||||
|
||||
def replacemissing(x):
|
||||
if x == -999999.0:
|
||||
return -9999.0
|
||||
return x
|
||||
|
||||
|
||||
# This is the standard boilerplate that runs this script as a main
|
||||
if __name__ == '__main__':
|
||||
# Run Test
|
||||
srv = 'edex-cloud.unidata.ucar.edu'
|
||||
tbl = 'grid'
|
||||
mdl = 'GFS20'
|
||||
cyc = '131227/0000'
|
||||
fcs = '43200'
|
||||
lv1 = '500'
|
||||
lv2 = '-1'
|
||||
vcd = 'PRES'
|
||||
prm = 'HGHT'
|
||||
nx = '720'
|
||||
ny = '361'
|
||||
|
||||
print(getheader(srv, tbl, mdl, cyc, fcs, lv1, lv2, vcd, prm, nx, ny))
|
||||
print(getgriddata(srv, tbl, mdl, cyc, fcs, lv1, lv2, vcd, prm, nx, ny))
|
||||
137
awips/gempak/GridInfoRetriever.py
Normal file
137
awips/gempak/GridInfoRetriever.py
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
import os
|
||||
from datetime import datetime
|
||||
from operator import itemgetter
|
||||
from awips import ThriftClient
|
||||
from dynamicserialize.dstypes.gov.noaa.nws.ncep.common.dataplugin.gempak.request import GetGridInfoRequest
|
||||
|
||||
|
||||
class GridInfoRetriever:
|
||||
|
||||
def __init__(self, server, pluginName, modelId, cycle=None, forecast=None):
|
||||
self.pluginName = pluginName
|
||||
self.modelId = modelId
|
||||
self.cycle = cycle
|
||||
self.forecast = forecast
|
||||
self.host = os.getenv("DEFAULT_HOST", server)
|
||||
self.port = os.getenv("DEFAULT_PORT", "9581")
|
||||
self.client = ThriftClient.ThriftClient(self.host, self.port)
|
||||
|
||||
def getInfo(self):
|
||||
""" Sends ThriftClient request and writes out received files."""
|
||||
req = GetGridInfoRequest()
|
||||
req.setPluginName(self.pluginName)
|
||||
req.setModelId(self.modelId)
|
||||
|
||||
req.setReftime(self.cycle)
|
||||
if len(self.cycle) > 2:
|
||||
dt = datetime.strptime(self.cycle, '%y%m%d/%H%M')
|
||||
ct = datetime.strftime(dt, '%Y-%m-%d %H:%M:%S')
|
||||
req.setReftime(ct)
|
||||
|
||||
req.setFcstsec(self.forecast)
|
||||
|
||||
resp = self.client.sendRequest(req)
|
||||
|
||||
sortresp = sorted(sorted(resp, key=itemgetter("reftime"), reverse=True), key=itemgetter("fcstsec"))
|
||||
|
||||
grids = []
|
||||
|
||||
count = 0
|
||||
for record in sortresp:
|
||||
s = '{:<12}'.format(record['param'])
|
||||
|
||||
if sys.byteorder == 'little':
|
||||
parm1 = (ord(s[3]) << 24) + (ord(s[2]) << 16) + (ord(s[1]) << 8) + ord(s[0])
|
||||
parm2 = (ord(s[7]) << 24) + (ord(s[6]) << 16) + (ord(s[5]) << 8) + ord(s[4])
|
||||
parm3 = (ord(s[11]) << 24) + (ord(s[10]) << 16) + (ord(s[9]) << 8) + ord(s[8])
|
||||
else:
|
||||
parm1 = (ord(s[0]) << 24) + (ord(s[1]) << 16) + (ord(s[2]) << 8) + ord(s[3])
|
||||
parm2 = (ord(s[4]) << 24) + (ord(s[5]) << 16) + (ord(s[6]) << 8) + ord(s[7])
|
||||
parm3 = (ord(s[8]) << 24) + (ord(s[9]) << 16) + (ord(s[10]) << 8) + ord(s[11])
|
||||
|
||||
dt = datetime.strptime(record['reftime'], '%Y-%m-%d %H:%M:%S.%f')
|
||||
dattim = dt.month * 100000000 + dt.day * 1000000 + (dt.year%100) * 10000 + dt.hour * 100 + dt.minute
|
||||
fcsth = (int(record['fcstsec']) / 60) / 60
|
||||
fcstm = (int(record['fcstsec']) / 60) % 60
|
||||
fcst = 100000 + fcsth * 100 + fcstm
|
||||
|
||||
lv1 = float(record['level1'])
|
||||
if lv1 == -999999.0:
|
||||
lv1 = -1.0
|
||||
lv2 = float(record['level2'])
|
||||
if lv2 == -999999.0:
|
||||
lv2 = -1.0
|
||||
|
||||
vcd = record['vcoord']
|
||||
if vcd == 'NONE':
|
||||
ivcd = 0
|
||||
elif vcd == 'PRES':
|
||||
ivcd = 1
|
||||
elif vcd == 'THTA':
|
||||
ivcd = 2
|
||||
elif vcd == 'HGHT':
|
||||
ivcd = 3
|
||||
elif vcd == 'SGMA':
|
||||
ivcd = 4
|
||||
if lv1 >= 0.0:
|
||||
lv1 = lv1 * 10000.0
|
||||
if lv2 >= 0.0:
|
||||
lv2 = lv2 * 10000.0
|
||||
elif vcd == 'DPTH':
|
||||
ivcd = 5
|
||||
if lv1 >= 0.0:
|
||||
lv1 = lv1 * 100.0
|
||||
if lv2 >= 0.0:
|
||||
lv2 = lv2 * 100.0
|
||||
elif vcd == 'HYBL':
|
||||
ivcd = 6
|
||||
else:
|
||||
v = '{:<4}'.format(vcd)
|
||||
if sys.byteorder == 'little':
|
||||
ivcd = (ord(v[3]) << 24) + (ord(v[2]) << 16) + (ord(v[1]) << 8) + ord(v[0])
|
||||
else:
|
||||
ivcd = (ord(v[0]) << 24) + (ord(v[1]) << 16) + (ord(v[2]) << 8) + ord(v[3])
|
||||
if vcd == 'POTV':
|
||||
if lv1 >= 0.0:
|
||||
lv1 = lv1 * 1000.0
|
||||
if lv2 >= 0.0:
|
||||
lv2 = lv2 * 1000.0
|
||||
grids.append(9999)
|
||||
grids.append(dattim)
|
||||
grids.append(fcst)
|
||||
grids.append(0)
|
||||
grids.append(0)
|
||||
grids.append(int(lv1))
|
||||
grids.append(int(lv2))
|
||||
grids.append(ivcd)
|
||||
grids.append(parm1)
|
||||
grids.append(parm2)
|
||||
grids.append(parm3)
|
||||
count += 1
|
||||
if count > 29998:
|
||||
break
|
||||
|
||||
return grids
|
||||
|
||||
|
||||
def getinfo(server, table, model, cycle, forecast):
|
||||
gir = GridInfoRetriever(server, table, model, cycle, forecast)
|
||||
return gir.getInfo()
|
||||
|
||||
|
||||
def getrow(server, table, model, cycle, forecast):
|
||||
idata = []
|
||||
idata.append(9999)
|
||||
idata.append(1)
|
||||
return idata
|
||||
|
||||
|
||||
# This is the standard boilerplate that runs this script as a main
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
# Run Test
|
||||
srv = 'edex-cloud.unidata.ucar.edu'
|
||||
tbl = 'grid'
|
||||
mdl = 'NAM40'
|
||||
print(getrow(srv, tbl, mdl))
|
||||
print(getinfo(srv, tbl, mdl))
|
||||
294
awips/gempak/GridNavRetriever.py
Normal file
294
awips/gempak/GridNavRetriever.py
Normal file
|
|
@ -0,0 +1,294 @@
|
|||
import os
|
||||
import math
|
||||
from awips import ThriftClient
|
||||
from dynamicserialize.dstypes.gov.noaa.nws.ncep.common.dataplugin.gempak.request import GetGridNavRequest
|
||||
from ctypes import *
|
||||
|
||||
EARTH_RADIUS = 6371200.0
|
||||
DEG_TO_RAD = math.pi / 180.0
|
||||
RAD_TO_DEG = 180.0 / math.pi
|
||||
TWOPI = math.pi * 2.0
|
||||
HALFPI = math.pi / 2.0
|
||||
PI4TH = math.pi / 4.0
|
||||
PI3RD = math.pi / 3.0
|
||||
|
||||
|
||||
def createPolar(nsflag, clon, lat1, lon1, dx, dy, unit, nx, ny):
|
||||
clonr = clon * DEG_TO_RAD
|
||||
latr = lat1 * DEG_TO_RAD
|
||||
lonr = lon1 * DEG_TO_RAD
|
||||
if nsflag == 'N':
|
||||
x1 = EARTH_RADIUS * math.tan(PI4TH - latr/2.0) * math.sin(lonr-clonr)
|
||||
y1 = -1 * EARTH_RADIUS * math.tan(PI4TH - latr/2.0) * math.cos(lonr-clonr)
|
||||
else:
|
||||
x1 = EARTH_RADIUS * math.tan(PI4TH + latr/2.0) * math.sin(lonr-clonr)
|
||||
y1 = EARTH_RADIUS * math.tan(PI4TH + latr/2.0) * math.cos(lonr-clonr)
|
||||
|
||||
if unit == 'm':
|
||||
tdx = dx / (1 + math.sin(PI3RD))
|
||||
tdy = dy / (1 + math.sin(PI3RD))
|
||||
else:
|
||||
tdx = (dx*1000.0) / (1 + math.sin(PI3RD))
|
||||
tdy = (dy*1000.0) / (1 + math.sin(PI3RD))
|
||||
|
||||
x2 = x1 + tdx * (nx-1)
|
||||
y2 = y1 + tdy * (ny-1)
|
||||
xll = min(x1, x2)
|
||||
yll = min(y1, y2)
|
||||
xur = max(x1, x2)
|
||||
yur = max(y1, y2)
|
||||
|
||||
if nsflag == 'N':
|
||||
latll = (HALFPI - 2*math.atan2(math.hypot(xll, yll), EARTH_RADIUS)) * RAD_TO_DEG
|
||||
rtemp = clonr + math.atan2(xll, -yll)
|
||||
else:
|
||||
latll = -1 * (HALFPI - 2*math.atan2(math.hypot(xll, yll), EARTH_RADIUS)) * RAD_TO_DEG
|
||||
rtemp = clonr + math.atan2(xll, yll)
|
||||
|
||||
if rtemp > math.pi:
|
||||
lonll = (rtemp-TWOPI) * RAD_TO_DEG
|
||||
elif rtemp < -math.pi:
|
||||
lonll = (rtemp+TWOPI) * RAD_TO_DEG
|
||||
else:
|
||||
lonll = rtemp * RAD_TO_DEG
|
||||
|
||||
if nsflag == 'N':
|
||||
latur = (HALFPI - 2*math.atan2(math.hypot(xur, yur), EARTH_RADIUS)) * RAD_TO_DEG
|
||||
rtemp = clonr + math.atan2(xur, -yur)
|
||||
else:
|
||||
latur = -1 * (HALFPI - 2*math.atan2(math.hypot(xur, yur), EARTH_RADIUS)) * RAD_TO_DEG
|
||||
rtemp = clonr + math.atan2(xur, yur)
|
||||
|
||||
if rtemp > math.pi:
|
||||
lonur = (rtemp-TWOPI) * RAD_TO_DEG
|
||||
elif rtemp < -math.pi:
|
||||
lonur = (rtemp+TWOPI) * RAD_TO_DEG
|
||||
else:
|
||||
lonur = rtemp * RAD_TO_DEG
|
||||
|
||||
return [latll, lonll, latur, lonur]
|
||||
|
||||
|
||||
def createConic(nsflag, clon, lat1, lon1, dx, dy, unit, nx, ny, ang1, ang3):
|
||||
clonr = clon * DEG_TO_RAD
|
||||
latr = lat1 * DEG_TO_RAD
|
||||
lonr = lon1 * DEG_TO_RAD
|
||||
|
||||
angle1 = HALFPI - (math.fabs(ang1) * DEG_TO_RAD)
|
||||
angle2 = HALFPI - (math.fabs(ang3) * DEG_TO_RAD)
|
||||
|
||||
if ang1 == ang3:
|
||||
cc = math.cos(angle1)
|
||||
else:
|
||||
cc = (math.log(math.sin(angle2)) - math.log(math.sin(angle1))) \
|
||||
/ (math.log(math.tan(angle2/2.0)) - math.log(math.tan(angle1/2.0)))
|
||||
|
||||
er = EARTH_RADIUS / cc
|
||||
|
||||
if nsflag == 'N':
|
||||
x1 = er * math.pow(math.tan((HALFPI-latr)/2.0), cc) * math.sin(cc*(lonr-clonr))
|
||||
y1 = -1.0 * er * math.pow(math.tan((HALFPI-latr)/2.0), cc) * math.cos(cc*(lonr-clonr))
|
||||
else:
|
||||
x1 = er * math.pow(math.tan((HALFPI+latr)/2.0), cc) * math.sin(cc*(lonr-clonr))
|
||||
y1 = er * math.pow(math.tan((HALFPI+latr)/2.0), cc) * math.cos(cc*(lonr-clonr))
|
||||
|
||||
alpha = math.pow(math.tan(angle1/2.0), cc) / math.sin(angle1)
|
||||
|
||||
if unit == 'm':
|
||||
x2 = x1 + (nx-1) * alpha * dx
|
||||
y2 = y1 + (ny-1) * alpha * dy
|
||||
else:
|
||||
x2 = x1 + (nx-1) * alpha * (dx*1000.0)
|
||||
y2 = y1 + (ny-1) * alpha * (dy*1000.0)
|
||||
|
||||
xll = min(x1, x2)
|
||||
yll = min(y1, y2)
|
||||
xur = max(x1, x2)
|
||||
yur = max(y1, y2)
|
||||
|
||||
if nsflag == 'N':
|
||||
latll = (HALFPI - 2.0 * math.atan(math.pow(math.hypot(xll, yll)/er, (1/cc)))) * RAD_TO_DEG
|
||||
rtemp = math.atan2(xll, -yll) * (1/cc) + clonr
|
||||
else:
|
||||
latll = (-1.0 * (HALFPI - 2.0 * math.atan(math.pow(math.hypot(xll, yll)/er, (1/cc))))) * RAD_TO_DEG
|
||||
rtemp = math.atan2(xll, yll) * (1/cc) + clonr
|
||||
|
||||
if rtemp > math.pi:
|
||||
lonll = (rtemp-TWOPI) * RAD_TO_DEG
|
||||
elif rtemp < -math.pi:
|
||||
lonll = (rtemp+TWOPI) * RAD_TO_DEG
|
||||
else:
|
||||
lonll = rtemp * RAD_TO_DEG
|
||||
|
||||
if nsflag == 'N':
|
||||
latur = (HALFPI - 2.0 * math.atan(math.pow(math.hypot(xur, yur)/er, (1/cc)))) * RAD_TO_DEG
|
||||
rtemp = math.atan2(xur, -yur) * (1/cc) + clonr
|
||||
else:
|
||||
latur = (-1.0 * (HALFPI - 2.0 * math.atan(math.pow(math.hypot(xur, yur)/er, (1/cc))))) * RAD_TO_DEG
|
||||
rtemp = math.atan2(xur, yur) * (1/cc) + clonr
|
||||
|
||||
if rtemp > math.pi:
|
||||
lonur = (rtemp-TWOPI) * RAD_TO_DEG
|
||||
elif rtemp < -math.pi:
|
||||
lonur = (rtemp+TWOPI) * RAD_TO_DEG
|
||||
else:
|
||||
lonur = rtemp * RAD_TO_DEG
|
||||
|
||||
return [latll, lonll, latur, lonur]
|
||||
|
||||
|
||||
class StringConverter(Union):
|
||||
_fields_ = [("char", c_char*4), ("int", c_int), ("float", c_float)]
|
||||
|
||||
|
||||
class GridNavRetriever:
|
||||
|
||||
def __init__(self, server, pluginName, modelId, arrayLen):
|
||||
self.pluginName = pluginName
|
||||
self.modelId = modelId
|
||||
self.arrayLen = arrayLen
|
||||
self.host = os.getenv("DEFAULT_HOST", server)
|
||||
self.port = os.getenv("DEFAULT_PORT", "9581")
|
||||
self.client = ThriftClient.ThriftClient(self.host, self.port)
|
||||
|
||||
def getNavBlk(self):
|
||||
""" Sends ThriftClient request and writes out received files."""
|
||||
req = GetGridNavRequest()
|
||||
req.setPluginName(self.pluginName)
|
||||
req.setModelId(self.modelId)
|
||||
resp = self.client.sendRequest(req)
|
||||
|
||||
nav = []
|
||||
|
||||
for record in resp:
|
||||
unit = record['spacingunit']
|
||||
sk = record['spatialkey']
|
||||
skarr = sk.split('/')
|
||||
|
||||
nx = float(skarr[1])
|
||||
ny = float(skarr[2])
|
||||
dx = float(skarr[3])
|
||||
dy = float(skarr[4])
|
||||
|
||||
sc = StringConverter()
|
||||
if record['projtype'] == 'LatLon':
|
||||
sc.char = 'CED '
|
||||
gemproj = 2.0
|
||||
ang1 = 0.0
|
||||
ang2 = 0.0
|
||||
ang3 = 0.0
|
||||
|
||||
lllat = float(record['lowerleftlat'])
|
||||
lllon = float(record['lowerleftlon'])
|
||||
urlat = lllat + (dy * (ny-1))
|
||||
urlon = lllon + (dx * (nx-1))
|
||||
if lllon > 180:
|
||||
lllon -= 360.0
|
||||
if urlon > 180:
|
||||
urlon -= 360.0
|
||||
|
||||
if record['projtype'] == 'Polar Stereographic':
|
||||
sc.char = 'STR '
|
||||
gemproj = 2.0
|
||||
if float(record['standard_parallel_1']) < 0.0:
|
||||
ang1 = -90.0
|
||||
nsflag = 'S'
|
||||
else:
|
||||
ang1 = 90.0
|
||||
nsflag = 'N'
|
||||
ang2 = float(record['central_meridian'])
|
||||
ang3 = 0.0
|
||||
|
||||
lat1 = float(record['lowerleftlat'])
|
||||
lon1 = float(record['lowerleftlon'])
|
||||
coords = createPolar(nsflag, ang2, lat1, lon1, dx, dy, unit, nx, ny)
|
||||
lllat = coords[0]
|
||||
lllon = coords[1]
|
||||
urlat = coords[2]
|
||||
urlon = coords[3]
|
||||
|
||||
if record['projtype'] == 'Lambert Conformal':
|
||||
sc.char = 'LCC '
|
||||
gemproj = 2.0
|
||||
|
||||
ang1 = float(skarr[7])
|
||||
ang2 = float(record['central_meridian'])
|
||||
ang3 = float(skarr[8])
|
||||
if ang1 < 0.0:
|
||||
nsflag = 'S'
|
||||
else:
|
||||
nsflag = 'N'
|
||||
|
||||
lat1 = float(record['lowerleftlat'])
|
||||
lon1 = float(record['lowerleftlon'])
|
||||
coords = createConic(nsflag, ang2, lat1, lon1, dx, dy, unit, nx, ny, ang1, ang3)
|
||||
lllat = coords[0]
|
||||
lllon = coords[1]
|
||||
urlat = coords[2]
|
||||
urlon = coords[3]
|
||||
|
||||
# Fill up the output array of floats
|
||||
nav.append(gemproj)
|
||||
nav.append(sc.float)
|
||||
nav.append(1.0)
|
||||
nav.append(1.0)
|
||||
nav.append(nx)
|
||||
nav.append(ny)
|
||||
nav.append(lllat)
|
||||
nav.append(lllon)
|
||||
nav.append(urlat)
|
||||
nav.append(urlon)
|
||||
nav.append(ang1)
|
||||
nav.append(ang2)
|
||||
nav.append(ang3)
|
||||
|
||||
for i in range(13, int(self.arrayLen)):
|
||||
nav.append(0.0)
|
||||
return nav
|
||||
|
||||
def getAnlBlk(self):
|
||||
anl = []
|
||||
# Type
|
||||
anl.append(2.0)
|
||||
# Delta
|
||||
anl.append(1.0)
|
||||
# Extend area
|
||||
anl.append(0.0)
|
||||
anl.append(0.0)
|
||||
anl.append(0.0)
|
||||
anl.append(0.0)
|
||||
# Grid area
|
||||
anl.append(-90.0)
|
||||
anl.append(-180.0)
|
||||
anl.append(90.0)
|
||||
anl.append(180.0)
|
||||
# Data area
|
||||
anl.append(-90.0)
|
||||
anl.append(-180.0)
|
||||
anl.append(90.0)
|
||||
anl.append(180.0)
|
||||
for i in range(18, int(self.arrayLen)):
|
||||
anl.append(0.0)
|
||||
return anl
|
||||
|
||||
|
||||
def getnavb(server, table, model, arrlen):
|
||||
gnr = GridNavRetriever(server, table, model, arrlen)
|
||||
return gnr.getNavBlk()
|
||||
|
||||
|
||||
def getanlb(server, table, model, arrlen):
|
||||
gnr = GridNavRetriever(server, table, model, arrlen)
|
||||
return gnr.getAnlBlk()
|
||||
|
||||
|
||||
# This is the standard boilerplate that runs this script as a main
|
||||
if __name__ == '__main__':
|
||||
# Run Test
|
||||
srv = 'edex-cloud.unidata.ucar.edu'
|
||||
tbl = 'grid_info'
|
||||
mdl = 'NAM40'
|
||||
navlen = '256'
|
||||
print(getnavb(srv, tbl, mdl, navlen))
|
||||
anllen = '128'
|
||||
print(getanlb(srv, tbl, mdl, anllen))
|
||||
136
awips/gempak/StationDataRetriever.py
Normal file
136
awips/gempak/StationDataRetriever.py
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
import os
|
||||
from datetime import datetime
|
||||
from awips import ThriftClient
|
||||
from dynamicserialize.dstypes.com.raytheon.uf.common.time import DataTime
|
||||
from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange
|
||||
from dynamicserialize.dstypes.gov.noaa.nws.ncep.common.dataplugin.gempak.request import StationDataRequest
|
||||
|
||||
|
||||
class StationDataRetriever:
|
||||
""" Retrieves all data for a requested station and time """
|
||||
|
||||
def __init__(self, server, pluginName, stationId, refTime, parmList, partNumber):
|
||||
self.pluginName = pluginName
|
||||
self.stationId = stationId
|
||||
self.refTime = refTime
|
||||
self.parmList = parmList
|
||||
self.partNumber = partNumber
|
||||
self.host = os.getenv("DEFAULT_HOST", server)
|
||||
self.port = os.getenv("DEFAULT_PORT", "9581")
|
||||
self.client = ThriftClient.ThriftClient(self.host, self.port)
|
||||
|
||||
def getStationData(self):
|
||||
""" Sends ThriftClient request and writes out received files."""
|
||||
dtime = datetime.strptime(self.refTime, "%y%m%d/%H%M")
|
||||
trange = TimeRange()
|
||||
trange.setStart(dtime)
|
||||
trange.setEnd(dtime)
|
||||
dataTime = DataTime(refTime=dtime, validPeriod=trange)
|
||||
req = StationDataRequest()
|
||||
req.setPluginName(self.pluginName)
|
||||
req.setStationId(self.stationId)
|
||||
req.setRefTime(dataTime)
|
||||
req.setParmList(self.parmList)
|
||||
req.setPartNumber(self.partNumber)
|
||||
resp = self.client.sendRequest(req)
|
||||
return resp
|
||||
|
||||
|
||||
def getstationdata(server, table, stationId, refTime, parmList, partNumber):
|
||||
sr = StationDataRetriever(server, table, stationId, refTime, parmList, partNumber)
|
||||
lcldict = sr.getStationData()
|
||||
|
||||
rdata = []
|
||||
|
||||
for substr in parmList.split(','):
|
||||
if substr in lcldict:
|
||||
rdata.append(lcldict[substr])
|
||||
else:
|
||||
rdata.append(-9999.00)
|
||||
|
||||
return rdata
|
||||
|
||||
|
||||
def getleveldata(server, table, stationId, refTime, parmList, partNumber):
|
||||
sr = StationDataRetriever(server, table, stationId, refTime, parmList, partNumber)
|
||||
lcldict = sr.getStationData()
|
||||
|
||||
numset = [1]
|
||||
for substr in parmList.split(','):
|
||||
if substr in lcldict:
|
||||
pnum = len(lcldict[substr]) - 1
|
||||
while pnum >= 0:
|
||||
if lcldict[substr][pnum] != -9999.00:
|
||||
break
|
||||
pnum = pnum - 1
|
||||
numset.append(pnum)
|
||||
|
||||
rdata = []
|
||||
|
||||
for jj in range(max(numset)):
|
||||
for substr in parmList.split(','):
|
||||
if substr in lcldict:
|
||||
if lcldict[substr][jj] == -9999998.0:
|
||||
rdata.append(-9999.0)
|
||||
else:
|
||||
rdata.append(lcldict[substr][jj])
|
||||
else:
|
||||
rdata.append(-9999.0)
|
||||
|
||||
return rdata
|
||||
|
||||
|
||||
def getstationtext(server, table, stationId, refTime, parmList, partNumber):
|
||||
sr = StationDataRetriever(server, table, stationId, refTime, parmList, partNumber)
|
||||
lcldict = sr.getStationData()
|
||||
|
||||
if parmList in lcldict:
|
||||
return lcldict[parmList]
|
||||
else:
|
||||
return ' '
|
||||
|
||||
|
||||
def getheader(server, table, stationId, refTime, parmList, partNumber):
|
||||
idata = []
|
||||
idata.append(0)
|
||||
return idata
|
||||
|
||||
|
||||
# This is the standard boilerplate that runs this script as a main
|
||||
if __name__ == '__main__':
|
||||
# Run Test
|
||||
srv = 'edex-cloud.unidata.ucar.edu'
|
||||
key = '-'
|
||||
print('OBS - METAR')
|
||||
tbl = 'obs'
|
||||
stn = 'KLGA'
|
||||
time = '130823/1700'
|
||||
parm = 'seaLevelPress,temperature,dewpoint,windSpeed,windDir'
|
||||
part = '0'
|
||||
print(getheader(srv, tbl, stn, time, parm, part))
|
||||
print(getstationdata(srv, tbl, stn, time, parm, part))
|
||||
parm = 'rawMETAR'
|
||||
print(getstationtext(srv, tbl, stn, time, parm, part))
|
||||
print('SFCOBS - SYNOP')
|
||||
tbl = 'sfcobs'
|
||||
stn = '72403'
|
||||
time = '130823/1800'
|
||||
parm = 'seaLevelPress,temperature,dewpoint,windSpeed,windDir'
|
||||
part = '0'
|
||||
print(getheader(srv, tbl, stn, time, parm, part))
|
||||
print(getstationdata(srv, tbl, stn, time, parm, part))
|
||||
parm = 'rawReport'
|
||||
print(getstationtext(srv, tbl, stn, time, parm, part))
|
||||
print('UAIR')
|
||||
tbl = 'bufrua'
|
||||
stn = '72469'
|
||||
time = '130823/1200'
|
||||
parm = 'prMan,htMan,tpMan,tdMan,wdMan,wsMan'
|
||||
part = '2020'
|
||||
print(getleveldata(srv, tbl, stn, time, parm, part))
|
||||
parm = 'prSigT,tpSigT,tdSigT'
|
||||
part = '2022'
|
||||
print(getleveldata(srv, tbl, stn, time, parm, part))
|
||||
parm = 'htSigW,wsSigW,wdSigW'
|
||||
part = '2021'
|
||||
print(getleveldata(srv, tbl, stn, time, parm, part))
|
||||
86
awips/gempak/StationRetriever.py
Normal file
86
awips/gempak/StationRetriever.py
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
import os
|
||||
import sys
|
||||
from awips import ThriftClient
|
||||
from dynamicserialize.dstypes.gov.noaa.nws.ncep.common.dataplugin.gempak.request import GetStationsRequest
|
||||
|
||||
|
||||
class StationRetriever:
|
||||
""" Retrieves all requested stations """
|
||||
|
||||
def __init__(self, server, pluginName):
|
||||
self.pluginName = pluginName
|
||||
self.outdir = os.getcwd()
|
||||
self.host = os.getenv("DEFAULT_HOST", server)
|
||||
self.port = os.getenv("DEFAULT_PORT", "9581")
|
||||
self.client = ThriftClient.ThriftClient(self.host, self.port)
|
||||
|
||||
def getStations(self):
|
||||
""" Sends ThriftClient request and writes out received files."""
|
||||
req = GetStationsRequest()
|
||||
req.setPluginName(self.pluginName)
|
||||
resp = self.client.sendRequest(req)
|
||||
|
||||
stns = []
|
||||
for item in resp:
|
||||
stationstr = '{:<8}'.format(item.getStationId())
|
||||
|
||||
if sys.byteorder == 'little':
|
||||
stnid = (ord(stationstr[3]) << 24) + (ord(stationstr[2]) << 16) + \
|
||||
(ord(stationstr[1]) << 8) + ord(stationstr[0])
|
||||
stnid2 = (ord(stationstr[7]) << 24) + (ord(stationstr[6]) << 16) + \
|
||||
(ord(stationstr[5]) << 8) + ord(stationstr[4])
|
||||
else:
|
||||
stnid = (ord(stationstr[0]) << 24) + (ord(stationstr[1]) << 16) + \
|
||||
(ord(stationstr[2]) << 8) + ord(stationstr[3])
|
||||
stnid2 = (ord(stationstr[4]) << 24) + (ord(stationstr[5]) << 16) + \
|
||||
(ord(stationstr[6]) << 8) + ord(stationstr[7])
|
||||
|
||||
if item.getState() is None:
|
||||
stationstr = ' '
|
||||
else:
|
||||
stationstr = '{:<4}'.format(item.getState())
|
||||
|
||||
if sys.byteorder == 'little':
|
||||
state = (ord(stationstr[3]) << 24) + (ord(stationstr[2]) << 16) \
|
||||
+ (ord(stationstr[1]) << 8) + ord(stationstr[0])
|
||||
else:
|
||||
state = (ord(stationstr[0]) << 24) + (ord(stationstr[1]) << 16) \
|
||||
+ (ord(stationstr[2]) << 8) + ord(stationstr[3])
|
||||
|
||||
stationstr = '{:<4}'.format(item.getCountry())
|
||||
if sys.byteorder == 'little':
|
||||
cntry = (ord(stationstr[3]) << 24) + (ord(stationstr[2]) << 16) \
|
||||
+ (ord(stationstr[1]) << 8) + ord(stationstr[0])
|
||||
else:
|
||||
cntry = (ord(stationstr[0]) << 24) + (ord(stationstr[1]) << 16) \
|
||||
+ (ord(stationstr[2]) << 8) + ord(stationstr[3])
|
||||
|
||||
stns.append(9999)
|
||||
stns.append(stnid)
|
||||
stns.append(item.getWmoIndex())
|
||||
stns.append(int(item.getLatitude()*100))
|
||||
stns.append(int(item.getLongitude()*100))
|
||||
stns.append(int(item.getElevation()))
|
||||
stns.append(state)
|
||||
stns.append(cntry)
|
||||
stns.append(stnid2)
|
||||
stns.append(0)
|
||||
return stns
|
||||
|
||||
|
||||
def getstations(server, table, key, dummy, dummy2):
|
||||
sr = StationRetriever(server, table)
|
||||
return sr.getStations()
|
||||
|
||||
|
||||
# This is the standard boilerplate that runs this script as a main
|
||||
if __name__ == '__main__':
|
||||
# Run Test
|
||||
srv = 'edex-cloud.unidata.ucar.edu'
|
||||
key = '-'
|
||||
print('OBS - METAR')
|
||||
tbl = 'obs'
|
||||
print(getstations(srv, tbl, key))
|
||||
print('SFCOBS - SYNOP')
|
||||
tbl = 'sfcobs'
|
||||
print(getstations(srv, tbl, key))
|
||||
68
awips/gempak/TimeRetriever.py
Normal file
68
awips/gempak/TimeRetriever.py
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
import os
|
||||
from datetime import datetime
|
||||
from awips import ThriftClient
|
||||
from dynamicserialize.dstypes.java.util import GregorianCalendar
|
||||
from dynamicserialize.dstypes.gov.noaa.nws.ncep.common.dataplugin.gempak.request import GetTimesRequest
|
||||
|
||||
|
||||
class TimeRetriever:
|
||||
""" Retrieves all requested times"""
|
||||
|
||||
def __init__(self, server, pluginName, timeField):
|
||||
self.pluginName = pluginName
|
||||
self.timeField = timeField
|
||||
self.outdir = os.getcwd()
|
||||
self.host = os.getenv("DEFAULT_HOST", server)
|
||||
self.port = os.getenv("DEFAULT_PORT", "9581")
|
||||
self.client = ThriftClient.ThriftClient(self.host, self.port)
|
||||
|
||||
def getTimes(self):
|
||||
""" Sends ThriftClient request and writes out received files."""
|
||||
req = GetTimesRequest()
|
||||
req.setPluginName(self.pluginName)
|
||||
req.setTimeField(self.timeField)
|
||||
resp = self.client.sendRequest(req)
|
||||
timelist = []
|
||||
for item in resp.getTimes():
|
||||
if isinstance(item, GregorianCalendar):
|
||||
tstamp = item.getTimeInMillis()
|
||||
else:
|
||||
tstamp = item.getTime()
|
||||
time = datetime.utcfromtimestamp(tstamp/1000)
|
||||
timelist.append(time)
|
||||
|
||||
timelist.sort(reverse=True)
|
||||
|
||||
times = []
|
||||
for time in timelist:
|
||||
times.append(9999)
|
||||
times.append((time.year % 100) * 10000 + (time.month * 100) + time.day)
|
||||
times.append((time.hour * 100) + time.minute)
|
||||
|
||||
# GEMPAK can only handle up to 200 times, which is 600 elements
|
||||
# in this array -- [9999, DATE, TIME] -- repeated
|
||||
return times[0:600]
|
||||
|
||||
|
||||
def gettimes(server, table, key, dummy, dummy2):
|
||||
tr = TimeRetriever(server, table, key)
|
||||
return tr.getTimes()
|
||||
|
||||
|
||||
# This is the standard boilerplate that runs this script as a main
|
||||
if __name__ == '__main__':
|
||||
srv = 'edex-cloud.unidata.ucar.edu'
|
||||
print('OBS - METAR')
|
||||
tbl = 'obs'
|
||||
key = 'refHour'
|
||||
print(gettimes(srv, tbl, key))
|
||||
|
||||
print('SFCOBS - SYNOP')
|
||||
tbl = 'sfcobs'
|
||||
key = 'refHour'
|
||||
print(gettimes(srv, tbl, key))
|
||||
|
||||
print('BUFRUA')
|
||||
tbl = 'bufrua'
|
||||
key = 'dataTime.refTime'
|
||||
print(gettimes(srv, tbl, key))
|
||||
99
awips/gempak/ncepGribTables.py
Executable file
99
awips/gempak/ncepGribTables.py
Executable file
|
|
@ -0,0 +1,99 @@
|
|||
#!/usr/bin/env python
|
||||
# Parse html tables from a given URL and output CSV.
|
||||
# Note: To install a missing python module foo do "easy_install foo"
|
||||
# (or the new way is "pip install foo" but you might have to do
|
||||
# "easy_install pip" first)
|
||||
|
||||
from BeautifulSoup import BeautifulSoup
|
||||
import scrape
|
||||
import urllib.request, urllib.error, urllib.parse
|
||||
import html.entities
|
||||
import re
|
||||
import sys
|
||||
import unicodedata
|
||||
|
||||
|
||||
# from http://stackoverflow.com/questions/1197981/convert-html-entities
|
||||
def asciify2(s):
|
||||
matches = re.findall("&#\d+;", s)
|
||||
if len(matches) > 0:
|
||||
hits = set(matches)
|
||||
for hit in hits:
|
||||
name = hit[2:-1]
|
||||
try:
|
||||
entnum = int(name)
|
||||
s = s.replace(hit, chr(entnum))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
matches = re.findall("&\w+;", s)
|
||||
hits = set(matches)
|
||||
amp = "&"
|
||||
if amp in hits:
|
||||
hits.remove(amp)
|
||||
for hit in hits:
|
||||
name = hit[1:-1]
|
||||
if name in html.entities.name2codepoint:
|
||||
s = s.replace(hit, "")
|
||||
s = s.replace(amp, "&")
|
||||
return s
|
||||
|
||||
|
||||
def opensoup(url):
|
||||
request = urllib.request.Request(url)
|
||||
request.add_header("User-Agent", "Mozilla/5.0")
|
||||
# To mimic a real browser's user-agent string more exactly, if necessary:
|
||||
# Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.14)
|
||||
# Gecko/20080418 Ubuntu/7.10 (gutsy) Firefox/2.0.0.14
|
||||
pagefile = urllib.request.urlopen(request)
|
||||
soup = BeautifulSoup(pagefile)
|
||||
pagefile.close()
|
||||
return soup
|
||||
|
||||
|
||||
def asciify(s):
|
||||
return unicodedata.normalize('NFKD', s).encode('ascii', 'ignore')
|
||||
|
||||
|
||||
# remove extra whitespace, including stripping leading and trailing whitespace.
|
||||
def condense(s):
|
||||
s = re.sub(r"\s+", " ", s, re.DOTALL)
|
||||
return s.strip()
|
||||
|
||||
|
||||
def stripurl(s):
|
||||
s = re.sub(r"\<span\s+style\s*\=\s*\"display\:none[^\"]*\"[^\>]*\>[^\<]*\<\/span\>", "", s)
|
||||
s = re.sub(r"\&\#160\;", " ", s)
|
||||
return condense(re.sub(r"\<[^\>]*\>", " ", s))
|
||||
|
||||
|
||||
# this gets rid of tags and condenses whitespace
|
||||
def striptags(s):
|
||||
s = re.sub(r"\<span\s+style\s*\=\s*\"display\:none[^\"]*\"[^\>]*\>[^\<]*\<\/span\>", "", s)
|
||||
s = re.sub(r"\&\#160\;", " ", s)
|
||||
return condense(s)
|
||||
|
||||
|
||||
def getUrlArgs(parseUrl):
|
||||
return re.search('grib2_table4-2-(\d+)-(\d+).shtml', parseUrl).groups()
|
||||
|
||||
|
||||
if len(sys.argv) == 1:
|
||||
print("Usage: ", sys.argv[0], " url [n]")
|
||||
print(" (where n indicates which html table to parse)")
|
||||
exit(1)
|
||||
|
||||
url = sys.argv[1]
|
||||
soup = opensoup(url)
|
||||
tables = soup.findAll("table")
|
||||
|
||||
for table in tables:
|
||||
for r in table.findAll('tr'):
|
||||
rl = []
|
||||
for c in r.findAll(re.compile('td|th')):
|
||||
rl.append(striptags(c.renderContents()))
|
||||
if len(rl) > 1 and "href" in rl[1]:
|
||||
print('! ' + stripurl(rl[1]))
|
||||
scrapeUrl = 'http://www.nco.ncep.noaa.gov/pmb/docs/grib2/grib2_table4-2-' + \
|
||||
getUrlArgs(rl[1])[0] + "-" + getUrlArgs(rl[1])[1] + '.shtml'
|
||||
scrape.run(scrapeUrl)
|
||||
106
awips/gempak/scrape.py
Executable file
106
awips/gempak/scrape.py
Executable file
|
|
@ -0,0 +1,106 @@
|
|||
#!/usr/bin/env python
|
||||
# Parse html tables from a given URL and output CSV.
|
||||
# Note: To install a missing python module foo do "easy_install foo"
|
||||
# (or the new way is "pip install foo" but you might have to do
|
||||
# "easy_install pip" first)
|
||||
|
||||
from BeautifulSoup import BeautifulSoup
|
||||
import urllib.request, urllib.error, urllib.parse
|
||||
import html.entities
|
||||
import re
|
||||
import sys
|
||||
import unicodedata
|
||||
|
||||
|
||||
# from http://stackoverflow.com/questions/1197981/convert-html-entities
|
||||
def asciify2(s):
|
||||
matches = re.findall("&#\d+;", s)
|
||||
if len(matches) > 0:
|
||||
hits = set(matches)
|
||||
for hit in hits:
|
||||
name = hit[2:-1]
|
||||
try:
|
||||
entnum = int(name)
|
||||
s = s.replace(hit, chr(entnum))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
matches = re.findall("&\w+;", s)
|
||||
hits = set(matches)
|
||||
amp = "&"
|
||||
if amp in hits:
|
||||
hits.remove(amp)
|
||||
for hit in hits:
|
||||
name = hit[1:-1]
|
||||
if name in html.entities.name2codepoint:
|
||||
s = s.replace(hit, "")
|
||||
s = s.replace(amp, "&")
|
||||
return s
|
||||
|
||||
|
||||
def opensoup(url):
|
||||
request = urllib.request.Request(url)
|
||||
request.add_header("User-Agent", "Mozilla/5.0")
|
||||
# To mimic a real browser's user-agent string more exactly, if necessary:
|
||||
# Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.14)
|
||||
# Gecko/20080418 Ubuntu/7.10 (gutsy) Firefox/2.0.0.14
|
||||
pagefile = urllib.request.urlopen(request)
|
||||
soup = BeautifulSoup(pagefile)
|
||||
pagefile.close()
|
||||
return soup
|
||||
|
||||
|
||||
def asciify(s):
|
||||
return unicodedata.normalize('NFKD', s).encode('ascii', 'ignore')
|
||||
|
||||
|
||||
# remove extra whitespace, including stripping leading and trailing whitespace.
|
||||
def condense(s):
|
||||
s = re.sub(r"\s+", " ", s, re.DOTALL)
|
||||
return s.strip()
|
||||
|
||||
|
||||
# this gets rid of tags and condenses whitespace
|
||||
def striptags(s):
|
||||
s = re.sub(r"\<span\s+style\s*\=\s*\"display\:none[^\"]*\"[^\>]*\>[^\<]*\<\/span\>", "", s)
|
||||
s = re.sub(r"\&\#160\;", " ", s)
|
||||
return condense(re.sub(r"\<[^\>]*\>", " ", s))
|
||||
|
||||
|
||||
if len(sys.argv) == 1: # called with no arguments
|
||||
print("Usage: ", sys.argv[0], " url [n]")
|
||||
print(" (where n indicates which html table to parse)")
|
||||
exit(1)
|
||||
|
||||
|
||||
def getUrlArgs(parseUrl):
|
||||
return re.search('grib2_table4-2-(\d+)-(\d+).shtml', parseUrl).groups()
|
||||
|
||||
|
||||
def run(url):
|
||||
soup = opensoup(url)
|
||||
tables = soup.findAll("table")
|
||||
for table in tables:
|
||||
ct = 0
|
||||
for r in table.findAll('tr'):
|
||||
rl = []
|
||||
for c in r.findAll(re.compile('td|th')):
|
||||
rl.append(striptags(c.renderContents()))
|
||||
if ct > 0:
|
||||
rl[0] = getUrlArgs(url)[0].zfill(3) + " " + \
|
||||
getUrlArgs(url)[1].zfill(3) + " " + rl[0].zfill(3) + " 000"
|
||||
if len(rl) > 1:
|
||||
if "Reserved" in rl[1]:
|
||||
rl[0] = '!' + rl[0]
|
||||
if "See Table" in rl[2] or "Code table" in rl[2]:
|
||||
rl[2] = "cat"
|
||||
rl[1] = rl[1][:32].ljust(32)
|
||||
rl[2] = rl[2].ljust(20)
|
||||
rl[3] = rl[3].ljust(12) + " 0 -9999.00"
|
||||
if ct:
|
||||
print(" ".join(rl))
|
||||
ct += 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(sys.argv[1])
|
||||
Loading…
Add table
Reference in a new issue