diff --git a/awips/gempak/GridDataRetriever.py b/awips/gempak/GridDataRetriever.py new file mode 100644 index 0000000..5525899 --- /dev/null +++ b/awips/gempak/GridDataRetriever.py @@ -0,0 +1,128 @@ +import os +import numpy +from datetime import datetime +from awips import ThriftClient +from dynamicserialize.dstypes.gov.noaa.nws.ncep.common.dataplugin.gempak.request import GetGridDataRequest + + +class GridDataRetriever: + + def __init__(self, server, pluginName, modelId, cycle, forecast, level1, level2, vcoord, param, nnx, nny): + self.pluginName = pluginName + self.modelId = modelId + self.cycle = cycle + self.forecast = forecast + self.level1 = level1 + self.level2 = level2 + self.vcoord = vcoord + self.param = param + self.nx = nnx + self.ny = nny + self.host = os.getenv("DEFAULT_HOST", server) + self.port = os.getenv("DEFAULT_PORT", "9581") + self.client = ThriftClient.ThriftClient(self.host, self.port) + + def getData(self): + """ Sends ThriftClient request and writes out received files.""" + req = GetGridDataRequest() + + req.setPluginName(self.pluginName) + req.setModelId(self.modelId) + + dt = datetime.strptime(self.cycle, '%y%m%d/%H%M') + ct = datetime.strftime(dt, '%Y-%m-%d %H:%M:%S') + req.setReftime(ct) + req.setFcstsec(self.forecast) + + if self.level1 == '-1': + f1 = -999999.0 + else: + f1 = float(self.level1) + + if self.level2 == '-1': + f2 = -999999.0 + else: + f2 = float(self.level2) + + vcoord = self.vcoord + if vcoord == 'SGMA': + if f1 >= 0.0: + f1 = f1 / 10000 + if f2 >= 0.0: + f2 = f2 / 10000 + elif vcoord == 'DPTH': + if f1 >= 0.0: + f1 = f1 / 100.0 + if f2 >= 0.0: + f2 = f2 / 100.0 + elif vcoord == 'POTV': + if f1 >= 0.0: + f1 = f1 / 1000.0 + if f2 >= 0.0: + f2 = f2 / 1000.0 + + req.setLevel1(str(f1)) + req.setLevel2(str(f2)) + req.setVcoord(vcoord) + + req.setParm(self.param) + + resp = self.client.sendRequest(req) + + # Get the dimensions of the grid + kx = int(self.nx) + ky = int(self.ny) + kxky = kx * ky + + # Put the data into a NUMPY array + grid = numpy.asarray(resp.getFloatData()) + + # All grids need to be flipped from a GEMPAK point of view + # Reshape the array into 2D + grid = numpy.reshape(grid, (ky, kx)) + # Flip the array in the up-down direction + grid = numpy.flipud(grid) + # Reshape the array back into 1D + grid = numpy.reshape(grid, kxky) + + return [replacemissing(x) for x in grid] + + +def getgriddata(server, table, model, cycle, forecast, level1, + level2, vcoord, param, nnx, nny): + gir = GridDataRetriever(server, table, model, cycle, forecast, + level1, level2, vcoord, param, nnx, nny) + return gir.getData() + + +def getheader(server, table, model, cycle, forecast, level1, + level2, vcoord, param, nnx, nny): + idata = [] + idata.append(0) + idata.append(0) + return idata + + +def replacemissing(x): + if x == -999999.0: + return -9999.0 + return x + + +# This is the standard boilerplate that runs this script as a main +if __name__ == '__main__': + # Run Test + srv = 'edex-cloud.unidata.ucar.edu' + tbl = 'grid' + mdl = 'GFS20' + cyc = '131227/0000' + fcs = '43200' + lv1 = '500' + lv2 = '-1' + vcd = 'PRES' + prm = 'HGHT' + nx = '720' + ny = '361' + + print(getheader(srv, tbl, mdl, cyc, fcs, lv1, lv2, vcd, prm, nx, ny)) + print(getgriddata(srv, tbl, mdl, cyc, fcs, lv1, lv2, vcd, prm, nx, ny)) diff --git a/awips/gempak/GridInfoRetriever.py b/awips/gempak/GridInfoRetriever.py new file mode 100644 index 0000000..af71e8a --- /dev/null +++ b/awips/gempak/GridInfoRetriever.py @@ -0,0 +1,137 @@ +import os +from datetime import datetime +from operator import itemgetter +from awips import ThriftClient +from dynamicserialize.dstypes.gov.noaa.nws.ncep.common.dataplugin.gempak.request import GetGridInfoRequest + + +class GridInfoRetriever: + + def __init__(self, server, pluginName, modelId, cycle=None, forecast=None): + self.pluginName = pluginName + self.modelId = modelId + self.cycle = cycle + self.forecast = forecast + self.host = os.getenv("DEFAULT_HOST", server) + self.port = os.getenv("DEFAULT_PORT", "9581") + self.client = ThriftClient.ThriftClient(self.host, self.port) + + def getInfo(self): + """ Sends ThriftClient request and writes out received files.""" + req = GetGridInfoRequest() + req.setPluginName(self.pluginName) + req.setModelId(self.modelId) + + req.setReftime(self.cycle) + if len(self.cycle) > 2: + dt = datetime.strptime(self.cycle, '%y%m%d/%H%M') + ct = datetime.strftime(dt, '%Y-%m-%d %H:%M:%S') + req.setReftime(ct) + + req.setFcstsec(self.forecast) + + resp = self.client.sendRequest(req) + + sortresp = sorted(sorted(resp, key=itemgetter("reftime"), reverse=True), key=itemgetter("fcstsec")) + + grids = [] + + count = 0 + for record in sortresp: + s = '{:<12}'.format(record['param']) + + if sys.byteorder == 'little': + parm1 = (ord(s[3]) << 24) + (ord(s[2]) << 16) + (ord(s[1]) << 8) + ord(s[0]) + parm2 = (ord(s[7]) << 24) + (ord(s[6]) << 16) + (ord(s[5]) << 8) + ord(s[4]) + parm3 = (ord(s[11]) << 24) + (ord(s[10]) << 16) + (ord(s[9]) << 8) + ord(s[8]) + else: + parm1 = (ord(s[0]) << 24) + (ord(s[1]) << 16) + (ord(s[2]) << 8) + ord(s[3]) + parm2 = (ord(s[4]) << 24) + (ord(s[5]) << 16) + (ord(s[6]) << 8) + ord(s[7]) + parm3 = (ord(s[8]) << 24) + (ord(s[9]) << 16) + (ord(s[10]) << 8) + ord(s[11]) + + dt = datetime.strptime(record['reftime'], '%Y-%m-%d %H:%M:%S.%f') + dattim = dt.month * 100000000 + dt.day * 1000000 + (dt.year%100) * 10000 + dt.hour * 100 + dt.minute + fcsth = (int(record['fcstsec']) / 60) / 60 + fcstm = (int(record['fcstsec']) / 60) % 60 + fcst = 100000 + fcsth * 100 + fcstm + + lv1 = float(record['level1']) + if lv1 == -999999.0: + lv1 = -1.0 + lv2 = float(record['level2']) + if lv2 == -999999.0: + lv2 = -1.0 + + vcd = record['vcoord'] + if vcd == 'NONE': + ivcd = 0 + elif vcd == 'PRES': + ivcd = 1 + elif vcd == 'THTA': + ivcd = 2 + elif vcd == 'HGHT': + ivcd = 3 + elif vcd == 'SGMA': + ivcd = 4 + if lv1 >= 0.0: + lv1 = lv1 * 10000.0 + if lv2 >= 0.0: + lv2 = lv2 * 10000.0 + elif vcd == 'DPTH': + ivcd = 5 + if lv1 >= 0.0: + lv1 = lv1 * 100.0 + if lv2 >= 0.0: + lv2 = lv2 * 100.0 + elif vcd == 'HYBL': + ivcd = 6 + else: + v = '{:<4}'.format(vcd) + if sys.byteorder == 'little': + ivcd = (ord(v[3]) << 24) + (ord(v[2]) << 16) + (ord(v[1]) << 8) + ord(v[0]) + else: + ivcd = (ord(v[0]) << 24) + (ord(v[1]) << 16) + (ord(v[2]) << 8) + ord(v[3]) + if vcd == 'POTV': + if lv1 >= 0.0: + lv1 = lv1 * 1000.0 + if lv2 >= 0.0: + lv2 = lv2 * 1000.0 + grids.append(9999) + grids.append(dattim) + grids.append(fcst) + grids.append(0) + grids.append(0) + grids.append(int(lv1)) + grids.append(int(lv2)) + grids.append(ivcd) + grids.append(parm1) + grids.append(parm2) + grids.append(parm3) + count += 1 + if count > 29998: + break + + return grids + + +def getinfo(server, table, model, cycle, forecast): + gir = GridInfoRetriever(server, table, model, cycle, forecast) + return gir.getInfo() + + +def getrow(server, table, model, cycle, forecast): + idata = [] + idata.append(9999) + idata.append(1) + return idata + + +# This is the standard boilerplate that runs this script as a main +if __name__ == '__main__': + import sys + # Run Test + srv = 'edex-cloud.unidata.ucar.edu' + tbl = 'grid' + mdl = 'NAM40' + print(getrow(srv, tbl, mdl)) + print(getinfo(srv, tbl, mdl)) diff --git a/awips/gempak/GridNavRetriever.py b/awips/gempak/GridNavRetriever.py new file mode 100644 index 0000000..c138d87 --- /dev/null +++ b/awips/gempak/GridNavRetriever.py @@ -0,0 +1,294 @@ +import os +import math +from awips import ThriftClient +from dynamicserialize.dstypes.gov.noaa.nws.ncep.common.dataplugin.gempak.request import GetGridNavRequest +from ctypes import * + +EARTH_RADIUS = 6371200.0 +DEG_TO_RAD = math.pi / 180.0 +RAD_TO_DEG = 180.0 / math.pi +TWOPI = math.pi * 2.0 +HALFPI = math.pi / 2.0 +PI4TH = math.pi / 4.0 +PI3RD = math.pi / 3.0 + + +def createPolar(nsflag, clon, lat1, lon1, dx, dy, unit, nx, ny): + clonr = clon * DEG_TO_RAD + latr = lat1 * DEG_TO_RAD + lonr = lon1 * DEG_TO_RAD + if nsflag == 'N': + x1 = EARTH_RADIUS * math.tan(PI4TH - latr/2.0) * math.sin(lonr-clonr) + y1 = -1 * EARTH_RADIUS * math.tan(PI4TH - latr/2.0) * math.cos(lonr-clonr) + else: + x1 = EARTH_RADIUS * math.tan(PI4TH + latr/2.0) * math.sin(lonr-clonr) + y1 = EARTH_RADIUS * math.tan(PI4TH + latr/2.0) * math.cos(lonr-clonr) + + if unit == 'm': + tdx = dx / (1 + math.sin(PI3RD)) + tdy = dy / (1 + math.sin(PI3RD)) + else: + tdx = (dx*1000.0) / (1 + math.sin(PI3RD)) + tdy = (dy*1000.0) / (1 + math.sin(PI3RD)) + + x2 = x1 + tdx * (nx-1) + y2 = y1 + tdy * (ny-1) + xll = min(x1, x2) + yll = min(y1, y2) + xur = max(x1, x2) + yur = max(y1, y2) + + if nsflag == 'N': + latll = (HALFPI - 2*math.atan2(math.hypot(xll, yll), EARTH_RADIUS)) * RAD_TO_DEG + rtemp = clonr + math.atan2(xll, -yll) + else: + latll = -1 * (HALFPI - 2*math.atan2(math.hypot(xll, yll), EARTH_RADIUS)) * RAD_TO_DEG + rtemp = clonr + math.atan2(xll, yll) + + if rtemp > math.pi: + lonll = (rtemp-TWOPI) * RAD_TO_DEG + elif rtemp < -math.pi: + lonll = (rtemp+TWOPI) * RAD_TO_DEG + else: + lonll = rtemp * RAD_TO_DEG + + if nsflag == 'N': + latur = (HALFPI - 2*math.atan2(math.hypot(xur, yur), EARTH_RADIUS)) * RAD_TO_DEG + rtemp = clonr + math.atan2(xur, -yur) + else: + latur = -1 * (HALFPI - 2*math.atan2(math.hypot(xur, yur), EARTH_RADIUS)) * RAD_TO_DEG + rtemp = clonr + math.atan2(xur, yur) + + if rtemp > math.pi: + lonur = (rtemp-TWOPI) * RAD_TO_DEG + elif rtemp < -math.pi: + lonur = (rtemp+TWOPI) * RAD_TO_DEG + else: + lonur = rtemp * RAD_TO_DEG + + return [latll, lonll, latur, lonur] + + +def createConic(nsflag, clon, lat1, lon1, dx, dy, unit, nx, ny, ang1, ang3): + clonr = clon * DEG_TO_RAD + latr = lat1 * DEG_TO_RAD + lonr = lon1 * DEG_TO_RAD + + angle1 = HALFPI - (math.fabs(ang1) * DEG_TO_RAD) + angle2 = HALFPI - (math.fabs(ang3) * DEG_TO_RAD) + + if ang1 == ang3: + cc = math.cos(angle1) + else: + cc = (math.log(math.sin(angle2)) - math.log(math.sin(angle1))) \ + / (math.log(math.tan(angle2/2.0)) - math.log(math.tan(angle1/2.0))) + + er = EARTH_RADIUS / cc + + if nsflag == 'N': + x1 = er * math.pow(math.tan((HALFPI-latr)/2.0), cc) * math.sin(cc*(lonr-clonr)) + y1 = -1.0 * er * math.pow(math.tan((HALFPI-latr)/2.0), cc) * math.cos(cc*(lonr-clonr)) + else: + x1 = er * math.pow(math.tan((HALFPI+latr)/2.0), cc) * math.sin(cc*(lonr-clonr)) + y1 = er * math.pow(math.tan((HALFPI+latr)/2.0), cc) * math.cos(cc*(lonr-clonr)) + + alpha = math.pow(math.tan(angle1/2.0), cc) / math.sin(angle1) + + if unit == 'm': + x2 = x1 + (nx-1) * alpha * dx + y2 = y1 + (ny-1) * alpha * dy + else: + x2 = x1 + (nx-1) * alpha * (dx*1000.0) + y2 = y1 + (ny-1) * alpha * (dy*1000.0) + + xll = min(x1, x2) + yll = min(y1, y2) + xur = max(x1, x2) + yur = max(y1, y2) + + if nsflag == 'N': + latll = (HALFPI - 2.0 * math.atan(math.pow(math.hypot(xll, yll)/er, (1/cc)))) * RAD_TO_DEG + rtemp = math.atan2(xll, -yll) * (1/cc) + clonr + else: + latll = (-1.0 * (HALFPI - 2.0 * math.atan(math.pow(math.hypot(xll, yll)/er, (1/cc))))) * RAD_TO_DEG + rtemp = math.atan2(xll, yll) * (1/cc) + clonr + + if rtemp > math.pi: + lonll = (rtemp-TWOPI) * RAD_TO_DEG + elif rtemp < -math.pi: + lonll = (rtemp+TWOPI) * RAD_TO_DEG + else: + lonll = rtemp * RAD_TO_DEG + + if nsflag == 'N': + latur = (HALFPI - 2.0 * math.atan(math.pow(math.hypot(xur, yur)/er, (1/cc)))) * RAD_TO_DEG + rtemp = math.atan2(xur, -yur) * (1/cc) + clonr + else: + latur = (-1.0 * (HALFPI - 2.0 * math.atan(math.pow(math.hypot(xur, yur)/er, (1/cc))))) * RAD_TO_DEG + rtemp = math.atan2(xur, yur) * (1/cc) + clonr + + if rtemp > math.pi: + lonur = (rtemp-TWOPI) * RAD_TO_DEG + elif rtemp < -math.pi: + lonur = (rtemp+TWOPI) * RAD_TO_DEG + else: + lonur = rtemp * RAD_TO_DEG + + return [latll, lonll, latur, lonur] + + +class StringConverter(Union): + _fields_ = [("char", c_char*4), ("int", c_int), ("float", c_float)] + + +class GridNavRetriever: + + def __init__(self, server, pluginName, modelId, arrayLen): + self.pluginName = pluginName + self.modelId = modelId + self.arrayLen = arrayLen + self.host = os.getenv("DEFAULT_HOST", server) + self.port = os.getenv("DEFAULT_PORT", "9581") + self.client = ThriftClient.ThriftClient(self.host, self.port) + + def getNavBlk(self): + """ Sends ThriftClient request and writes out received files.""" + req = GetGridNavRequest() + req.setPluginName(self.pluginName) + req.setModelId(self.modelId) + resp = self.client.sendRequest(req) + + nav = [] + + for record in resp: + unit = record['spacingunit'] + sk = record['spatialkey'] + skarr = sk.split('/') + + nx = float(skarr[1]) + ny = float(skarr[2]) + dx = float(skarr[3]) + dy = float(skarr[4]) + + sc = StringConverter() + if record['projtype'] == 'LatLon': + sc.char = 'CED ' + gemproj = 2.0 + ang1 = 0.0 + ang2 = 0.0 + ang3 = 0.0 + + lllat = float(record['lowerleftlat']) + lllon = float(record['lowerleftlon']) + urlat = lllat + (dy * (ny-1)) + urlon = lllon + (dx * (nx-1)) + if lllon > 180: + lllon -= 360.0 + if urlon > 180: + urlon -= 360.0 + + if record['projtype'] == 'Polar Stereographic': + sc.char = 'STR ' + gemproj = 2.0 + if float(record['standard_parallel_1']) < 0.0: + ang1 = -90.0 + nsflag = 'S' + else: + ang1 = 90.0 + nsflag = 'N' + ang2 = float(record['central_meridian']) + ang3 = 0.0 + + lat1 = float(record['lowerleftlat']) + lon1 = float(record['lowerleftlon']) + coords = createPolar(nsflag, ang2, lat1, lon1, dx, dy, unit, nx, ny) + lllat = coords[0] + lllon = coords[1] + urlat = coords[2] + urlon = coords[3] + + if record['projtype'] == 'Lambert Conformal': + sc.char = 'LCC ' + gemproj = 2.0 + + ang1 = float(skarr[7]) + ang2 = float(record['central_meridian']) + ang3 = float(skarr[8]) + if ang1 < 0.0: + nsflag = 'S' + else: + nsflag = 'N' + + lat1 = float(record['lowerleftlat']) + lon1 = float(record['lowerleftlon']) + coords = createConic(nsflag, ang2, lat1, lon1, dx, dy, unit, nx, ny, ang1, ang3) + lllat = coords[0] + lllon = coords[1] + urlat = coords[2] + urlon = coords[3] + + # Fill up the output array of floats + nav.append(gemproj) + nav.append(sc.float) + nav.append(1.0) + nav.append(1.0) + nav.append(nx) + nav.append(ny) + nav.append(lllat) + nav.append(lllon) + nav.append(urlat) + nav.append(urlon) + nav.append(ang1) + nav.append(ang2) + nav.append(ang3) + + for i in range(13, int(self.arrayLen)): + nav.append(0.0) + return nav + + def getAnlBlk(self): + anl = [] + # Type + anl.append(2.0) + # Delta + anl.append(1.0) + # Extend area + anl.append(0.0) + anl.append(0.0) + anl.append(0.0) + anl.append(0.0) + # Grid area + anl.append(-90.0) + anl.append(-180.0) + anl.append(90.0) + anl.append(180.0) + # Data area + anl.append(-90.0) + anl.append(-180.0) + anl.append(90.0) + anl.append(180.0) + for i in range(18, int(self.arrayLen)): + anl.append(0.0) + return anl + + +def getnavb(server, table, model, arrlen): + gnr = GridNavRetriever(server, table, model, arrlen) + return gnr.getNavBlk() + + +def getanlb(server, table, model, arrlen): + gnr = GridNavRetriever(server, table, model, arrlen) + return gnr.getAnlBlk() + + +# This is the standard boilerplate that runs this script as a main +if __name__ == '__main__': + # Run Test + srv = 'edex-cloud.unidata.ucar.edu' + tbl = 'grid_info' + mdl = 'NAM40' + navlen = '256' + print(getnavb(srv, tbl, mdl, navlen)) + anllen = '128' + print(getanlb(srv, tbl, mdl, anllen)) diff --git a/awips/gempak/StationDataRetriever.py b/awips/gempak/StationDataRetriever.py new file mode 100644 index 0000000..6b16d02 --- /dev/null +++ b/awips/gempak/StationDataRetriever.py @@ -0,0 +1,136 @@ +import os +from datetime import datetime +from awips import ThriftClient +from dynamicserialize.dstypes.com.raytheon.uf.common.time import DataTime +from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange +from dynamicserialize.dstypes.gov.noaa.nws.ncep.common.dataplugin.gempak.request import StationDataRequest + + +class StationDataRetriever: + """ Retrieves all data for a requested station and time """ + + def __init__(self, server, pluginName, stationId, refTime, parmList, partNumber): + self.pluginName = pluginName + self.stationId = stationId + self.refTime = refTime + self.parmList = parmList + self.partNumber = partNumber + self.host = os.getenv("DEFAULT_HOST", server) + self.port = os.getenv("DEFAULT_PORT", "9581") + self.client = ThriftClient.ThriftClient(self.host, self.port) + + def getStationData(self): + """ Sends ThriftClient request and writes out received files.""" + dtime = datetime.strptime(self.refTime, "%y%m%d/%H%M") + trange = TimeRange() + trange.setStart(dtime) + trange.setEnd(dtime) + dataTime = DataTime(refTime=dtime, validPeriod=trange) + req = StationDataRequest() + req.setPluginName(self.pluginName) + req.setStationId(self.stationId) + req.setRefTime(dataTime) + req.setParmList(self.parmList) + req.setPartNumber(self.partNumber) + resp = self.client.sendRequest(req) + return resp + + +def getstationdata(server, table, stationId, refTime, parmList, partNumber): + sr = StationDataRetriever(server, table, stationId, refTime, parmList, partNumber) + lcldict = sr.getStationData() + + rdata = [] + + for substr in parmList.split(','): + if substr in lcldict: + rdata.append(lcldict[substr]) + else: + rdata.append(-9999.00) + + return rdata + + +def getleveldata(server, table, stationId, refTime, parmList, partNumber): + sr = StationDataRetriever(server, table, stationId, refTime, parmList, partNumber) + lcldict = sr.getStationData() + + numset = [1] + for substr in parmList.split(','): + if substr in lcldict: + pnum = len(lcldict[substr]) - 1 + while pnum >= 0: + if lcldict[substr][pnum] != -9999.00: + break + pnum = pnum - 1 + numset.append(pnum) + + rdata = [] + + for jj in range(max(numset)): + for substr in parmList.split(','): + if substr in lcldict: + if lcldict[substr][jj] == -9999998.0: + rdata.append(-9999.0) + else: + rdata.append(lcldict[substr][jj]) + else: + rdata.append(-9999.0) + + return rdata + + +def getstationtext(server, table, stationId, refTime, parmList, partNumber): + sr = StationDataRetriever(server, table, stationId, refTime, parmList, partNumber) + lcldict = sr.getStationData() + + if parmList in lcldict: + return lcldict[parmList] + else: + return ' ' + + +def getheader(server, table, stationId, refTime, parmList, partNumber): + idata = [] + idata.append(0) + return idata + + +# This is the standard boilerplate that runs this script as a main +if __name__ == '__main__': + # Run Test + srv = 'edex-cloud.unidata.ucar.edu' + key = '-' + print('OBS - METAR') + tbl = 'obs' + stn = 'KLGA' + time = '130823/1700' + parm = 'seaLevelPress,temperature,dewpoint,windSpeed,windDir' + part = '0' + print(getheader(srv, tbl, stn, time, parm, part)) + print(getstationdata(srv, tbl, stn, time, parm, part)) + parm = 'rawMETAR' + print(getstationtext(srv, tbl, stn, time, parm, part)) + print('SFCOBS - SYNOP') + tbl = 'sfcobs' + stn = '72403' + time = '130823/1800' + parm = 'seaLevelPress,temperature,dewpoint,windSpeed,windDir' + part = '0' + print(getheader(srv, tbl, stn, time, parm, part)) + print(getstationdata(srv, tbl, stn, time, parm, part)) + parm = 'rawReport' + print(getstationtext(srv, tbl, stn, time, parm, part)) + print('UAIR') + tbl = 'bufrua' + stn = '72469' + time = '130823/1200' + parm = 'prMan,htMan,tpMan,tdMan,wdMan,wsMan' + part = '2020' + print(getleveldata(srv, tbl, stn, time, parm, part)) + parm = 'prSigT,tpSigT,tdSigT' + part = '2022' + print(getleveldata(srv, tbl, stn, time, parm, part)) + parm = 'htSigW,wsSigW,wdSigW' + part = '2021' + print(getleveldata(srv, tbl, stn, time, parm, part)) diff --git a/awips/gempak/StationRetriever.py b/awips/gempak/StationRetriever.py new file mode 100644 index 0000000..960e423 --- /dev/null +++ b/awips/gempak/StationRetriever.py @@ -0,0 +1,86 @@ +import os +import sys +from awips import ThriftClient +from dynamicserialize.dstypes.gov.noaa.nws.ncep.common.dataplugin.gempak.request import GetStationsRequest + + +class StationRetriever: + """ Retrieves all requested stations """ + + def __init__(self, server, pluginName): + self.pluginName = pluginName + self.outdir = os.getcwd() + self.host = os.getenv("DEFAULT_HOST", server) + self.port = os.getenv("DEFAULT_PORT", "9581") + self.client = ThriftClient.ThriftClient(self.host, self.port) + + def getStations(self): + """ Sends ThriftClient request and writes out received files.""" + req = GetStationsRequest() + req.setPluginName(self.pluginName) + resp = self.client.sendRequest(req) + + stns = [] + for item in resp: + stationstr = '{:<8}'.format(item.getStationId()) + + if sys.byteorder == 'little': + stnid = (ord(stationstr[3]) << 24) + (ord(stationstr[2]) << 16) + \ + (ord(stationstr[1]) << 8) + ord(stationstr[0]) + stnid2 = (ord(stationstr[7]) << 24) + (ord(stationstr[6]) << 16) + \ + (ord(stationstr[5]) << 8) + ord(stationstr[4]) + else: + stnid = (ord(stationstr[0]) << 24) + (ord(stationstr[1]) << 16) + \ + (ord(stationstr[2]) << 8) + ord(stationstr[3]) + stnid2 = (ord(stationstr[4]) << 24) + (ord(stationstr[5]) << 16) + \ + (ord(stationstr[6]) << 8) + ord(stationstr[7]) + + if item.getState() is None: + stationstr = ' ' + else: + stationstr = '{:<4}'.format(item.getState()) + + if sys.byteorder == 'little': + state = (ord(stationstr[3]) << 24) + (ord(stationstr[2]) << 16) \ + + (ord(stationstr[1]) << 8) + ord(stationstr[0]) + else: + state = (ord(stationstr[0]) << 24) + (ord(stationstr[1]) << 16) \ + + (ord(stationstr[2]) << 8) + ord(stationstr[3]) + + stationstr = '{:<4}'.format(item.getCountry()) + if sys.byteorder == 'little': + cntry = (ord(stationstr[3]) << 24) + (ord(stationstr[2]) << 16) \ + + (ord(stationstr[1]) << 8) + ord(stationstr[0]) + else: + cntry = (ord(stationstr[0]) << 24) + (ord(stationstr[1]) << 16) \ + + (ord(stationstr[2]) << 8) + ord(stationstr[3]) + + stns.append(9999) + stns.append(stnid) + stns.append(item.getWmoIndex()) + stns.append(int(item.getLatitude()*100)) + stns.append(int(item.getLongitude()*100)) + stns.append(int(item.getElevation())) + stns.append(state) + stns.append(cntry) + stns.append(stnid2) + stns.append(0) + return stns + + +def getstations(server, table, key, dummy, dummy2): + sr = StationRetriever(server, table) + return sr.getStations() + + +# This is the standard boilerplate that runs this script as a main +if __name__ == '__main__': + # Run Test + srv = 'edex-cloud.unidata.ucar.edu' + key = '-' + print('OBS - METAR') + tbl = 'obs' + print(getstations(srv, tbl, key)) + print('SFCOBS - SYNOP') + tbl = 'sfcobs' + print(getstations(srv, tbl, key)) diff --git a/awips/gempak/TimeRetriever.py b/awips/gempak/TimeRetriever.py new file mode 100644 index 0000000..5d7aff6 --- /dev/null +++ b/awips/gempak/TimeRetriever.py @@ -0,0 +1,68 @@ +import os +from datetime import datetime +from awips import ThriftClient +from dynamicserialize.dstypes.java.util import GregorianCalendar +from dynamicserialize.dstypes.gov.noaa.nws.ncep.common.dataplugin.gempak.request import GetTimesRequest + + +class TimeRetriever: + """ Retrieves all requested times""" + + def __init__(self, server, pluginName, timeField): + self.pluginName = pluginName + self.timeField = timeField + self.outdir = os.getcwd() + self.host = os.getenv("DEFAULT_HOST", server) + self.port = os.getenv("DEFAULT_PORT", "9581") + self.client = ThriftClient.ThriftClient(self.host, self.port) + + def getTimes(self): + """ Sends ThriftClient request and writes out received files.""" + req = GetTimesRequest() + req.setPluginName(self.pluginName) + req.setTimeField(self.timeField) + resp = self.client.sendRequest(req) + timelist = [] + for item in resp.getTimes(): + if isinstance(item, GregorianCalendar): + tstamp = item.getTimeInMillis() + else: + tstamp = item.getTime() + time = datetime.utcfromtimestamp(tstamp/1000) + timelist.append(time) + + timelist.sort(reverse=True) + + times = [] + for time in timelist: + times.append(9999) + times.append((time.year % 100) * 10000 + (time.month * 100) + time.day) + times.append((time.hour * 100) + time.minute) + + # GEMPAK can only handle up to 200 times, which is 600 elements + # in this array -- [9999, DATE, TIME] -- repeated + return times[0:600] + + +def gettimes(server, table, key, dummy, dummy2): + tr = TimeRetriever(server, table, key) + return tr.getTimes() + + +# This is the standard boilerplate that runs this script as a main +if __name__ == '__main__': + srv = 'edex-cloud.unidata.ucar.edu' + print('OBS - METAR') + tbl = 'obs' + key = 'refHour' + print(gettimes(srv, tbl, key)) + + print('SFCOBS - SYNOP') + tbl = 'sfcobs' + key = 'refHour' + print(gettimes(srv, tbl, key)) + + print('BUFRUA') + tbl = 'bufrua' + key = 'dataTime.refTime' + print(gettimes(srv, tbl, key)) diff --git a/awips/gempak/ncepGribTables.py b/awips/gempak/ncepGribTables.py new file mode 100755 index 0000000..9423513 --- /dev/null +++ b/awips/gempak/ncepGribTables.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# Parse html tables from a given URL and output CSV. +# Note: To install a missing python module foo do "easy_install foo" +# (or the new way is "pip install foo" but you might have to do +# "easy_install pip" first) + +from BeautifulSoup import BeautifulSoup +import scrape +import urllib.request, urllib.error, urllib.parse +import html.entities +import re +import sys +import unicodedata + + +# from http://stackoverflow.com/questions/1197981/convert-html-entities +def asciify2(s): + matches = re.findall("&#\d+;", s) + if len(matches) > 0: + hits = set(matches) + for hit in hits: + name = hit[2:-1] + try: + entnum = int(name) + s = s.replace(hit, chr(entnum)) + except ValueError: + pass + + matches = re.findall("&\w+;", s) + hits = set(matches) + amp = "&" + if amp in hits: + hits.remove(amp) + for hit in hits: + name = hit[1:-1] + if name in html.entities.name2codepoint: + s = s.replace(hit, "") + s = s.replace(amp, "&") + return s + + +def opensoup(url): + request = urllib.request.Request(url) + request.add_header("User-Agent", "Mozilla/5.0") + # To mimic a real browser's user-agent string more exactly, if necessary: + # Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.14) + # Gecko/20080418 Ubuntu/7.10 (gutsy) Firefox/2.0.0.14 + pagefile = urllib.request.urlopen(request) + soup = BeautifulSoup(pagefile) + pagefile.close() + return soup + + +def asciify(s): + return unicodedata.normalize('NFKD', s).encode('ascii', 'ignore') + + +# remove extra whitespace, including stripping leading and trailing whitespace. +def condense(s): + s = re.sub(r"\s+", " ", s, re.DOTALL) + return s.strip() + + +def stripurl(s): + s = re.sub(r"\]*\>[^\<]*\<\/span\>", "", s) + s = re.sub(r"\&\#160\;", " ", s) + return condense(re.sub(r"\<[^\>]*\>", " ", s)) + + +# this gets rid of tags and condenses whitespace +def striptags(s): + s = re.sub(r"\]*\>[^\<]*\<\/span\>", "", s) + s = re.sub(r"\&\#160\;", " ", s) + return condense(s) + + +def getUrlArgs(parseUrl): + return re.search('grib2_table4-2-(\d+)-(\d+).shtml', parseUrl).groups() + + +if len(sys.argv) == 1: + print("Usage: ", sys.argv[0], " url [n]") + print(" (where n indicates which html table to parse)") + exit(1) + +url = sys.argv[1] +soup = opensoup(url) +tables = soup.findAll("table") + +for table in tables: + for r in table.findAll('tr'): + rl = [] + for c in r.findAll(re.compile('td|th')): + rl.append(striptags(c.renderContents())) + if len(rl) > 1 and "href" in rl[1]: + print('! ' + stripurl(rl[1])) + scrapeUrl = 'http://www.nco.ncep.noaa.gov/pmb/docs/grib2/grib2_table4-2-' + \ + getUrlArgs(rl[1])[0] + "-" + getUrlArgs(rl[1])[1] + '.shtml' + scrape.run(scrapeUrl) diff --git a/awips/gempak/scrape.py b/awips/gempak/scrape.py new file mode 100755 index 0000000..06989f4 --- /dev/null +++ b/awips/gempak/scrape.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +# Parse html tables from a given URL and output CSV. +# Note: To install a missing python module foo do "easy_install foo" +# (or the new way is "pip install foo" but you might have to do +# "easy_install pip" first) + +from BeautifulSoup import BeautifulSoup +import urllib.request, urllib.error, urllib.parse +import html.entities +import re +import sys +import unicodedata + + +# from http://stackoverflow.com/questions/1197981/convert-html-entities +def asciify2(s): + matches = re.findall("&#\d+;", s) + if len(matches) > 0: + hits = set(matches) + for hit in hits: + name = hit[2:-1] + try: + entnum = int(name) + s = s.replace(hit, chr(entnum)) + except ValueError: + pass + + matches = re.findall("&\w+;", s) + hits = set(matches) + amp = "&" + if amp in hits: + hits.remove(amp) + for hit in hits: + name = hit[1:-1] + if name in html.entities.name2codepoint: + s = s.replace(hit, "") + s = s.replace(amp, "&") + return s + + +def opensoup(url): + request = urllib.request.Request(url) + request.add_header("User-Agent", "Mozilla/5.0") + # To mimic a real browser's user-agent string more exactly, if necessary: + # Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.14) + # Gecko/20080418 Ubuntu/7.10 (gutsy) Firefox/2.0.0.14 + pagefile = urllib.request.urlopen(request) + soup = BeautifulSoup(pagefile) + pagefile.close() + return soup + + +def asciify(s): + return unicodedata.normalize('NFKD', s).encode('ascii', 'ignore') + + +# remove extra whitespace, including stripping leading and trailing whitespace. +def condense(s): + s = re.sub(r"\s+", " ", s, re.DOTALL) + return s.strip() + + +# this gets rid of tags and condenses whitespace +def striptags(s): + s = re.sub(r"\]*\>[^\<]*\<\/span\>", "", s) + s = re.sub(r"\&\#160\;", " ", s) + return condense(re.sub(r"\<[^\>]*\>", " ", s)) + + +if len(sys.argv) == 1: # called with no arguments + print("Usage: ", sys.argv[0], " url [n]") + print(" (where n indicates which html table to parse)") + exit(1) + + +def getUrlArgs(parseUrl): + return re.search('grib2_table4-2-(\d+)-(\d+).shtml', parseUrl).groups() + + +def run(url): + soup = opensoup(url) + tables = soup.findAll("table") + for table in tables: + ct = 0 + for r in table.findAll('tr'): + rl = [] + for c in r.findAll(re.compile('td|th')): + rl.append(striptags(c.renderContents())) + if ct > 0: + rl[0] = getUrlArgs(url)[0].zfill(3) + " " + \ + getUrlArgs(url)[1].zfill(3) + " " + rl[0].zfill(3) + " 000" + if len(rl) > 1: + if "Reserved" in rl[1]: + rl[0] = '!' + rl[0] + if "See Table" in rl[2] or "Code table" in rl[2]: + rl[2] = "cat" + rl[1] = rl[1][:32].ljust(32) + rl[2] = rl[2].ljust(20) + rl[3] = rl[3].ljust(12) + " 0 -9999.00" + if ct: + print(" ".join(rl)) + ct += 1 + + +if __name__ == '__main__': + run(sys.argv[1])