add classes from GEMPAK7/scripts/python/

awips/gempak/GridDataRetriever.py

@@ -0,0 +1,128 @@
+import os
+import numpy
+from datetime import datetime
+from awips import ThriftClient
+from dynamicserialize.dstypes.gov.noaa.nws.ncep.common.dataplugin.gempak.request import GetGridDataRequest
+
+
+class GridDataRetriever:
+
+    def __init__(self, server, pluginName, modelId, cycle, forecast, level1, level2, vcoord, param, nnx, nny):
+        self.pluginName = pluginName
+        self.modelId = modelId
+        self.cycle = cycle
+        self.forecast = forecast
+        self.level1 = level1
+        self.level2 = level2
+        self.vcoord = vcoord
+        self.param = param
+        self.nx = nnx
+        self.ny = nny
+        self.host = os.getenv("DEFAULT_HOST", server)
+        self.port = os.getenv("DEFAULT_PORT", "9581")
+        self.client = ThriftClient.ThriftClient(self.host, self.port)
+
+    def getData(self):
+        """ Sends ThriftClient request and writes out received files."""
+        req = GetGridDataRequest()
+
+        req.setPluginName(self.pluginName)
+        req.setModelId(self.modelId)
+
+        dt = datetime.strptime(self.cycle, '%y%m%d/%H%M')
+        ct = datetime.strftime(dt, '%Y-%m-%d %H:%M:%S')
+        req.setReftime(ct)
+        req.setFcstsec(self.forecast)
+
+        if self.level1 == '-1':
+            f1 = -999999.0
+        else:
+            f1 = float(self.level1)
+
+        if self.level2 == '-1':
+            f2 = -999999.0
+        else:
+            f2 = float(self.level2)
+
+        vcoord = self.vcoord
+        if vcoord == 'SGMA':
+            if f1 >= 0.0:
+                f1 = f1 / 10000
+            if f2 >= 0.0:
+                f2 = f2 / 10000
+        elif vcoord == 'DPTH':
+            if f1 >= 0.0:
+                f1 = f1 / 100.0
+            if f2 >= 0.0:
+                f2 = f2 / 100.0
+        elif vcoord == 'POTV':
+            if f1 >= 0.0:
+                f1 = f1 / 1000.0
+            if f2 >= 0.0:
+                f2 = f2 / 1000.0
+
+        req.setLevel1(str(f1))
+        req.setLevel2(str(f2))
+        req.setVcoord(vcoord)
+
+        req.setParm(self.param)
+
+        resp = self.client.sendRequest(req)
+
+        # Get the dimensions of the grid
+        kx = int(self.nx)
+        ky = int(self.ny)
+        kxky = kx * ky
+
+        # Put the data into a NUMPY array
+        grid = numpy.asarray(resp.getFloatData())
+
+        # All grids need to be flipped from a GEMPAK point of view
+        # Reshape the array into 2D
+        grid = numpy.reshape(grid, (ky, kx))
+        # Flip the array in the up-down direction
+        grid = numpy.flipud(grid)
+        # Reshape the array back into 1D
+        grid = numpy.reshape(grid, kxky)
+
+        return [replacemissing(x) for x in grid]
+
+
+def getgriddata(server, table, model, cycle, forecast, level1,
+                level2, vcoord, param, nnx, nny):
+    gir = GridDataRetriever(server, table, model, cycle, forecast,
+                            level1, level2, vcoord, param, nnx, nny)
+    return gir.getData()
+
+
+def getheader(server, table, model, cycle, forecast, level1,
+              level2, vcoord, param, nnx, nny):
+    idata = []
+    idata.append(0)
+    idata.append(0)
+    return idata
+
+
+def replacemissing(x):
+    if x == -999999.0:
+        return -9999.0
+    return x
+
+
+# This is the standard boilerplate that runs this script as a main
+if __name__ == '__main__':
+    # Run Test
+    srv = 'edex-cloud.unidata.ucar.edu'
+    tbl = 'grid'
+    mdl = 'GFS20'
+    cyc = '131227/0000'
+    fcs = '43200'
+    lv1 = '500'
+    lv2 = '-1'
+    vcd = 'PRES'
+    prm = 'HGHT'
+    nx = '720'
+    ny = '361'
+
+    print(getheader(srv, tbl, mdl, cyc, fcs, lv1, lv2, vcd, prm, nx, ny))
+    print(getgriddata(srv, tbl, mdl, cyc, fcs, lv1, lv2, vcd, prm, nx, ny))

awips/gempak/GridInfoRetriever.py

@@ -0,0 +1,137 @@
+import os
+from datetime import datetime
+from operator import itemgetter
+from awips import ThriftClient
+from dynamicserialize.dstypes.gov.noaa.nws.ncep.common.dataplugin.gempak.request import GetGridInfoRequest
+
+
+class GridInfoRetriever:
+
+    def __init__(self, server, pluginName, modelId, cycle=None, forecast=None):
+        self.pluginName = pluginName
+        self.modelId = modelId
+        self.cycle = cycle
+        self.forecast = forecast
+        self.host = os.getenv("DEFAULT_HOST", server)
+        self.port = os.getenv("DEFAULT_PORT", "9581")
+        self.client = ThriftClient.ThriftClient(self.host, self.port)
+
+    def getInfo(self):
+        """ Sends ThriftClient request and writes out received files."""
+        req = GetGridInfoRequest()
+        req.setPluginName(self.pluginName)
+        req.setModelId(self.modelId)
+
+        req.setReftime(self.cycle)
+        if len(self.cycle) > 2:
+            dt = datetime.strptime(self.cycle, '%y%m%d/%H%M')
+            ct = datetime.strftime(dt, '%Y-%m-%d %H:%M:%S')
+            req.setReftime(ct)
+
+        req.setFcstsec(self.forecast)
+
+        resp = self.client.sendRequest(req)
+
+        sortresp = sorted(sorted(resp, key=itemgetter("reftime"), reverse=True), key=itemgetter("fcstsec"))
+
+        grids = []
+
+        count = 0
+        for record in sortresp:
+            s = '{:<12}'.format(record['param'])
+
+            if sys.byteorder == 'little':
+                parm1 = (ord(s[3]) << 24) + (ord(s[2]) << 16) + (ord(s[1]) << 8) + ord(s[0])
+                parm2 = (ord(s[7]) << 24) + (ord(s[6]) << 16) + (ord(s[5]) << 8) + ord(s[4])
+                parm3 = (ord(s[11]) << 24) + (ord(s[10]) << 16) + (ord(s[9]) << 8) + ord(s[8])
+            else:
+                parm1 = (ord(s[0]) << 24) + (ord(s[1]) << 16) + (ord(s[2]) << 8) + ord(s[3])
+                parm2 = (ord(s[4]) << 24) + (ord(s[5]) << 16) + (ord(s[6]) << 8) + ord(s[7])
+                parm3 = (ord(s[8]) << 24) + (ord(s[9]) << 16) + (ord(s[10]) << 8) + ord(s[11])
+
+            dt = datetime.strptime(record['reftime'], '%Y-%m-%d %H:%M:%S.%f')
+            dattim = dt.month * 100000000 + dt.day * 1000000 + (dt.year%100) * 10000 + dt.hour * 100 + dt.minute
+            fcsth = (int(record['fcstsec']) / 60) / 60
+            fcstm = (int(record['fcstsec']) / 60) % 60
+            fcst = 100000 + fcsth * 100 + fcstm
+
+            lv1 = float(record['level1'])
+            if lv1 == -999999.0:
+                lv1 = -1.0
+            lv2 = float(record['level2'])
+            if lv2 == -999999.0:
+                lv2 = -1.0
+
+            vcd = record['vcoord']
+            if vcd == 'NONE':
+                ivcd = 0
+            elif vcd == 'PRES':
+                ivcd = 1
+            elif vcd == 'THTA':
+                ivcd = 2
+            elif vcd == 'HGHT':
+                ivcd = 3
+            elif vcd == 'SGMA':
+                ivcd = 4
+                if lv1 >= 0.0:
+                    lv1 = lv1 * 10000.0
+                if lv2 >= 0.0:
+                    lv2 = lv2 * 10000.0
+            elif vcd == 'DPTH':
+                ivcd = 5
+                if lv1 >= 0.0:
+                    lv1 = lv1 * 100.0
+                if lv2 >= 0.0:
+                    lv2 = lv2 * 100.0
+            elif vcd == 'HYBL':
+                ivcd = 6
+            else:
+                v = '{:<4}'.format(vcd)
+                if sys.byteorder == 'little':
+                    ivcd = (ord(v[3]) << 24) + (ord(v[2]) << 16) + (ord(v[1]) << 8) + ord(v[0])
+                else:
+                    ivcd = (ord(v[0]) << 24) + (ord(v[1]) << 16) + (ord(v[2]) << 8) + ord(v[3])
+                if vcd == 'POTV':
+                    if lv1 >= 0.0:
+                        lv1 = lv1 * 1000.0
+                    if lv2 >= 0.0:
+                        lv2 = lv2 * 1000.0
+            grids.append(9999)
+            grids.append(dattim)
+            grids.append(fcst)
+            grids.append(0)
+            grids.append(0)
+            grids.append(int(lv1))
+            grids.append(int(lv2))
+            grids.append(ivcd)
+            grids.append(parm1)
+            grids.append(parm2)
+            grids.append(parm3)
+            count += 1
+            if count > 29998:
+                break
+
+        return grids
+             
+
+def getinfo(server, table, model, cycle, forecast):
+    gir = GridInfoRetriever(server, table, model, cycle, forecast)
+    return gir.getInfo()
+
+
+def getrow(server, table, model, cycle, forecast):
+    idata = []
+    idata.append(9999)
+    idata.append(1)
+    return idata
+
+
+# This is the standard boilerplate that runs this script as a main
+if __name__ == '__main__':
+    import sys 
+    # Run Test
+    srv = 'edex-cloud.unidata.ucar.edu'
+    tbl = 'grid'
+    mdl = 'NAM40'
+    print(getrow(srv, tbl, mdl))
+    print(getinfo(srv, tbl, mdl))

awips/gempak/GridNavRetriever.py

@@ -0,0 +1,294 @@
+import os
+import math
+from awips import ThriftClient
+from dynamicserialize.dstypes.gov.noaa.nws.ncep.common.dataplugin.gempak.request import GetGridNavRequest
+from ctypes import *
+
+EARTH_RADIUS = 6371200.0
+DEG_TO_RAD = math.pi / 180.0
+RAD_TO_DEG = 180.0 / math.pi
+TWOPI = math.pi * 2.0
+HALFPI = math.pi / 2.0
+PI4TH = math.pi / 4.0
+PI3RD = math.pi / 3.0
+
+
+def createPolar(nsflag, clon, lat1, lon1, dx, dy, unit, nx, ny):
+    clonr = clon * DEG_TO_RAD
+    latr = lat1 * DEG_TO_RAD
+    lonr = lon1 * DEG_TO_RAD
+    if nsflag == 'N':
+        x1 = EARTH_RADIUS * math.tan(PI4TH - latr/2.0) * math.sin(lonr-clonr)
+        y1 = -1 * EARTH_RADIUS * math.tan(PI4TH - latr/2.0) * math.cos(lonr-clonr)
+    else:
+        x1 = EARTH_RADIUS * math.tan(PI4TH + latr/2.0) * math.sin(lonr-clonr)
+        y1 = EARTH_RADIUS * math.tan(PI4TH + latr/2.0) * math.cos(lonr-clonr)
+
+    if unit == 'm':
+        tdx = dx / (1 + math.sin(PI3RD))
+        tdy = dy / (1 + math.sin(PI3RD))
+    else:
+        tdx = (dx*1000.0) / (1 + math.sin(PI3RD))
+        tdy = (dy*1000.0) / (1 + math.sin(PI3RD))
+
+    x2 = x1 + tdx * (nx-1)
+    y2 = y1 + tdy * (ny-1)
+    xll = min(x1, x2)
+    yll = min(y1, y2)
+    xur = max(x1, x2)
+    yur = max(y1, y2)
+
+    if nsflag == 'N':
+        latll = (HALFPI - 2*math.atan2(math.hypot(xll, yll), EARTH_RADIUS)) * RAD_TO_DEG
+        rtemp = clonr + math.atan2(xll, -yll)
+    else:
+        latll = -1 * (HALFPI - 2*math.atan2(math.hypot(xll, yll), EARTH_RADIUS)) * RAD_TO_DEG
+        rtemp = clonr + math.atan2(xll, yll)
+
+    if rtemp > math.pi:
+        lonll = (rtemp-TWOPI) * RAD_TO_DEG
+    elif rtemp < -math.pi:
+        lonll = (rtemp+TWOPI) * RAD_TO_DEG
+    else:
+        lonll = rtemp * RAD_TO_DEG
+
+    if nsflag == 'N':
+        latur = (HALFPI - 2*math.atan2(math.hypot(xur, yur), EARTH_RADIUS)) * RAD_TO_DEG
+        rtemp = clonr + math.atan2(xur, -yur)
+    else:
+        latur = -1 * (HALFPI - 2*math.atan2(math.hypot(xur, yur), EARTH_RADIUS)) * RAD_TO_DEG
+        rtemp = clonr + math.atan2(xur, yur)
+
+    if rtemp > math.pi:
+        lonur = (rtemp-TWOPI) * RAD_TO_DEG
+    elif rtemp < -math.pi:
+        lonur = (rtemp+TWOPI) * RAD_TO_DEG
+    else:
+        lonur = rtemp * RAD_TO_DEG
+
+    return [latll, lonll, latur, lonur]
+
+
+def createConic(nsflag, clon, lat1, lon1, dx, dy, unit, nx, ny, ang1, ang3):
+    clonr = clon * DEG_TO_RAD
+    latr = lat1 * DEG_TO_RAD
+    lonr = lon1 * DEG_TO_RAD
+
+    angle1 = HALFPI - (math.fabs(ang1) * DEG_TO_RAD)
+    angle2 = HALFPI - (math.fabs(ang3) * DEG_TO_RAD)
+
+    if ang1 == ang3:
+        cc = math.cos(angle1)
+    else:
+        cc = (math.log(math.sin(angle2)) - math.log(math.sin(angle1))) \
+             / (math.log(math.tan(angle2/2.0)) - math.log(math.tan(angle1/2.0)))
+
+    er = EARTH_RADIUS / cc
+
+    if nsflag == 'N':
+        x1 = er * math.pow(math.tan((HALFPI-latr)/2.0), cc) * math.sin(cc*(lonr-clonr))
+        y1 = -1.0 * er * math.pow(math.tan((HALFPI-latr)/2.0), cc) * math.cos(cc*(lonr-clonr))
+    else:
+        x1 = er * math.pow(math.tan((HALFPI+latr)/2.0), cc) * math.sin(cc*(lonr-clonr))
+        y1 = er * math.pow(math.tan((HALFPI+latr)/2.0), cc) * math.cos(cc*(lonr-clonr))
+
+    alpha = math.pow(math.tan(angle1/2.0), cc) / math.sin(angle1)
+
+    if unit == 'm':
+        x2 = x1 + (nx-1) * alpha * dx
+        y2 = y1 + (ny-1) * alpha * dy
+    else:
+        x2 = x1 + (nx-1) * alpha * (dx*1000.0)
+        y2 = y1 + (ny-1) * alpha * (dy*1000.0)
+
+    xll = min(x1, x2)
+    yll = min(y1, y2)
+    xur = max(x1, x2)
+    yur = max(y1, y2)
+
+    if nsflag == 'N':
+        latll = (HALFPI - 2.0 * math.atan(math.pow(math.hypot(xll, yll)/er, (1/cc)))) * RAD_TO_DEG
+        rtemp = math.atan2(xll, -yll) * (1/cc) + clonr
+    else:
+        latll = (-1.0 * (HALFPI - 2.0 * math.atan(math.pow(math.hypot(xll, yll)/er, (1/cc))))) * RAD_TO_DEG
+        rtemp = math.atan2(xll, yll) * (1/cc) + clonr
+
+    if rtemp > math.pi:
+        lonll = (rtemp-TWOPI) * RAD_TO_DEG
+    elif rtemp < -math.pi:
+        lonll = (rtemp+TWOPI) * RAD_TO_DEG
+    else:
+        lonll = rtemp * RAD_TO_DEG
+
+    if nsflag == 'N':
+        latur = (HALFPI - 2.0 * math.atan(math.pow(math.hypot(xur, yur)/er, (1/cc)))) * RAD_TO_DEG
+        rtemp = math.atan2(xur, -yur) * (1/cc) + clonr
+    else:
+        latur = (-1.0 * (HALFPI - 2.0 * math.atan(math.pow(math.hypot(xur, yur)/er, (1/cc))))) * RAD_TO_DEG
+        rtemp = math.atan2(xur, yur) * (1/cc) + clonr
+
+    if rtemp > math.pi:
+        lonur = (rtemp-TWOPI) * RAD_TO_DEG
+    elif rtemp < -math.pi:
+        lonur = (rtemp+TWOPI) * RAD_TO_DEG
+    else:
+        lonur = rtemp * RAD_TO_DEG
+
+    return [latll, lonll, latur, lonur]
+
+
+class StringConverter(Union):
+    _fields_ = [("char", c_char*4), ("int", c_int), ("float", c_float)]
+
+
+class GridNavRetriever:
+
+    def __init__(self, server, pluginName, modelId, arrayLen):
+        self.pluginName = pluginName
+        self.modelId = modelId
+        self.arrayLen = arrayLen
+        self.host = os.getenv("DEFAULT_HOST", server)
+        self.port = os.getenv("DEFAULT_PORT", "9581")
+        self.client = ThriftClient.ThriftClient(self.host, self.port)
+
+    def getNavBlk(self):
+        """ Sends ThriftClient request and writes out received files."""
+        req = GetGridNavRequest()
+        req.setPluginName(self.pluginName)
+        req.setModelId(self.modelId)
+        resp = self.client.sendRequest(req)
+
+        nav = []
+
+        for record in resp:
+            unit = record['spacingunit']
+            sk = record['spatialkey']
+            skarr = sk.split('/')
+
+            nx = float(skarr[1])
+            ny = float(skarr[2])
+            dx = float(skarr[3])
+            dy = float(skarr[4])
+
+            sc = StringConverter()
+            if record['projtype'] == 'LatLon':
+                sc.char = 'CED '
+                gemproj = 2.0
+                ang1 = 0.0
+                ang2 = 0.0
+                ang3 = 0.0
+
+                lllat = float(record['lowerleftlat'])
+                lllon = float(record['lowerleftlon'])
+                urlat = lllat + (dy * (ny-1))
+                urlon = lllon + (dx * (nx-1))
+                if lllon > 180:
+                    lllon -= 360.0
+                if urlon > 180:
+                    urlon -= 360.0
+
+            if record['projtype'] == 'Polar Stereographic':
+                sc.char = 'STR '
+                gemproj = 2.0
+                if float(record['standard_parallel_1']) < 0.0:
+                    ang1 = -90.0
+                    nsflag = 'S'
+                else:
+                    ang1 = 90.0
+                    nsflag = 'N'
+                ang2 = float(record['central_meridian'])
+                ang3 = 0.0
+
+                lat1 = float(record['lowerleftlat'])
+                lon1 = float(record['lowerleftlon'])
+                coords = createPolar(nsflag, ang2, lat1, lon1, dx, dy, unit, nx, ny)
+                lllat = coords[0]
+                lllon = coords[1]
+                urlat = coords[2]
+                urlon = coords[3]
+
+            if record['projtype'] == 'Lambert Conformal':
+                sc.char = 'LCC '
+                gemproj = 2.0
+
+                ang1 = float(skarr[7])
+                ang2 = float(record['central_meridian'])
+                ang3 = float(skarr[8])
+                if ang1 < 0.0:
+                    nsflag = 'S'
+                else:
+                    nsflag = 'N'
+
+                lat1 = float(record['lowerleftlat'])
+                lon1 = float(record['lowerleftlon'])
+                coords = createConic(nsflag, ang2, lat1, lon1, dx, dy, unit, nx, ny, ang1, ang3)
+                lllat = coords[0]
+                lllon = coords[1]
+                urlat = coords[2]
+                urlon = coords[3]
+
+            # Fill up the output array of floats
+            nav.append(gemproj)
+            nav.append(sc.float)
+            nav.append(1.0)
+            nav.append(1.0)
+            nav.append(nx)
+            nav.append(ny)
+            nav.append(lllat)
+            nav.append(lllon)
+            nav.append(urlat)
+            nav.append(urlon)
+            nav.append(ang1)
+            nav.append(ang2)
+            nav.append(ang3)
+
+        for i in range(13, int(self.arrayLen)):
+            nav.append(0.0)
+        return nav
+
+    def getAnlBlk(self):
+        anl = []
+        # Type
+        anl.append(2.0)
+        # Delta
+        anl.append(1.0)
+        # Extend area
+        anl.append(0.0)
+        anl.append(0.0)
+        anl.append(0.0)
+        anl.append(0.0)
+        # Grid area
+        anl.append(-90.0)
+        anl.append(-180.0)
+        anl.append(90.0)
+        anl.append(180.0)
+        # Data area
+        anl.append(-90.0)
+        anl.append(-180.0)
+        anl.append(90.0)
+        anl.append(180.0)
+        for i in range(18, int(self.arrayLen)):
+            anl.append(0.0)
+        return anl
+
+
+def getnavb(server, table, model, arrlen):
+    gnr = GridNavRetriever(server, table, model, arrlen)
+    return gnr.getNavBlk()
+
+
+def getanlb(server, table, model, arrlen):
+    gnr = GridNavRetriever(server, table, model, arrlen)
+    return gnr.getAnlBlk()
+
+
+# This is the standard boilerplate that runs this script as a main
+if __name__ == '__main__':
+    # Run Test
+    srv = 'edex-cloud.unidata.ucar.edu'
+    tbl = 'grid_info'
+    mdl = 'NAM40'
+    navlen = '256'
+    print(getnavb(srv, tbl, mdl, navlen))
+    anllen = '128'
+    print(getanlb(srv, tbl, mdl, anllen))

awips/gempak/StationDataRetriever.py

@@ -0,0 +1,136 @@
+import os
+from datetime import datetime
+from awips import ThriftClient
+from dynamicserialize.dstypes.com.raytheon.uf.common.time import DataTime
+from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange
+from dynamicserialize.dstypes.gov.noaa.nws.ncep.common.dataplugin.gempak.request import StationDataRequest
+
+
+class StationDataRetriever:
+    """ Retrieves all data for a requested station and time """
+
+    def __init__(self, server, pluginName, stationId, refTime, parmList, partNumber):
+        self.pluginName = pluginName
+        self.stationId = stationId
+        self.refTime = refTime
+        self.parmList = parmList
+        self.partNumber = partNumber
+        self.host = os.getenv("DEFAULT_HOST", server)
+        self.port = os.getenv("DEFAULT_PORT", "9581")
+        self.client = ThriftClient.ThriftClient(self.host, self.port)
+
+    def getStationData(self):
+        """ Sends ThriftClient request and writes out received files."""
+        dtime = datetime.strptime(self.refTime, "%y%m%d/%H%M")
+        trange = TimeRange()
+        trange.setStart(dtime)
+        trange.setEnd(dtime)
+        dataTime = DataTime(refTime=dtime, validPeriod=trange)
+        req = StationDataRequest()
+        req.setPluginName(self.pluginName)
+        req.setStationId(self.stationId)
+        req.setRefTime(dataTime)
+        req.setParmList(self.parmList)
+        req.setPartNumber(self.partNumber)
+        resp = self.client.sendRequest(req)
+        return resp
+
+
+def getstationdata(server, table, stationId, refTime, parmList, partNumber):
+    sr = StationDataRetriever(server, table, stationId, refTime, parmList, partNumber)
+    lcldict = sr.getStationData()
+
+    rdata = []
+    
+    for substr in parmList.split(','):
+        if substr in lcldict:
+            rdata.append(lcldict[substr])
+        else:
+            rdata.append(-9999.00)
+
+    return rdata
+
+
+def getleveldata(server, table, stationId, refTime, parmList, partNumber):
+    sr = StationDataRetriever(server, table, stationId, refTime, parmList, partNumber)
+    lcldict = sr.getStationData()
+
+    numset = [1]
+    for substr in parmList.split(','):
+        if substr in lcldict:
+            pnum = len(lcldict[substr]) - 1
+            while pnum >= 0:
+                if lcldict[substr][pnum] != -9999.00:
+                    break
+                pnum = pnum - 1
+            numset.append(pnum)
+
+    rdata = []
+
+    for jj in range(max(numset)):
+        for substr in parmList.split(','):
+            if substr in lcldict:
+                if lcldict[substr][jj] == -9999998.0:
+                    rdata.append(-9999.0)
+                else:
+                    rdata.append(lcldict[substr][jj])
+            else:
+                rdata.append(-9999.0)
+
+    return rdata
+
+
+def getstationtext(server, table, stationId, refTime, parmList, partNumber):
+    sr = StationDataRetriever(server, table, stationId, refTime, parmList, partNumber)
+    lcldict = sr.getStationData()
+
+    if parmList in lcldict:
+        return lcldict[parmList]
+    else:
+        return ' '
+
+
+def getheader(server, table, stationId, refTime, parmList, partNumber):
+    idata = []
+    idata.append(0)
+    return idata
+
+
+# This is the standard boilerplate that runs this script as a main
+if __name__ == '__main__':
+    # Run Test
+    srv = 'edex-cloud.unidata.ucar.edu'
+    key = '-'
+    print('OBS - METAR')
+    tbl = 'obs'
+    stn = 'KLGA'
+    time = '130823/1700'
+    parm = 'seaLevelPress,temperature,dewpoint,windSpeed,windDir'
+    part = '0'
+    print(getheader(srv, tbl, stn, time, parm, part))
+    print(getstationdata(srv, tbl, stn, time, parm, part))
+    parm = 'rawMETAR'
+    print(getstationtext(srv, tbl, stn, time, parm, part))
+    print('SFCOBS - SYNOP')
+    tbl = 'sfcobs'
+    stn = '72403'
+    time = '130823/1800'
+    parm = 'seaLevelPress,temperature,dewpoint,windSpeed,windDir'
+    part = '0'
+    print(getheader(srv, tbl, stn, time, parm, part))
+    print(getstationdata(srv, tbl, stn, time, parm, part))
+    parm = 'rawReport'
+    print(getstationtext(srv, tbl, stn, time, parm, part))
+    print('UAIR')
+    tbl = 'bufrua'
+    stn = '72469'
+    time = '130823/1200'
+    parm = 'prMan,htMan,tpMan,tdMan,wdMan,wsMan'
+    part = '2020'
+    print(getleveldata(srv, tbl, stn, time, parm, part))
+    parm = 'prSigT,tpSigT,tdSigT'
+    part = '2022'
+    print(getleveldata(srv, tbl, stn, time, parm, part))
+    parm = 'htSigW,wsSigW,wdSigW'
+    part = '2021'
+    print(getleveldata(srv, tbl, stn, time, parm, part))

awips/gempak/StationRetriever.py

@@ -0,0 +1,86 @@
+import os
+import sys
+from awips import ThriftClient
+from dynamicserialize.dstypes.gov.noaa.nws.ncep.common.dataplugin.gempak.request import GetStationsRequest
+
+
+class StationRetriever:
+    """ Retrieves all requested stations """
+
+    def __init__(self, server, pluginName):
+        self.pluginName = pluginName
+        self.outdir = os.getcwd()
+        self.host = os.getenv("DEFAULT_HOST", server)
+        self.port = os.getenv("DEFAULT_PORT", "9581")
+        self.client = ThriftClient.ThriftClient(self.host, self.port)
+
+    def getStations(self):
+        """ Sends ThriftClient request and writes out received files."""
+        req = GetStationsRequest()
+        req.setPluginName(self.pluginName)
+        resp = self.client.sendRequest(req)
+
+        stns = []
+        for item in resp:
+            stationstr = '{:<8}'.format(item.getStationId())
+
+            if sys.byteorder == 'little':
+                stnid = (ord(stationstr[3]) << 24) + (ord(stationstr[2]) << 16) + \
+                        (ord(stationstr[1]) << 8) + ord(stationstr[0])
+                stnid2 = (ord(stationstr[7]) << 24) + (ord(stationstr[6]) << 16) + \
+                         (ord(stationstr[5]) << 8) + ord(stationstr[4])
+            else:
+                stnid = (ord(stationstr[0]) << 24) + (ord(stationstr[1]) << 16) + \
+                        (ord(stationstr[2]) << 8) + ord(stationstr[3])
+                stnid2 = (ord(stationstr[4]) << 24) + (ord(stationstr[5]) << 16) + \
+                         (ord(stationstr[6]) << 8) + ord(stationstr[7])
+
+            if item.getState() is None:
+                stationstr = '    '
+            else:
+                stationstr = '{:<4}'.format(item.getState())
+
+            if sys.byteorder == 'little':
+                state = (ord(stationstr[3]) << 24) + (ord(stationstr[2]) << 16) \
+                        + (ord(stationstr[1]) << 8) + ord(stationstr[0])
+            else:
+                state = (ord(stationstr[0]) << 24) + (ord(stationstr[1]) << 16) \
+                        + (ord(stationstr[2]) << 8) + ord(stationstr[3])
+
+            stationstr = '{:<4}'.format(item.getCountry())
+            if sys.byteorder == 'little':
+                cntry = (ord(stationstr[3]) << 24) + (ord(stationstr[2]) << 16) \
+                        + (ord(stationstr[1]) << 8) + ord(stationstr[0])
+            else:
+                cntry = (ord(stationstr[0]) << 24) + (ord(stationstr[1]) << 16) \
+                        + (ord(stationstr[2]) << 8) + ord(stationstr[3])
+
+            stns.append(9999)
+            stns.append(stnid)
+            stns.append(item.getWmoIndex())
+            stns.append(int(item.getLatitude()*100))
+            stns.append(int(item.getLongitude()*100))
+            stns.append(int(item.getElevation()))
+            stns.append(state)
+            stns.append(cntry)
+            stns.append(stnid2)
+            stns.append(0)
+        return stns
+
+
+def getstations(server, table, key, dummy, dummy2):
+    sr = StationRetriever(server, table)
+    return sr.getStations()
+
+
+# This is the standard boilerplate that runs this script as a main
+if __name__ == '__main__':
+    # Run Test
+    srv = 'edex-cloud.unidata.ucar.edu'
+    key = '-'
+    print('OBS - METAR')
+    tbl = 'obs'
+    print(getstations(srv, tbl, key))
+    print('SFCOBS - SYNOP')
+    tbl = 'sfcobs'
+    print(getstations(srv, tbl, key))

awips/gempak/TimeRetriever.py

@@ -0,0 +1,68 @@
+import os
+from datetime import datetime
+from awips import ThriftClient
+from dynamicserialize.dstypes.java.util import GregorianCalendar
+from dynamicserialize.dstypes.gov.noaa.nws.ncep.common.dataplugin.gempak.request import GetTimesRequest
+
+
+class TimeRetriever:
+    """ Retrieves all requested times"""
+
+    def __init__(self, server, pluginName, timeField):
+        self.pluginName = pluginName
+        self.timeField = timeField
+        self.outdir = os.getcwd()
+        self.host = os.getenv("DEFAULT_HOST", server)
+        self.port = os.getenv("DEFAULT_PORT", "9581")
+        self.client = ThriftClient.ThriftClient(self.host, self.port)
+
+    def getTimes(self):
+        """ Sends ThriftClient request and writes out received files."""
+        req = GetTimesRequest()
+        req.setPluginName(self.pluginName)
+        req.setTimeField(self.timeField)
+        resp = self.client.sendRequest(req)
+        timelist = []
+        for item in resp.getTimes():
+            if isinstance(item, GregorianCalendar):
+                tstamp = item.getTimeInMillis()
+            else:
+                tstamp = item.getTime()
+            time = datetime.utcfromtimestamp(tstamp/1000)
+            timelist.append(time)
+
+        timelist.sort(reverse=True)
+
+        times = []
+        for time in timelist:
+            times.append(9999)
+            times.append((time.year % 100) * 10000 + (time.month * 100) + time.day)
+            times.append((time.hour * 100) + time.minute)
+
+        # GEMPAK can only handle up to 200 times, which is 600 elements
+        # in this array -- [9999, DATE, TIME] -- repeated
+        return times[0:600]
+
+
+def gettimes(server, table, key, dummy, dummy2):
+    tr = TimeRetriever(server, table, key)
+    return tr.getTimes()
+
+
+# This is the standard boilerplate that runs this script as a main
+if __name__ == '__main__':
+    srv = 'edex-cloud.unidata.ucar.edu'
+    print('OBS - METAR')
+    tbl = 'obs'
+    key = 'refHour'
+    print(gettimes(srv, tbl, key))
+
+    print('SFCOBS - SYNOP')
+    tbl = 'sfcobs'
+    key = 'refHour'
+    print(gettimes(srv, tbl, key))
+
+    print('BUFRUA')
+    tbl = 'bufrua'
+    key = 'dataTime.refTime'
+    print(gettimes(srv, tbl, key))

awips/gempak/ncepGribTables.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+# Parse html tables from a given URL and output CSV.
+# Note: To install a missing python module foo do "easy_install foo"
+#   (or the new way is "pip install foo" but you might have to do 
+#    "easy_install pip" first)
+
+from BeautifulSoup import BeautifulSoup
+import scrape
+import urllib.request, urllib.error, urllib.parse
+import html.entities
+import re
+import sys
+import unicodedata
+
+
+# from http://stackoverflow.com/questions/1197981/convert-html-entities
+def asciify2(s):
+    matches = re.findall("&#\d+;", s)
+    if len(matches) > 0:
+        hits = set(matches)
+        for hit in hits:
+            name = hit[2:-1]
+            try:
+                entnum = int(name)
+                s = s.replace(hit, chr(entnum))
+            except ValueError:
+                pass
+
+    matches = re.findall("&\w+;", s)
+    hits = set(matches)
+    amp = "&"
+    if amp in hits:
+        hits.remove(amp)
+    for hit in hits:
+        name = hit[1:-1]
+        if name in html.entities.name2codepoint:
+            s = s.replace(hit, "")
+    s = s.replace(amp, "&")
+    return s
+
+
+def opensoup(url):
+    request = urllib.request.Request(url)
+    request.add_header("User-Agent", "Mozilla/5.0")
+    # To mimic a real browser's user-agent string more exactly, if necessary:
+    #   Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.14)
+    #   Gecko/20080418 Ubuntu/7.10 (gutsy) Firefox/2.0.0.14
+    pagefile = urllib.request.urlopen(request)
+    soup = BeautifulSoup(pagefile)
+    pagefile.close()
+    return soup
+
+
+def asciify(s):
+    return unicodedata.normalize('NFKD', s).encode('ascii', 'ignore')
+
+
+# remove extra whitespace, including stripping leading and trailing whitespace.
+def condense(s):
+    s = re.sub(r"\s+", " ", s, re.DOTALL)
+    return s.strip()
+
+
+def stripurl(s):
+    s = re.sub(r"\<span\s+style\s*\=\s*\"display\:none[^\"]*\"[^\>]*\>[^\<]*\<\/span\>", "", s)
+    s = re.sub(r"\&\#160\;", " ", s)
+    return condense(re.sub(r"\<[^\>]*\>", " ", s))
+
+
+# this gets rid of tags and condenses whitespace
+def striptags(s):
+    s = re.sub(r"\<span\s+style\s*\=\s*\"display\:none[^\"]*\"[^\>]*\>[^\<]*\<\/span\>", "", s)
+    s = re.sub(r"\&\#160\;", " ", s)
+    return condense(s)
+
+
+def getUrlArgs(parseUrl):
+    return re.search('grib2_table4-2-(\d+)-(\d+).shtml', parseUrl).groups()
+
+
+if len(sys.argv) == 1:
+    print("Usage: ", sys.argv[0], " url [n]")
+    print("  (where n indicates which html table to parse)")
+    exit(1)
+
+url = sys.argv[1]
+soup = opensoup(url)
+tables = soup.findAll("table")
+
+for table in tables:
+    for r in table.findAll('tr'):
+        rl = []
+        for c in r.findAll(re.compile('td|th')):
+            rl.append(striptags(c.renderContents()))
+        if len(rl) > 1 and "href" in rl[1]:
+            print('! ' + stripurl(rl[1]))
+            scrapeUrl = 'http://www.nco.ncep.noaa.gov/pmb/docs/grib2/grib2_table4-2-' + \
+                        getUrlArgs(rl[1])[0] + "-" + getUrlArgs(rl[1])[1] + '.shtml'
+            scrape.run(scrapeUrl)

awips/gempak/scrape.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+# Parse html tables from a given URL and output CSV.
+# Note: To install a missing python module foo do "easy_install foo"
+#   (or the new way is "pip install foo" but you might have to do 
+#    "easy_install pip" first)
+
+from BeautifulSoup import BeautifulSoup
+import urllib.request, urllib.error, urllib.parse
+import html.entities
+import re
+import sys
+import unicodedata
+
+
+# from http://stackoverflow.com/questions/1197981/convert-html-entities
+def asciify2(s):
+    matches = re.findall("&#\d+;", s)
+    if len(matches) > 0:
+        hits = set(matches)
+        for hit in hits:
+            name = hit[2:-1]
+            try:
+                entnum = int(name)
+                s = s.replace(hit, chr(entnum))
+            except ValueError:
+                pass
+
+    matches = re.findall("&\w+;", s)
+    hits = set(matches)
+    amp = "&"
+    if amp in hits:
+        hits.remove(amp)
+    for hit in hits:
+        name = hit[1:-1]
+        if name in html.entities.name2codepoint:
+            s = s.replace(hit, "")
+    s = s.replace(amp, "&")
+    return s
+
+
+def opensoup(url):
+    request = urllib.request.Request(url)
+    request.add_header("User-Agent", "Mozilla/5.0")
+    # To mimic a real browser's user-agent string more exactly, if necessary:
+    #   Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.14)
+    #   Gecko/20080418 Ubuntu/7.10 (gutsy) Firefox/2.0.0.14
+    pagefile = urllib.request.urlopen(request)
+    soup = BeautifulSoup(pagefile)
+    pagefile.close()
+    return soup
+
+
+def asciify(s):
+    return unicodedata.normalize('NFKD', s).encode('ascii', 'ignore')
+
+
+# remove extra whitespace, including stripping leading and trailing whitespace.
+def condense(s):
+    s = re.sub(r"\s+", " ", s, re.DOTALL)
+    return s.strip()
+
+
+# this gets rid of tags and condenses whitespace
+def striptags(s):
+    s = re.sub(r"\<span\s+style\s*\=\s*\"display\:none[^\"]*\"[^\>]*\>[^\<]*\<\/span\>", "", s)
+    s = re.sub(r"\&\#160\;", " ", s)
+    return condense(re.sub(r"\<[^\>]*\>", " ", s))
+
+
+if len(sys.argv) == 1:  # called with no arguments
+    print("Usage: ", sys.argv[0], " url [n]")
+    print("  (where n indicates which html table to parse)")
+    exit(1)
+
+
+def getUrlArgs(parseUrl):
+    return re.search('grib2_table4-2-(\d+)-(\d+).shtml', parseUrl).groups()
+
+
+def run(url):
+    soup = opensoup(url)
+    tables = soup.findAll("table")
+    for table in tables:
+        ct = 0
+        for r in table.findAll('tr'):
+            rl = []
+            for c in r.findAll(re.compile('td|th')):
+                rl.append(striptags(c.renderContents()))
+        if ct > 0:
+            rl[0] = getUrlArgs(url)[0].zfill(3) + " " + \
+                    getUrlArgs(url)[1].zfill(3) + " " + rl[0].zfill(3) + " 000"
+        if len(rl) > 1:
+            if "Reserved" in rl[1]:
+                rl[0] = '!' + rl[0]
+            if "See Table" in rl[2] or "Code table" in rl[2]:
+                rl[2] = "cat"
+            rl[1] = rl[1][:32].ljust(32)
+            rl[2] = rl[2].ljust(20)
+            rl[3] = rl[3].ljust(12) + "     0  -9999.00"
+            if ct:
+                print(" ".join(rl))
+        ct += 1
+
+
+if __name__ == '__main__':
+    run(sys.argv[1])