From a2a6f0dde11ca991ee86b5a30bd170d79f80a60f Mon Sep 17 00:00:00 2001 From: XANTRONIX Industrial Date: Wed, 19 Feb 2025 20:45:42 -0500 Subject: [PATCH] Split VTEC string parsing from AFOS message parsing --- db/nexrad.sql | 10 +- lib/nexrad/afos.py | 183 +++++++++++++++++++++++++++++ lib/nexrad/vtec.py | 284 +++++++++++---------------------------------- 3 files changed, 253 insertions(+), 224 deletions(-) create mode 100644 lib/nexrad/afos.py diff --git a/db/nexrad.sql b/db/nexrad.sql index 2f4616a..c05e3b7 100644 --- a/db/nexrad.sql +++ b/db/nexrad.sql @@ -43,7 +43,7 @@ select AddGeometryColumn('nexrad_storm_event', 'coord_end', 4326, 'POINT', 'XY', 0), CreateSpatialIndex('nexrad_storm_event', 'coord_end'); -create table nexrad_vtec_event ( +create table nexrad_afos_message ( id INTEGER PRIMARY KEY NOT NULL, timestamp_issued TIMESTAMP NOT NULL, timestamp_start TIMESTAMP NOT NULL, @@ -61,11 +61,11 @@ create table nexrad_vtec_event ( ); select - AddGeometryColumn('nexrad_vtec_event', 'location', 4326, 'POINT'), - CreateSpatialIndex('nexrad_vtec_event', 'location'); + AddGeometryColumn('nexrad_afos_message', 'location', 4326, 'POINT'), + CreateSpatialIndex('nexrad_afos_message', 'location'); select - AddGeometryColumn('nexrad_vtec_event', 'poly', 4326, 'POLYGON'), - CreateSpatialIndex('nexrad_vtec_event', 'poly'); + AddGeometryColumn('nexrad_afos_message', 'poly', 4326, 'POLYGON'), + CreateSpatialIndex('nexrad_afos_message', 'poly'); commit; diff --git a/lib/nexrad/afos.py b/lib/nexrad/afos.py new file mode 100644 index 0000000..03fb515 --- /dev/null +++ b/lib/nexrad/afos.py @@ -0,0 +1,183 @@ +import re +import enum +import datetime +import shapely + +from nexrad.db import DatabaseTable +from nexrad.coord import COORD_SYSTEM +from nexrad.vtec import VTECEvent + +RE_ID = re.compile(r'^(\d+)$') + +RE_ISSUANCE = re.compile(r''' + ^ (WF[A-Z]{2}\d{2}) + [ ]{1} (?P[A-Z]{4}) + [ ]{1} (?P\d{2}) + (?P\d{2}) (?P\d{2}) + $ +''', re.X) + +RE_POLY = re.compile(r'^LAT\.\.\.LON (?P\d+(?: \d+)+)') + +RE_MOTION = re.compile(r''' + ^ TIME + \.\.\. MOT + \.\.\. LOC + [ ]{1} (?P\d{2})(?P\d{2})Z + [ ]{1} (?P\d+)DEG + [ ]{1} (?P\d+)KT + [ ]{1} (?P\d+) + [ ]{1} (?P\d+) + $ +''', re.X) + +def parse_lon(text: str): + size = len(text) + return 0 - float(text[0:size-2]) + (float(text[size-2:size]) / 100) + +def parse_lat(text: str): + size = len(text) + return float(text[0:size-2]) + (float(text[size-2:size]) / 100) + +def parse_location(lon: str, lat: str): + return shapely.Point(parse_lon(lon), parse_lat(lat)) + +def parse_shape(text: str): + points = list() + coords = text.split(' ') + + for i in range(0, len(coords), 2): + lat = coords[i] + lon = coords[i+1] + + points.append([parse_lon(lon), parse_lat(lat)]) + + points.append([parse_lon(coords[0]), parse_lat(coords[1])]) + + return shapely.Polygon(points) + +class AFOSMessageParserState(enum.Enum): + NONE = 1 + HEADER = enum.auto() + ISSUANCE = enum.auto() + META = enum.auto() + TYPEOFFICE = enum.auto() + VTEC = enum.auto() + BODY_SEP = enum.auto() + BODY = enum.auto() + TAGS = enum.auto() + FOOTER = enum.auto() + +class AFOSMessage(DatabaseTable): + __table__ = 'nexrad_afos_messsage' + __key__ = 'id' + + __columns__ = ( + 'id', 'timestamp_issued', 'timestamp_start', 'timestamp_end', + 'typeof', 'etn', 'actions', 'wfo', 'phenom', 'sig', 'body', + 'azimuth', 'speed', 'location', 'forecaster', 'poly', + ) + + __columns_read__ = { + 'poly': 'ST_AsText(poly) as poly' + } + + __values_write__ = { + 'poly': shapely.from_wkt + } + + __columns_write__ = { + 'poly': 'ST_GeomFromText(:poly, {crs})'.format(crs=COORD_SYSTEM) + } + + __values_write__ = { + 'poly': lambda v: {'poly': shapely.to_wkt(v)} + } + + id: int + timestamp_issued: datetime.datetime + timestamp_start: datetime.datetime + timestamp_end: datetime.datetime + typeof: str + actions: str + wfo: str + phenom: str + sig: str + etn: int + body: str + azimuth: int + speed: int + location: shapely.Point + forecaster: str + poly: shapely.Geometry + + @staticmethod + def parse(text: str): + event = AFOSMessage() + state = AFOSMessageParserState.NONE + + issuance = None + + for line in text.split('\n'): + line = line.rstrip() + + if state == AFOSMessageParserState.NONE: + match = RE_ID.match(line) + + if match is not None: + event.id = int(match[1]) + state = AFOSMessageParserState.HEADER + elif state == AFOSMessageParserState.HEADER: + match = RE_ISSUANCE.match(line) + + if match is not None: + issuance = match + state = AFOSMessageParserState.ISSUANCE + elif state == AFOSMessageParserState.ISSUANCE: + state = AFOSMessageParserState.META + elif state == AFOSMessageParserState.META: + vtec = VTECEvent.parse(line) + + if vtec is not None: + event.timestamp_start = vtec.timestamp_start + event.timestamp_end = vtec.timestamp_end + + event.typeof = vtec.typeof + event.actions = vtec.actions + event.wfo = vtec.wfo + event.phenom = vtec.phenom + event.sig = vtec.sig + event.etn = vtec.etn + + state = AFOSMessageParserState.VTEC + elif state == AFOSMessageParserState.VTEC: + if line == '': + state = AFOSMessageParserState.BODY_SEP + elif state == AFOSMessageParserState.BODY_SEP: + event.body = line + state = AFOSMessageParserState.BODY + elif state == AFOSMessageParserState.BODY: + if line == '&&': + state = AFOSMessageParserState.TAGS + else: + event.body += '\n' + line + elif state == AFOSMessageParserState.TAGS: + if line == '$$': + state = AFOSMessageParserState.FOOTER + else: + match = RE_POLY.match(line) + + if match is not None: + event.poly = parse_shape(match['coords']) + + match = RE_MOTION.match(line) + + if match is not None: + event.azimuth = int(match['azimuth']) + event.speed = int(match['speed']) + event.location = parse_location(match['lon'], match['lat']) + elif state == AFOSMessageParserState.FOOTER: + if line != '': + event.forecaster = line + + return event diff --git a/lib/nexrad/vtec.py b/lib/nexrad/vtec.py index 6863d27..8a23230 100644 --- a/lib/nexrad/vtec.py +++ b/lib/nexrad/vtec.py @@ -1,247 +1,93 @@ import re import enum import datetime -import shapely - -from nexrad.db import DatabaseTable -from nexrad.coord import COORD_SYSTEM - -RE_ID = re.compile(r'^(\d+)$') - -RE_ISSUANCE = re.compile(r''' - ^ - (WF[A-Z]{2}\d{2}) - [ ]{1} - (?P[A-Z]{4}) - [ ]{1} - (?P\d{2}) - (?P\d{2}) - (?P\d{2}) - $ -''', re.X) RE_PHENOM = re.compile(r''' - ^/ - (?P[OTEX]) - \. - (?P[A-Z]{3}) - \. - (?P[A-Z]{4}) - \. - (?P[A-Z]{2}) - \. - (?P[A-Z]) - \. - (?P\d{4}) - \. - (?P\d{6}T\d{4}Z) - - - (?P\d{6}T\d{4}Z) + ^/ (?P[OTEX]) + \. (?P[A-Z]{3}) + \. (?P[A-Z]{4}) + \. (?P[A-Z]{2}) + \. (?P[A-Z]) + \. (?P\d{4}) + \. (?P\d{6}T\d{4}Z) + - (?P\d{6}T\d{4}Z) /$ ''', re.X) RE_HYDRO = re.compile(r''' - ^/ - (?P[0N1]) - \. - (?P[A-Z]{2}) - \. - (?P\d{6}T\d{4}Z) - - - (?P\d{6}T\d{4}Z) - \. - (?P[A-Z]{2}) + ^/ (?P[0N1]) + \. (?P[A-Z]{2}) + \. (?P\d{6}T\d{4}Z) + - (?P\d{6}T\d{4}Z) + \. (?P[A-Z]{2}) /$ ''', re.X) -RE_POLY = re.compile(r'^LAT\.\.\.LON (?P\d+(?: \d+)+)') - -RE_MOTION = re.compile(r''' - ^ - TIME - \.\.\. - MOT - \.\.\. - LOC - [ ]{1} - (?P\d{2})(?P\d{2})Z - [ ]{1} - (?P\d+)DEG - [ ]{1} - (?P\d+)KT - [ ]{1} - (?P\d+) - [ ]{1} - (?P\d+) - $ -''', re.X) - -def parse_timestamp(text: str, post_2016_05_11: bool): +def parse_timestamp(text: str): return datetime.datetime.strptime( text, '%y%m%dT%H%M%SZ' ).astimezone(datetime.UTC) -def parse_lon(text: str): - size = len(text) - return 0 - float(text[0:size-2]) + (float(text[size-2:size]) / 100) - -def parse_lat(text: str): - size = len(text) - return float(text[0:size-2]) + (float(text[size-2:size]) / 100) - -def parse_location(lon: str, lat: str): - return shapely.Point(parse_lon(lon), parse_lat(lat)) - -def parse_shape(text: str): - points = list() - coords = text.split(' ') - - for i in range(0, len(coords), 2): - lat = coords[i] - lon = coords[i+1] - - points.append([parse_lon(lon), parse_lat(lat)]) - - points.append([parse_lon(coords[0]), parse_lat(coords[1])]) - - return shapely.Polygon(points) - class VTECEventType(enum.StrEnum): OPERATIONAL = 'O' TEST = 'T' EXPERIMENTAL = 'E' EXPERIMENTAL_VTEC = 'X' -class VTECEventParserState(enum.Enum): - NONE = 1 - HEADER = enum.auto() - ISSUANCE = enum.auto() - META = enum.auto() - TYPEOFFICE = enum.auto() - VTEC = enum.auto() - BODY_SEP = enum.auto() - BODY = enum.auto() - TAGS = enum.auto() - FOOTER = enum.auto() - -class VTECEvent(DatabaseTable): - __table__ = 'nexrad_vtec_event' - __key__ = 'id' - - __columns__ = ( - 'id', 'timestamp_issued', 'timestamp_start', 'timestamp_end', - 'typeof', 'etn', 'actions', 'wfo', 'phenom', 'sig', 'body', - 'azimuth', 'speed', 'location', 'forecaster', 'poly', - ) - - __columns_read__ = { - 'poly': 'ST_AsText(poly) as poly' - } - - __values_write__ = { - 'poly': shapely.from_wkt - } - - __columns_write__ = { - 'poly': 'ST_GeomFromText(:poly, {crs})'.format(crs=COORD_SYSTEM) - } - - __values_write__ = { - 'poly': lambda v: {'poly': shapely.to_wkt(v)} - } - - id: int - timestamp_issued: datetime.datetime - timestamp_start: datetime.datetime - timestamp_end: datetime.datetime - typeof: str - actions: str - wfo: str - phenom: str - sig: str - etn: int - body: str - azimuth: int - speed: int - location: shapely.Point - forecaster: str - poly: shapely.Geometry +class VTECEvent(): + typeof: str + actions: str + wfo: str + phenom: str + sig: str + etn: int + timestamp_start: datetime.datetime + timestamp_end: datetime.datetime @staticmethod - def parse(text: str): + def parse(text: str) -> __class__: + match = RE_PHENOM.match(text) + + if match is None: + return + event = VTECEvent() - state = VTECEventParserState.NONE + event.typeof = match['typeof'] + event.actions = match['actions'] + event.wfo = match['wfo'] + event.phenom = match['phenom'] + event.sig = match['sig'] + event.etn = int(match['etn']) - # - # A timestamp post 11 May 2016 can be detected based on the - # presence of lowercase letters in bulletin text, as per: - # - # https://www.noaa.gov/media-release/national-weather-service-will-stop-using-all-caps-in-its-forecasts - # - post_2016_05_11 = any(c for c in text if c.islower()) - - issuance = None - - for line in text.split('\n'): - line = line.rstrip() - - if state == VTECEventParserState.NONE: - match = RE_ID.match(line) - - if match is not None: - event.id = int(match[1]) - state = VTECEventParserState.HEADER - elif state == VTECEventParserState.HEADER: - match = RE_ISSUANCE.match(line) - - if match is not None: - issuance = match - state = VTECEventParserState.ISSUANCE - elif state == VTECEventParserState.ISSUANCE: - state = VTECEventParserState.META - elif state == VTECEventParserState.META: - match = RE_PHENOM.match(line) - - if match is not None: - event.timestamp_start = parse_timestamp(match['time_start'], post_2016_05_11) - event.timestamp_end = parse_timestamp(match['time_end'], post_2016_05_11) - - event.typeof = match['typeof'] - event.actions = match['actions'] - event.wfo = match['wfo'] - event.phenom = match['phenom'] - event.sig = match['sig'] - event.etn = int(match['etn']) - - state = VTECEventParserState.VTEC - elif state == VTECEventParserState.VTEC: - if line == '': - state = VTECEventParserState.BODY_SEP - elif state == VTECEventParserState.BODY_SEP: - event.body = line - state = VTECEventParserState.BODY - elif state == VTECEventParserState.BODY: - if line == '&&': - state = VTECEventParserState.TAGS - else: - event.body += '\n' + line - elif state == VTECEventParserState.TAGS: - if line == '$$': - state = VTECEventParserState.FOOTER - else: - match = RE_POLY.match(line) - - if match is not None: - event.poly = parse_shape(match['coords']) - - match = RE_MOTION.match(line) - - if match is not None: - event.azimuth = int(match['azimuth']) - event.speed = int(match['speed']) - event.location = parse_location(match['lon'], match['lat']) - elif state == VTECEventParserState.FOOTER: - if line != '': - event.forecaster = line + event.timestamp_start = parse_timestamp(match['time_start']) + event.timestamp_end = parse_timestamp(match['time_end']) + + return event + +class VTECHydroEvent(): + __slots__ = ( + 'severity', 'cause', 'record', + 'timestamp_start', 'timestamp_end' + ) + + severity: str + cause: str + record: str + timestamp_start: datetime.datetime + timestamp_end: datetime.datetime + + @staticmethod + def parse(text: str) -> __class__: + match = RE_HYDRO.match(text) + + if match is None: + return + + event = VTECHydroEvent() + event.severity = match['severity'] + event.cause = match['cause'] + + event.timestamp_start = parse_timestamp(match['time_start']) + event.timestamp_end = parse_timestamp(match['time_end']) return event