diff --git a/Dockerfile b/Dockerfile index 0c24bb0..1814a1b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,21 +5,21 @@ COPY requirements.txt /root RUN apk add build-base geos geos-dev sqlite libspatialite RUN pip3 install -r /root/requirements.txt -RUN mkdir -p /var/opt/nexrad-archive/lib/nexrad -RUN mkdir -p /var/opt/nexrad-archive/bin -RUN mkdir -p /var/lib/nexrad-archive +RUN mkdir -p /var/opt/xmet/lib/xmet +RUN mkdir -p /var/opt/xmet/bin +RUN mkdir -p /var/lib/xmet -COPY db/nexrad.sql doc/radars.tsv doc/wfo.tsv /tmp -COPY lib/nexrad/*.py /var/opt/nexrad-archive/lib/nexrad -COPY bin/nexrad-archive bin/nexrad-archive-init /var/opt/nexrad-archive/bin +COPY db/xmet.sql doc/radars.tsv doc/wfo.tsv /tmp +COPY lib/xmet/*.py /var/opt/xmet/lib/xmet +COPY bin/xmet-nexrad-archive bin/xmet-db-init /var/opt/xmet/bin -ENV PYTHONPATH=/var/opt/nexrad-archive/lib +ENV PYTHONPATH=/var/opt/xmet/lib -RUN sqlite3 -init /tmp/nexrad.sql /var/lib/nexrad-archive/nexrad.db .quit +RUN sqlite3 -init /tmp/xmet.sql /var/lib/xmet/xmet.db .quit -RUN /var/opt/nexrad-archive/bin/nexrad-archive-init \ - /var/lib/nexrad-archive/nexrad.db \ +RUN /var/opt/xmet/bin/xmet-db-init \ + /var/lib/xmet/xmet.db \ /tmp/radars.tsv \ /tmp/wfo.tsv -ENTRYPOINT ["/var/opt/nexrad-archive/bin/nexrad-archive", "/var/lib/nexrad-archive/nexrad.db"] +ENTRYPOINT ["/var/opt/xmet/bin/xmet-nexrad-archive", "/var/lib/xmet/xmet.db"] diff --git a/Makefile b/Makefile index 2cf21e7..f02af31 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ DOCKER = docker -DOCKER_TAG = nexrad-archive:latest +DOCKER_TAG = xmet:latest all: $(DOCKER) image build --tag $(DOCKER_TAG) . diff --git a/README.md b/README.md index 372bdb1..7fe5c6d 100644 --- a/README.md +++ b/README.md @@ -1,58 +1,6 @@ ![WSR-88D: Come And Take It](doc/come-and-take-it.png) -# NEXRAD Level II Archive Tool +# XMET: XANTRONIX Meteorological Suite -This tool uses -[Spatialite](https://www.gaia-gis.it/fossil/libspatialite/index) -in conjunction with the -[NCEI Storm Events Database](https://www.ncei.noaa.gov/pub/data/swdi/stormevents/csvfiles/) -to archive only NEXRAD Level II data for which storms were recorded -by the National Weather Service. Using the start and end times and -coordinates of a given storm event, this tool is able to query and -filter objects in the -[NEXRAD Level II Amazon bucket](https://registry.opendata.aws/noaa-nexrad/), -allowing one to only archive volume scans for which storms were noted, -from the appropriate radar sites. - -## Building - -Simply run `make` to produce a Docker image pre-populated with the -NEXRAD radar site database. - -## Running - -Invoke `sh run.sh` to run this archival tool, passing two arguments: - -* The path to a storm events database CSV export file, whose name - as obtained from the NCEI begins with `StormEvents_details_`. Only - these files contain the start and end coordinates of a storm which - are used to select data from the correct radar sites. - -* A path to a destination directory where archived data shall be - downloaded. Note that files downloaded under this directory will - be organized in the `YYYY/MM/DD/SITE/` hierarchy. - -Other optional arguments will be passed through to the -`bin/nexrad-archive` executable within the Docker image: - -* `--dry-run` - - This will avoid actually performing any archiving; instead, the - tool will indicate what actions it would take. - -* `--quiet` - - Suppresses standard output. - -* `--exclude="Event Type"` - - Exclude one or more event types from consideration for archival. - These event types are - [documented](https://www.ncei.noaa.gov/pub/data/swdi/stormevents/csvfiles/Storm-Data-Bulk-csv-Format.pdf) - by the NCEI. Cannot be specified alongside `--type`. Can be - specified more than once. - -* `--type="Event Type"` - - Explicitly specify which event types to archive. Cannot be - specified alongside `--exclude`. Can be specified more than once. +The beginnings of a meteorological suite, forked from +[nexrad-archive](https://dev.xantronix.net/xan/nexrad-archive). diff --git a/bin/nexrad-archive-afos-ingest b/bin/xmet-afos-ingest similarity index 91% rename from bin/nexrad-archive-afos-ingest rename to bin/xmet-afos-ingest index f23eef3..f5462a9 100755 --- a/bin/nexrad-archive-afos-ingest +++ b/bin/xmet-afos-ingest @@ -2,8 +2,8 @@ import sys -from nexrad.db import Database -from nexrad.afos import AFOSMessageParser +from xmet.db import Database +from xmet.afos import AFOSMessageParser CHUNK_SIZE = 4096 diff --git a/bin/nexrad-archive-init b/bin/xmet-db-init similarity index 78% rename from bin/nexrad-archive-init rename to bin/xmet-db-init index b576437..717e718 100755 --- a/bin/nexrad-archive-init +++ b/bin/xmet-db-init @@ -2,12 +2,12 @@ import argparse -from nexrad.db import Database -from nexrad.radar import Radar -from nexrad.wfo import WFO +from xmet.db import Database +from xmet.radar import Radar +from xmet.wfo import WFO parser = argparse.ArgumentParser( - description = 'Initialize NEXRAD radar site database' + description = 'Initialize NEXRAD radar site database table' ) parser.add_argument('db', help='Path to SQLite3 database') diff --git a/bin/nexrad-archive-event-ingest b/bin/xmet-event-ingest similarity index 82% rename from bin/nexrad-archive-event-ingest rename to bin/xmet-event-ingest index 2bf3828..4bb8090 100755 --- a/bin/nexrad-archive-event-ingest +++ b/bin/xmet-event-ingest @@ -2,8 +2,8 @@ import argparse -from nexrad.db import Database -from nexrad.storm import StormEvent +from xmet.db import Database +from xmet.storm import StormEvent parser = argparse.ArgumentParser( description = 'Ingest events from StormEvent_details_*.csv.gz files' @@ -11,7 +11,7 @@ parser = argparse.ArgumentParser( parser.add_argument('--quiet', action='store_true', help='Suppress output') parser.add_argument('--dry-run', action='store_true', help='Do not actually ingest events') -parser.add_argument('db', help='SQLite3 NEXRAD radar site database') +parser.add_argument('db', help='XMET SQLite3 database') parser.add_argument('csv-event-details', nargs='+', help='Compressed storm event details CSV file') args = parser.parse_args() diff --git a/bin/nexrad-archive b/bin/xmet-nexrad-archive similarity index 90% rename from bin/nexrad-archive rename to bin/xmet-nexrad-archive index 310998f..1211252 100755 --- a/bin/nexrad-archive +++ b/bin/xmet-nexrad-archive @@ -2,10 +2,10 @@ import argparse -from nexrad.db import Database -from nexrad.s3 import S3Bucket -from nexrad.storm import StormEvent -from nexrad.archive import Archive +from xmet.db import Database +from xmet.s3 import S3Bucket +from xmet.storm import StormEvent +from xmet.archive import Archive parser = argparse.ArgumentParser( description = 'Archive NEXRAD Level II data from Amazon S3' @@ -18,7 +18,7 @@ group = parser.add_mutually_exclusive_group() group.add_argument('--exclude', action='append', type=str, help='Exclude types of events from ingest') group.add_argument('--type', action='append', type=str, help='Specify only given types of events to ingest') -parser.add_argument('db', help='SQLite3 NEXRAD radar site database') +parser.add_argument('db', help='XMET SQLite3 database') parser.add_argument('csv-event-details', nargs='+', help='Compressed storm event details CSV file') parser.add_argument('archive-dir', help='Target archive directory') diff --git a/db/nexrad.sql b/db/nexrad.sql deleted file mode 100644 index b16de4c..0000000 --- a/db/nexrad.sql +++ /dev/null @@ -1,93 +0,0 @@ -select load_extension('mod_spatialite.so.8'); -select InitSpatialMetadata(1); - -begin transaction; - -create table nexrad_wfo ( - code TEXT PRIMARY KEY NOT NULL, - city TEXT NOT NULL, - state TEXT NOT NULL, - address TEXT NOT NULL -); - -select - AddGeometryColumn('nexrad_wfo', 'coord', 4326, 'POINT', 'XY', 1), - CreateSpatialIndex('nexrad_wfo', 'coord'); - -create table nexrad_radar ( - call TEXT PRIMARY KEY NOT NULL, - name TEXT NOT NULL, - wban INTEGER, - site_elevation FLOAT NOT NULL, - tower_height FLOAT NOT NULL -); - -select - AddGeometryColumn('nexrad_radar', 'coord', 4326, 'POINT', 'XY'), - CreateSpatialIndex('nexrad_radar', 'coord'); - -create table nexrad_storm_event ( - id INTEGER PRIMARY KEY NOT NULL, - episode_id INTEGER, - timestamp_start TIMESTAMP NOT NULL, - timestamp_end TIMESTAMP NOT NULL, - state TEXT NOT NULL, - event_type TEXT NOT NULL, - wfo TEXT NOT NULL, - locale_start TEXT NOT NULL, - locale_end TEXT NOT NULL, - tornado_f_rating TEXT -); - -create index nexrad_storm_event_episode_id_idx on nexrad_storm_event (episode_id); -create index nexrad_storm_event_event_type_idx on nexrad_storm_event (event_type); -create index nexrad_storm_event_wfo_idx on nexrad_storm_event (wfo); -create index nexrad_storm_event_timestamp_start_idx on nexrad_storm_event (timestamp_start); -create index nexrad_storm_event_timestamp_end_idx on nexrad_storm_event (timestamp_end); - -select - AddGeometryColumn('nexrad_storm_event', 'coord_start', 4326, 'POINT', 'XY', 0), - CreateSpatialIndex('nexrad_storm_event', 'coord_start'); - -select - AddGeometryColumn('nexrad_storm_event', 'coord_end', 4326, 'POINT', 'XY', 0), - CreateSpatialIndex('nexrad_storm_event', 'coord_end'); - -create table nexrad_afos_message ( - id INTEGER PRIMARY KEY NOT NULL, - timestamp_issued TIMESTAMP NOT NULL, - serial INTEGER NOT NULL, - text_raw TEXT NOT NULL, - product TEXT NOT NULL, - wfo TEXT NOT NULL, - vtec_start TIMESTAMP, - vtec_end TIMESTAMP, - vtec_type TEXT, - actions TEXT, - phenom TEXT, - sig TEXT, - etn INTEGER, - hydro_severity TEXT, - hydro_cause TEXT, - hydro_record TEXT, - azimuth FLOAT, - speed FLOAT, - forecaster TEXT NOT NULL -); - -create index nexrad_afos_message_timestamp_idx on nexrad_afos_message (timestamp_issued); -create index nexrad_afos_message_vtec_timestamp_idx on nexrad_afos_message (vtec_start, vtec_end); -create index nexrad_afos_message_product_idx on nexrad_afos_message (product); -create index nexrad_afos_message_wfo_idx on nexrad_afos_message (wfo); -create index nexrad_afos_message_phenom_idx on nexrad_afos_message (phenom); -create index nexrad_afos_message_sig_idx on nexrad_afos_message (sig); - -select - AddGeometryColumn('nexrad_afos_message', 'location', 4326, 'POINT'), - CreateSpatialIndex('nexrad_afos_message', 'location'); - -select - AddGeometryColumn('nexrad_afos_message', 'poly', 4326, 'POLYGON'), - CreateSpatialIndex('nexrad_afos_message', 'poly'); - -commit; diff --git a/db/xmet.sql b/db/xmet.sql new file mode 100644 index 0000000..1cc52ed --- /dev/null +++ b/db/xmet.sql @@ -0,0 +1,93 @@ +select load_extension('mod_spatialite.so.8'); +select InitSpatialMetadata(1); + +begin transaction; + +create table xmet_wfo ( + code TEXT PRIMARY KEY NOT NULL, + city TEXT NOT NULL, + state TEXT NOT NULL, + address TEXT NOT NULL +); + +select + AddGeometryColumn('xmet_wfo', 'coord', 4326, 'POINT', 'XY', 1), + CreateSpatialIndex('xmet_wfo', 'coord'); + +create table xmet_nexrad_radar ( + call TEXT PRIMARY KEY NOT NULL, + name TEXT NOT NULL, + wban INTEGER, + site_elevation FLOAT NOT NULL, + tower_height FLOAT NOT NULL +); + +select + AddGeometryColumn('xmet_nexrad_radar', 'coord', 4326, 'POINT', 'XY'), + CreateSpatialIndex('xmet_nexrad_radar', 'coord'); + +create table xmet_storm_event ( + id INTEGER PRIMARY KEY NOT NULL, + episode_id INTEGER, + timestamp_start TIMESTAMP NOT NULL, + timestamp_end TIMESTAMP NOT NULL, + state TEXT NOT NULL, + event_type TEXT NOT NULL, + wfo TEXT NOT NULL, + locale_start TEXT NOT NULL, + locale_end TEXT NOT NULL, + tornado_f_rating TEXT +); + +create index xmet_storm_event_episode_id_idx on xmet_storm_event (episode_id); +create index xmet_storm_event_event_type_idx on xmet_storm_event (event_type); +create index xmet_storm_event_wfo_idx on xmet_storm_event (wfo); +create index xmet_storm_event_timestamp_start_idx on xmet_storm_event (timestamp_start); +create index xmet_storm_event_timestamp_end_idx on xmet_storm_event (timestamp_end); + +select + AddGeometryColumn('xmet_storm_event', 'coord_start', 4326, 'POINT', 'XY', 0), + CreateSpatialIndex('xmet_storm_event', 'coord_start'); + +select + AddGeometryColumn('xmet_storm_event', 'coord_end', 4326, 'POINT', 'XY', 0), + CreateSpatialIndex('xmet_storm_event', 'coord_end'); + +create table xmet_afos_message ( + id INTEGER PRIMARY KEY NOT NULL, + timestamp_issued TIMESTAMP NOT NULL, + serial INTEGER NOT NULL, + text_raw TEXT NOT NULL, + product TEXT NOT NULL, + wfo TEXT NOT NULL, + vtec_start TIMESTAMP, + vtec_end TIMESTAMP, + vtec_type TEXT, + actions TEXT, + phenom TEXT, + sig TEXT, + etn INTEGER, + hydro_severity TEXT, + hydro_cause TEXT, + hydro_record TEXT, + azimuth FLOAT, + speed FLOAT, + forecaster TEXT NOT NULL +); + +create index xmet_afos_message_timestamp_idx on xmet_afos_message (timestamp_issued); +create index xmet_afos_message_vtec_timestamp_idx on xmet_afos_message (vtec_start, vtec_end); +create index xmet_afos_message_product_idx on xmet_afos_message (product); +create index xmet_afos_message_wfo_idx on xmet_afos_message (wfo); +create index xmet_afos_message_phenom_idx on xmet_afos_message (phenom); +create index xmet_afos_message_sig_idx on xmet_afos_message (sig); + +select + AddGeometryColumn('xmet_afos_message', 'location', 4326, 'POINT'), + CreateSpatialIndex('xmet_afos_message', 'location'); + +select + AddGeometryColumn('xmet_afos_message', 'poly', 4326, 'POLYGON'), + CreateSpatialIndex('xmet_afos_message', 'poly'); + +commit; diff --git a/lib/nexrad/afos.py b/lib/xmet/afos.py similarity index 98% rename from lib/nexrad/afos.py rename to lib/xmet/afos.py index 0ab716b..582c01a 100644 --- a/lib/nexrad/afos.py +++ b/lib/xmet/afos.py @@ -3,9 +3,9 @@ import enum import datetime import shapely -from nexrad.db import DatabaseTable -from nexrad.coord import COORD_SYSTEM -from nexrad.vtec import VTECEvent, VTECHydroEvent +from xmet.db import DatabaseTable +from xmet.coord import COORD_SYSTEM +from xmet.vtec import VTECEvent, VTECHydroEvent RE_ID = re.compile(r'^(\d+)$') @@ -86,7 +86,7 @@ def poly_from_coords(coords: list) -> shapely.Polygon: return shapely.Polygon([*coords, [coords[0][0], coords[0][1]]]) class AFOSMessage(DatabaseTable): - __table__ = 'nexrad_afos_message' + __table__ = 'xmet_afos_message' __key__ = 'id' __columns__ = ( diff --git a/lib/nexrad/archive.py b/lib/xmet/archive.py similarity index 96% rename from lib/nexrad/archive.py rename to lib/xmet/archive.py index 52627ef..800f392 100644 --- a/lib/nexrad/archive.py +++ b/lib/xmet/archive.py @@ -3,9 +3,9 @@ import re import enum import datetime -from nexrad.db import Database -from nexrad.s3 import S3Bucket, S3_KEY_RE -from nexrad.radar import RADAR_RANGE +from xmet.db import Database +from xmet.s3 import S3Bucket, S3_KEY_RE +from xmet.radar import RADAR_RANGE class ArchiveDateError(Exception): def __init__(self, supplied: str, missing: str): @@ -98,8 +98,8 @@ class ArchiveProduct(): radar.coord, true) as distance from - nexrad_storm_event as event, - nexrad_radar as radar + xmet_storm_event as event, + xmet_nexrad_radar as radar where distance <= :radius and :timestamp between event.timestamp_start and event.timestamp_end diff --git a/lib/nexrad/coord.py b/lib/xmet/coord.py similarity index 100% rename from lib/nexrad/coord.py rename to lib/xmet/coord.py diff --git a/lib/nexrad/db.py b/lib/xmet/db.py similarity index 100% rename from lib/nexrad/db.py rename to lib/xmet/db.py diff --git a/lib/nexrad/radar.py b/lib/xmet/radar.py similarity index 95% rename from lib/nexrad/radar.py rename to lib/xmet/radar.py index 451f2e7..3a6cf11 100644 --- a/lib/nexrad/radar.py +++ b/lib/xmet/radar.py @@ -2,8 +2,8 @@ import csv import re import shapely -from nexrad.db import DatabaseTable -from nexrad.coord import COORD_SYSTEM +from xmet.db import DatabaseTable +from xmet.coord import COORD_SYSTEM """ Implements a parser and wrapper class for the WSR-88D radar list @@ -47,7 +47,7 @@ class Radar(DatabaseTable): 'call', 'wban', 'name', 'coord', 'site_elevation', 'tower_height', ) - __table__ = 'nexrad_radar' + __table__ = 'xmet_nexrad_radar' __key__ = 'call' __columns__ = ( diff --git a/lib/nexrad/s3.py b/lib/xmet/s3.py similarity index 100% rename from lib/nexrad/s3.py rename to lib/xmet/s3.py diff --git a/lib/nexrad/storm.py b/lib/xmet/storm.py similarity index 97% rename from lib/nexrad/storm.py rename to lib/xmet/storm.py index d7d34a3..15f9db7 100644 --- a/lib/nexrad/storm.py +++ b/lib/xmet/storm.py @@ -4,9 +4,9 @@ import csv import datetime import shapely -from nexrad.db import DatabaseTable -from nexrad.coord import COORD_SYSTEM -from nexrad.radar import RADAR_RANGE +from xmet.db import DatabaseTable +from xmet.coord import COORD_SYSTEM +from xmet.radar import RADAR_RANGE def time_from_str(time: str): size = len(time) @@ -88,7 +88,7 @@ class StormEvent(DatabaseTable): 'tornado_f_rating', 'coord_start', 'coord_end' ) - __table__ = 'nexrad_storm_event' + __table__ = 'xmet_storm_event' __key__ = 'id' __columns__ = ( @@ -206,7 +206,7 @@ class StormEvent(DatabaseTable): 'timestamp': str(timestamp) }) - sql = "select " + ", ".join(columns) + " from nexrad_storm_event" + sql = "select " + ", ".join(columns) + " from xmet_storm_event" if len(clauses) > 0: sql += " where " + " and ".join(clauses) @@ -262,7 +262,7 @@ class StormEvent(DatabaseTable): MakePoint(?, ?, {csr})), true) as distance from - nexrad_radar + xmet_nexrad_radar where distance <= ? order by diff --git a/lib/nexrad/vtec.py b/lib/xmet/vtec.py similarity index 100% rename from lib/nexrad/vtec.py rename to lib/xmet/vtec.py diff --git a/lib/nexrad/wfo.py b/lib/xmet/wfo.py similarity index 92% rename from lib/nexrad/wfo.py rename to lib/xmet/wfo.py index 5485c8a..3bf9329 100644 --- a/lib/nexrad/wfo.py +++ b/lib/xmet/wfo.py @@ -1,15 +1,15 @@ import csv import shapely -from nexrad.db import DatabaseTable -from nexrad.coord import COORD_SYSTEM +from xmet.db import DatabaseTable +from xmet.coord import COORD_SYSTEM class WFO(DatabaseTable): __slots__ = ( 'code', 'city', 'state', 'address', 'coord' ) - __table__ = 'nexrad_wfo' + __table__ = 'xmet_wfo' __key__ = 'code' __columns__ = ( diff --git a/run.sh b/run.sh index 72f6c48..b6bcb92 100644 --- a/run.sh +++ b/run.sh @@ -8,4 +8,4 @@ shift 2 docker run --rm -it \ --volume "$events_csv_gz:/tmp/events.csv.gz" \ --volume "$dest:/dest" \ - nexrad-archive:latest /tmp/events.csv.gz /dest "$@" + xmet:latest /tmp/events.csv.gz /dest "$@"