xan/xmet
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rename from nexrad-archive to xmet

Browse source
This commit is contained in:
XANTRONIX Industrial 2025-02-22 13:53:54 -05:00
parent 4f91172984
commit 6d784cb100
19 changed files with 144 additions and 196 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
Dockerfile
View file

@ -5,21 +5,21 @@ COPY requirements.txt /root
RUN apk add build-base geos geos-dev sqlite libspatialite RUN apk add build-base geos geos-dev sqlite libspatialite
RUN pip3 install -r /root/requirements.txt RUN pip3 install -r /root/requirements.txt
RUN mkdir -p /var/opt/nexrad-archive/lib/nexrad RUN mkdir -p /var/opt/xmet/lib/xmet
RUN mkdir -p /var/opt/nexrad-archive/bin RUN mkdir -p /var/opt/xmet/bin
RUN mkdir -p /var/lib/nexrad-archive RUN mkdir -p /var/lib/xmet
COPY db/nexrad.sql doc/radars.tsv doc/wfo.tsv /tmp COPY db/xmet.sql doc/radars.tsv doc/wfo.tsv /tmp
COPY lib/nexrad/*.py /var/opt/nexrad-archive/lib/nexrad COPY lib/xmet/*.py /var/opt/xmet/lib/xmet
COPY bin/nexrad-archive bin/nexrad-archive-init /var/opt/nexrad-archive/bin COPY bin/xmet-nexrad-archive bin/xmet-db-init /var/opt/xmet/bin
ENV PYTHONPATH=/var/opt/nexrad-archive/lib ENV PYTHONPATH=/var/opt/xmet/lib
RUN sqlite3 -init /tmp/nexrad.sql /var/lib/nexrad-archive/nexrad.db .quit RUN sqlite3 -init /tmp/xmet.sql /var/lib/xmet/xmet.db .quit
RUN /var/opt/nexrad-archive/bin/nexrad-archive-init \ RUN /var/opt/xmet/bin/xmet-db-init \
/var/lib/nexrad-archive/nexrad.db \ /var/lib/xmet/xmet.db \
/tmp/radars.tsv \ /tmp/radars.tsv \
/tmp/wfo.tsv /tmp/wfo.tsv
ENTRYPOINT ["/var/opt/nexrad-archive/bin/nexrad-archive", "/var/lib/nexrad-archive/nexrad.db"] ENTRYPOINT ["/var/opt/xmet/bin/xmet-nexrad-archive", "/var/lib/xmet/xmet.db"]

2
Makefile
View file

@ -1,5 +1,5 @@
DOCKER = docker DOCKER = docker
DOCKER_TAG = nexrad-archive:latest DOCKER_TAG = xmet:latest
all: all:
$(DOCKER) image build --tag $(DOCKER_TAG) . $(DOCKER) image build --tag $(DOCKER_TAG) .

58
README.md
View file

@ -1,58 +1,6 @@
![WSR-88D: Come And Take It](doc/come-and-take-it.png) ![WSR-88D: Come And Take It](doc/come-and-take-it.png)
# NEXRAD Level II Archive Tool # XMET: XANTRONIX Meteorological Suite
This tool uses The beginnings of a meteorological suite, forked from
[Spatialite](https://www.gaia-gis.it/fossil/libspatialite/index) [nexrad-archive](https://dev.xantronix.net/xan/nexrad-archive).
in conjunction with the
[NCEI Storm Events Database](https://www.ncei.noaa.gov/pub/data/swdi/stormevents/csvfiles/)
to archive only NEXRAD Level II data for which storms were recorded
by the National Weather Service. Using the start and end times and
coordinates of a given storm event, this tool is able to query and
filter objects in the
[NEXRAD Level II Amazon bucket](https://registry.opendata.aws/noaa-nexrad/),
allowing one to only archive volume scans for which storms were noted,
from the appropriate radar sites.
## Building
Simply run `make` to produce a Docker image pre-populated with the
NEXRAD radar site database.
## Running
Invoke `sh run.sh` to run this archival tool, passing two arguments:
* The path to a storm events database CSV export file, whose name
as obtained from the NCEI begins with `StormEvents_details_`. Only
these files contain the start and end coordinates of a storm which
are used to select data from the correct radar sites.
* A path to a destination directory where archived data shall be
downloaded. Note that files downloaded under this directory will
be organized in the `YYYY/MM/DD/SITE/` hierarchy.
Other optional arguments will be passed through to the
`bin/nexrad-archive` executable within the Docker image:
* `--dry-run`
This will avoid actually performing any archiving; instead, the
tool will indicate what actions it would take.
* `--quiet`
Suppresses standard output.
* `--exclude="Event Type"`
Exclude one or more event types from consideration for archival.
These event types are
[documented](https://www.ncei.noaa.gov/pub/data/swdi/stormevents/csvfiles/Storm-Data-Bulk-csv-Format.pdf)
by the NCEI. Cannot be specified alongside `--type`. Can be
specified more than once.
* `--type="Event Type"`
Explicitly specify which event types to archive. Cannot be
specified alongside `--exclude`. Can be specified more than once.

4
bin/nexrad-archive-afos-ingest → bin/xmet-afos-ingest
View file

@ -2,8 +2,8 @@
import sys import sys
from nexrad.db import Database from xmet.db import Database
from nexrad.afos import AFOSMessageParser from xmet.afos import AFOSMessageParser
CHUNK_SIZE = 4096 CHUNK_SIZE = 4096

8
bin/nexrad-archive-init → bin/xmet-db-init
View file

@ -2,12 +2,12 @@
import argparse import argparse
from nexrad.db import Database from xmet.db import Database
from nexrad.radar import Radar from xmet.radar import Radar
from nexrad.wfo import WFO from xmet.wfo import WFO
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description = 'Initialize NEXRAD radar site database' description = 'Initialize NEXRAD radar site database table'
) )
parser.add_argument('db', help='Path to SQLite3 database') parser.add_argument('db', help='Path to SQLite3 database')

6
bin/nexrad-archive-event-ingest → bin/xmet-event-ingest
View file

@ -2,8 +2,8 @@
import argparse import argparse
from nexrad.db import Database from xmet.db import Database
from nexrad.storm import StormEvent from xmet.storm import StormEvent
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description = 'Ingest events from StormEvent_details_*.csv.gz files' description = 'Ingest events from StormEvent_details_*.csv.gz files'
@ -11,7 +11,7 @@ parser = argparse.ArgumentParser(
parser.add_argument('--quiet', action='store_true', help='Suppress output') parser.add_argument('--quiet', action='store_true', help='Suppress output')
parser.add_argument('--dry-run', action='store_true', help='Do not actually ingest events') parser.add_argument('--dry-run', action='store_true', help='Do not actually ingest events')
parser.add_argument('db', help='SQLite3 NEXRAD radar site database') parser.add_argument('db', help='XMET SQLite3 database')
parser.add_argument('csv-event-details', nargs='+', help='Compressed storm event details CSV file') parser.add_argument('csv-event-details', nargs='+', help='Compressed storm event details CSV file')
args = parser.parse_args() args = parser.parse_args()

10
bin/nexrad-archive → bin/xmet-nexrad-archive
View file

@ -2,10 +2,10 @@
import argparse import argparse
from nexrad.db import Database from xmet.db import Database
from nexrad.s3 import S3Bucket from xmet.s3 import S3Bucket
from nexrad.storm import StormEvent from xmet.storm import StormEvent
from nexrad.archive import Archive from xmet.archive import Archive
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description = 'Archive NEXRAD Level II data from Amazon S3' description = 'Archive NEXRAD Level II data from Amazon S3'
@ -18,7 +18,7 @@ group = parser.add_mutually_exclusive_group()
group.add_argument('--exclude', action='append', type=str, help='Exclude types of events from ingest') group.add_argument('--exclude', action='append', type=str, help='Exclude types of events from ingest')
group.add_argument('--type', action='append', type=str, help='Specify only given types of events to ingest') group.add_argument('--type', action='append', type=str, help='Specify only given types of events to ingest')
parser.add_argument('db', help='SQLite3 NEXRAD radar site database') parser.add_argument('db', help='XMET SQLite3 database')
parser.add_argument('csv-event-details', nargs='+', help='Compressed storm event details CSV file') parser.add_argument('csv-event-details', nargs='+', help='Compressed storm event details CSV file')
parser.add_argument('archive-dir', help='Target archive directory') parser.add_argument('archive-dir', help='Target archive directory')

93
db/nexrad.sql
View file

@ -1,93 +0,0 @@
select load_extension('mod_spatialite.so.8');
select InitSpatialMetadata(1);
begin transaction;
create table nexrad_wfo (
code TEXT PRIMARY KEY NOT NULL,
city TEXT NOT NULL,
state TEXT NOT NULL,
address TEXT NOT NULL
);
select
AddGeometryColumn('nexrad_wfo', 'coord', 4326, 'POINT', 'XY', 1),
CreateSpatialIndex('nexrad_wfo', 'coord');
create table nexrad_radar (
call TEXT PRIMARY KEY NOT NULL,
name TEXT NOT NULL,
wban INTEGER,
site_elevation FLOAT NOT NULL,
tower_height FLOAT NOT NULL
);
select
AddGeometryColumn('nexrad_radar', 'coord', 4326, 'POINT', 'XY'),
CreateSpatialIndex('nexrad_radar', 'coord');
create table nexrad_storm_event (
id INTEGER PRIMARY KEY NOT NULL,
episode_id INTEGER,
timestamp_start TIMESTAMP NOT NULL,
timestamp_end TIMESTAMP NOT NULL,
state TEXT NOT NULL,
event_type TEXT NOT NULL,
wfo TEXT NOT NULL,
locale_start TEXT NOT NULL,
locale_end TEXT NOT NULL,
tornado_f_rating TEXT
);
create index nexrad_storm_event_episode_id_idx on nexrad_storm_event (episode_id);
create index nexrad_storm_event_event_type_idx on nexrad_storm_event (event_type);
create index nexrad_storm_event_wfo_idx on nexrad_storm_event (wfo);
create index nexrad_storm_event_timestamp_start_idx on nexrad_storm_event (timestamp_start);
create index nexrad_storm_event_timestamp_end_idx on nexrad_storm_event (timestamp_end);
select
AddGeometryColumn('nexrad_storm_event', 'coord_start', 4326, 'POINT', 'XY', 0),
CreateSpatialIndex('nexrad_storm_event', 'coord_start');
select
AddGeometryColumn('nexrad_storm_event', 'coord_end', 4326, 'POINT', 'XY', 0),
CreateSpatialIndex('nexrad_storm_event', 'coord_end');
create table nexrad_afos_message (
id INTEGER PRIMARY KEY NOT NULL,
timestamp_issued TIMESTAMP NOT NULL,
serial INTEGER NOT NULL,
text_raw TEXT NOT NULL,
product TEXT NOT NULL,
wfo TEXT NOT NULL,
vtec_start TIMESTAMP,
vtec_end TIMESTAMP,
vtec_type TEXT,
actions TEXT,
phenom TEXT,
sig TEXT,
etn INTEGER,
hydro_severity TEXT,
hydro_cause TEXT,
hydro_record TEXT,
azimuth FLOAT,
speed FLOAT,
forecaster TEXT NOT NULL
);
create index nexrad_afos_message_timestamp_idx on nexrad_afos_message (timestamp_issued);
create index nexrad_afos_message_vtec_timestamp_idx on nexrad_afos_message (vtec_start, vtec_end);
create index nexrad_afos_message_product_idx on nexrad_afos_message (product);
create index nexrad_afos_message_wfo_idx on nexrad_afos_message (wfo);
create index nexrad_afos_message_phenom_idx on nexrad_afos_message (phenom);
create index nexrad_afos_message_sig_idx on nexrad_afos_message (sig);
select
AddGeometryColumn('nexrad_afos_message', 'location', 4326, 'POINT'),
CreateSpatialIndex('nexrad_afos_message', 'location');
select
AddGeometryColumn('nexrad_afos_message', 'poly', 4326, 'POLYGON'),
CreateSpatialIndex('nexrad_afos_message', 'poly');
commit;

93
db/xmet.sql Normal file
View file

@ -0,0 +1,93 @@
select load_extension('mod_spatialite.so.8');
select InitSpatialMetadata(1);
begin transaction;
create table xmet_wfo (
code TEXT PRIMARY KEY NOT NULL,
city TEXT NOT NULL,
state TEXT NOT NULL,
address TEXT NOT NULL
);
select
AddGeometryColumn('xmet_wfo', 'coord', 4326, 'POINT', 'XY', 1),
CreateSpatialIndex('xmet_wfo', 'coord');
create table xmet_nexrad_radar (
call TEXT PRIMARY KEY NOT NULL,
name TEXT NOT NULL,
wban INTEGER,
site_elevation FLOAT NOT NULL,
tower_height FLOAT NOT NULL
);
select
AddGeometryColumn('xmet_nexrad_radar', 'coord', 4326, 'POINT', 'XY'),
CreateSpatialIndex('xmet_nexrad_radar', 'coord');
create table xmet_storm_event (
id INTEGER PRIMARY KEY NOT NULL,
episode_id INTEGER,
timestamp_start TIMESTAMP NOT NULL,
timestamp_end TIMESTAMP NOT NULL,
state TEXT NOT NULL,
event_type TEXT NOT NULL,
wfo TEXT NOT NULL,
locale_start TEXT NOT NULL,
locale_end TEXT NOT NULL,
tornado_f_rating TEXT
);
create index xmet_storm_event_episode_id_idx on xmet_storm_event (episode_id);
create index xmet_storm_event_event_type_idx on xmet_storm_event (event_type);
create index xmet_storm_event_wfo_idx on xmet_storm_event (wfo);
create index xmet_storm_event_timestamp_start_idx on xmet_storm_event (timestamp_start);
create index xmet_storm_event_timestamp_end_idx on xmet_storm_event (timestamp_end);
select
AddGeometryColumn('xmet_storm_event', 'coord_start', 4326, 'POINT', 'XY', 0),
CreateSpatialIndex('xmet_storm_event', 'coord_start');
select
AddGeometryColumn('xmet_storm_event', 'coord_end', 4326, 'POINT', 'XY', 0),
CreateSpatialIndex('xmet_storm_event', 'coord_end');
create table xmet_afos_message (
id INTEGER PRIMARY KEY NOT NULL,
timestamp_issued TIMESTAMP NOT NULL,
serial INTEGER NOT NULL,
text_raw TEXT NOT NULL,
product TEXT NOT NULL,
wfo TEXT NOT NULL,
vtec_start TIMESTAMP,
vtec_end TIMESTAMP,
vtec_type TEXT,
actions TEXT,
phenom TEXT,
sig TEXT,
etn INTEGER,
hydro_severity TEXT,
hydro_cause TEXT,
hydro_record TEXT,
azimuth FLOAT,
speed FLOAT,
forecaster TEXT NOT NULL
);
create index xmet_afos_message_timestamp_idx on xmet_afos_message (timestamp_issued);
create index xmet_afos_message_vtec_timestamp_idx on xmet_afos_message (vtec_start, vtec_end);
create index xmet_afos_message_product_idx on xmet_afos_message (product);
create index xmet_afos_message_wfo_idx on xmet_afos_message (wfo);
create index xmet_afos_message_phenom_idx on xmet_afos_message (phenom);
create index xmet_afos_message_sig_idx on xmet_afos_message (sig);
select
AddGeometryColumn('xmet_afos_message', 'location', 4326, 'POINT'),
CreateSpatialIndex('xmet_afos_message', 'location');
select
AddGeometryColumn('xmet_afos_message', 'poly', 4326, 'POLYGON'),
CreateSpatialIndex('xmet_afos_message', 'poly');
commit;

8
lib/nexrad/afos.py → lib/xmet/afos.py
View file

@ -3,9 +3,9 @@ import enum
import datetime import datetime
import shapely import shapely
from nexrad.db import DatabaseTable from xmet.db import DatabaseTable
from nexrad.coord import COORD_SYSTEM from xmet.coord import COORD_SYSTEM
from nexrad.vtec import VTECEvent, VTECHydroEvent from xmet.vtec import VTECEvent, VTECHydroEvent
RE_ID = re.compile(r'^(\d+)$') RE_ID = re.compile(r'^(\d+)$')
@ -86,7 +86,7 @@ def poly_from_coords(coords: list) -> shapely.Polygon:
return shapely.Polygon([*coords, [coords[0][0], coords[0][1]]]) return shapely.Polygon([*coords, [coords[0][0], coords[0][1]]])
class AFOSMessage(DatabaseTable): class AFOSMessage(DatabaseTable):
__table__ = 'nexrad_afos_message' __table__ = 'xmet_afos_message'
__key__ = 'id' __key__ = 'id'
__columns__ = ( __columns__ = (

10
lib/nexrad/archive.py → lib/xmet/archive.py
View file

@ -3,9 +3,9 @@ import re
import enum import enum
import datetime import datetime
from nexrad.db import Database from xmet.db import Database
from nexrad.s3 import S3Bucket, S3_KEY_RE from xmet.s3 import S3Bucket, S3_KEY_RE
from nexrad.radar import RADAR_RANGE from xmet.radar import RADAR_RANGE
class ArchiveDateError(Exception): class ArchiveDateError(Exception):
def __init__(self, supplied: str, missing: str): def __init__(self, supplied: str, missing: str):
@ -98,8 +98,8 @@ class ArchiveProduct():
radar.coord, radar.coord,
true) as distance true) as distance
from from
nexrad_storm_event as event, xmet_storm_event as event,
nexrad_radar as radar xmet_nexrad_radar as radar
where where
distance <= :radius distance <= :radius
and :timestamp between event.timestamp_start and event.timestamp_end and :timestamp between event.timestamp_start and event.timestamp_end

0
lib/nexrad/coord.py → lib/xmet/coord.py
View file

0
lib/nexrad/db.py → lib/xmet/db.py
View file

6
lib/nexrad/radar.py → lib/xmet/radar.py
View file

@ -2,8 +2,8 @@ import csv
import re import re
import shapely import shapely
from nexrad.db import DatabaseTable from xmet.db import DatabaseTable
from nexrad.coord import COORD_SYSTEM from xmet.coord import COORD_SYSTEM
""" """
Implements a parser and wrapper class for the WSR-88D radar list Implements a parser and wrapper class for the WSR-88D radar list
@ -47,7 +47,7 @@ class Radar(DatabaseTable):
'call', 'wban', 'name', 'coord', 'site_elevation', 'tower_height', 'call', 'wban', 'name', 'coord', 'site_elevation', 'tower_height',
) )
__table__ = 'nexrad_radar' __table__ = 'xmet_nexrad_radar'
__key__ = 'call' __key__ = 'call'
__columns__ = ( __columns__ = (

0
lib/nexrad/s3.py → lib/xmet/s3.py
View file

12
lib/nexrad/storm.py → lib/xmet/storm.py
View file

@ -4,9 +4,9 @@ import csv
import datetime import datetime
import shapely import shapely
from nexrad.db import DatabaseTable from xmet.db import DatabaseTable
from nexrad.coord import COORD_SYSTEM from xmet.coord import COORD_SYSTEM
from nexrad.radar import RADAR_RANGE from xmet.radar import RADAR_RANGE
def time_from_str(time: str): def time_from_str(time: str):
size = len(time) size = len(time)
@ -88,7 +88,7 @@ class StormEvent(DatabaseTable):
'tornado_f_rating', 'coord_start', 'coord_end' 'tornado_f_rating', 'coord_start', 'coord_end'
) )
__table__ = 'nexrad_storm_event' __table__ = 'xmet_storm_event'
__key__ = 'id' __key__ = 'id'
__columns__ = ( __columns__ = (
@ -206,7 +206,7 @@ class StormEvent(DatabaseTable):
'timestamp': str(timestamp) 'timestamp': str(timestamp)
}) })
sql = "select " + ", ".join(columns) + " from nexrad_storm_event" sql = "select " + ", ".join(columns) + " from xmet_storm_event"
if len(clauses) > 0: if len(clauses) > 0:
sql += " where " + " and ".join(clauses) sql += " where " + " and ".join(clauses)
@ -262,7 +262,7 @@ class StormEvent(DatabaseTable):
MakePoint(?, ?, {csr})), MakePoint(?, ?, {csr})),
true) as distance true) as distance
from from
nexrad_radar xmet_nexrad_radar
where where
distance <= ? distance <= ?
order by order by

0
lib/nexrad/vtec.py → lib/xmet/vtec.py
View file

6
lib/nexrad/wfo.py → lib/xmet/wfo.py
View file

@ -1,15 +1,15 @@
import csv import csv
import shapely import shapely
from nexrad.db import DatabaseTable from xmet.db import DatabaseTable
from nexrad.coord import COORD_SYSTEM from xmet.coord import COORD_SYSTEM
class WFO(DatabaseTable): class WFO(DatabaseTable):
__slots__ = ( __slots__ = (
'code', 'city', 'state', 'address', 'coord' 'code', 'city', 'state', 'address', 'coord'
) )
__table__ = 'nexrad_wfo' __table__ = 'xmet_wfo'
__key__ = 'code' __key__ = 'code'
__columns__ = ( __columns__ = (

2
run.sh
View file

@ -8,4 +8,4 @@ shift 2
docker run --rm -it \ docker run --rm -it \
--volume "$events_csv_gz:/tmp/events.csv.gz" \ --volume "$events_csv_gz:/tmp/events.csv.gz" \
--volume "$dest:/dest" \ --volume "$dest:/dest" \
nexrad-archive:latest /tmp/events.csv.gz /dest "$@" xmet:latest /tmp/events.csv.gz /dest "$@"