xan/xmet
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implement --type to only archive certain types of data

Browse source
This commit is contained in:
XANTRONIX Industrial 2025-02-16 11:13:41 -05:00
parent 0046c0c4fb
commit d72e639ba7
2 changed files with 22 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
README.md
View file

@ -49,4 +49,9 @@ Other optional arguments will be passed through to the
Exclude one or more event types from consideration for archival. Exclude one or more event types from consideration for archival.
These event types are These event types are
[documented](https://www.ncei.noaa.gov/pub/data/swdi/stormevents/csvfiles/Storm-Data-Bulk-csv-Format.pdf) [documented](https://www.ncei.noaa.gov/pub/data/swdi/stormevents/csvfiles/Storm-Data-Bulk-csv-Format.pdf)
by the NCEI. by the NCEI. Must not be specified alongside `--type`.
* `--type="Event Type"`
Explicitly specify which event types to archive. Must not be
specified alongside `--exclude`.

21
bin/nexrad-archive
View file

@ -13,7 +13,11 @@ parser = argparse.ArgumentParser(
parser.add_argument('--quiet', action='store_true', help='Suppress output') parser.add_argument('--quiet', action='store_true', help='Suppress output')
parser.add_argument('--dry-run', action='store_true', help='Do not actually archive data') parser.add_argument('--dry-run', action='store_true', help='Do not actually archive data')
parser.add_argument('--exclude', action='append', type=str, help='Exclude types of reports from ingest')
group = parser.add_mutually_exclusive_group()
group.add_argument('--exclude', action='append', type=str, help='Exclude types of reports from ingest')
group.add_argument('--type', action='append', type=str, help='Specify only given types of reports to ingest')
parser.add_argument('db', help='SQLite3 NEXRAD radar site database') parser.add_argument('db', help='SQLite3 NEXRAD radar site database')
parser.add_argument('csv-report-details', nargs='+', help='Compressed storm report details CSV file') parser.add_argument('csv-report-details', nargs='+', help='Compressed storm report details CSV file')
parser.add_argument('archive-dir', help='Target archive directory') parser.add_argument('archive-dir', help='Target archive directory')
@ -23,14 +27,21 @@ args = parser.parse_args()
db = Database.connect(args.db) db = Database.connect(args.db)
bucket = S3Bucket() bucket = S3Bucket()
archive = Archive(getattr(args, 'archive-dir'), bucket) archive = Archive(getattr(args, 'archive-dir'), bucket)
exclude = dict() exclude = None
types = None
for event_type in args.exclude: if args.exclude is not None:
exclude[event_type] = True exclude = {s: True for s in args.exclude}
if args.type is not None:
types = {s: True for s in args.type}
for path in getattr(args, 'csv-report-details'): for path in getattr(args, 'csv-report-details'):
for report in StormReport.each_from_csv_file(path): for report in StormReport.each_from_csv_file(path):
if report.event_type in exclude: if args.exclude is not None and report.event_type in exclude:
continue
if args.type is not None and report.event_type not in types:
continue continue
if report.coord_start is None or report.coord_end is None: if report.coord_start is None or report.coord_end is None: