From d72e639ba7e54659fa414c9d6be0d2520835ebaf Mon Sep 17 00:00:00 2001 From: XANTRONIX Industrial Date: Sun, 16 Feb 2025 11:13:41 -0500 Subject: [PATCH] Implement --type to only archive certain types of data --- README.md | 7 ++++++- bin/nexrad-archive | 21 ++++++++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 6664b54..82694ef 100644 --- a/README.md +++ b/README.md @@ -49,4 +49,9 @@ Other optional arguments will be passed through to the Exclude one or more event types from consideration for archival. These event types are [documented](https://www.ncei.noaa.gov/pub/data/swdi/stormevents/csvfiles/Storm-Data-Bulk-csv-Format.pdf) - by the NCEI. + by the NCEI. Must not be specified alongside `--type`. + +* `--type="Event Type"` + + Explicitly specify which event types to archive. Must not be + specified alongside `--exclude`. diff --git a/bin/nexrad-archive b/bin/nexrad-archive index fcdc0e9..b1240c6 100755 --- a/bin/nexrad-archive +++ b/bin/nexrad-archive @@ -13,7 +13,11 @@ parser = argparse.ArgumentParser( parser.add_argument('--quiet', action='store_true', help='Suppress output') parser.add_argument('--dry-run', action='store_true', help='Do not actually archive data') -parser.add_argument('--exclude', action='append', type=str, help='Exclude types of reports from ingest') + +group = parser.add_mutually_exclusive_group() +group.add_argument('--exclude', action='append', type=str, help='Exclude types of reports from ingest') +group.add_argument('--type', action='append', type=str, help='Specify only given types of reports to ingest') + parser.add_argument('db', help='SQLite3 NEXRAD radar site database') parser.add_argument('csv-report-details', nargs='+', help='Compressed storm report details CSV file') parser.add_argument('archive-dir', help='Target archive directory') @@ -23,14 +27,21 @@ args = parser.parse_args() db = Database.connect(args.db) bucket = S3Bucket() archive = Archive(getattr(args, 'archive-dir'), bucket) -exclude = dict() +exclude = None +types = None -for event_type in args.exclude: - exclude[event_type] = True +if args.exclude is not None: + exclude = {s: True for s in args.exclude} + +if args.type is not None: + types = {s: True for s in args.type} for path in getattr(args, 'csv-report-details'): for report in StormReport.each_from_csv_file(path): - if report.event_type in exclude: + if args.exclude is not None and report.event_type in exclude: + continue + + if args.type is not None and report.event_type not in types: continue if report.coord_start is None or report.coord_end is None: