#! /usr/bin/env python3 import argparse from nexrad.db import Database from nexrad.s3 import S3Bucket from nexrad.storm import StormReport from nexrad.archive import Archive parser = argparse.ArgumentParser( description = 'Archive NEXRAD Level II data from Amazon S3' ) parser.add_argument('--quiet', action='store_true', help='Suppress output') parser.add_argument('--dry-run', action='store_true', help='Do not actually archive data') group = parser.add_mutually_exclusive_group() group.add_argument('--exclude', action='append', type=str, help='Exclude types of reports from ingest') group.add_argument('--type', action='append', type=str, help='Specify only given types of reports to ingest') parser.add_argument('db', help='SQLite3 NEXRAD radar site database') parser.add_argument('csv-report-details', nargs='+', help='Compressed storm report details CSV file') parser.add_argument('archive-dir', help='Target archive directory') args = parser.parse_args() db = Database.connect(args.db) bucket = S3Bucket() archive = Archive(getattr(args, 'archive-dir'), bucket) exclude = None types = None if args.exclude is not None: exclude = {s: True for s in args.exclude} if args.type is not None: types = {s: True for s in args.type} for path in getattr(args, 'csv-report-details'): for report in StormReport.each_from_csv_file(path): if args.exclude is not None and report.event_type in exclude: continue if args.type is not None and report.event_type not in types: continue if report.coord_start is None or report.coord_end is None: continue if not report.is_radar_significant(): continue radars = report.nearby_radars(db) for key in bucket.each_matching_key(radars, report.timestamp_start, report.timestamp_end): if archive.is_downloaded(key): if not args.quiet: print(f"event {report.id} key {key} type {report.event_type} already archived") else: if not args.quiet: if args.dry_run: print(f"event {report.id} key {key} type {report.event_type} would archive") else: print(f"event {report.id} key {key} type {report.event_type} archiving") if not args.dry_run: archive.download(key)