67 lines
2.4 KiB
Python
Executable file
67 lines
2.4 KiB
Python
Executable file
#! /usr/bin/env python3
|
|
|
|
import argparse
|
|
|
|
from nexrad.db import Database
|
|
from nexrad.s3 import S3Bucket
|
|
from nexrad.storm import StormReport
|
|
from nexrad.archive import Archive
|
|
|
|
parser = argparse.ArgumentParser(
|
|
description = 'Archive NEXRAD Level II data from Amazon S3'
|
|
)
|
|
|
|
parser.add_argument('--quiet', action='store_true', help='Suppress output')
|
|
parser.add_argument('--dry-run', action='store_true', help='Do not actually archive data')
|
|
|
|
group = parser.add_mutually_exclusive_group()
|
|
group.add_argument('--exclude', action='append', type=str, help='Exclude types of reports from ingest')
|
|
group.add_argument('--type', action='append', type=str, help='Specify only given types of reports to ingest')
|
|
|
|
parser.add_argument('db', help='SQLite3 NEXRAD radar site database')
|
|
parser.add_argument('csv-report-details', nargs='+', help='Compressed storm report details CSV file')
|
|
parser.add_argument('archive-dir', help='Target archive directory')
|
|
|
|
args = parser.parse_args()
|
|
|
|
db = Database.connect(args.db)
|
|
bucket = S3Bucket()
|
|
archive = Archive(getattr(args, 'archive-dir'), bucket)
|
|
exclude = None
|
|
types = None
|
|
|
|
if args.exclude is not None:
|
|
exclude = {s: True for s in args.exclude}
|
|
|
|
if args.type is not None:
|
|
types = {s: True for s in args.type}
|
|
|
|
for path in getattr(args, 'csv-report-details'):
|
|
for report in StormReport.each_from_csv_file(path):
|
|
if args.exclude is not None and report.event_type in exclude:
|
|
continue
|
|
|
|
if args.type is not None and report.event_type not in types:
|
|
continue
|
|
|
|
if report.coord_start is None or report.coord_end is None:
|
|
continue
|
|
|
|
if not report.is_radar_significant():
|
|
continue
|
|
|
|
radars = report.nearby_radars(db)
|
|
|
|
for key in bucket.each_matching_key(radars, report.timestamp_start, report.timestamp_end):
|
|
if archive.is_downloaded(key):
|
|
if not args.quiet:
|
|
print(f"event {report.id} key {key} type {report.event_type} already archived")
|
|
else:
|
|
if not args.quiet:
|
|
if args.dry_run:
|
|
print(f"event {report.id} key {key} type {report.event_type} would archive")
|
|
else:
|
|
print(f"event {report.id} key {key} type {report.event_type} archiving")
|
|
|
|
if not args.dry_run:
|
|
archive.download(key)
|