diff --git a/bin/xenu-nntp-import b/bin/xenu-nntp-import new file mode 100755 index 0000000..5cd553e --- /dev/null +++ b/bin/xenu-nntp-import @@ -0,0 +1,75 @@ +#! /usr/bin/env python3 + +import sys +import argparse +import traceback + +from xenu_nntp.config import Config +from xenu_nntp.db import Database +from xenu_nntp.newsgroup import Newsgroup +from xenu_nntp.mbox import MBoxReader + +class ImportException(Exception): + def __init__(self, path: str, line: int, error: str): + self.path = path + self.line = line + self.error = error + + def __str__(self): + return "Error importing %s at line %d: %s" % ( + self.path, self.line, self.error + ) + +class Importer(): + def __init__(self, db: Database, newsgroup: Newsgroup, args): + self.db = db + self.newsgroup = newsgroup + self.args = args + + def import_mbox_file(db: Database, newsgroup: Newsgroup, mboxfile: str): + reader = MBoxReader(mboxfile) + + try: + for message in reader.messages(): + if self.args.verbose and not self.args.quiet: + print(f"Importing message {message.id}") + + db.add(message) + + db.execute(f'insert into newsgroup_message values (%s, %s)', + (newsgroup.id, message.id)) + except Exception as e: + raise ImportException(mboxfile, reader.line, str(e)) + +parser = argparse.ArgumentParser(description='Create new account') +parser.add_argument('--dry-run', action='store_true', help='Do not commit to database') +parser.add_argument('--quiet', action='store_true', help='Do not print errors') +parser.add_argument('--verbose', action='store_true', help='Print messages when importing files and messages') +parser.add_argument('--ignore-errors', action='store_true', help='Do not exit on encountering errors') +parser.add_argument('--config-file', '-f', type=str, help='Specify a configuration file location') +parser.add_argument('newsgroup', type=str, help='Name of newsgroup to ingest articles into') +parser.add_argument('mboxfile', type=str, nargs='+', help='mbox file to ingest') + +args = parser.parse_args() + +config = Config.load(args.config_file) +db = Database.from_config(config) +newsgroup = db.get(Newsgroup, {'name': args.newsgroup}) + +db.execute("begin transaction") + +for mboxfile in args.mboxfile: + if args.verbose and not args.quiet: + print(f"Importing mbox file {mboxfile}") + + try: + import_mbox_file(db, newsgroup, mboxfile) + except ImportException as e: + if not args.quiet: + print(str(e), file=sys.stderr) + + if not args.ignore_errors: + exit(1) + +if not args.dry_run: + db.commit()