65 lines
2.5 KiB
Python
Executable file
65 lines
2.5 KiB
Python
Executable file
#! /usr/bin/env python3
|
|
|
|
import sys
|
|
import argparse
|
|
import traceback
|
|
|
|
from xenu_nntp.config import Config
|
|
from xenu_nntp.db import Database
|
|
from xenu_nntp.newsgroup import Newsgroup
|
|
from xenu_nntp.mbox import MBoxReader
|
|
|
|
class Importer():
|
|
def __init__(self, db: Database, newsgroup: Newsgroup, args):
|
|
self.db = db
|
|
self.newsgroup = newsgroup
|
|
self.args = args
|
|
|
|
def print_error(self, mboxfile: str, reader: MBoxReader, e: Exception):
|
|
print(f"Error importing file {mboxfile} on line {reader.line} to newsgroup {self.newsgroup.name}: {e}")
|
|
|
|
def import_mbox_file(self, mboxfile: str):
|
|
if self.args.verbose and not self.args.quiet:
|
|
print(f"Reading file {mboxfile}")
|
|
|
|
reader = MBoxReader(mboxfile)
|
|
|
|
for message in reader.messages():
|
|
if self.args.verbose and not self.args.quiet:
|
|
print(f"Importing message {message.message_id} to newsgroup {self.newsgroup.name}")
|
|
|
|
try:
|
|
self.db.add(message)
|
|
|
|
self.db.execute(f'insert into newsgroup_message values (%s, %s)',
|
|
(self.newsgroup.id, message.id))
|
|
except Exception as e:
|
|
if not self.args.quiet:
|
|
self.print_error(mboxfile, reader, e)
|
|
|
|
if not self.args.ignore_errors:
|
|
exit(1)
|
|
|
|
parser = argparse.ArgumentParser(description='Create new account')
|
|
parser.add_argument('--dry-run', action='store_true', help='Do not commit to database')
|
|
parser.add_argument('--quiet', action='store_true', help='Do not print errors')
|
|
parser.add_argument('--verbose', action='store_true', help='Print messages when importing files and messages')
|
|
parser.add_argument('--ignore-errors', action='store_true', help='Do not exit on encountering errors')
|
|
parser.add_argument('--config-file', '-f', type=str, help='Specify a configuration file location')
|
|
parser.add_argument('newsgroup', type=str, help='Name of newsgroup to ingest articles into')
|
|
parser.add_argument('mboxfile', type=str, nargs='+', help='mbox file to ingest')
|
|
|
|
args = parser.parse_args()
|
|
|
|
config = Config.load(args.config_file)
|
|
db = Database.from_config(config)
|
|
newsgroup = db.get(Newsgroup, {'name': args.newsgroup})
|
|
importer = Importer(db, newsgroup, args)
|
|
|
|
db.execute("begin transaction")
|
|
|
|
for mboxfile in args.mboxfile:
|
|
importer.import_mbox_file(mboxfile)
|
|
|
|
if not args.dry_run:
|
|
db.commit()
|