xmet/bin/xmet-afos-ingest

#! /usr/bin/env python3

import sys

from xmet.db   import Database
from xmet.afos import AFOSMessageParser

CHUNK_SIZE = 4096

def each_chunk(fh, sep: str):
    buf = ''

    while True:
        chunk = fh.read(CHUNK_SIZE)

        if chunk == '' or chunk is None:
            yield buf.strip()
            break

        buf += chunk

        while True:
            try:
                part, buf = buf.split(sep, 1)
            except ValueError:
                break
            else:
                yield part.strip()

db = Database.connect(sys.argv[1])
db.execute('begin transaction')

parser = AFOSMessageParser()

for path in sys.argv[2:]:
    with open(path, 'r') as fh:
        for data in each_chunk(fh, '\x01'):
            if len(data) == 0:
                continue

            try:
                message = parser.parse(data)

                db.add(message)
            except:
                pass

db.commit()