diff --git a/bin/xmet-afos-ingest b/bin/xmet-afos-ingest index 24f7a05..548021d 100755 --- a/bin/xmet-afos-ingest +++ b/bin/xmet-afos-ingest @@ -4,30 +4,8 @@ import argparse from xmet.db import Database from xmet.afos import AFOSMessageParser +from xmet.util import each_chunk -CHUNK_SIZE = 4096 - -def each_chunk(fh, sep: str): - buf = '' - - while True: - chunk = fh.read(CHUNK_SIZE) - - if chunk == '' or chunk is None: - yield buf.strip() - break - - buf += chunk - - while True: - try: - part, buf = buf.split(sep, 1) - except ValueError: - break - else: - yield part.strip() - -parser = argparse.ArgumentParser( description = 'Ingest National Weather Service text bulletin products' ) diff --git a/lib/xmet/util.py b/lib/xmet/util.py new file mode 100644 index 0000000..f6ccf12 --- /dev/null +++ b/lib/xmet/util.py @@ -0,0 +1,31 @@ +import io + +CHUNK_SIZE = 4096 +CHUNK_STRIP = "\x01\x03\x0a\x20" + +def each_chunk(fh: io.TextIOBase, sep: str, strip=None): + buf = '' + + while True: + chunk = fh.read(CHUNK_SIZE) + + if chunk == '': + ret = buf.strip(strip) + + if ret != '': + yield ret + + break + + buf += chunk + + while True: + try: + part, buf = buf.split(sep, 1) + except ValueError: + break + else: + ret = part.strip(strip) + + if ret != '': + yield ret