Move each_chunk into util.py

bin/xmet-afos-ingest

@@ -4,30 +4,8 @@ import argparse
 
 from xmet.db   import Database
 from xmet.afos import AFOSMessageParser
+from xmet.util import each_chunk
 
-CHUNK_SIZE = 4096
-
-def each_chunk(fh, sep: str):
-    buf = ''
-
-    while True:
-        chunk = fh.read(CHUNK_SIZE)
-
-        if chunk == '' or chunk is None:
-            yield buf.strip()
-            break
-
-        buf += chunk
-
-        while True:
-            try:
-                part, buf = buf.split(sep, 1)
-            except ValueError:
-                break
-            else:
-                yield part.strip()
-
-parser = argparse.ArgumentParser(
     description = 'Ingest National Weather Service text bulletin products'
 )
 

lib/xmet/afos.py

@@ -10,7 +10,7 @@ from xmet.vtec  import VTECEvent, VTECHydroEvent
 RE_ID = re.compile(r'^(\d+)$')
 
 RE_ISSUANCE = re.compile(r'''
-    ^ ([A-Z]{4}\d+)
+      ^ ([A-Z]{4}\d+)
     \s+ (?P<wfo>[A-Z]{4})
     \s+ (?P<day>\d{2}) (?P<hour>\d{2}) (?P<minute>\d{2})
 ''', re.X)

lib/xmet/util.py

@@ -0,0 +1,31 @@
+import io
+
+CHUNK_SIZE  = 4096
+CHUNK_STRIP = "\x01\x03\x0a\x20"
+
+def each_chunk(fh: io.TextIOBase, sep: str, strip=None):
+    buf = ''
+
+    while True:
+        chunk = fh.read(CHUNK_SIZE)
+
+        if chunk == '':
+            ret = buf.strip(strip)
+
+            if ret != '':
+                yield ret
+
+            break
+
+        buf += chunk
+
+        while True:
+            try:
+                part, buf = buf.split(sep, 1)
+            except ValueError:
+                break
+            else:
+                ret = part.strip(strip)
+
+                if ret != '':
+                    yield ret