Update data_extraction.py

data_extraction.py

@@ -43,33 +43,90 @@ INSTRUCTIONS = {
 # --------------------------------------------------------------------------
 # Loading
 # --------------------------------------------------------------------------
+def _absorb(obj, records):
+    """Given a parsed JSON value, pull out record dicts from known shapes."""
+    if isinstance(obj, dict) and "response" in obj:
+        records.extend(obj["response"])
+    elif isinstance(obj, list):
+        records.extend(obj)
+    elif isinstance(obj, dict):
+        records.append(obj)
+
+
+def _parse_concatenated_json(raw):
+    """
+    Parses a string containing one or more back-to-back JSON values
+    (e.g. multiple {"response": [...]} wrapper objects with no separator,
+    or no enclosing list) using a streaming decoder.
+    Returns a list of parsed top-level JSON values.
+    """
+    decoder = json.JSONDecoder()
+    values = []
+    idx = 0
+    n = len(raw)
+    while idx < n:
+        while idx < n and raw[idx].isspace():
+            idx += 1
+        if idx >= n:
+            break
+        obj, end = decoder.raw_decode(raw, idx)
+        values.append(obj)
+        idx = end
+    return values
+
+
 def load_records(filepaths):
     """
-    Handles three shapes:
+    Handles several shapes robustly:
       1) {"response": [ {...}, {...}, ... ]}   (paginated API wrapper)
       2) [ {...}, {...}, ... ]                 (plain list of records)
       3) JSON Lines: one record dict per line
-    Returns a flat list of record dicts.
+      4) Multiple JSON values concatenated back-to-back in one file
+    Also strips BOM and skips empty files, reporting problems per-file
+    instead of crashing the whole run.
     """
     records = []
     for fp in filepaths:
-        with open(fp, "r", encoding="utf-8") as fh:
+        with open(fp, "r", encoding="utf-8-sig") as fh:  # utf-8-sig strips BOM if present
             raw = fh.read().strip()
+
+        if not raw:
+            print(f"  [skip] {fp} is empty")
+            continue
+
         try:
             obj = json.loads(raw)
-            if isinstance(obj, dict) and "response" in obj:
-                records.extend(obj["response"])
-            elif isinstance(obj, list):
-                records.extend(obj)
-            elif isinstance(obj, dict):
-                records.append(obj)
+            _absorb(obj, records)
+            continue
+        except json.JSONDecodeError:
+            pass
+
+        try:
+            values = _parse_concatenated_json(raw)
+            if values:
+                for v in values:
+                    _absorb(v, records)
+                print(f"  [ok] {fp}: parsed as {len(values)} concatenated JSON value(s)")
+                continue
         except json.JSONDecodeError:
-            # fall back to JSON Lines
+            pass
+
+        try:
+            line_records = []
             for line in raw.splitlines():
                 line = line.strip()
                 if not line:
                     continue
-                records.append(json.loads(line))
+                line_records.append(json.loads(line))
+            for v in line_records:
+                _absorb(v, records)
+            print(f"  [ok] {fp}: parsed as JSON Lines ({len(line_records)} lines)")
+            continue
+        except json.JSONDecodeError as e:
+            print(f"  [FAILED] {fp}: could not parse as JSON, concatenated JSON, or JSON Lines.")
+            print(f"           First 200 chars: {raw[:200]!r}")
+            print(f"           Error: {e}")
+
     return records