Updated app.py with EOF Error Handling logic for ghost characters

app.py

@@ -243,13 +243,56 @@ def detect_delimiter(path, candidates=(",", "|", "^", "\t", ";", "\x1f", "\x1e")
     return best
 
 
+_CTRL_DELETE = {c: None for c in range(32)}
+for _keep in (9, 10, 13):            # keep tab, newline, carriage return
+    _CTRL_DELETE.pop(_keep, None)
+_CTRL_DELETE[127] = None             # DEL
+
+
+def _strip_ctrl(s):
+    """Remove C0 control bytes (NUL, SUB/'EOF', and the rest) except tab/LF/CR.
+    A stray control byte embedded in a text field (e.g. an address or legal
+    description from the source system) otherwise either aborts pandas' C parser
+    or silently truncates the field at a fake end-of-file."""
+    return s.translate(_CTRL_DELETE)
+
+
+class _CleanReader:
+    """Text file-like wrapper that strips control bytes as the file is read, so a
+    stray control character in one field can't break the parse. Works with pandas'
+    chunked and whole-file reads alike; opened with errors='replace' so stray
+    non-UTF-8 bytes don't abort the read either."""
+    def __init__(self, path, encoding="utf-8", errors="replace"):
+        self._f = open(path, "r", encoding=encoding, errors=errors, newline="")
+
+    def read(self, size=-1):
+        return _strip_ctrl(self._f.read(size))
+
+    def readline(self, size=-1):
+        return _strip_ctrl(self._f.readline(size))
+
+    def __iter__(self):
+        for line in self._f:
+            yield _strip_ctrl(line)
+
+    def close(self):
+        self._f.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc):
+        self.close()
+
+
 def _read_csv_auto(path, **kwargs):
     """pd.read_csv with the delimiter auto-detected (unless caller passed sep),
     tolerating malformed rows (a stray delimiter in one row won't abort the read)."""
     if "sep" not in kwargs:
         kwargs["sep"] = detect_delimiter(path)
     kwargs.setdefault("on_bad_lines", "skip")
-    return pd.read_csv(path, **kwargs)
+    with _CleanReader(path) as fh:
+        return pd.read_csv(fh, **kwargs)
 
 
 def scan_bad_rows(path, sep, out_dir):
@@ -262,12 +305,12 @@ def scan_bad_rows(path, sep, out_dir):
     bw = None
     try:
         with open(path, "r", encoding="utf-8", errors="replace", newline="") as fin:
-            header = fin.readline()
+            header = _strip_ctrl(fin.readline())
             if not header:
                 return 0, None
             base = header.rstrip("\r\n").count(sep)   # header delimiter count
             for lineno, line in enumerate(fin, start=2):
-                s = line.rstrip("\r\n")
+                s = _strip_ctrl(line).rstrip("\r\n")
                 if not s:
                     continue
                 if s.count(sep) > base:               # too many fields -> skipped by pandas
@@ -850,8 +893,9 @@ def run_pipeline(uploaded_files, reference_file, blank_cities_not_in_list,
         try:
             if str(name).lower().endswith(".csv"):
                 sep = detect_delimiter(name)
-                dfs.append(pd.read_csv(name, low_memory=False, sep=sep,
-                                       on_bad_lines="skip"))
+                with _CleanReader(name) as _fh:
+                    dfs.append(pd.read_csv(_fh, low_memory=False, sep=sep,
+                                           on_bad_lines="skip"))
                 _step(log, f"Loaded {Path(name).name}  ({len(dfs[-1]):,} rows, "
                            f"delimiter {sep!r})")
             else:
@@ -1501,8 +1545,9 @@ def _split_by_county(uploaded_files, out_dir, log, chunksize=STREAM_CHUNK_ROWS):
     # detect the FIPS-county column from a small sample of the first file
     first = uploaded_files[0].name
     if str(first).lower().endswith(".csv"):
-        sample = pd.read_csv(first, nrows=50_000, low_memory=False,
-                             sep=detect_delimiter(first), on_bad_lines="skip")
+        with _CleanReader(first) as _fh:
+            sample = pd.read_csv(_fh, nrows=50_000, low_memory=False,
+                                 sep=detect_delimiter(first), on_bad_lines="skip")
     else:
         sample = pd.read_excel(first)
     fips_col, reason = detect_fips_county_column(sample)
@@ -1536,11 +1581,12 @@ def _split_by_county(uploaded_files, out_dir, log, chunksize=STREAM_CHUNK_ROWS):
         if str(nm).lower().endswith(".csv"):
             sep = detect_delimiter(nm)
             _step(log, f"Reading {Path(nm).name} (delimiter {sep!r})")
-            for chunk in pd.read_csv(nm, chunksize=chunksize, low_memory=False,
-                                     sep=sep, on_bad_lines="skip"):
-                route(chunk)
-                total += len(chunk)
-                _step(log, f"Split: routed {total:,} rows into {len(county_paths)} counties")
+            with _CleanReader(nm) as _fh:
+                for chunk in pd.read_csv(_fh, chunksize=chunksize, low_memory=False,
+                                         sep=sep, on_bad_lines="skip"):
+                    route(chunk)
+                    total += len(chunk)
+                    _step(log, f"Split: routed {total:,} rows into {len(county_paths)} counties")
         else:
             chunk = pd.read_excel(nm)
             route(chunk)