Added Delimiter-Sniffer Logic

app.py

@@ -56,7 +56,7 @@ FILTER_DOMINANT_COUNTY = False  # process ALL counties (multi-county / full-stat
 #   "csv"   -> Master_Dataset.zip, every result bundled as a CSV (faster: no Excel
 #              writing -- use this for very large / state-sized runs)
 # The individual per-category files are CSV in both modes.
-MASTER_OUTPUT = "csv"
+MASTER_OUTPUT = "excel"
 
 # How records are processed:
 #   "stream" -> ONE COUNTY AT A TIME. The input is split into per-county temp
@@ -117,10 +117,49 @@ def get_safety_list_path() -> str:
 _SAFETY_CACHE: dict | None = None
 
 
+def detect_delimiter(path, candidates=(",", "|", "^", "\t", ";")):
+    """Sniff a text file's field delimiter from its first lines. SQL exports here
+    are sometimes '^' or '|' rather than ','. Returns the delimiter char (default
+    ','). Detects once from a small header sample so the actual (possibly chunked)
+    read can still use pandas' fast C parser with an explicit sep."""
+    try:
+        with open(path, "r", encoding="utf-8", errors="replace", newline="") as fh:
+            lines = []
+            for _ in range(8):
+                ln = fh.readline()
+                if not ln:
+                    break
+                ln = ln.rstrip("\r\n")
+                if ln:
+                    lines.append(ln)
+    except Exception:
+        return ","
+    if not lines:
+        return ","
+    best, best_score = ",", -1
+    for d in candidates:
+        header_count = lines[0].count(d)
+        if header_count == 0:
+            continue
+        # reward a high, consistent field count across the sampled rows
+        consistent = sum(1 for ln in lines if ln.count(d) == header_count)
+        score = header_count * 100 + consistent
+        if score > best_score:
+            best, best_score = d, score
+    return best
+
+
+def _read_csv_auto(path, **kwargs):
+    """pd.read_csv with the delimiter auto-detected (unless caller passed sep)."""
+    if "sep" not in kwargs:
+        kwargs["sep"] = detect_delimiter(path)
+    return pd.read_csv(path, **kwargs)
+
+
 def build_whitelist_from_path(path):
     """Parse a safety-list file (xlsx or csv) into {pairs, cities, rows, fips_name}."""
     if str(path).lower().endswith(".csv"):
-        safety_df = pd.read_csv(path)
+        safety_df = _read_csv_auto(path)
     else:
         safety_df = pd.read_excel(path)
     acceptable_cols = [c for c in safety_df.columns if str(c).startswith("Acceptable FIPS")]
@@ -661,10 +700,13 @@ def run_pipeline(uploaded_files, reference_file, blank_cities_not_in_list,
         name = f.name
         try:
             if str(name).lower().endswith(".csv"):
-                dfs.append(pd.read_csv(name, low_memory=False))
+                sep = detect_delimiter(name)
+                dfs.append(pd.read_csv(name, low_memory=False, sep=sep))
+                _step(log, f"Loaded {Path(name).name}  ({len(dfs[-1]):,} rows, "
+                           f"delimiter {sep!r})")
             else:
                 dfs.append(pd.read_excel(name))
-            _step(log, f"Loaded {Path(name).name}  ({len(dfs[-1]):,} rows)")
+                _step(log, f"Loaded {Path(name).name}  ({len(dfs[-1]):,} rows)")
         except Exception as e:
             _step(log, f"ERROR loading {Path(name).name}: {e}")
     if not dfs:
@@ -1235,7 +1277,7 @@ def _split_by_county(uploaded_files, out_dir, log, chunksize=STREAM_CHUNK_ROWS):
     # detect the FIPS-county column from a small sample of the first file
     first = uploaded_files[0].name
     if str(first).lower().endswith(".csv"):
-        sample = pd.read_csv(first, nrows=50_000, low_memory=False)
+        sample = pd.read_csv(first, nrows=50_000, low_memory=False, sep=detect_delimiter(first))
     else:
         sample = pd.read_excel(first)
     fips_col, reason = detect_fips_county_column(sample)
@@ -1263,7 +1305,9 @@ def _split_by_county(uploaded_files, out_dir, log, chunksize=STREAM_CHUNK_ROWS):
     for f in uploaded_files:
         nm = f.name
         if str(nm).lower().endswith(".csv"):
-            for chunk in pd.read_csv(nm, chunksize=chunksize, low_memory=False):
+            sep = detect_delimiter(nm)
+            _step(log, f"Reading {Path(nm).name} (delimiter {sep!r})")
+            for chunk in pd.read_csv(nm, chunksize=chunksize, low_memory=False, sep=sep):
                 route(chunk)
                 total += len(chunk)
                 _step(log, f"Split: routed {total:,} rows into {len(county_paths)} counties")