Enabled CSV type file as input

app.py

@@ -50,6 +50,14 @@ CLUSTER_THRESHOLD = 95        # RapidFuzz similarity for LegalSubDivision cluste
 REPLACE_THRESHOLD = 70        # majority-vote % required to fill a blank
 FILTER_DOMINANT_COUNTY = False  # process ALL counties (multi-county / full-state files)
 
+# Master bundle format:
+#   "excel" -> Master_Dataset.xlsx, every result on its own sheet (full output
+#              split across Output_..._1/_2/... if it exceeds Excel's row cap)
+#   "csv"   -> Master_Dataset.zip, every result bundled as a CSV (faster: no Excel
+#              writing -- use this for very large / state-sized runs)
+# The individual per-category files are CSV in both modes.
+MASTER_OUTPUT = "excel"
+
 
 def get_safety_list_path() -> str:
     local_name = os.environ.get("SAFETY_LIST_FILE", DEFAULT_SAFETY_LIST_NAME)
@@ -472,6 +480,54 @@ def save_csv_in_parts(df, base_filename, out_dir, row_limit=1_000_000):
     return created
 
 
+def write_master_workbook(path, sheets_in_order, row_limit=1_048_575):
+    """Write a multi-sheet .xlsx from [(sheet_name, df), ...]. Any sheet larger than
+    Excel's 1,048,576-row cap is split into <name>_1, <name>_2, ... so nothing is
+    dropped. Uses xlsxwriter in constant_memory mode, driven row-by-row (low RAM --
+    important for the full-output sheet on state-sized files). We write the rows
+    ourselves because the pandas + constant_memory path corrupts rows that contain
+    NaN cells. Falls back to openpyxl if xlsxwriter isn't installed."""
+    try:
+        import xlsxwriter
+    except Exception:
+        xlsxwriter = None
+
+    if xlsxwriter is not None:
+        wb = xlsxwriter.Workbook(str(path), {"constant_memory": True, "nan_inf_to_errors": True})
+
+        def _write(name, dfx):
+            ws = wb.add_worksheet(str(name)[:31])
+            ws.write_row(0, 0, [str(c) for c in dfx.columns])
+            clean = dfx.where(pd.notna(dfx), None)   # NaN -> blank cell
+            for r, rowvals in enumerate(clean.itertuples(index=False, name=None), start=1):
+                ws.write_row(r, 0, rowvals)
+
+        for name, dfx in sheets_in_order:
+            if dfx is None:
+                continue
+            n = len(dfx)
+            if n <= row_limit:
+                _write(name, dfx)
+            else:
+                for i in range(math.ceil(n / row_limit)):
+                    _write(f"{name}_{i + 1}", dfx.iloc[i * row_limit:(i + 1) * row_limit])
+                    gc.collect()
+        wb.close()
+    else:
+        with pd.ExcelWriter(path, engine="openpyxl") as writer:
+            for name, dfx in sheets_in_order:
+                if dfx is None:
+                    continue
+                n = len(dfx)
+                if n <= row_limit:
+                    dfx.to_excel(writer, sheet_name=str(name)[:31], index=False)
+                else:
+                    for i in range(math.ceil(n / row_limit)):
+                        dfx.iloc[i * row_limit:(i + 1) * row_limit].to_excel(
+                            writer, sheet_name=f"{name}_{i + 1}"[:31], index=False)
+    return str(path)
+
+
 # =====================================================================
 # MAIN PIPELINE
 # =====================================================================
@@ -500,14 +556,18 @@ def run_pipeline(uploaded_files, reference_file, blank_cities_not_in_list,
     except Exception as e:
         return "\n".join(log) + f"\n\nCould not parse reference list: {e}", []
 
-    # ---- Load & merge uploads ----------------------------------------
+    # ---- Load & merge uploads (Excel or CSV) -------------------------
     dfs = []
     for f in uploaded_files:
+        name = f.name
         try:
-            dfs.append(pd.read_excel(f.name))
-            log.append(f"Loaded {Path(f.name).name}")
+            if str(name).lower().endswith(".csv"):
+                dfs.append(pd.read_csv(name))
+            else:
+                dfs.append(pd.read_excel(name))
+            log.append(f"Loaded {Path(name).name}")
         except Exception as e:
-            log.append(f"ERROR loading {Path(f.name).name}: {e}")
+            log.append(f"ERROR loading {Path(name).name}: {e}")
     if not dfs:
         return "\n".join(log) + "\nNo files could be read.", []
     df = pd.concat(dfs, ignore_index=True)
@@ -753,20 +813,31 @@ def run_pipeline(uploaded_files, reference_file, blank_cities_not_in_list,
     summary_df.to_csv(summary_path, index=False)
     outputs.append(str(summary_path))
 
-    # ---- Master bundle: one ZIP of CSVs (no Excel row limit) ---------
-    # A single CSV can hold only one table, so the "master workbook" becomes a
-    # ZIP that bundles every result as its own CSV -- including the full output
-    # (chunked into parts), which no longer hits Excel's 1,048,576-row ceiling.
-    # CSV writing also avoids openpyxl's heavy memory use on huge files.
-    master_path = out_dir / "Master_Dataset.zip"
-    member_names = ["FIPS_Validation_Audit_ChangedRecords.csv", "cluster_df.csv",
-                    "final_cities.csv", "final_zips.csv", "final_census_tracts.csv",
-                    "Summary.csv"]
-    members = [out_dir / n for n in member_names if (out_dir / n).exists()]
-    members += sorted(out_dir.glob("OutputFile_*.csv"))   # full output (chunked)
-    with zipfile.ZipFile(master_path, "w", zipfile.ZIP_DEFLATED) as zf:
-        for p in members:
-            zf.write(p, arcname=p.name)
+    # ---- Master bundle: Excel workbook OR zip of CSVs (see MASTER_OUTPUT) -
+    if MASTER_OUTPUT == "csv":
+        # Bundle the already-written CSVs into one zip. No Excel writing -> fast.
+        master_path = out_dir / "Master_Dataset.zip"
+        member_names = ["FIPS_Validation_Audit_ChangedRecords.csv", "cluster_df.csv",
+                        "final_cities.csv", "final_zips.csv", "final_census_tracts.csv",
+                        "Summary.csv"]
+        members = [out_dir / n for n in member_names if (out_dir / n).exists()]
+        members += sorted(out_dir.glob("OutputFile_*.csv"))   # full output (chunked)
+        with zipfile.ZipFile(master_path, "w", zipfile.ZIP_DEFLATED) as zf:
+            for p in members:
+                zf.write(p, arcname=p.name)
+    else:
+        # Every result on its own sheet; the full output splits across
+        # Output_70_95_RapidFuzz_1, _2, ... if it exceeds Excel's row cap.
+        master_path = out_dir / "Master_Dataset.xlsx"
+        write_master_workbook(master_path, [
+            ("Output_70_95_RapidFuzz", output_df),
+            ("ValidationAuditResults", audit),
+            ("cluster_df", cluster_df),
+            ("final_cities", final_cities),
+            ("final_zips", final_zips),
+            ("final_census_tracts", final_ct),
+            ("Summary", summary_df),
+        ])
     outputs.append(str(master_path))
 
     elapsed_min = (time.time() - t_start) / 60
@@ -813,9 +884,9 @@ with gr.Blocks() as demo:
         with gr.Column(scale=1, elem_id="sidebar", min_width=270):
             gr.Markdown("### Upload Files")
 
-            gr.HTML('<div class="step-label">1. Upload Excel Files</div>')
-            files_in = gr.File(file_count="multiple", file_types=[".xlsx", ".xls"],
-                               label="200MB per file • XLSX", show_label=True)
+            gr.HTML('<div class="step-label">1. Upload Files</div>')
+            files_in = gr.File(file_count="multiple", file_types=[".xlsx", ".xls", ".csv"],
+                               label="200MB per file • XLSX, XLS, CSV", show_label=True)
 
             gr.HTML('<div class="step-label">2. Upload Reference File (Optional)</div>')
             reference_in = gr.File(file_count="single", file_types=[".xlsx", ".csv"],