Added logic to handle big datasets with private HF Dataset

app.py

@@ -847,6 +847,80 @@ def run_pipeline(uploaded_files, reference_file, blank_cities_not_in_list,
     return "\n".join(log), outputs
 
 
+# =====================================================================
+# DATASET I/O  (for large inputs that time out on browser upload -> HTTP 408)
+# ---------------------------------------------------------------------
+# Instead of uploading a multi-GB file through the browser, put it in a private
+# HF *dataset* repo and let the Space read it over HF's internal network. Outputs
+# are written back to a dataset the same way (large downloads also 408 otherwise).
+# Configure as Space Variables/Secrets:
+#   INPUT_DATASET_REPO   e.g. "DDPLLeads/subdivision-inputs"
+#   OUTPUT_DATASET_REPO  e.g. "DDPLLeads/subdivision-outputs"  (optional)
+#   HF_TOKEN             a token with read on input + write on output
+# =====================================================================
+INPUT_DATASET_REPO = os.environ.get("INPUT_DATASET_REPO", "")
+OUTPUT_DATASET_REPO = os.environ.get("OUTPUT_DATASET_REPO", "")
+
+
+class _LocalFile:
+    """Adapter so downloaded dataset files look like uploaded files to run_pipeline."""
+    def __init__(self, path):
+        self.name = str(path)
+
+
+def list_dataset_inputs(repo_id, token=None):
+    from huggingface_hub import list_repo_files
+    token = token or os.environ.get("HF_TOKEN")
+    files = list_repo_files(repo_id, repo_type="dataset", token=token)
+    return sorted(f for f in files if f.lower().endswith((".csv", ".xlsx", ".xls")))
+
+
+def run_from_dataset(input_repo, output_repo):
+    """Download every supported file from input_repo, run the pipeline, and (if
+    output_repo is set) upload the results back to that dataset. Returns
+    (log_text, output_paths) just like run_pipeline, so it wires to the same UI."""
+    token = os.environ.get("HF_TOKEN")
+    input_repo = (input_repo or "").strip()
+    output_repo = (output_repo or "").strip()
+    if not input_repo:
+        return "Set an input dataset repo (or the INPUT_DATASET_REPO variable).", []
+    if not token:
+        return "An HF_TOKEN secret is required to read/write datasets.", []
+
+    from huggingface_hub import hf_hub_download, HfApi
+    try:
+        filenames = list_dataset_inputs(input_repo, token)
+    except Exception as e:
+        return f"Could not list dataset '{input_repo}': {e}", []
+    if not filenames:
+        return f"No .csv/.xlsx/.xls files found in dataset '{input_repo}'.", []
+
+    local = []
+    for fn in filenames:
+        try:
+            local.append(_LocalFile(
+                hf_hub_download(repo_id=input_repo, filename=fn, repo_type="dataset", token=token)))
+        except Exception as e:
+            return f"Failed to download '{fn}' from '{input_repo}': {e}", []
+
+    log, outputs = run_pipeline(local, None, STRICT_BLANK_CITIES, FILTER_DOMINANT_COUNTY,
+                                CLUSTER_THRESHOLD, REPLACE_THRESHOLD)
+
+    if output_repo and outputs:
+        try:
+            api = HfApi(token=token)
+            stamp = time.strftime("%Y%m%d_%H%M%S")
+            for p in outputs:
+                api.upload_file(path_or_fileobj=p, path_in_repo=f"runs/{stamp}/{Path(p).name}",
+                                repo_id=output_repo, repo_type="dataset")
+            log += (f"\nUploaded {len(outputs)} file(s) to dataset '{output_repo}' "
+                    f"under runs/{stamp}/. Retrieve them there:\n"
+                    f"https://huggingface.co/datasets/{output_repo}/tree/main/runs/{stamp}")
+        except Exception as e:
+            log += f"\nWARNING: processed OK but upload to '{output_repo}' failed: {e}"
+    return log, outputs
+
+
 # =====================================================================
 # STARTUP HEALTH CHECK
 # =====================================================================
@@ -886,22 +960,31 @@ with gr.Blocks() as demo:
 
             gr.HTML('<div class="step-label">1. Upload Files</div>')
             files_in = gr.File(file_count="multiple", file_types=[".xlsx", ".xls", ".csv"],
-                               label="200MB per file • XLSX, XLS, CSV", show_label=True)
+                               label="XLSX, XLS, or CSV — upload several and they're merged",
+                               show_label=True)
 
             gr.HTML('<div class="step-label">2. Upload Reference File (Optional)</div>')
             reference_in = gr.File(file_count="single", file_types=[".xlsx", ".csv"],
-                                   label="200MB per file • XLSX, CSV", show_label=True)
+                                   label="XLSX or CSV", show_label=True)
 
             run_btn = gr.Button("Run pipeline", variant="primary")
 
+            gr.HTML('<div class="step-label">Large files: process from a HF Dataset '
+                    '(no upload — avoids the upload timeout)</div>')
+            ds_in = gr.Textbox(value=INPUT_DATASET_REPO, label="Input dataset repo",
+                               placeholder="user/dataset-name")
+            ds_out = gr.Textbox(value=OUTPUT_DATASET_REPO, label="Output dataset repo (optional)",
+                                placeholder="user/dataset-name")
+            ds_btn = gr.Button("Run from dataset")
+
         # -------- MAIN AREA: title, info, results -----------------------
         with gr.Column(scale=4):
             gr.HTML('<div id="main-title">Subdivision Cluster &amp; FIPS Validation</div>')
             gr.HTML(
-                '<div class="info-banner">Please upload your Excel file(s) using the '
-                'sidebar, then click <b>Run pipeline</b>. The reference safety list is '
-                'applied automatically; upload one on the sidebar only to override it for '
-                'a single run.</div>'
+                '<div class="info-banner">Upload your file(s) on the sidebar and click '
+                '<b>Run pipeline</b>. For files too large to upload (multi-GB / full-state), '
+                'put them in a private HF Dataset and use <b>Run from dataset</b> instead — '
+                'the results are written back to your output dataset.</div>'
             )
             status = gr.Textbox(label="Log", lines=14)
             files_out = gr.File(label="Output files", file_count="multiple")
@@ -916,6 +999,14 @@ with gr.Blocks() as demo:
         inputs=[files_in, reference_in],
         outputs=[status, files_out],
     )
+    ds_btn.click(
+        run_from_dataset,
+        inputs=[ds_in, ds_out],
+        outputs=[status, files_out],
+    )
 
 if __name__ == "__main__":
-    demo.launch()
+    # max_file_size=None -> no per-file upload cap imposed by the app itself.
+    # (Very large uploads are still limited by HF's proxy/RAM -- prefer splitting
+    # the input into several files, which this app merges, or run big jobs in Colab.)
+    demo.launch(max_file_size=None)