Add shard limit setting (1-15) to side panel and wire through crawl config

app.py

@@ -37,6 +37,7 @@ import gradio as gr
 
 from crawler import (
     MAX_SHARD_ROWS,
+    MAX_SHARDS,
     NORMAL_TOTAL_WORKERS,
     SUPER_TOTAL_WORKERS,
     AsyncCrawler,
@@ -352,6 +353,7 @@ SETTING_HELP_JS = """
     ["Theme", "Switch between visual color themes."],
     ["Seed URL List (one URL per line)", "Provide crawl entry points. Put one URL per line; duplicates are ignored."],
     ["Shard Size Rows", "Rows written per parquet shard before a full shard is emitted."],
+    ["Shard Limit", "Maximum number of shards to produce for a run (1 to 15)."],
     ["Max Links Per Page", "Maximum discovered links to enqueue from each parsed page."],
     ["Request Timeout (seconds)", "HTTP request timeout per URL."],
     ["Max Response Bytes", "Maximum response body bytes to read per page."],
@@ -533,6 +535,7 @@ def build_crawler_config(
     request_timeout_seconds: float,
     max_response_bytes: int,
     shard_size_rows: int,
+    max_shards: int,
     enable_hf_upload: bool,
     upload_incomplete_shards: bool,
     hf_repo_id: str,
@@ -550,6 +553,7 @@ def build_crawler_config(
         request_timeout_seconds=float(request_timeout_seconds),
         max_response_bytes=int(max_response_bytes),
         shard_size_rows=int(shard_size_rows),
+        max_shards=int(max_shards),
         output_dir=Path(__file__).resolve().parent / "shards",
         enable_hf_upload=bool(enable_hf_upload),
         upload_incomplete_shards=bool(upload_incomplete_shards),
@@ -769,6 +773,7 @@ def _start_crawl(
     request_timeout_seconds: float,
     max_response_bytes: int,
     shard_size_rows: int,
+    max_shards: int,
     enable_hf_upload: bool,
     upload_incomplete_shards: bool,
     hf_repo_id: str,
@@ -783,6 +788,7 @@ def _start_crawl(
             request_timeout_seconds=request_timeout_seconds,
             max_response_bytes=max_response_bytes,
             shard_size_rows=shard_size_rows,
+            max_shards=max_shards,
             enable_hf_upload=enable_hf_upload,
             upload_incomplete_shards=upload_incomplete_shards,
             hf_repo_id=hf_repo_id,
@@ -805,6 +811,7 @@ def start_crawl_standard(
     request_timeout_seconds: float,
     max_response_bytes: int,
     shard_size_rows: int,
+    max_shards: int,
     enable_hf_upload: bool,
     upload_incomplete_shards: bool,
     hf_repo_id: str,
@@ -819,6 +826,7 @@ def start_crawl_standard(
         request_timeout_seconds=request_timeout_seconds,
         max_response_bytes=max_response_bytes,
         shard_size_rows=shard_size_rows,
+        max_shards=max_shards,
         enable_hf_upload=enable_hf_upload,
         upload_incomplete_shards=upload_incomplete_shards,
         hf_repo_id=hf_repo_id,
@@ -834,6 +842,7 @@ def start_crawl_super(
     request_timeout_seconds: float,
     max_response_bytes: int,
     shard_size_rows: int,
+    max_shards: int,
     enable_hf_upload: bool,
     upload_incomplete_shards: bool,
     hf_repo_id: str,
@@ -848,6 +857,7 @@ def start_crawl_super(
         request_timeout_seconds=request_timeout_seconds,
         max_response_bytes=max_response_bytes,
         shard_size_rows=shard_size_rows,
+        max_shards=max_shards,
         enable_hf_upload=enable_hf_upload,
         upload_incomplete_shards=upload_incomplete_shards,
         hf_repo_id=hf_repo_id,
@@ -947,6 +957,13 @@ def build_ui() -> gr.Blocks:
                     step=100,
                     value=min(defaults.shard_size_rows, MAX_SHARD_ROWS),
                 )
+                max_shards = gr.Slider(
+                    label=f"Shard Limit (1-{MAX_SHARDS})",
+                    minimum=1,
+                    maximum=MAX_SHARDS,
+                    step=1,
+                    value=min(defaults.max_shards, MAX_SHARDS),
+                )
                 max_links_per_page = gr.Slider(
                     label="Max Links Per Page",
                     minimum=10,
@@ -1017,6 +1034,7 @@ def build_ui() -> gr.Blocks:
             request_timeout_seconds,
             max_response_bytes,
             shard_size_rows,
+            max_shards,
             enable_hf_upload,
             upload_incomplete_shards,
             hf_repo_id,