Remove timed incomplete flush config

crawler/config.py

@@ -52,7 +52,6 @@ class CrawlerConfig:
 
     enable_hf_upload: bool = False
     upload_incomplete_shards: bool = False
-    incomplete_shard_flush_seconds: float = 30.0
     hf_repo_id: str = ""
     hf_token: str = ""
     hf_repo_type: str = "dataset"
@@ -92,9 +91,6 @@ class CrawlerConfig:
         self.hf_repo_id = self.hf_repo_id.strip()
         self.hf_token = self.hf_token.strip()
         self.hf_path_prefix = self.hf_path_prefix.strip() or "crawl_shards"
-        self.incomplete_shard_flush_seconds = float(self.incomplete_shard_flush_seconds)
-        if self.incomplete_shard_flush_seconds <= 0:
-            raise ValueError("incomplete_shard_flush_seconds must be > 0.")
 
         if self.enable_hf_upload:
             if not self.hf_repo_id: