Update scripts/hf_runner.py

scripts/hf_runner.py

@@ -1,5 +1,7 @@
 import os
 import subprocess
+import multiprocessing
+from huggingface_hub import HfApi, login
 from huggingface_hub import HfApi, login
 
 # 1. Configuration
@@ -24,9 +26,10 @@ def main():
         print(f"Failed to create/check repo: {e}")
 
     # 2. Run the heavy pipeline
-    print("\n--- STARTING DAHS PIPELINE (3000 Scenarios) ---")
+    cores = str(multiprocessing.cpu_count())
+    print(f"\n--- STARTING DAHS PIPELINE (6000 Scenarios on {cores} Workers) ---")
     # Using subprocess to run the pipeline exactly as you would locally
-    result = subprocess.run(["python", "scripts/run_pipeline.py", "--scenarios", "3000"])
+    result = subprocess.run(["python", "scripts/run_pipeline.py", "--scenarios", "6000", "--workers", cores])
     
     if result.returncode != 0:
         print("\nPipeline failed! Aborting upload.")