fix(config): remove mem_per_worker_gib from config files and calculate dynamically in resource_probe script

README.md

@@ -131,7 +131,6 @@ The pipeline respects your resource limits and adapts processing strategy by dat
 resources:
   max_memory_gib: 240      # Total memory available
   max_workers: 42          # Maximum parallel workers
-  mem_per_worker_gib: 5.5  # Memory per worker
   chunk_size: 12288        # Matrix chunk size
 
 dataset_thresholds:

configs/eda_config_template.yaml

@@ -5,7 +5,6 @@
 resources:
   max_memory_gib: 256  # Total memory available
   max_workers: 32      # Maximum concurrent workers
-  mem_per_worker_gib: 8.0  # Memory per worker process
   chunk_size: 8192     # Chunk size for reading X matrix
 
 # Input/Output Paths

configs/eda_optimized.yaml

@@ -4,7 +4,7 @@
 
 resources:
   max_memory_gib: 200  # Leave ~10 GB buffer for system
-  max_workers: 32      # Based on actual RAM availability
+  max_workers: 24      # Based on actual RAM availability
   chunk_size: 12288    # Good balance for large matrices
 
 paths:

scripts/resource_probe.py

@@ -1,5 +1,9 @@
 #!/usr/bin/env python3
-"""Probe local HPC resources and suggest safe EDA concurrency settings."""
+"""Probe local HPC resources and suggest safe EDA concurrency settings.
+
+This script dynamically calculates memory per worker from the EDA config file
+(configs/eda_optimized.yaml) using max_memory_gib / max_workers.
+"""
 
 from __future__ import annotations
 
@@ -9,6 +13,7 @@ import os
 import platform
 import shutil
 from pathlib import Path
+import yaml
 
 
 def _mem_available_gib() -> float:
@@ -48,14 +53,16 @@ def main() -> None:
         default=Path("/project/GOV108018"),
         help="Path to check disk usage for.",
     )
-    parser.add_argument(
-        "--mem-per-worker-gib",
-        type=float,
-        default=8.0,
-        help="Memory budget per EDA worker to compute a safe recommendation.",
-    )
     args = parser.parse_args()
 
+    # Read config to calculate dynamic mem_per_worker_gib
+    config_path = Path(__file__).parent.parent / "configs" / "eda_optimized.yaml"
+    with open(config_path) as f:
+        config = yaml.safe_load(f)
+    max_memory_gib = config['resources']['max_memory_gib']
+    max_workers = config['resources']['max_workers']
+    mem_per_worker_gib = max_memory_gib / max_workers
+
     cpu_count = os.cpu_count() or 1
     mem_total_gib = _mem_total_gib()
     mem_available_gib = _mem_available_gib()
@@ -64,7 +71,7 @@ def main() -> None:
     recommended_workers = _recommend_workers(
         cpu_count=cpu_count,
         mem_available_gib=mem_available_gib,
-        mem_per_worker_gib=args.mem_per_worker_gib,
+        mem_per_worker_gib=mem_per_worker_gib,
     )
     recommended_shards = max(1, min(8, cpu_count // max(1, recommended_workers)))
 
@@ -77,7 +84,7 @@ def main() -> None:
         "disk_total_gib": round(disk_total / (1024**3), 2),
         "disk_used_gib": round(disk_used / (1024**3), 2),
         "disk_free_gib": round(disk_free / (1024**3), 2),
-        "assumptions": {"mem_per_worker_gib": args.mem_per_worker_gib},
+        "assumptions": {"mem_per_worker_gib": mem_per_worker_gib},
         "recommendation": {
             "workers_per_node": recommended_workers,
             "num_shards_suggestion": recommended_shards,