cuda: sleep 30s instead of poll-fail; let model load handle lazy init

scripts/bold_pipeline.py

@@ -22,28 +22,19 @@ import random
 import sys
 from pathlib import Path
 
-def _wait_for_cuda(max_tries: int = 90):
+def _wait_for_cuda():
     import time as _t
+    print("[cuda] sleeping 30s for driver init...", flush=True)
+    _t.sleep(30)
     try:
         import torch
+        n = torch.cuda.device_count() if torch.cuda.is_available() else 0
+        if n > 0:
+            print(f"[cuda] ready: {n} device(s), {torch.cuda.get_device_name(0)}", flush=True)
+        else:
+            print("[cuda] device_count=0 after sleep — letting model load trigger lazy init", flush=True)
     except Exception as e:
-        raise RuntimeError(f"torch missing: {e}")
-    for i in range(max_tries):
-        try:
-            try:
-                torch.cuda.init()
-            except Exception:
-                pass
-            n = torch.cuda.device_count()
-            if n > 0:
-                _ = torch.zeros(1, device="cuda")
-                print(f"[cuda] ready after {i*2}s: {n} device(s), {torch.cuda.get_device_name(0)}", flush=True)
-                return
-        except Exception as e:
-            if i % 5 == 0:
-                print(f"[cuda] wait {i}/{max_tries}: {type(e).__name__}: {str(e)[:120]}", flush=True)
-        _t.sleep(2)
-    raise RuntimeError(f"CUDA never became ready after {max_tries*2}s — likely bad H200 node, retry job")
+        print(f"[cuda] probe error (non-fatal, model load will retry): {e}", flush=True)
 
 
 def _setup_workdir():