eval: skip unfinished runs; PCI GPU order

experiments/evaluate.py

@@ -15,6 +15,7 @@ Outputs (experiments/results/eval/ by default; existing rows of other models are
 """
 import argparse
 import glob
+import json
 import os
 import time
 
@@ -63,8 +64,13 @@ def stage1_model(path, device):
 def run_models(device, pattern):
     out = {}
     for p in sorted(glob.glob(pattern)):
+        h = os.path.join(os.path.dirname(p), "history.json")
         ck = torch.load(p, map_location="cpu", weights_only=False)
         cfg = ck["config"]
+        done = os.path.exists(h) and len(json.load(open(h))) >= cfg.get("epochs", 0)
+        if not done and not os.environ.get("S2F_EVAL_UNFINISHED"):
+            print(f"  skipping unfinished run {os.path.basename(os.path.dirname(p))} (set S2F_EVAL_UNFINISHED=1 to include)")
+            continue
         g = build_generator(**cfg); g.load_state_dict(ck["generator_state_dict"], strict=True)
         name = os.path.basename(os.path.dirname(p))
         out[name] = dict(model=g.to(device).eval(), kind="singlecell", in_channels=cfg["in_channels"], tanh=True,

experiments/run_all.py

@@ -52,7 +52,7 @@ def gpu_queue(queue, gpus, num_workers, extra=()):
                 run_dir = os.path.join(RUNS_DIR, f"{v}_seed{seed}")
                 os.makedirs(run_dir, exist_ok=True)
                 log = open(os.path.join(run_dir, "train.log"), "a")
-                env = dict(os.environ, CUDA_VISIBLE_DEVICES=gpu, PYTHONUNBUFFERED="1")
+                env = dict(os.environ, CUDA_DEVICE_ORDER="PCI_BUS_ID", CUDA_VISIBLE_DEVICES=gpu, PYTHONUNBUFFERED="1")  # same numbering as nvidia-smi
                 cmd = [sys.executable, "-m", "experiments.train", "--variant", v, "--seed", seed, "--resume",
                        "--num-workers", num_workers] + list(extra)
                 running[gpu] = (subprocess.Popen(cmd, cwd=S2F_ROOT, env=env, stdout=log, stderr=subprocess.STDOUT), v, seed, log)