Upload folder using huggingface_hub

code/scripts/gen_hires_512_manifest.py

@@ -0,0 +1,42 @@
+"""Rebuild the 768 recap manifest at R=512 for fives/refuge2/idridd (user chose to
+drop 768->512: still high-res & resolution-fair, 300ep/3seed protocol intact, but
+~2.3x less compute -> ~6-8 days instead of ~12-18). Reuses the capped 768 manifest's
+36 jobs (thread caps already injected; cv2.setNumThreads(1) is in the code) and only:
+  --img_size 768 -> 512,  --batch_size 4 -> 8  (batch 8 matches finished kvasir/busi@512).
+expected_output paths unchanged (metrics.json gets written at whatever res trained).
+Run ON a100:  TS=$(date -u +%Y%m%dT%H%M%SZ) python3 scripts/gen_hires_512_manifest.py
+"""
+import json, os
+
+SRC = os.path.expanduser("~/.aris_queue/runs/hires768_20260611T004855Z/manifest.json")
+
+man = json.load(open(SRC))
+top = {k: v for k, v in man.items() if k != "phases"}
+src_jobs = man.get("jobs") or man["phases"][0]["jobs"]
+
+jobs = []
+for j in src_jobs:
+    c = j["cmd"]
+    if "--img_size 768" not in c or "--batch_size 4" not in c:
+        raise SystemExit("anchor not found in " + j["id"])
+    j2 = dict(j)
+    j2["id"] = j["id"].replace("rc_hr_", "r512_")
+    j2["cmd"] = c.replace("--img_size 768", "--img_size 512").replace("--batch_size 4", "--batch_size 8")
+    jobs.append(j2)
+
+if not jobs:
+    raise SystemExit("no jobs")
+
+man2 = dict(top)
+man2["project"] = "baselines_hires_512"
+# GPU5 hosts a non-campaign DDIM job (~43GB); lower the free-mem gate so the queue can
+# still place light 512 jobs (~16GB) in GPU5's spare room instead of pinning to GPU4.
+man2["gpu_free_threshold_mib"] = 30000
+man2["phases"] = [{"name": "r512", "depends_on": [], "jobs": jobs}]
+
+RUN = "hires512_" + os.environ["TS"]
+rd = os.path.expanduser("~/.aris_queue/runs/" + RUN)
+os.makedirs(rd + "/logs", exist_ok=True)
+json.dump(man2, open(rd + "/manifest.json", "w"), indent=2)
+print("RUN=" + RUN, "jobs=" + str(len(jobs)))
+print("sample:", jobs[0]["cmd"][:200])

code/scripts/gen_hires_768_recap_manifest.py

@@ -0,0 +1,50 @@
+"""Rebuild a queue manifest for ONLY the 768-res datasets (fives/refuge2/idridd),
+injecting CPU thread caps to fix the dataloader thread-oversubscription that starved
+the GPU (epoch ~750s -> ~180s). Reuses the original hires manifest's jobs verbatim
+(img_size 768, batch 4, num_workers 8, PCI_BUS_ID) and only:
+  (1) keeps the 3 R=768 datasets' jobs,
+  (2) prepends OMP/MKL/OPENBLAS/NUMEXPR/VECLIB_NUM_THREADS=8 to each export line.
+The already-finished cvc/kvasir/busi (512/384) cells are untouched.
+Run ON a100:  TS=$(date -u +%Y%m%dT%H%M%SZ) python3 scripts/gen_hires_768_recap_manifest.py
+"""
+import json, os, glob
+
+SRC = os.path.expanduser("~/.aris_queue/runs/hires_20260610T021920Z/manifest.json")
+DS768_PREFIXES = ("hr_fives_", "hr_refuge2_", "hr_idridd_segmentation_")
+CAP = ("OMP_NUM_THREADS=8 MKL_NUM_THREADS=8 OPENBLAS_NUM_THREADS=8 "
+       "NUMEXPR_NUM_THREADS=8 VECLIB_MAXIMUM_THREADS=8")
+OLD = "export CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=${GPU}"
+NEW = "export CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=${GPU} " + CAP
+
+man = json.load(open(SRC))
+top = {k: v for k, v in man.items() if k != "phases"}
+src_jobs = man.get("jobs") or man["phases"][0]["jobs"]
+
+jobs = []
+for j in src_jobs:
+    if not j["id"].startswith(DS768_PREFIXES):
+        continue
+    # skip any that somehow already finished
+    if os.path.isfile(os.path.join(top.get("cwd", "."), j["expected_output"])):
+        continue
+    c = j["cmd"]
+    if OLD not in c:
+        raise SystemExit("export anchor not found in: " + j["id"])
+    j2 = dict(j)
+    j2["id"] = "rc_" + j["id"]            # rc_ = recapped
+    j2["cmd"] = c.replace(OLD, NEW)
+    jobs.append(j2)
+
+if not jobs:
+    raise SystemExit("no 768 jobs to (re)run")
+
+man2 = dict(top)
+man2["project"] = "baselines_hires_768_recap"
+man2["phases"] = [{"name": "rc768", "depends_on": [], "jobs": jobs}]
+
+RUN = "hires768_" + os.environ["TS"]
+rd = os.path.expanduser("~/.aris_queue/runs/" + RUN)
+os.makedirs(rd + "/logs", exist_ok=True)
+json.dump(man2, open(rd + "/manifest.json", "w"), indent=2)
+print("RUN=" + RUN, "jobs=" + str(len(jobs)))
+print("sample cmd:", jobs[0]["cmd"][:160])

code/scripts/gen_hires_manifest.py

@@ -0,0 +1,55 @@
+"""Resolution-aligned RE-TRAIN manifest: the 4 fully-conv framework methods on the
+6 resolution-mismatched datasets, at a per-dataset higher img_size (matching ~nnU-Net's
+working resolution). Reuses each arch's original command, swaps in --img_size R +
+smaller batch for the bigger inputs, injects CUDA_DEVICE_ORDER=PCI_BUS_ID (else jobs
+land on busy L40s). 4 archs x 6 datasets x 3 seeds = 72 jobs."""
+import json, glob, re, os
+
+# (dataset, protocol, R, batch)
+DS = [
+    ("cvc_clinicdb", "official", 384, 16),
+    ("kvasir_seg", "official", 512, 8),
+    ("busi", "fold01", 512, 8),
+    ("fives", "official", 768, 4),
+    ("refuge2", "official", 768, 4),
+    ("idridd_segmentation", "fold01", 768, 4),
+]
+ARCHS = ["unet", "unetpp", "deeplabv3plus", "attention_unet"]  # SwinUNet/TransUNet are res-locked
+
+base = {}  # (dataset, arch) -> seed0 cmd
+for p in sorted(glob.glob(os.path.expanduser("~/.aris_queue/runs/20260605T13*/manifest.json"))):
+    d = json.load(open(p)); js = d.get("jobs") or d.get("phases", [{}])[0].get("jobs", [])
+    for j in js:
+        c = j.get("cmd", "")
+        m = re.search(r"--dataset (\S+).*?--arch (\S+)", c)
+        if m and "--seed 0" in c:
+            base.setdefault((m.group(1), m.group(2)), c)
+
+jobs, missing = [], []
+for ds, proto, R, B in DS:
+    for a in ARCHS:
+        bc = base.get((ds, a))
+        if not bc:
+            missing.append((ds, a)); continue
+        for s in (0, 1, 2):
+            c = bc
+            c = c.replace("export CUDA_VISIBLE_DEVICES=${GPU}",
+                          "export CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=${GPU}")
+            c = c.replace("--batch_size 16", "--batch_size %d" % B)
+            c = c.replace("--seed 0", "--img_size %d --seed %d" % (R, s))  # hits train.py AND test.py
+            out = "results/baselines/%s_%s/%s/seed%d/metrics.json" % (ds, proto, a, s)
+            jobs.append({"id": "hr_%s_%s_s%d" % (ds, a, s), "cmd": c, "expected_output": out})
+
+if missing:
+    raise SystemExit("missing base cmds: %s" % missing)
+
+manifest = {"project": "baselines_hires", "cwd": "/home/wzhang/LSC/Code/NPJ", "conda": "seggen",
+            "ssh": "a100", "gpus": [4, 5], "jobs_per_gpu": 2, "max_parallel": 4,
+            "gpu_free_threshold_mib": 40000, "oom_retry": {"delay": 300, "max_attempts": 3},
+            "phases": [{"name": "hr", "depends_on": [], "jobs": jobs}]}
+
+RUN = "hires_" + os.environ["TS"]
+rd = os.path.expanduser("~/.aris_queue/runs/" + RUN)
+os.makedirs(rd + "/logs", exist_ok=True)
+json.dump(manifest, open(rd + "/manifest.json", "w"), indent=2)
+print("RUN=" + RUN, "jobs=" + str(len(jobs)))

code/scripts/gen_pannuke_fw_manifest.py

@@ -0,0 +1,39 @@
+"""Generate the framework PanNuke fold02/fold03 manifest (12 jobs = 6 archs x 2
+folds, seed0) by reusing the EXACT fold01 commands (preserves per-arch encoder /
+pretrained ckpt / epochs), swapping only the protocol. Writes to a run dir."""
+import json, glob, re, os
+
+base = {}
+for p in sorted(glob.glob(os.path.expanduser("~/.aris_queue/runs/20260605T13*/manifest.json"))):
+    d = json.load(open(p)); js = d.get("jobs") or d.get("phases", [{}])[0].get("jobs", [])
+    for j in js:
+        c = j.get("cmd", "")
+        if "pannuke" in c and "--protocol fold01" in c and "--seed 0" in c:
+            m = re.search(r"--arch (\S+)", c)
+            if m:
+                base.setdefault(m.group(1), c)
+
+archs = ["unet", "unetpp", "deeplabv3plus", "attention_unet", "transunet", "swinunet"]
+assert all(a in base for a in archs), "missing base cmd: %s" % [a for a in archs if a not in base]
+
+jobs = []
+for proto in ["fold02", "fold03"]:
+    for a in archs:
+        c = base[a].replace("--protocol fold01", "--protocol " + proto)
+        # CRITICAL: without PCI_BUS_ID, CVD=4/5 map to L40s (44GB, other users) not
+        # the A100s -> ECC/OOM. Force PCI bus order so 4/5 == nvidia-smi A100 #4/#5.
+        c = c.replace("export CUDA_VISIBLE_DEVICES=${GPU}",
+                      "export CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=${GPU}")
+        out = "results/baselines/pannuke_semantic_%s/%s/seed0/metrics.json" % (proto, a)
+        jobs.append({"id": "fw_%s_%s" % (proto, a), "cmd": c, "expected_output": out})
+
+manifest = {"project": "pannuke_fw_cv", "cwd": "/home/wzhang/LSC/Code/NPJ", "conda": "seggen",
+            "ssh": "a100", "gpus": [4, 5], "jobs_per_gpu": 3, "max_parallel": 6,
+            "gpu_free_threshold_mib": 60000, "oom_retry": {"delay": 180, "max_attempts": 3},
+            "phases": [{"name": "fw", "depends_on": [], "jobs": jobs}]}
+
+RUN = "pannuke_fw_" + os.environ["TS"]
+rd = os.path.expanduser("~/.aris_queue/runs/" + RUN)
+os.makedirs(rd + "/logs", exist_ok=True)
+json.dump(manifest, open(rd + "/manifest.json", "w"), indent=2)
+print("RUN=" + RUN, "jobs=" + str(len(jobs)))

code/scripts/gen_pannuke_fw_rerun.py

@@ -0,0 +1,33 @@
+"""Re-run manifest for the 10 framework PanNuke fold02/03 jobs that died on a
+transient GPU4 uncorrectable-ECC burst. Excludes attention_unet (those 2 are
+finishing in the original queue). Lower concurrency (jobs_per_gpu=2)."""
+import json, glob, re, os
+
+base = {}
+for p in sorted(glob.glob(os.path.expanduser("~/.aris_queue/runs/20260605T13*/manifest.json"))):
+    d = json.load(open(p)); js = d.get("jobs") or d.get("phases", [{}])[0].get("jobs", [])
+    for j in js:
+        c = j.get("cmd", "")
+        if "pannuke" in c and "--protocol fold01" in c and "--seed 0" in c:
+            m = re.search(r"--arch (\S+)", c)
+            if m:
+                base.setdefault(m.group(1), c)
+
+archs = ["unet", "unetpp", "deeplabv3plus", "transunet", "swinunet"]  # attention_unet excluded
+jobs = []
+for proto in ["fold02", "fold03"]:
+    for a in archs:
+        c = base[a].replace("--protocol fold01", "--protocol " + proto)
+        out = "results/baselines/pannuke_semantic_%s/%s/seed0/metrics.json" % (proto, a)
+        jobs.append({"id": "fw_%s_%s" % (proto, a), "cmd": c, "expected_output": out})
+
+manifest = {"project": "pannuke_fw_rerun", "cwd": "/home/wzhang/LSC/Code/NPJ", "conda": "seggen",
+            "ssh": "a100", "gpus": [4, 5], "jobs_per_gpu": 2, "max_parallel": 4,
+            "gpu_free_threshold_mib": 60000, "oom_retry": {"delay": 240, "max_attempts": 3},
+            "phases": [{"name": "fw", "depends_on": [], "jobs": jobs}]}
+
+RUN = "pannuke_fwre_" + os.environ["TS"]
+rd = os.path.expanduser("~/.aris_queue/runs/" + RUN)
+os.makedirs(rd + "/logs", exist_ok=True)
+json.dump(manifest, open(rd + "/manifest.json", "w"), indent=2)
+print("RUN=" + RUN, "jobs=" + str(len(jobs)))

code/scripts/hf_upload_baselines.py

@@ -0,0 +1,166 @@
+"""Upload the BASELINE snapshot (code + confirmed results + curated weights) to a
+private HF model repo. SegGen WIP is excluded. Run ON a100 (where code/results/weights live).
+
+  # dry-run: print manifest, upload nothing
+  HF_TOKEN=... python scripts/hf_upload_baselines.py --dry-run
+  # real:
+  HF_TOKEN=... python scripts/hf_upload_baselines.py
+
+Curation: code = framework (minus synth/__pycache__) + scripts + envs/{seggen,nnunet,umamba}.yml.
+results = all metrics.json + summary.{html,csv,md,tex} + efficiency.md. weights = best seed per
+(dataset,arch) for framework + best fold per dataset for nnU-Net/U-Mamba.
+"""
+import os, glob, json, argparse, sys
+
+REPO = "MaybeRichard/GenSeg-Baselines"
+ROOT = "/home/wzhang/LSC/Code/NPJ"
+NNRAW_IDS = {  # results/baselines cell name -> nnU-Net Dataset id
+    "cvc_clinicdb_official":1, "kvasir_seg_official":2, "fives_official":3,
+    "refuge2_official":4, "busi_fold01":5, "idridd_segmentation_fold01":6,
+    "acdc_png_official":7, "pannuke_semantic_fold01":8,
+    "medsegdb_isic2018_holdout":9, "medsegdb_kits19_fold01":10,
+}
+
+def sz(p):
+    try: return os.path.getsize(p)
+    except OSError: return 0
+
+def human(n): return f"{n/1024**3:.2f} GB" if n>=1024**3 else f"{n/1024**2:.1f} MB"
+
+def curate_framework():
+    """best seed per (cell,arch) -> best.pth"""
+    best = {}
+    for mj in glob.glob(f"{ROOT}/results/baselines/*/*/seed*/metrics.json"):
+        parts = mj.split("/"); cell, arch = parts[-4], parts[-3]
+        if arch in ("nnunet","umamba"): continue            # handled separately
+        try: dice = json.load(open(mj)).get("metrics",{}).get("dice_mean",0)
+        except Exception: continue
+        pth = os.path.join(os.path.dirname(mj),"best.pth")
+        if not os.path.isfile(pth): continue
+        k = (cell,arch)
+        if k not in best or dice > best[k][0]:
+            best[k] = (dice, pth, f"weights/framework/{cell}/{arch}.pth")
+    return best
+
+def curate_nn(method, results_dir):
+    """best fold per dataset -> checkpoint_best.pth (matched via results/baselines metrics)"""
+    out = {}
+    for cell, did in NNRAW_IDS.items():
+        # best fold by our scored metrics
+        folds = []
+        for mj in glob.glob(f"{ROOT}/results/baselines/{cell}/{method}/seed*/metrics.json"):
+            f = int(mj.split("/seed")[-1].split("/")[0])
+            try: d = json.load(open(mj)).get("metrics",{}).get("dice_mean",0)
+            except Exception: d = 0
+            folds.append((d,f))
+        if not folds: continue
+        _, bf = max(folds)
+        cks = glob.glob(f"{results_dir}/Dataset{did:03d}_*/**/fold_{bf}/checkpoint_best.pth", recursive=True)
+        if not cks: continue
+        ck = max(cks, key=sz)
+        out[cell] = (ck, f"weights/{method}/{cell}_fold{bf}.pth")
+    return out
+
+def list_code():
+    inc = []
+    for r,_,fs in os.walk(f"{ROOT}/framework"):
+        if "/synth" in r or "__pycache__" in r: continue
+        for f in fs:
+            if f.endswith(".pyc"): continue
+            inc.append(os.path.join(r,f))
+    for r,_,fs in os.walk(f"{ROOT}/scripts"):
+        if "__pycache__" in r: continue
+        for f in fs: inc.append(os.path.join(r,f))
+    for y in ("seggen","nnunet","umamba"):
+        p=f"{ROOT}/envs/{y}.yml"
+        if os.path.isfile(p): inc.append(p)
+    return inc
+
+def list_results():
+    out = glob.glob(f"{ROOT}/results/baselines/**/metrics.json", recursive=True)
+    for pat in ("summary.html","summary.csv","summary.md","summary.tex","efficiency.md"):
+        out += glob.glob(f"{ROOT}/results/baselines/{pat}")
+    return out
+
+def main():
+    ap = argparse.ArgumentParser(); ap.add_argument("--dry-run",action="store_true"); a=ap.parse_args()
+    code, res = list_code(), list_results()
+    fw = curate_framework()
+    nn = curate_nn("nnunet", f"{ROOT}/nnunet_workspace/results_nnunet")
+    um = curate_nn("umamba", f"{ROOT}/nnunet_workspace/results_umamba")
+    w_fw = sum(sz(v[1]) for v in fw.values())
+    w_nn = sum(sz(v[0]) for v in nn.values()); w_um = sum(sz(v[0]) for v in um.values())
+    code_sz = sum(sz(p) for p in code); res_sz = sum(sz(p) for p in res)
+    print("="*60)
+    print(f"REPO: {REPO} (private)")
+    print(f"CODE   : {len(code):4d} files  {human(code_sz)}  (framework w/o synth + scripts + 3 envs)")
+    print(f"RESULTS: {len(res):4d} files  {human(res_sz)}  (metrics.json + summary.* + efficiency.md)")
+    print(f"WEIGHTS framework: {len(fw):3d} cells  {human(w_fw)}  (best seed per dataset x arch)")
+    print(f"WEIGHTS nnU-Net  : {len(nn):3d} dsets  {human(w_nn)}  (best fold)")
+    print(f"WEIGHTS U-Mamba  : {len(um):3d} dsets  {human(w_um)}  (best fold)")
+    print(f"TOTAL  : {human(code_sz+res_sz+w_fw+w_nn+w_um)}")
+    print("="*60)
+    print("sample framework weights:")
+    for k in list(fw)[:3]: print("  ", fw[k][2], "<-", os.path.relpath(fw[k][1],ROOT))
+    print("sample nnU-Net/U-Mamba weights:")
+    for d in (nn,um):
+        for k in list(d)[:2]: print("  ", d[k][1], "<-", os.path.relpath(d[k][0],ROOT))
+    miss = [c for c in NNRAW_IDS if c not in nn] + [c for c in NNRAW_IDS if c not in um]
+    if miss: print("NOTE missing nn/um ckpts for:", sorted(set(miss)))
+    if a.dry_run:
+        print("\n[dry-run] nothing uploaded.")
+        return
+    # ---- real upload ----
+    from huggingface_hub import HfApi, create_repo
+    api = HfApi()
+    create_repo(REPO, repo_type="model", private=True, exist_ok=True)
+    readme = """---
+license: cc-by-nc-4.0
+tags: [medical-imaging, segmentation, benchmark]
+---
+
+# GenSeg-Baselines
+
+Baseline benchmark for 2D medical image segmentation: **8 methods x 10 datasets x 3 seeds/folds, 7 metrics**.
+Companion to the [GenSegDataset](https://huggingface.co/datasets/MaybeRichard/GenSegDataset).
+
+**Methods:** UNet, UNet++, DeepLabV3+ (ResNet-50/ImageNet), Attention-UNet (scratch),
+TransUNet (R50-ViT-B/16), Swin-UNet (Swin-Tiny), nnU-Net v2 (250ep), U-Mamba (UMambaBot, 100ep).
+
+**Datasets:** cvc_clinicdb, kvasir_seg, fives, busi, refuge2, acdc, idridd, pannuke, isic2018, kits19.
+
+**Metrics:** Dice, IoU, HD95, ASSD, Sensitivity, Specificity, Precision (+ efficiency).
+
+## Layout
+- `code/` - baseline framework (train/test/aggregate), scripts, conda envs. *(Generative SegGen code excluded.)*
+- `results/` - per-run `metrics.json` + aggregated `summary.{html,csv,md,tex}` + `efficiency.md`.
+- `weights/` - curated checkpoints: best seed per (dataset, arch) for framework; best fold for nnU-Net / U-Mamba.
+
+## Note
+These are the **256-px baseline** (confirmed). A resolution-fair re-evaluation (conv methods retrained at a
+higher per-dataset resolution; all methods scored at a common R so HD95 is comparable) is in progress and
+will be added later.
+"""
+    import tempfile
+    tmp = os.path.join(tempfile.gettempdir(), "GENSEG_README.md")
+    open(tmp,"w").write(readme)
+    api.upload_file(path_or_fileobj=tmp, repo_id=REPO, path_in_repo="README.md")
+    print("repo ready; uploading code+results ...")
+    api.upload_folder(folder_path=f"{ROOT}/framework", repo_id=REPO, path_in_repo="code/framework",
+                      ignore_patterns=["synth/*","**/__pycache__/*","*.pyc"])
+    api.upload_folder(folder_path=f"{ROOT}/scripts", repo_id=REPO, path_in_repo="code/scripts",
+                      ignore_patterns=["**/__pycache__/*","*.pyc"])
+    for y in ("seggen","nnunet","umamba"):
+        api.upload_file(path_or_fileobj=f"{ROOT}/envs/{y}.yml", repo_id=REPO, path_in_repo=f"code/envs/{y}.yml")
+    api.upload_folder(folder_path=f"{ROOT}/results/baselines", repo_id=REPO, path_in_repo="results",
+                      allow_patterns=["**/metrics.json","summary.*","efficiency.md"])
+    print("uploading weights ...")
+    for v in fw.values():
+        api.upload_file(path_or_fileobj=v[1], repo_id=REPO, path_in_repo=v[2])
+    for d in (nn,um):
+        for v in d.values():
+            api.upload_file(path_or_fileobj=v[0], repo_id=REPO, path_in_repo=v[1])
+    print("DONE:", f"https://huggingface.co/{REPO}")
+
+if __name__ == "__main__":
+    main()

code/scripts/nnunet_eval_only_seggen.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+# Re-score ALL 10 nnU-Net datasets (fold 0/1/2) from cached test predictions,
+# running in the *seggen* env (which has MONAI+medpy) so HD95/ASSD are real, not
+# NaN. Idempotent: overwrites results/baselines/<ds>/nnunet/seed<f>/metrics.json.
+set -u
+cd /home/wzhang/LSC/Code/NPJ
+source /opt/anaconda3/etc/profile.d/conda.sh
+conda activate seggen
+DATA_ROOT=/home/wzhang/LSC/Dataset/Segmentation/processed_unified
+RAW=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/raw
+PRED=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/predTs
+
+SPECS=(
+  "1:cvc_clinicdb:official"
+  "2:kvasir_seg:official"
+  "3:fives:official"
+  "4:refuge2:official"
+  "5:busi:fold01"
+  "6:idridd_segmentation:fold01"
+  "7:acdc_png:official"
+  "8:pannuke_semantic:fold01"
+  "9:medsegdb_isic2018:holdout"
+  "10:medsegdb_kits19:fold01"
+)
+for spec in "${SPECS[@]}"; do
+  IFS=: read -r id ds proto <<< "$spec"
+  for f in 0 1 2; do
+    outdir=$PRED/d${id}_f${f}
+    if [ ! -d "$outdir" ] || [ -z "$(ls -A "$outdir"/*.png 2>/dev/null)" ]; then
+      echo "[skip] $ds fold$f: no predictions in $outdir"; continue
+    fi
+    python framework/nnunet_eval.py --data_root "$DATA_ROOT" \
+      --dataset "$ds" --protocol "$proto" --raw "$RAW" \
+      --dataset_id "$id" --fold "$f" --pred_dir "$outdir" \
+      --arch nnunet --exp_name baselines
+  done
+done
+echo EVAL_ONLY_ALL_DONE

code/scripts/nnunet_eval_par_heavy.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Parallel HD95/ASSD re-score for the heavy nnU-Net datasets (IDs 6-10), all folds,
+# seggen env. Datasets 1-5 are already scored correctly and are left untouched.
+# Each (dataset,fold) runs as its own process; 384 cores easily absorb 15 at once.
+set -u
+cd /home/wzhang/LSC/Code/NPJ
+source /opt/anaconda3/etc/profile.d/conda.sh
+conda activate seggen
+# Cap intra-op threads: MONAI/torch otherwise grab all 384 cores per process, so
+# 15 procs oversubscribe (load ~880) and thrash. 8 threads x 15 procs = ~120 << 384.
+export OMP_NUM_THREADS=8 MKL_NUM_THREADS=8 OPENBLAS_NUM_THREADS=8 NUMEXPR_NUM_THREADS=8
+DATA_ROOT=/home/wzhang/LSC/Dataset/Segmentation/processed_unified
+RAW=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/raw
+PRED=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/predTs
+LOGD=nnunet_workspace/eval_par_logs
+mkdir -p "$LOGD"
+
+for id in 6 7 8 9 10; do
+  case $id in
+    6)  ds=idridd_segmentation; proto=fold01;;
+    7)  ds=acdc_png;            proto=official;;
+    8)  ds=pannuke_semantic;    proto=fold01;;
+    9)  ds=medsegdb_isic2018;   proto=holdout;;
+    10) ds=medsegdb_kits19;     proto=fold01;;
+  esac
+  for f in 0 1 2; do
+    (
+      python framework/nnunet_eval.py --data_root "$DATA_ROOT" \
+        --dataset "$ds" --protocol "$proto" --raw "$RAW" \
+        --dataset_id "$id" --fold "$f" --pred_dir "$PRED/d${id}_f${f}" \
+        --arch nnunet --exp_name baselines > "$LOGD/d${id}_f${f}.log" 2>&1
+      echo "done d${id}_f${f}: $(tail -1 "$LOGD/d${id}_f${f}.log")"
+    ) &
+  done
+done
+wait
+echo PAR_EVAL_DONE

code/scripts/nnunet_predict_eval_remaining.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+# Bulk predict-on-test + 7-metric eval for the 6 remaining nnU-Net datasets
+# (IDs 005-010, each fold 0/1/2). Splits work across the two idle A100s (#4,#5).
+set -u
+cd /home/wzhang/LSC/Code/NPJ
+source /opt/anaconda3/etc/profile.d/conda.sh
+conda activate nnunet
+export CUDA_DEVICE_ORDER=PCI_BUS_ID
+export nnUNet_raw=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/raw
+export nnUNet_preprocessed=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/preprocessed
+export nnUNet_results=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/results_nnunet
+DATA_ROOT=/home/wzhang/LSC/Dataset/Segmentation/processed_unified
+PRED=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/predTs
+
+run_ds () {
+  local gpu=$1; shift
+  for spec in "$@"; do
+    IFS=: read -r id ds proto <<< "$spec"
+    dsname=$(printf "Dataset%03d_%s_%s" "$id" "$ds" "$proto")
+    for f in 0 1 2; do
+      outdir=$PRED/d${id}_f${f}
+      mkdir -p "$outdir"
+      echo "[predict] gpu=$gpu $dsname fold$f -> $outdir"
+      CUDA_VISIBLE_DEVICES=$gpu nnUNetv2_predict \
+        -i "$nnUNet_raw/$dsname/imagesTs" -o "$outdir" \
+        -d "$id" -c 2d -f "$f" -tr nnUNetTrainer_250epochs --disable_tta \
+        > "$outdir/predict.log" 2>&1
+      echo "[eval] $dsname fold$f"
+      python framework/nnunet_eval.py --data_root "$DATA_ROOT" \
+        --dataset "$ds" --protocol "$proto" --raw "$nnUNet_raw" \
+        --dataset_id "$id" --fold "$f" --pred_dir "$outdir" \
+        --arch nnunet --exp_name baselines > "$outdir/eval.log" 2>&1
+      tail -1 "$outdir/eval.log"
+    done
+  done
+}
+
+run_ds 4 "5:busi:fold01" "6:idridd_segmentation:fold01" "7:acdc_png:official" &
+run_ds 5 "8:pannuke_semantic:fold01" "9:medsegdb_isic2018:holdout" "10:medsegdb_kits19:fold01" &
+wait
+echo ALL_NNUNET_PREDICT_EVAL_DONE

code/scripts/pannuke_cv_nnunet.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+# nnU-Net PanNuke fold02(d11)+fold03(d12): train fold0 on both A100s, predict, score.
+set -u
+cd /home/wzhang/LSC/Code/NPJ
+source /opt/anaconda3/etc/profile.d/conda.sh
+RAW=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/raw
+PRED=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/predTs
+DATA_ROOT=/home/wzhang/LSC/Dataset/Segmentation/processed_unified
+export CUDA_DEVICE_ORDER=PCI_BUS_ID
+
+conda activate nnunet
+export nnUNet_raw=$RAW
+export nnUNet_preprocessed=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/preprocessed_nnunet
+export nnUNet_results=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/results_nnunet
+export nnUNet_n_proc_DA=8 OMP_NUM_THREADS=4
+CUDA_VISIBLE_DEVICES=4 nnUNetv2_train 11 2d 0 -tr nnUNetTrainer_250epochs > nnunet_workspace/train_nnunet_d11.log 2>&1 &
+CUDA_VISIBLE_DEVICES=5 nnUNetv2_train 12 2d 0 -tr nnUNetTrainer_250epochs > nnunet_workspace/train_nnunet_d12.log 2>&1 &
+wait
+echo NNUNET_CV_TRAIN_DONE
+CUDA_VISIBLE_DEVICES=4 nnUNetv2_predict -i "$RAW/Dataset011_pannuke_semantic_fold02/imagesTs" -o "$PRED/d11_f0" -d 11 -c 2d -f 0 -tr nnUNetTrainer_250epochs --disable_tta > nnunet_workspace/pred_nnunet_d11.log 2>&1 &
+CUDA_VISIBLE_DEVICES=5 nnUNetv2_predict -i "$RAW/Dataset012_pannuke_semantic_fold03/imagesTs" -o "$PRED/d12_f0" -d 12 -c 2d -f 0 -tr nnUNetTrainer_250epochs --disable_tta > nnunet_workspace/pred_nnunet_d12.log 2>&1 &
+wait
+
+conda deactivate; conda activate seggen
+export OMP_NUM_THREADS=8 MKL_NUM_THREADS=8 OPENBLAS_NUM_THREADS=8
+python framework/nnunet_eval.py --data_root "$DATA_ROOT" --dataset pannuke_semantic --protocol fold02 --raw "$RAW" --dataset_id 11 --fold 0 --pred_dir "$PRED/d11_f0" --arch nnunet --exp_name baselines
+python framework/nnunet_eval.py --data_root "$DATA_ROOT" --dataset pannuke_semantic --protocol fold03 --raw "$RAW" --dataset_id 12 --fold 0 --pred_dir "$PRED/d12_f0" --arch nnunet --exp_name baselines
+echo NNUNET_CV_DONE

code/scripts/pannuke_cv_preprocess.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+# Preprocess pannuke fold02 (d11) + fold03 (d12) for BOTH nnU-Net and U-Mamba
+# (separate preprocessed dirs, separate envs), CPU-only, thread-capped. Copies the
+# fixed splits into each preprocessed dataset.
+set -u
+cd /home/wzhang/LSC/Code/NPJ
+source /opt/anaconda3/etc/profile.d/conda.sh
+RAW=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/raw
+export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 OPENBLAS_NUM_THREADS=4
+
+( conda activate nnunet
+  export nnUNet_raw=$RAW
+  export nnUNet_preprocessed=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/preprocessed_nnunet
+  export nnUNet_results=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/results_nnunet
+  for id in 11 12; do
+    nnUNetv2_plan_and_preprocess -d "$id" -c 2d -np 12 > "nnunet_workspace/pp_nnunet_d${id}.log" 2>&1
+    name=$(basename "$(ls -d $RAW/Dataset0${id}_*)")
+    cp "$RAW/$name/splits_final.json" "$nnUNet_preprocessed/$name/splits_final.json"
+    echo "nnunet d$id preprocessed ($name)"
+  done ) &
+
+( conda activate umamba
+  export nnUNet_raw=$RAW
+  export nnUNet_preprocessed=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/preprocessed_umamba
+  export nnUNet_results=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/results_umamba
+  for id in 11 12; do
+    nnUNetv2_plan_and_preprocess -d "$id" -c 2d -np 12 > "nnunet_workspace/pp_umamba_d${id}.log" 2>&1
+    name=$(basename "$(ls -d $RAW/Dataset0${id}_*)")
+    cp "$RAW/$name/splits_final.json" "$nnUNet_preprocessed/$name/splits_final.json"
+    echo "umamba d$id preprocessed ($name)"
+  done ) &
+
+wait
+echo PANNUKE_PP_DONE

code/scripts/pannuke_cv_umamba.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+# U-Mamba PanNuke fold02(d11)+fold03(d12): train fold0 (100ep) on both A100s, predict, score.
+set -u
+cd /home/wzhang/LSC/Code/NPJ
+source /opt/anaconda3/etc/profile.d/conda.sh
+RAW=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/raw
+PRED=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/predTs_umamba
+DATA_ROOT=/home/wzhang/LSC/Dataset/Segmentation/processed_unified
+TR=nnUNetTrainerUMambaBot_100epochs
+export CUDA_DEVICE_ORDER=PCI_BUS_ID
+mkdir -p "$PRED"
+
+conda activate umamba
+export nnUNet_raw=$RAW
+export nnUNet_preprocessed=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/preprocessed_umamba
+export nnUNet_results=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/results_umamba
+export nnUNet_n_proc_DA=8 OMP_NUM_THREADS=4
+CUDA_VISIBLE_DEVICES=4 nnUNetv2_train 11 2d 0 -tr "$TR" > nnunet_workspace/train_umamba_d11.log 2>&1 &
+CUDA_VISIBLE_DEVICES=5 nnUNetv2_train 12 2d 0 -tr "$TR" > nnunet_workspace/train_umamba_d12.log 2>&1 &
+wait
+echo UMAMBA_CV_TRAIN_DONE
+CUDA_VISIBLE_DEVICES=4 nnUNetv2_predict -i "$RAW/Dataset011_pannuke_semantic_fold02/imagesTs" -o "$PRED/d11_f0" -d 11 -c 2d -f 0 -tr "$TR" --disable_tta > nnunet_workspace/pred_umamba_d11.log 2>&1 &
+CUDA_VISIBLE_DEVICES=5 nnUNetv2_predict -i "$RAW/Dataset012_pannuke_semantic_fold03/imagesTs" -o "$PRED/d12_f0" -d 12 -c 2d -f 0 -tr "$TR" --disable_tta > nnunet_workspace/pred_umamba_d12.log 2>&1 &
+wait
+
+conda deactivate; conda activate seggen
+export OMP_NUM_THREADS=8 MKL_NUM_THREADS=8 OPENBLAS_NUM_THREADS=8
+python framework/nnunet_eval.py --data_root "$DATA_ROOT" --dataset pannuke_semantic --protocol fold02 --raw "$RAW" --dataset_id 11 --fold 0 --pred_dir "$PRED/d11_f0" --arch umamba --exp_name baselines
+python framework/nnunet_eval.py --data_root "$DATA_ROOT" --dataset pannuke_semantic --protocol fold03 --raw "$RAW" --dataset_id 12 --fold 0 --pred_dir "$PRED/d12_f0" --arch umamba --exp_name baselines
+echo UMAMBA_CV_DONE

code/scripts/umamba_predict_eval.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+# U-Mamba finale: predict test sets (umamba env, both A100s) then score the 7
+# framework metrics (seggen env, thread-capped). Mirrors the nnU-Net predict+eval
+# flow but with results_umamba / preprocessed_umamba / trainer 100ep / arch=umamba.
+set -u
+cd /home/wzhang/LSC/Code/NPJ
+source /opt/anaconda3/etc/profile.d/conda.sh
+export CUDA_DEVICE_ORDER=PCI_BUS_ID
+RAW=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/raw
+PRE=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/preprocessed_umamba
+RES=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/results_umamba
+PRED=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/predTs_umamba
+DATA_ROOT=/home/wzhang/LSC/Dataset/Segmentation/processed_unified
+TR=nnUNetTrainerUMambaBot_100epochs
+mkdir -p "$PRED"
+
+dsname () { case $1 in
+  1) echo cvc_clinicdb official;;     2) echo kvasir_seg official;;  3) echo fives official;;
+  4) echo refuge2 official;;          5) echo busi fold01;;          6) echo idridd_segmentation fold01;;
+  7) echo acdc_png official;;         8) echo pannuke_semantic fold01;; 9) echo medsegdb_isic2018 holdout;;
+  10) echo medsegdb_kits19 fold01;;
+esac; }
+rawname () { printf "Dataset%03d_%s_%s" "$1" "$2" "$3"; }
+
+# ---------- Phase 1: predict (umamba env), ids split across GPU4 / GPU5 ----------
+conda activate umamba
+export nnUNet_raw=$RAW nnUNet_preprocessed=$PRE nnUNet_results=$RES
+export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4
+predict_ids () {
+  local gpu=$1; shift
+  for id in "$@"; do
+    read -r ds proto < <(dsname "$id"); dn=$(rawname "$id" "$ds" "$proto")
+    for f in 0 1 2; do
+      out=$PRED/d${id}_f${f}; mkdir -p "$out"
+      echo "[predict gpu$gpu] $dn f$f"
+      CUDA_VISIBLE_DEVICES=$gpu nnUNetv2_predict -i "$RAW/$dn/imagesTs" -o "$out" \
+        -d "$id" -c 2d -f "$f" -tr "$TR" --disable_tta > "$out/predict.log" 2>&1
+    done
+  done
+}
+predict_ids 4 1 2 3 4 5 &
+predict_ids 5 6 7 8 9 10 &
+wait
+echo PREDICT_DONE
+
+# ---------- Phase 2: eval (seggen env), parallel, thread-capped, max 10 ----------
+conda deactivate; conda activate seggen
+export OMP_NUM_THREADS=8 MKL_NUM_THREADS=8 OPENBLAS_NUM_THREADS=8 NUMEXPR_NUM_THREADS=8
+for id in 1 2 3 4 5 6 7 8 9 10; do
+  read -r ds proto < <(dsname "$id")
+  for f in 0 1 2; do
+    while (( $(jobs -rp | wc -l) >= 10 )); do wait -n; done
+    (
+      python framework/nnunet_eval.py --data_root "$DATA_ROOT" \
+        --dataset "$ds" --protocol "$proto" --raw "$RAW" \
+        --dataset_id "$id" --fold "$f" --pred_dir "$PRED/d${id}_f${f}" \
+        --arch umamba --exp_name baselines > "$PRED/d${id}_f${f}/eval.log" 2>&1
+      echo "evaled d${id}_f${f}: $(tail -1 "$PRED/d${id}_f${f}/eval.log")"
+    ) &
+  done
+done
+wait
+echo EVAL_DONE

code/scripts/umamba_preprocess.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# U-Mamba preprocessing for datasets 2-10 (cvc/d1 already done) into a SEPARATE
+# preprocessed_umamba dir (U-Mamba's nnunetv2 2.1.x plans differ from the 2.7.0
+# nnunet env). Runs in the umamba env. Sequential + thread-capped so the planner's
+# BLAS threads don't oversubscribe the 384-core box. Copies the 3-identical-fold
+# splits_final.json into each preprocessed dataset afterward.
+set -u
+cd /home/wzhang/LSC/Code/NPJ
+source /opt/anaconda3/etc/profile.d/conda.sh
+conda activate umamba
+export nnUNet_raw=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/raw
+export nnUNet_preprocessed=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/preprocessed_umamba
+export nnUNet_results=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/results_umamba
+export OMP_NUM_THREADS=2 MKL_NUM_THREADS=2 OPENBLAS_NUM_THREADS=2 NUMEXPR_NUM_THREADS=2
+mkdir -p "$nnUNet_preprocessed" "$nnUNet_results"
+LOGD=nnunet_workspace/umamba_preprocess_logs; mkdir -p "$LOGD"
+
+dsname () {
+  case $1 in
+    2) echo Dataset002_kvasir_seg_official;;
+    3) echo Dataset003_fives_official;;
+    4) echo Dataset004_refuge2_official;;
+    5) echo Dataset005_busi_fold01;;
+    6) echo Dataset006_idridd_segmentation_fold01;;
+    7) echo Dataset007_acdc_png_official;;
+    8) echo Dataset008_pannuke_semantic_fold01;;
+    9) echo Dataset009_medsegdb_isic2018_holdout;;
+    10) echo Dataset010_medsegdb_kits19_fold01;;
+  esac
+}
+
+for id in 2 3 4 5 6 7 8 9 10; do
+  name=$(dsname "$id")
+  echo "=== preprocess d$id ($name) ==="
+  nnUNetv2_plan_and_preprocess -d "$id" -c 2d -np 16 > "$LOGD/d${id}.log" 2>&1
+  rc=$?
+  if [ -d "$nnUNet_preprocessed/$name" ]; then
+    cp "nnunet_workspace/raw/$name/splits_final.json" "$nnUNet_preprocessed/$name/splits_final.json"
+    echo "d$id rc=$rc splits_copied=$( [ -f "$nnUNet_preprocessed/$name/splits_final.json" ] && echo Y || echo N )"
+  else
+    echo "d$id rc=$rc FAILED_no_preprocessed_dir"
+  fi
+done
+echo UMAMBA_PREPROCESS_DONE