l

configs/train_config.yaml

@@ -34,7 +34,7 @@ data:
   #                     generated findings is not implemented yet.
   # When switching modes, the on-disk instruct JSON must be rebuilt (delete it
   # to trigger auto_build, or rerun iu_xray_builder.py with --report_mode).
-  report_mode: "split"
+  report_mode: "split_cascade"
 
   # How multiple images per study are handled. A study (one XML / one MIMIC
   # report) often has multiple views (frontal PA/AP + lateral). Three options:

scripts/vertex_job.yaml

@@ -14,7 +14,9 @@
 
 workerPoolSpecs:
 - machineSpec:
-    machineType: g2-standard-8           # 8 vCPU, 32GB RAM, 1×L4 24GB
+    machineType: g2-standard-12          # 12 vCPU, 48GB RAM, 1×L4 24GB
+    # ↑ Bumped from g2-standard-8 (32GB) — OOM with ITC batch=192 + persistent
+    # dataloader workers. 48GB has comfortable headroom.
     acceleratorType: NVIDIA_L4
     acceleratorCount: 1
   replicaCount: 1

training/train.py

@@ -394,10 +394,17 @@ def get_trainer(
         data_collator   = collator,
     )
     trainer.remove_callback(PrinterCallback)
-    # Replace default ProgressCallback with one that skips the eval per-batch
-    # bar — see _NoEvalTqdmCallback docstring for the Colab-subprocess rationale.
-    trainer.remove_callback(ProgressCallback)
-    trainer.add_callback(_NoEvalTqdmCallback())
+    # ── Eval progress bar policy ─────────────────────────────────────────
+    # Default: KEEP the eval per-batch tqdm bar so users can see eval
+    # actually running on long val sets (22k MIMIC-CXR samples → ~25-45 min
+    # of silent eval is hard to distinguish from a hang).
+    # Set DISABLE_EVAL_TQDM=1 to restore the suppressor (originally added
+    # because Colab's text renderer spams \r-updates as new lines — only
+    # matters in a Colab `!python -m ...` subprocess; native notebooks,
+    # Lightning, and Vertex handle \r fine).
+    if os.environ.get("DISABLE_EVAL_TQDM", "0") == "1":
+        trainer.remove_callback(ProgressCallback)
+        trainer.add_callback(_NoEvalTqdmCallback())
     return trainer