f

scripts/gcp_finetune_vqa_entrypoint.py

@@ -103,6 +103,14 @@ CKPT_ROOT     = WORK / "ckpt"
 for d in (DATA_SRC, RUN_PULL_ROOT, CKPT_ROOT):
     d.mkdir(parents=True, exist_ok=True)
 
+# Make project modules (utils.*, data.*, model.*) importable from this script
+# itself. Python's sys.path includes the SCRIPT's dir (/workspace/code/scripts)
+# but NOT the project root (/workspace/code) when invoked directly. The
+# subprocess we spawn later uses `python -m training.train` which resolves
+# modules relative to cwd, so it's fine — but the pre-flight VQA check below
+# needs `from utils.dataset_resolver import ...` to work in-kernel.
+sys.path.insert(0, str(PROJECT))
+
 print(f"[gcp_finetune_vqa] PROJECT       = {PROJECT}")
 print(f"[gcp_finetune_vqa] WORK          = {WORK}")
 print(f"[gcp_finetune_vqa] DATASET_NAME  = {DATASET_NAME}")
@@ -220,6 +228,14 @@ TGT_S1_DIR.mkdir(parents=True, exist_ok=True)
 shutil.copy2(SRC_PROJ, TGT_S1_DIR / "stage1_final_projection.pt")
 print(f"[gcp_finetune_vqa] seeded projection -> {TGT_S1_DIR / 'stage1_final_projection.pt'}")
 
+# train.py:1043 gates on the LITERAL file `stage1_final.pt` existing (it then
+# passes that path's stem to load_checkpoint, which derives the actual
+# `*_projection.pt` + `*_lora/` paths from it). Without this sentinel the
+# gate fails silently and Stage 2 starts with a RANDOM projection — defeats
+# the entire seed step. The file's content is never read; it's a marker only.
+(TGT_S1_DIR / "stage1_final.pt").touch()
+print(f"[gcp_finetune_vqa] seeded sentinel   -> {TGT_S1_DIR / 'stage1_final.pt'}  (marker for train.py gate)")
+
 tgt_lora = TGT_S1_DIR / "stage1_final_lora"
 if tgt_lora.exists():
     shutil.rmtree(tgt_lora)

scripts/vertex_finetune_vqa_job.yaml

@@ -52,7 +52,7 @@ workerPoolSpecs:
     env:
     # ── Required ────────────────────────────────────────────────────────────
     - name: HF_TOKEN
-      value: 
+      value:
     - name: DATASET_NAME
       value: MIMIC-CXR_resized
     - name: SOURCE_RUN_ID
@@ -85,3 +85,11 @@ scheduling:
   # Stage 2 finetune (3 epochs, mixed batch ~80k samples) on L4 ≈ 3-4h.
   # 6h ceiling leaves slack for data pull + builder.
   timeout: 21600s
+  # Don't auto-retry on worker failure. When entrypoint asserts (bad path,
+  # missing source run, etc.) the retry just re-pulls 40GB tar shards and
+  # crashes the same way — wasteful. Set to 1 to fail fast.
+  restartJobOnWorkerRestart: false
+  # restartJobOnWorkerRestart is for preemption; for hard errors we want
+  # explicit no-retry which is the default (replicaCount=1, no replicas
+  # spec), but the explicit `disableRetries: true` makes intent clear.
+  disableRetries: true