f

data/mimic_cxr_resized_builder.py

@@ -289,15 +289,16 @@ def build_mimic_cxr_resized_instruct_json(
                                         "structured_findings": structured})
 
     # ── Pass 2: optional VQA attach ─────────────────────────────────────────
-    # Match policy: VQA `image_path` and manifest `image_relpath` should be the
-    # same string, but historical subset-build scripts differ in whether the
-    # `files/` prefix is included. Build a normalized image_index that maps
-    # BOTH the original key and its files/-stripped variant back to the
-    # canonical manifest path. On a lookup miss, also try stripping/adding
-    # `files/` from the VQA path. This handles the three known formats:
-    #   manifest "files/p10/...", vqa "files/p10/..."    → exact match
-    #   manifest "files/p10/...", vqa "p10/..."          → +files/ on VQA
-    #   manifest "p10/...",       vqa "files/p10/..."    → -files/ on VQA
+    # Match strategy (in order of preference):
+    #   1. EXACT path match  — image_path == image_relpath (ideal case).
+    #   2. FILES/ heuristic  — strip/add `files/` prefix on either side.
+    #   3. (subj, study) join — when paths are structurally unrelated (e.g.
+    #      the subset was rebuilt with flat filenames like `Study_N.jpg` but
+    #      VQA was generated against the ORIGINAL PhysioNet paths). Look up
+    #      the study's image_relpath via the manifest's (subject_id, study_id)
+    #      key instead of by path. Always falls back to the study's FIRST
+    #      available image_relpath — fine when image_mode collapsed multi-view
+    #      studies to one frontal anyway.
     def _strip_files(p: str) -> str:
         return p[len("files/"):] if p.startswith("files/") else p
 
@@ -306,18 +307,40 @@ def build_mimic_cxr_resized_instruct_json(
         norm_index[k] = k                  # exact
         norm_index[_strip_files(k)] = k    # without files/
 
-    def _resolve_vqa_path(p: str) -> Optional[str]:
+    # (subj, study) → first image_relpath of that study (after on-disk filter)
+    study_to_image: Dict[Tuple[str, str], str] = {}
+    # Reverse pass: walk samples we already appended (findings/impression) and
+    # extract one image_relpath per study. Faster + more reliable than
+    # re-walking manifests because we already know which images survived the
+    # on-disk filter. Handles all 3 image_modes: single-image samples carry
+    # `image_path`; multi_image_merged samples carry `image_paths` (list).
+    for s in samples:
+        ip = s.get("image_path") or (
+            s["image_paths"][0] if s.get("image_paths") else None
+        )
+        if ip:
+            key = (s["subject_id"].lstrip("p"), s["study_id"].lstrip("s"))
+            study_to_image.setdefault(key, ip)
+
+    def _resolve_vqa_path(p: str, subj: str, study: str) -> Optional[str]:
+        # 1) exact
         if p in norm_index:
             return norm_index[p]
+        # 2) files/ heuristic
         stripped = _strip_files(p)
         if stripped in norm_index:
             return norm_index[stripped]
         with_prefix = f"files/{p}" if not p.startswith("files/") else p
         if with_prefix in norm_index:
             return norm_index[with_prefix]
+        # 3) (subj, study) join — robust when paths are unrelated
+        key = (subj.lstrip("p"), study.lstrip("s"))
+        if key in study_to_image:
+            return study_to_image[key]
         return None
 
     n_vqa = n_vqa_dropped = 0
+    n_resolved_by_path = n_resolved_by_study = 0
     _logged_diag = False
     _dropped_samples: List[str] = []     # for diagnostic on total drop
     if vqa_dir:
@@ -338,22 +361,35 @@ def build_mimic_cxr_resized_instruct_json(
                 _idx_samples = list(image_index.keys())[:3]
                 _vqa_samples = [str(r.get("image_path", "")).lstrip("/")
                                 for r in rows[:3]]
+                _vqa_keys    = [(str(r.get("subject_id", "")).strip(),
+                                 str(r.get("study_id",   "")).strip())
+                                for r in rows[:3]]
+                _idx_keys    = list(study_to_image.keys())[:3]
                 print(f"  manifest image_relpath samples : {_idx_samples}")
                 print(f"  vqa      image_path     samples : {_vqa_samples}")
+                print(f"  manifest (subj,study)   samples : {_idx_keys}")
+                print(f"  vqa      (subj,study)   samples : {_vqa_keys}")
 
             for row in rows:
                 img_path = str(row.get("image_path", "")).lstrip("/")
-                canonical = _resolve_vqa_path(img_path)
+                subj  = str(row.get("subject_id", "")).strip()
+                study = str(row.get("study_id",   "")).strip()
+                canonical = _resolve_vqa_path(img_path, subj, study)
                 if canonical is None:
                     n_vqa_dropped += 1
                     if len(_dropped_samples) < 10:
-                        _dropped_samples.append(img_path)
+                        _dropped_samples.append(f"path={img_path!r} subj={subj!r} study={study!r}")
                     continue
+                # Track which strategy resolved it so the log shows the mix.
+                if img_path in norm_index or _strip_files(img_path) in norm_index \
+                        or (f"files/{img_path}" if not img_path.startswith("files/") else img_path) in norm_index:
+                    n_resolved_by_path += 1
+                else:
+                    n_resolved_by_study += 1
+
                 ans = row.get("answer", [])
                 answer = (", ".join(map(str, ans)) if isinstance(ans, list)
                           else str(ans)) or "No."
-                subj  = str(row.get("subject_id", "")).strip()
-                study = str(row.get("study_id",   "")).strip()
                 # Abnormality-guided VQA: reuse the manifest's CheXpert PNU
                 # for this study (same context as findings/impression). None
                 # if the study wasn't in any manifest — should not happen
@@ -372,20 +408,24 @@ def build_mimic_cxr_resized_instruct_json(
                 })
                 n_vqa += 1
 
-    # Loud warning if ALL vqa rows were dropped — this almost always means a
-    # path-format mismatch the heuristic above didn't cover, e.g. different
-    # file extension (.dcm vs .jpg) or different study/dicom id encoding.
+    if vqa_dir:
+        print(f"[mimic_cxr_resized_builder] VQA resolution: by_path={n_resolved_by_path:,}  "
+              f"by_(subj,study)={n_resolved_by_study:,}  dropped={n_vqa_dropped:,}")
+
+    # Loud warning if ALL vqa rows were dropped — by now both path AND
+    # (subj,study) join failed. Means the manifest and VQA file describe
+    # different studies entirely (wrong subset, wrong split, etc.).
     if vqa_dir and n_vqa == 0 and n_vqa_dropped > 0:
         print(f"[mimic_cxr_resized_builder] !! ALL {n_vqa_dropped:,} VQA rows "
-              f"dropped — path mismatch not handled by the heuristic.")
-        print(f"  First 10 unresolved VQA image_paths:")
+              f"dropped — neither path nor (subj,study) match anything in the manifest.")
+        print(f"  First 10 unresolved VQA rows:")
         for p in _dropped_samples:
-            print(f"    {p!r}")
-        print(f"  First 5 manifest image_relpath keys:")
-        for k in list(image_index.keys())[:5]:
-            print(f"    {k!r}")
-        print(f"  Inspect both sets and adjust _resolve_vqa_path() in "
-              f"mimic_cxr_resized_builder.py.")
+            print(f"    {p}")
+        print(f"  First 5 manifest (subj,study) keys:")
+        for k in list(study_to_image.keys())[:5]:
+            print(f"    {k}")
+        print(f"  Most likely cause: VQA file was generated against a different "
+              f"subject/study pool than the current manifest_*.csv.")
 
     # ── Write ───────────────────────────────────────────────────────────────
     output_path.parent.mkdir(parents=True, exist_ok=True)

scripts/vertex_finetune_vqa_job.yaml

@@ -52,7 +52,7 @@ workerPoolSpecs:
     env:
     # ── Required ────────────────────────────────────────────────────────────
     - name: HF_TOKEN
-      value:
+      value: 
     - name: DATASET_NAME
       value: MIMIC-CXR_resized
     - name: SOURCE_RUN_ID