ablations/mc2-20261011/evaluation/evaluation.patch

--- Ablation-no-imagelogits/stage1_eva_i2t.py
+++ Ablation-no-imagelogits/stage1_eva_i2t.py
@@ -11,6 +11,9 @@
 import argparse
 import json
 import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
 
 import numpy as np
 import pandas as pd
@@ -18,11 +21,16 @@
 from torch.utils.data import DataLoader, Subset
 from transformers import AutoModel, AutoTokenizer
 
-from Rad_IR.radir import lorentz as L
-from Rad_IR.radir.RadIR import RADIR, l2norm
-from Rad_IR.radir.data.hmclip_mimic_box import HyCoClipMimicBoxDataset_JPG, collate_hmclip_box
+ROOT = os.path.dirname(os.path.abspath(__file__))
+sys.path[:0] = [os.path.join(ROOT, "Rad_IR"), os.path.join(ROOT, "transformer_maskgit")]
+RUNTIME_ROOT = os.environ.get("HMCLIP_RUN_ROOT", "/shared/yue/hmclip_ablation_20261011")
+ASSET_ROOT = os.path.join(RUNTIME_ROOT, "assets/restored")
+
+from radir import lorentz as L
+from radir.RadIR import RADIR, l2norm
+from radir.data.hmclip_mimic_box import HyCoClipMimicBoxDataset_JPG, collate_hmclip_box
 from retrieval_metric import compute_ndcg_uncon, soft_retrieval_uncon
-from transformer_maskgit.transformer_maskgit.ctvit import CTViT
+from transformer_maskgit.ctvit import CTViT
 
 ROOT = os.path.dirname(os.path.abspath(__file__))
 
@@ -104,6 +112,8 @@
     """Slice full RaTEScore matrix to match eval sample order."""
     gt = np.load(gt_path)
     idx = np.asarray(csv_indices, dtype=np.int64)
+    if gt.ndim != 2 or gt.shape[0] != gt.shape[1] or not np.isfinite(gt).all():
+        raise ValueError("RaTEScore GT must be a finite square matrix")
     gt_sub = gt[np.ix_(idx, idx)].astype(np.float32)
     return torch.from_numpy(gt_sub / 100.0)
 
@@ -132,7 +142,7 @@
 
     model.eval()
     with torch.no_grad():
-        for batch in dataloader:
+        for batch_idx, batch in enumerate(dataloader, 1):
             imgs = preprocess_image_like_forward(batch["imgs"].to(device, non_blocking=True))
 
             enc_image = model.visual_transformer(
@@ -159,6 +169,8 @@
 
             all_hyp_img.append(model.project_img(image_global).cpu())
             all_hyp_txt.append(model.project_txt(text_global).cpu())
+            if batch_idx % 25 == 0 or batch_idx == len(dataloader):
+                print(f"FEATURES batch={batch_idx}/{len(dataloader)}", flush=True)
 
     return torch.cat(all_hyp_img, dim=0), torch.cat(all_hyp_txt, dim=0)
 
@@ -224,6 +236,7 @@
 # Main
 # ---------------------------------------------------------------------------
 
+@torch.inference_mode()
 def main():
     parser = argparse.ArgumentParser(
         description="I2T/T2I evaluation with RaTEScore-aligned GT (valid.jsonl subset)"
@@ -232,25 +245,25 @@
     parser.add_argument(
         "--model_path",
         default=os.path.join(
-            ROOT, "hf_models/microsoft/BiomedVLP-CXR-BERT-specialized"
+            ASSET_ROOT, "hf_models/microsoft/BiomedVLP-CXR-BERT-specialized"
         ),
     )
-    parser.add_argument("--radir_ckpt", default=os.path.join(ROOT, "outputs/ablation/CXRCT_stage1_no_cross_text_entail_20260703_153255/radir_best.pt"))
-    parser.add_argument("--dataset_root", default=os.path.join(ROOT, "dataset"))
+    parser.add_argument("--radir_ckpt", default=os.path.join(RUNTIME_ROOT, "results/noimage/radir_best.pt"))
+    parser.add_argument("--dataset_root", default=os.path.join(ASSET_ROOT, "dataset"))
     parser.add_argument("--caption_file", default="valid.jsonl")
     parser.add_argument(
         "--csv_file",
-        default=os.path.join(ROOT, "dataset/val_caption.csv"),
+        default=os.path.join(ASSET_ROOT, "dataset/val_caption.csv"),
         help="Original RadIR caption list (defines val_ratescore row order)",
     )
     parser.add_argument(
         "--gt_path",
-        default=os.path.join(ROOT, "dataset/val_ratescore.npy"),
-    )
-    parser.add_argument("--cig_matched_dir", default="dataset/processed/matched")
+        default=os.path.join(ASSET_ROOT, "dataset/val_ratescore.npy"),
+    )
+    parser.add_argument("--cig_matched_dir", default=os.path.join(ASSET_ROOT, "dataset/processed/matched"))
     parser.add_argument("--split", default="valid", type=str)
     parser.add_argument("--batch_size", default=16, type=int)
-    parser.add_argument("--num_workers", default=8, type=int)
+    parser.add_argument("--num_workers", default=4, type=int)
     parser.add_argument("--imsize", default=224, type=int)
     parser.add_argument("--max_words", default=128, type=int)
     parser.add_argument(
@@ -269,6 +282,7 @@
         nargs="+",
         default=[1, 5, 10, 20, 50, 100],
     )
+    parser.add_argument("--output_json", default=os.path.join(RUNTIME_ROOT, "evaluation/results/noimage-best.json"))
     args = parser.parse_args()
     device = args.device
 
@@ -314,13 +328,14 @@
         image_encoder=image_encoder,
         text_encoder=text_model,
         dim_text=768,
-        dim_image=768,
-        dim_latent=768,
+        dim_image=512,
+        dim_latent=512,
         use_mlm=False,
         use_all_token_embeds=False,
     )
     print(f"\nLoading checkpoint: {args.radir_ckpt}")
-    model.load(args.radir_ckpt)
+    model.load(args.radir_ckpt, allow_partial_load=False)
+    print("STRICT_CHECKPOINT_LOAD_PASSED", flush=True)
     model.to(device)
 
     full_dataset = HyCoClipMimicBoxDataset_JPG(
@@ -333,6 +348,8 @@
         max_words=args.max_words,
         transform=None,
     )
+    if len(full_dataset) != stats["jsonl_rows"]:
+        raise RuntimeError("Dataset row count differs from JSONL; alignment requires unique image paths")
     dataset = Subset(full_dataset, keep_rows)
     dataloader = DataLoader(
         dataset,
@@ -389,5 +406,34 @@
         )
 
 
+    if not torch.isfinite(sim_i2t).all():
+        raise RuntimeError("Non-finite retrieval similarities")
+    checkpoint = torch.load(args.radir_ckpt, map_location="cpu", weights_only=False, mmap=True)
+    payload = {
+        "completed_at_utc": datetime.now(timezone.utc).isoformat(),
+        "checkpoint_path": args.radir_ckpt,
+        "checkpoint_equivalent_step": checkpoint.get("step"),
+        "checkpoint_best_validation_loss": checkpoint.get("best_val_loss"),
+        "strict_checkpoint_load": True,
+        "protocol": "stage1_eva_i2t.py; full aligned valid.jsonl; no study deduplication unless requested; RaTEScore / 100; soft positive > 0.9",
+        "split": args.split, "alignment": stats,
+        "arguments": vars(args), "curvature": float(curv.item()),
+        "metrics": {},
+    }
+    for direction in ("i2t", "t2i"):
+        payload["metrics"][direction] = {
+            str(k): {"paired_recall": float(res[direction]["paired_r"][k]),
+                     "soft_recall": float(res[direction]["soft_r"][k]),
+                     "ndcg": float(res[direction]["ndcg"][i])}
+            for i, k in enumerate(ks)
+        }
+    destination = Path(args.output_json)
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    temporary = destination.with_suffix(".json.tmp")
+    temporary.write_text(json.dumps(payload, ensure_ascii=False, indent=2, allow_nan=False) + "\n")
+    temporary.replace(destination)
+    print(f"EVALUATION_COMPLETE {destination}", flush=True)
+
+
 if __name__ == "__main__":
     main()
--- Ablation-no-textlogits/stage1_eva_i2t.py
+++ Ablation-no-textlogits/stage1_eva_i2t.py
@@ -11,6 +11,9 @@
 import argparse
 import json
 import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
 
 import numpy as np
 import pandas as pd
@@ -18,11 +21,16 @@
 from torch.utils.data import DataLoader, Subset
 from transformers import AutoModel, AutoTokenizer
 
-from Rad_IR.radir import lorentz as L
-from Rad_IR.radir.RadIR import RADIR, l2norm
-from Rad_IR.radir.data.hmclip_mimic_box import HyCoClipMimicBoxDataset_JPG, collate_hmclip_box
+ROOT = os.path.dirname(os.path.abspath(__file__))
+sys.path[:0] = [os.path.join(ROOT, "Rad_IR"), os.path.join(ROOT, "transformer_maskgit")]
+RUNTIME_ROOT = os.environ.get("HMCLIP_RUN_ROOT", "/shared/yue/hmclip_ablation_20261011")
+ASSET_ROOT = os.path.join(RUNTIME_ROOT, "assets/restored")
+
+from radir import lorentz as L
+from radir.RadIR import RADIR, l2norm
+from radir.data.hmclip_mimic_box import HyCoClipMimicBoxDataset_JPG, collate_hmclip_box
 from retrieval_metric import compute_ndcg_uncon, soft_retrieval_uncon
-from transformer_maskgit.transformer_maskgit.ctvit import CTViT
+from transformer_maskgit.ctvit import CTViT
 
 ROOT = os.path.dirname(os.path.abspath(__file__))
 
@@ -104,6 +112,8 @@
     """Slice full RaTEScore matrix to match eval sample order."""
     gt = np.load(gt_path)
     idx = np.asarray(csv_indices, dtype=np.int64)
+    if gt.ndim != 2 or gt.shape[0] != gt.shape[1] or not np.isfinite(gt).all():
+        raise ValueError("RaTEScore GT must be a finite square matrix")
     gt_sub = gt[np.ix_(idx, idx)].astype(np.float32)
     return torch.from_numpy(gt_sub / 100.0)
 
@@ -132,7 +142,7 @@
 
     model.eval()
     with torch.no_grad():
-        for batch in dataloader:
+        for batch_idx, batch in enumerate(dataloader, 1):
             imgs = preprocess_image_like_forward(batch["imgs"].to(device, non_blocking=True))
 
             enc_image = model.visual_transformer(
@@ -159,6 +169,8 @@
 
             all_hyp_img.append(model.project_img(image_global).cpu())
             all_hyp_txt.append(model.project_txt(text_global).cpu())
+            if batch_idx % 25 == 0 or batch_idx == len(dataloader):
+                print(f"FEATURES batch={batch_idx}/{len(dataloader)}", flush=True)
 
     return torch.cat(all_hyp_img, dim=0), torch.cat(all_hyp_txt, dim=0)
 
@@ -224,6 +236,7 @@
 # Main
 # ---------------------------------------------------------------------------
 
+@torch.inference_mode()
 def main():
     parser = argparse.ArgumentParser(
         description="I2T/T2I evaluation with RaTEScore-aligned GT (valid.jsonl subset)"
@@ -232,25 +245,25 @@
     parser.add_argument(
         "--model_path",
         default=os.path.join(
-            ROOT, "hf_models/microsoft/BiomedVLP-CXR-BERT-specialized"
+            ASSET_ROOT, "hf_models/microsoft/BiomedVLP-CXR-BERT-specialized"
         ),
     )
-    parser.add_argument("--radir_ckpt", default=os.path.join(ROOT, "outputs/ablation/CXRCT_stage1_no_cross_text_entail_20260703_153255/radir_best.pt"))
-    parser.add_argument("--dataset_root", default=os.path.join(ROOT, "dataset"))
+    parser.add_argument("--radir_ckpt", default=os.path.join(RUNTIME_ROOT, "results/notext/radir_best.pt"))
+    parser.add_argument("--dataset_root", default=os.path.join(ASSET_ROOT, "dataset"))
     parser.add_argument("--caption_file", default="valid.jsonl")
     parser.add_argument(
         "--csv_file",
-        default=os.path.join(ROOT, "dataset/val_caption.csv"),
+        default=os.path.join(ASSET_ROOT, "dataset/val_caption.csv"),
         help="Original RadIR caption list (defines val_ratescore row order)",
     )
     parser.add_argument(
         "--gt_path",
-        default=os.path.join(ROOT, "dataset/val_ratescore.npy"),
-    )
-    parser.add_argument("--cig_matched_dir", default="dataset/processed/matched")
+        default=os.path.join(ASSET_ROOT, "dataset/val_ratescore.npy"),
+    )
+    parser.add_argument("--cig_matched_dir", default=os.path.join(ASSET_ROOT, "dataset/processed/matched"))
     parser.add_argument("--split", default="valid", type=str)
     parser.add_argument("--batch_size", default=16, type=int)
-    parser.add_argument("--num_workers", default=8, type=int)
+    parser.add_argument("--num_workers", default=4, type=int)
     parser.add_argument("--imsize", default=224, type=int)
     parser.add_argument("--max_words", default=128, type=int)
     parser.add_argument(
@@ -269,6 +282,7 @@
         nargs="+",
         default=[1, 5, 10, 20, 50, 100],
     )
+    parser.add_argument("--output_json", default=os.path.join(RUNTIME_ROOT, "evaluation/results/notext-best.json"))
     args = parser.parse_args()
     device = args.device
 
@@ -314,13 +328,14 @@
         image_encoder=image_encoder,
         text_encoder=text_model,
         dim_text=768,
-        dim_image=768,
-        dim_latent=768,
+        dim_image=512,
+        dim_latent=512,
         use_mlm=False,
         use_all_token_embeds=False,
     )
     print(f"\nLoading checkpoint: {args.radir_ckpt}")
-    model.load(args.radir_ckpt)
+    model.load(args.radir_ckpt, allow_partial_load=False)
+    print("STRICT_CHECKPOINT_LOAD_PASSED", flush=True)
     model.to(device)
 
     full_dataset = HyCoClipMimicBoxDataset_JPG(
@@ -333,6 +348,8 @@
         max_words=args.max_words,
         transform=None,
     )
+    if len(full_dataset) != stats["jsonl_rows"]:
+        raise RuntimeError("Dataset row count differs from JSONL; alignment requires unique image paths")
     dataset = Subset(full_dataset, keep_rows)
     dataloader = DataLoader(
         dataset,
@@ -389,5 +406,34 @@
         )
 
 
+    if not torch.isfinite(sim_i2t).all():
+        raise RuntimeError("Non-finite retrieval similarities")
+    checkpoint = torch.load(args.radir_ckpt, map_location="cpu", weights_only=False, mmap=True)
+    payload = {
+        "completed_at_utc": datetime.now(timezone.utc).isoformat(),
+        "checkpoint_path": args.radir_ckpt,
+        "checkpoint_equivalent_step": checkpoint.get("step"),
+        "checkpoint_best_validation_loss": checkpoint.get("best_val_loss"),
+        "strict_checkpoint_load": True,
+        "protocol": "stage1_eva_i2t.py; full aligned valid.jsonl; no study deduplication unless requested; RaTEScore / 100; soft positive > 0.9",
+        "split": args.split, "alignment": stats,
+        "arguments": vars(args), "curvature": float(curv.item()),
+        "metrics": {},
+    }
+    for direction in ("i2t", "t2i"):
+        payload["metrics"][direction] = {
+            str(k): {"paired_recall": float(res[direction]["paired_r"][k]),
+                     "soft_recall": float(res[direction]["soft_r"][k]),
+                     "ndcg": float(res[direction]["ndcg"][i])}
+            for i, k in enumerate(ks)
+        }
+    destination = Path(args.output_json)
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    temporary = destination.with_suffix(".json.tmp")
+    temporary.write_text(json.dumps(payload, ensure_ascii=False, indent=2, allow_nan=False) + "\n")
+    temporary.replace(destination)
+    print(f"EVALUATION_COMPLETE {destination}", flush=True)
+
+
 if __name__ == "__main__":
     main()