--- Ablation-no-imagelogits/stage1_eva_i2t.py +++ Ablation-no-imagelogits/stage1_eva_i2t.py @@ -11,6 +11,9 @@ import argparse import json import os +import sys +from datetime import datetime, timezone +from pathlib import Path import numpy as np import pandas as pd @@ -18,11 +21,16 @@ from torch.utils.data import DataLoader, Subset from transformers import AutoModel, AutoTokenizer -from Rad_IR.radir import lorentz as L -from Rad_IR.radir.RadIR import RADIR, l2norm -from Rad_IR.radir.data.hmclip_mimic_box import HyCoClipMimicBoxDataset_JPG, collate_hmclip_box +ROOT = os.path.dirname(os.path.abspath(__file__)) +sys.path[:0] = [os.path.join(ROOT, "Rad_IR"), os.path.join(ROOT, "transformer_maskgit")] +RUNTIME_ROOT = os.environ.get("HMCLIP_RUN_ROOT", "/shared/yue/hmclip_ablation_20261011") +ASSET_ROOT = os.path.join(RUNTIME_ROOT, "assets/restored") + +from radir import lorentz as L +from radir.RadIR import RADIR, l2norm +from radir.data.hmclip_mimic_box import HyCoClipMimicBoxDataset_JPG, collate_hmclip_box from retrieval_metric import compute_ndcg_uncon, soft_retrieval_uncon -from transformer_maskgit.transformer_maskgit.ctvit import CTViT +from transformer_maskgit.ctvit import CTViT ROOT = os.path.dirname(os.path.abspath(__file__)) @@ -104,6 +112,8 @@ """Slice full RaTEScore matrix to match eval sample order.""" gt = np.load(gt_path) idx = np.asarray(csv_indices, dtype=np.int64) + if gt.ndim != 2 or gt.shape[0] != gt.shape[1] or not np.isfinite(gt).all(): + raise ValueError("RaTEScore GT must be a finite square matrix") gt_sub = gt[np.ix_(idx, idx)].astype(np.float32) return torch.from_numpy(gt_sub / 100.0) @@ -132,7 +142,7 @@ model.eval() with torch.no_grad(): - for batch in dataloader: + for batch_idx, batch in enumerate(dataloader, 1): imgs = preprocess_image_like_forward(batch["imgs"].to(device, non_blocking=True)) enc_image = model.visual_transformer( @@ -159,6 +169,8 @@ all_hyp_img.append(model.project_img(image_global).cpu()) all_hyp_txt.append(model.project_txt(text_global).cpu()) + if batch_idx % 25 == 0 or batch_idx == len(dataloader): + print(f"FEATURES batch={batch_idx}/{len(dataloader)}", flush=True) return torch.cat(all_hyp_img, dim=0), torch.cat(all_hyp_txt, dim=0) @@ -224,6 +236,7 @@ # Main # --------------------------------------------------------------------------- +@torch.inference_mode() def main(): parser = argparse.ArgumentParser( description="I2T/T2I evaluation with RaTEScore-aligned GT (valid.jsonl subset)" @@ -232,25 +245,25 @@ parser.add_argument( "--model_path", default=os.path.join( - ROOT, "hf_models/microsoft/BiomedVLP-CXR-BERT-specialized" + ASSET_ROOT, "hf_models/microsoft/BiomedVLP-CXR-BERT-specialized" ), ) - parser.add_argument("--radir_ckpt", default=os.path.join(ROOT, "outputs/ablation/CXRCT_stage1_no_cross_text_entail_20260703_153255/radir_best.pt")) - parser.add_argument("--dataset_root", default=os.path.join(ROOT, "dataset")) + parser.add_argument("--radir_ckpt", default=os.path.join(RUNTIME_ROOT, "results/noimage/radir_best.pt")) + parser.add_argument("--dataset_root", default=os.path.join(ASSET_ROOT, "dataset")) parser.add_argument("--caption_file", default="valid.jsonl") parser.add_argument( "--csv_file", - default=os.path.join(ROOT, "dataset/val_caption.csv"), + default=os.path.join(ASSET_ROOT, "dataset/val_caption.csv"), help="Original RadIR caption list (defines val_ratescore row order)", ) parser.add_argument( "--gt_path", - default=os.path.join(ROOT, "dataset/val_ratescore.npy"), - ) - parser.add_argument("--cig_matched_dir", default="dataset/processed/matched") + default=os.path.join(ASSET_ROOT, "dataset/val_ratescore.npy"), + ) + parser.add_argument("--cig_matched_dir", default=os.path.join(ASSET_ROOT, "dataset/processed/matched")) parser.add_argument("--split", default="valid", type=str) parser.add_argument("--batch_size", default=16, type=int) - parser.add_argument("--num_workers", default=8, type=int) + parser.add_argument("--num_workers", default=4, type=int) parser.add_argument("--imsize", default=224, type=int) parser.add_argument("--max_words", default=128, type=int) parser.add_argument( @@ -269,6 +282,7 @@ nargs="+", default=[1, 5, 10, 20, 50, 100], ) + parser.add_argument("--output_json", default=os.path.join(RUNTIME_ROOT, "evaluation/results/noimage-best.json")) args = parser.parse_args() device = args.device @@ -314,13 +328,14 @@ image_encoder=image_encoder, text_encoder=text_model, dim_text=768, - dim_image=768, - dim_latent=768, + dim_image=512, + dim_latent=512, use_mlm=False, use_all_token_embeds=False, ) print(f"\nLoading checkpoint: {args.radir_ckpt}") - model.load(args.radir_ckpt) + model.load(args.radir_ckpt, allow_partial_load=False) + print("STRICT_CHECKPOINT_LOAD_PASSED", flush=True) model.to(device) full_dataset = HyCoClipMimicBoxDataset_JPG( @@ -333,6 +348,8 @@ max_words=args.max_words, transform=None, ) + if len(full_dataset) != stats["jsonl_rows"]: + raise RuntimeError("Dataset row count differs from JSONL; alignment requires unique image paths") dataset = Subset(full_dataset, keep_rows) dataloader = DataLoader( dataset, @@ -389,5 +406,34 @@ ) + if not torch.isfinite(sim_i2t).all(): + raise RuntimeError("Non-finite retrieval similarities") + checkpoint = torch.load(args.radir_ckpt, map_location="cpu", weights_only=False, mmap=True) + payload = { + "completed_at_utc": datetime.now(timezone.utc).isoformat(), + "checkpoint_path": args.radir_ckpt, + "checkpoint_equivalent_step": checkpoint.get("step"), + "checkpoint_best_validation_loss": checkpoint.get("best_val_loss"), + "strict_checkpoint_load": True, + "protocol": "stage1_eva_i2t.py; full aligned valid.jsonl; no study deduplication unless requested; RaTEScore / 100; soft positive > 0.9", + "split": args.split, "alignment": stats, + "arguments": vars(args), "curvature": float(curv.item()), + "metrics": {}, + } + for direction in ("i2t", "t2i"): + payload["metrics"][direction] = { + str(k): {"paired_recall": float(res[direction]["paired_r"][k]), + "soft_recall": float(res[direction]["soft_r"][k]), + "ndcg": float(res[direction]["ndcg"][i])} + for i, k in enumerate(ks) + } + destination = Path(args.output_json) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = destination.with_suffix(".json.tmp") + temporary.write_text(json.dumps(payload, ensure_ascii=False, indent=2, allow_nan=False) + "\n") + temporary.replace(destination) + print(f"EVALUATION_COMPLETE {destination}", flush=True) + + if __name__ == "__main__": main() --- Ablation-no-textlogits/stage1_eva_i2t.py +++ Ablation-no-textlogits/stage1_eva_i2t.py @@ -11,6 +11,9 @@ import argparse import json import os +import sys +from datetime import datetime, timezone +from pathlib import Path import numpy as np import pandas as pd @@ -18,11 +21,16 @@ from torch.utils.data import DataLoader, Subset from transformers import AutoModel, AutoTokenizer -from Rad_IR.radir import lorentz as L -from Rad_IR.radir.RadIR import RADIR, l2norm -from Rad_IR.radir.data.hmclip_mimic_box import HyCoClipMimicBoxDataset_JPG, collate_hmclip_box +ROOT = os.path.dirname(os.path.abspath(__file__)) +sys.path[:0] = [os.path.join(ROOT, "Rad_IR"), os.path.join(ROOT, "transformer_maskgit")] +RUNTIME_ROOT = os.environ.get("HMCLIP_RUN_ROOT", "/shared/yue/hmclip_ablation_20261011") +ASSET_ROOT = os.path.join(RUNTIME_ROOT, "assets/restored") + +from radir import lorentz as L +from radir.RadIR import RADIR, l2norm +from radir.data.hmclip_mimic_box import HyCoClipMimicBoxDataset_JPG, collate_hmclip_box from retrieval_metric import compute_ndcg_uncon, soft_retrieval_uncon -from transformer_maskgit.transformer_maskgit.ctvit import CTViT +from transformer_maskgit.ctvit import CTViT ROOT = os.path.dirname(os.path.abspath(__file__)) @@ -104,6 +112,8 @@ """Slice full RaTEScore matrix to match eval sample order.""" gt = np.load(gt_path) idx = np.asarray(csv_indices, dtype=np.int64) + if gt.ndim != 2 or gt.shape[0] != gt.shape[1] or not np.isfinite(gt).all(): + raise ValueError("RaTEScore GT must be a finite square matrix") gt_sub = gt[np.ix_(idx, idx)].astype(np.float32) return torch.from_numpy(gt_sub / 100.0) @@ -132,7 +142,7 @@ model.eval() with torch.no_grad(): - for batch in dataloader: + for batch_idx, batch in enumerate(dataloader, 1): imgs = preprocess_image_like_forward(batch["imgs"].to(device, non_blocking=True)) enc_image = model.visual_transformer( @@ -159,6 +169,8 @@ all_hyp_img.append(model.project_img(image_global).cpu()) all_hyp_txt.append(model.project_txt(text_global).cpu()) + if batch_idx % 25 == 0 or batch_idx == len(dataloader): + print(f"FEATURES batch={batch_idx}/{len(dataloader)}", flush=True) return torch.cat(all_hyp_img, dim=0), torch.cat(all_hyp_txt, dim=0) @@ -224,6 +236,7 @@ # Main # --------------------------------------------------------------------------- +@torch.inference_mode() def main(): parser = argparse.ArgumentParser( description="I2T/T2I evaluation with RaTEScore-aligned GT (valid.jsonl subset)" @@ -232,25 +245,25 @@ parser.add_argument( "--model_path", default=os.path.join( - ROOT, "hf_models/microsoft/BiomedVLP-CXR-BERT-specialized" + ASSET_ROOT, "hf_models/microsoft/BiomedVLP-CXR-BERT-specialized" ), ) - parser.add_argument("--radir_ckpt", default=os.path.join(ROOT, "outputs/ablation/CXRCT_stage1_no_cross_text_entail_20260703_153255/radir_best.pt")) - parser.add_argument("--dataset_root", default=os.path.join(ROOT, "dataset")) + parser.add_argument("--radir_ckpt", default=os.path.join(RUNTIME_ROOT, "results/notext/radir_best.pt")) + parser.add_argument("--dataset_root", default=os.path.join(ASSET_ROOT, "dataset")) parser.add_argument("--caption_file", default="valid.jsonl") parser.add_argument( "--csv_file", - default=os.path.join(ROOT, "dataset/val_caption.csv"), + default=os.path.join(ASSET_ROOT, "dataset/val_caption.csv"), help="Original RadIR caption list (defines val_ratescore row order)", ) parser.add_argument( "--gt_path", - default=os.path.join(ROOT, "dataset/val_ratescore.npy"), - ) - parser.add_argument("--cig_matched_dir", default="dataset/processed/matched") + default=os.path.join(ASSET_ROOT, "dataset/val_ratescore.npy"), + ) + parser.add_argument("--cig_matched_dir", default=os.path.join(ASSET_ROOT, "dataset/processed/matched")) parser.add_argument("--split", default="valid", type=str) parser.add_argument("--batch_size", default=16, type=int) - parser.add_argument("--num_workers", default=8, type=int) + parser.add_argument("--num_workers", default=4, type=int) parser.add_argument("--imsize", default=224, type=int) parser.add_argument("--max_words", default=128, type=int) parser.add_argument( @@ -269,6 +282,7 @@ nargs="+", default=[1, 5, 10, 20, 50, 100], ) + parser.add_argument("--output_json", default=os.path.join(RUNTIME_ROOT, "evaluation/results/notext-best.json")) args = parser.parse_args() device = args.device @@ -314,13 +328,14 @@ image_encoder=image_encoder, text_encoder=text_model, dim_text=768, - dim_image=768, - dim_latent=768, + dim_image=512, + dim_latent=512, use_mlm=False, use_all_token_embeds=False, ) print(f"\nLoading checkpoint: {args.radir_ckpt}") - model.load(args.radir_ckpt) + model.load(args.radir_ckpt, allow_partial_load=False) + print("STRICT_CHECKPOINT_LOAD_PASSED", flush=True) model.to(device) full_dataset = HyCoClipMimicBoxDataset_JPG( @@ -333,6 +348,8 @@ max_words=args.max_words, transform=None, ) + if len(full_dataset) != stats["jsonl_rows"]: + raise RuntimeError("Dataset row count differs from JSONL; alignment requires unique image paths") dataset = Subset(full_dataset, keep_rows) dataloader = DataLoader( dataset, @@ -389,5 +406,34 @@ ) + if not torch.isfinite(sim_i2t).all(): + raise RuntimeError("Non-finite retrieval similarities") + checkpoint = torch.load(args.radir_ckpt, map_location="cpu", weights_only=False, mmap=True) + payload = { + "completed_at_utc": datetime.now(timezone.utc).isoformat(), + "checkpoint_path": args.radir_ckpt, + "checkpoint_equivalent_step": checkpoint.get("step"), + "checkpoint_best_validation_loss": checkpoint.get("best_val_loss"), + "strict_checkpoint_load": True, + "protocol": "stage1_eva_i2t.py; full aligned valid.jsonl; no study deduplication unless requested; RaTEScore / 100; soft positive > 0.9", + "split": args.split, "alignment": stats, + "arguments": vars(args), "curvature": float(curv.item()), + "metrics": {}, + } + for direction in ("i2t", "t2i"): + payload["metrics"][direction] = { + str(k): {"paired_recall": float(res[direction]["paired_r"][k]), + "soft_recall": float(res[direction]["soft_r"][k]), + "ndcg": float(res[direction]["ndcg"][i])} + for i, k in enumerate(ks) + } + destination = Path(args.output_json) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = destination.with_suffix(".json.tmp") + temporary.write_text(json.dumps(payload, ensure_ascii=False, indent=2, allow_nan=False) + "\n") + temporary.replace(destination) + print(f"EVALUATION_COMPLETE {destination}", flush=True) + + if __name__ == "__main__": main()