Hugging Face's logo

AIGeeksGroup
/
HMCLIP

Model card Files
xet
 Community
HMCLIP / ablations /mc2-20261011 /evaluation /evaluation.patch
Zhongrui136's picture
Zhongrui136
Add complete validation retrieval results and evaluated entrypoint scripts
4aa2c96 verified
Raw History Blame Contribute Delete
14.5 kB
--- Ablation-no-imagelogits/stage1_eva_i2t.py
+++ Ablation-no-imagelogits/stage1_eva_i2t.py
@@ -11,6 +11,9 @@
import argparse
import json
import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
import numpy as np
import pandas as pd
@@ -18,11 +21,16 @@
from torch.utils.data import DataLoader, Subset
from transformers import AutoModel, AutoTokenizer
-from Rad_IR.radir import lorentz as L
-from Rad_IR.radir.RadIR import RADIR, l2norm
-from Rad_IR.radir.data.hmclip_mimic_box import HyCoClipMimicBoxDataset_JPG, collate_hmclip_box
+ROOT = os.path.dirname(os.path.abspath(__file__))
+sys.path[:0] = [os.path.join(ROOT, "Rad_IR"), os.path.join(ROOT, "transformer_maskgit")]
+RUNTIME_ROOT = os.environ.get("HMCLIP_RUN_ROOT", "/shared/yue/hmclip_ablation_20261011")
+ASSET_ROOT = os.path.join(RUNTIME_ROOT, "assets/restored")
+
+from radir import lorentz as L
+from radir.RadIR import RADIR, l2norm
+from radir.data.hmclip_mimic_box import HyCoClipMimicBoxDataset_JPG, collate_hmclip_box
from retrieval_metric import compute_ndcg_uncon, soft_retrieval_uncon
-from transformer_maskgit.transformer_maskgit.ctvit import CTViT
+from transformer_maskgit.ctvit import CTViT
ROOT = os.path.dirname(os.path.abspath(__file__))
@@ -104,6 +112,8 @@
"""Slice full RaTEScore matrix to match eval sample order."""
gt = np.load(gt_path)
idx = np.asarray(csv_indices, dtype=np.int64)
+ if gt.ndim != 2 or gt.shape[0] != gt.shape[1] or not np.isfinite(gt).all():
+ raise ValueError("RaTEScore GT must be a finite square matrix")
gt_sub = gt[np.ix_(idx, idx)].astype(np.float32)
return torch.from_numpy(gt_sub / 100.0)
@@ -132,7 +142,7 @@
model.eval()
with torch.no_grad():
- for batch in dataloader:
+ for batch_idx, batch in enumerate(dataloader, 1):
imgs = preprocess_image_like_forward(batch["imgs"].to(device, non_blocking=True))
enc_image = model.visual_transformer(
@@ -159,6 +169,8 @@
all_hyp_img.append(model.project_img(image_global).cpu())
all_hyp_txt.append(model.project_txt(text_global).cpu())
+ if batch_idx % 25 == 0 or batch_idx == len(dataloader):
+ print(f"FEATURES batch={batch_idx}/{len(dataloader)}", flush=True)
return torch.cat(all_hyp_img, dim=0), torch.cat(all_hyp_txt, dim=0)
@@ -224,6 +236,7 @@
# Main
# ---------------------------------------------------------------------------
+@torch.inference_mode()
def main():
parser = argparse.ArgumentParser(
description="I2T/T2I evaluation with RaTEScore-aligned GT (valid.jsonl subset)"
@@ -232,25 +245,25 @@
parser.add_argument(
"--model_path",
default=os.path.join(
- ROOT, "hf_models/microsoft/BiomedVLP-CXR-BERT-specialized"
+ ASSET_ROOT, "hf_models/microsoft/BiomedVLP-CXR-BERT-specialized"
),
)
- parser.add_argument("--radir_ckpt", default=os.path.join(ROOT, "outputs/ablation/CXRCT_stage1_no_cross_text_entail_20260703_153255/radir_best.pt"))
- parser.add_argument("--dataset_root", default=os.path.join(ROOT, "dataset"))
+ parser.add_argument("--radir_ckpt", default=os.path.join(RUNTIME_ROOT, "results/noimage/radir_best.pt"))
+ parser.add_argument("--dataset_root", default=os.path.join(ASSET_ROOT, "dataset"))
parser.add_argument("--caption_file", default="valid.jsonl")
parser.add_argument(
"--csv_file",
- default=os.path.join(ROOT, "dataset/val_caption.csv"),
+ default=os.path.join(ASSET_ROOT, "dataset/val_caption.csv"),
help="Original RadIR caption list (defines val_ratescore row order)",
)
parser.add_argument(
"--gt_path",
- default=os.path.join(ROOT, "dataset/val_ratescore.npy"),
- )
- parser.add_argument("--cig_matched_dir", default="dataset/processed/matched")
+ default=os.path.join(ASSET_ROOT, "dataset/val_ratescore.npy"),
+ )
+ parser.add_argument("--cig_matched_dir", default=os.path.join(ASSET_ROOT, "dataset/processed/matched"))
parser.add_argument("--split", default="valid", type=str)
parser.add_argument("--batch_size", default=16, type=int)
- parser.add_argument("--num_workers", default=8, type=int)
+ parser.add_argument("--num_workers", default=4, type=int)
parser.add_argument("--imsize", default=224, type=int)
parser.add_argument("--max_words", default=128, type=int)
parser.add_argument(
@@ -269,6 +282,7 @@
nargs="+",
default=[1, 5, 10, 20, 50, 100],
)
+ parser.add_argument("--output_json", default=os.path.join(RUNTIME_ROOT, "evaluation/results/noimage-best.json"))
args = parser.parse_args()
device = args.device
@@ -314,13 +328,14 @@
image_encoder=image_encoder,
text_encoder=text_model,
dim_text=768,
- dim_image=768,
- dim_latent=768,
+ dim_image=512,
+ dim_latent=512,
use_mlm=False,
use_all_token_embeds=False,
)
print(f"\nLoading checkpoint: {args.radir_ckpt}")
- model.load(args.radir_ckpt)
+ model.load(args.radir_ckpt, allow_partial_load=False)
+ print("STRICT_CHECKPOINT_LOAD_PASSED", flush=True)
model.to(device)
full_dataset = HyCoClipMimicBoxDataset_JPG(
@@ -333,6 +348,8 @@
max_words=args.max_words,
transform=None,
)
+ if len(full_dataset) != stats["jsonl_rows"]:
+ raise RuntimeError("Dataset row count differs from JSONL; alignment requires unique image paths")
dataset = Subset(full_dataset, keep_rows)
dataloader = DataLoader(
dataset,
@@ -389,5 +406,34 @@
)
+ if not torch.isfinite(sim_i2t).all():
+ raise RuntimeError("Non-finite retrieval similarities")
+ checkpoint = torch.load(args.radir_ckpt, map_location="cpu", weights_only=False, mmap=True)
+ payload = {
+ "completed_at_utc": datetime.now(timezone.utc).isoformat(),
+ "checkpoint_path": args.radir_ckpt,
+ "checkpoint_equivalent_step": checkpoint.get("step"),
+ "checkpoint_best_validation_loss": checkpoint.get("best_val_loss"),
+ "strict_checkpoint_load": True,
+ "protocol": "stage1_eva_i2t.py; full aligned valid.jsonl; no study deduplication unless requested; RaTEScore / 100; soft positive > 0.9",
+ "split": args.split, "alignment": stats,
+ "arguments": vars(args), "curvature": float(curv.item()),
+ "metrics": {},
+ }
+ for direction in ("i2t", "t2i"):
+ payload["metrics"][direction] = {
+ str(k): {"paired_recall": float(res[direction]["paired_r"][k]),
+ "soft_recall": float(res[direction]["soft_r"][k]),
+ "ndcg": float(res[direction]["ndcg"][i])}
+ for i, k in enumerate(ks)
+ }
+ destination = Path(args.output_json)
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ temporary = destination.with_suffix(".json.tmp")
+ temporary.write_text(json.dumps(payload, ensure_ascii=False, indent=2, allow_nan=False) + "\n")
+ temporary.replace(destination)
+ print(f"EVALUATION_COMPLETE {destination}", flush=True)
+
+
if __name__ == "__main__":
main()
--- Ablation-no-textlogits/stage1_eva_i2t.py
+++ Ablation-no-textlogits/stage1_eva_i2t.py
@@ -11,6 +11,9 @@
import argparse
import json
import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
import numpy as np
import pandas as pd
@@ -18,11 +21,16 @@
from torch.utils.data import DataLoader, Subset
from transformers import AutoModel, AutoTokenizer
-from Rad_IR.radir import lorentz as L
-from Rad_IR.radir.RadIR import RADIR, l2norm
-from Rad_IR.radir.data.hmclip_mimic_box import HyCoClipMimicBoxDataset_JPG, collate_hmclip_box
+ROOT = os.path.dirname(os.path.abspath(__file__))
+sys.path[:0] = [os.path.join(ROOT, "Rad_IR"), os.path.join(ROOT, "transformer_maskgit")]
+RUNTIME_ROOT = os.environ.get("HMCLIP_RUN_ROOT", "/shared/yue/hmclip_ablation_20261011")
+ASSET_ROOT = os.path.join(RUNTIME_ROOT, "assets/restored")
+
+from radir import lorentz as L
+from radir.RadIR import RADIR, l2norm
+from radir.data.hmclip_mimic_box import HyCoClipMimicBoxDataset_JPG, collate_hmclip_box
from retrieval_metric import compute_ndcg_uncon, soft_retrieval_uncon
-from transformer_maskgit.transformer_maskgit.ctvit import CTViT
+from transformer_maskgit.ctvit import CTViT
ROOT = os.path.dirname(os.path.abspath(__file__))
@@ -104,6 +112,8 @@
"""Slice full RaTEScore matrix to match eval sample order."""
gt = np.load(gt_path)
idx = np.asarray(csv_indices, dtype=np.int64)
+ if gt.ndim != 2 or gt.shape[0] != gt.shape[1] or not np.isfinite(gt).all():
+ raise ValueError("RaTEScore GT must be a finite square matrix")
gt_sub = gt[np.ix_(idx, idx)].astype(np.float32)
return torch.from_numpy(gt_sub / 100.0)
@@ -132,7 +142,7 @@
model.eval()
with torch.no_grad():
- for batch in dataloader:
+ for batch_idx, batch in enumerate(dataloader, 1):
imgs = preprocess_image_like_forward(batch["imgs"].to(device, non_blocking=True))
enc_image = model.visual_transformer(
@@ -159,6 +169,8 @@
all_hyp_img.append(model.project_img(image_global).cpu())
all_hyp_txt.append(model.project_txt(text_global).cpu())
+ if batch_idx % 25 == 0 or batch_idx == len(dataloader):
+ print(f"FEATURES batch={batch_idx}/{len(dataloader)}", flush=True)
return torch.cat(all_hyp_img, dim=0), torch.cat(all_hyp_txt, dim=0)
@@ -224,6 +236,7 @@
# Main
# ---------------------------------------------------------------------------
+@torch.inference_mode()
def main():
parser = argparse.ArgumentParser(
description="I2T/T2I evaluation with RaTEScore-aligned GT (valid.jsonl subset)"
@@ -232,25 +245,25 @@
parser.add_argument(
"--model_path",
default=os.path.join(
- ROOT, "hf_models/microsoft/BiomedVLP-CXR-BERT-specialized"
+ ASSET_ROOT, "hf_models/microsoft/BiomedVLP-CXR-BERT-specialized"
),
)
- parser.add_argument("--radir_ckpt", default=os.path.join(ROOT, "outputs/ablation/CXRCT_stage1_no_cross_text_entail_20260703_153255/radir_best.pt"))
- parser.add_argument("--dataset_root", default=os.path.join(ROOT, "dataset"))
+ parser.add_argument("--radir_ckpt", default=os.path.join(RUNTIME_ROOT, "results/notext/radir_best.pt"))
+ parser.add_argument("--dataset_root", default=os.path.join(ASSET_ROOT, "dataset"))
parser.add_argument("--caption_file", default="valid.jsonl")
parser.add_argument(
"--csv_file",
- default=os.path.join(ROOT, "dataset/val_caption.csv"),
+ default=os.path.join(ASSET_ROOT, "dataset/val_caption.csv"),
help="Original RadIR caption list (defines val_ratescore row order)",
)
parser.add_argument(
"--gt_path",
- default=os.path.join(ROOT, "dataset/val_ratescore.npy"),
- )
- parser.add_argument("--cig_matched_dir", default="dataset/processed/matched")
+ default=os.path.join(ASSET_ROOT, "dataset/val_ratescore.npy"),
+ )
+ parser.add_argument("--cig_matched_dir", default=os.path.join(ASSET_ROOT, "dataset/processed/matched"))
parser.add_argument("--split", default="valid", type=str)
parser.add_argument("--batch_size", default=16, type=int)
- parser.add_argument("--num_workers", default=8, type=int)
+ parser.add_argument("--num_workers", default=4, type=int)
parser.add_argument("--imsize", default=224, type=int)
parser.add_argument("--max_words", default=128, type=int)
parser.add_argument(
@@ -269,6 +282,7 @@
nargs="+",
default=[1, 5, 10, 20, 50, 100],
)
+ parser.add_argument("--output_json", default=os.path.join(RUNTIME_ROOT, "evaluation/results/notext-best.json"))
args = parser.parse_args()
device = args.device
@@ -314,13 +328,14 @@
image_encoder=image_encoder,
text_encoder=text_model,
dim_text=768,
- dim_image=768,
- dim_latent=768,
+ dim_image=512,
+ dim_latent=512,
use_mlm=False,
use_all_token_embeds=False,
)
print(f"\nLoading checkpoint: {args.radir_ckpt}")
- model.load(args.radir_ckpt)
+ model.load(args.radir_ckpt, allow_partial_load=False)
+ print("STRICT_CHECKPOINT_LOAD_PASSED", flush=True)
model.to(device)
full_dataset = HyCoClipMimicBoxDataset_JPG(
@@ -333,6 +348,8 @@
max_words=args.max_words,
transform=None,
)
+ if len(full_dataset) != stats["jsonl_rows"]:
+ raise RuntimeError("Dataset row count differs from JSONL; alignment requires unique image paths")
dataset = Subset(full_dataset, keep_rows)
dataloader = DataLoader(
dataset,
@@ -389,5 +406,34 @@
)
+ if not torch.isfinite(sim_i2t).all():
+ raise RuntimeError("Non-finite retrieval similarities")
+ checkpoint = torch.load(args.radir_ckpt, map_location="cpu", weights_only=False, mmap=True)
+ payload = {
+ "completed_at_utc": datetime.now(timezone.utc).isoformat(),
+ "checkpoint_path": args.radir_ckpt,
+ "checkpoint_equivalent_step": checkpoint.get("step"),
+ "checkpoint_best_validation_loss": checkpoint.get("best_val_loss"),
+ "strict_checkpoint_load": True,
+ "protocol": "stage1_eva_i2t.py; full aligned valid.jsonl; no study deduplication unless requested; RaTEScore / 100; soft positive > 0.9",
+ "split": args.split, "alignment": stats,
+ "arguments": vars(args), "curvature": float(curv.item()),
+ "metrics": {},
+ }
+ for direction in ("i2t", "t2i"):
+ payload["metrics"][direction] = {
+ str(k): {"paired_recall": float(res[direction]["paired_r"][k]),
+ "soft_recall": float(res[direction]["soft_r"][k]),
+ "ndcg": float(res[direction]["ndcg"][i])}
+ for i, k in enumerate(ks)
+ }
+ destination = Path(args.output_json)
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ temporary = destination.with_suffix(".json.tmp")
+ temporary.write_text(json.dumps(payload, ensure_ascii=False, indent=2, allow_nan=False) + "\n")
+ temporary.replace(destination)
+ print(f"EVALUATION_COMPLETE {destination}", flush=True)
+
+
if __name__ == "__main__":
main()