#!/usr/bin/env python3 from __future__ import annotations import argparse import csv import sys import time from dataclasses import dataclass from pathlib import Path import json from common import cfg_path, load_config, media_dir_for_model, select_models, utc_now, write_json @dataclass(frozen=True) class Sample: model_name: str prompt_idx: int seed_idx: int video_path: str audio_path: str def configure_javis_paths(config: dict) -> None: javis_root = cfg_path(config, "javis_root") imagebind_root = javis_root / "eval/javisbench/src/ImageBind" for path in (javis_root, imagebind_root): path_str = str(path) if path_str not in sys.path: sys.path.insert(0, path_str) def sample_paths(config: dict, model: dict) -> list[Sample]: subset = config.get("prompt_subset", {}) num_prompts = int(subset.get("num_prompts", 200)) num_seeds = int(subset.get("num_seeds", 8)) out_dir = media_dir_for_model(config, model) samples = [] for prompt_idx in range(num_prompts): for seed_idx in range(num_seeds): stem = f"sample_{prompt_idx:04d}_seed{seed_idx:04d}" video_path = out_dir / f"{stem}.mp4" audio_path = out_dir / f"{stem}.wav" if (not video_path.is_file() or not audio_path.is_file() or video_path.stat().st_size == 0 or audio_path.stat().st_size == 0): raise FileNotFoundError(f"{model['name']}: missing {video_path} or {audio_path}") sidecar = out_dir / f"{stem}.json" if sidecar.is_file(): record = json.loads(sidecar.read_text(encoding="utf-8")) if record.get("index") != prompt_idx or record.get("seed_idx") != seed_idx: raise ValueError(f"Generated index/seed mismatch: {sidecar}") samples.append(Sample(model["name"], prompt_idx, seed_idx, str(video_path), str(audio_path))) return samples def complete_model(config: dict, model: dict) -> bool: try: sample_paths(config, model) return True except FileNotFoundError: return False def make_dataset_class(config: dict): import torch from torch.utils.data import Dataset class AVPathDataset(Dataset): def __init__(self, samples: list[Sample]): self.samples = samples def __len__(self): return len(self.samples) def __getitem__(self, index: int): configure_javis_paths(config) from eval.javisbench.src.ImageBind.imagebind import data as imagebind_data from eval.javisbench.src.ImageBind.imagebind.models.imagebind_model import ModalityType sample = self.samples[index] return { "prompt_idx": sample.prompt_idx, "seed_idx": sample.seed_idx, "video_path": sample.video_path, "audio_path": sample.audio_path, ModalityType.VISION: imagebind_data.load_and_transform_video_data([sample.video_path], "cpu")[0], ModalityType.AUDIO: imagebind_data.load_and_transform_audio_data([sample.audio_path], "cpu")[0], } return AVPathDataset def collate(batch): import torch from eval.javisbench.src.ImageBind.imagebind.models.imagebind_model import ModalityType return { "prompt_idx": torch.tensor([item["prompt_idx"] for item in batch], dtype=torch.long), "seed_idx": torch.tensor([item["seed_idx"] for item in batch], dtype=torch.long), "video_path": [item["video_path"] for item in batch], "audio_path": [item["audio_path"] for item in batch], ModalityType.VISION: torch.stack([item[ModalityType.VISION] for item in batch], dim=0), ModalityType.AUDIO: torch.stack([item[ModalityType.AUDIO] for item in batch], dim=0), } def load_model(config: dict, device): import os javis_root = cfg_path(config, "javis_root") os.chdir(javis_root) configure_javis_paths(config) from eval.javisbench.src.ImageBind.imagebind.models import imagebind_model model = imagebind_model.imagebind_huge(pretrained=True) model.eval().to(device) return model def embed_path(config: dict, model: dict) -> Path: return cfg_path(config, "diversity_metric_root") / "embeddings" / f"{model['name']}.pt" def upper_triangle_values(matrix): import torch n = matrix.shape[0] idx = torch.triu_indices(n, n, offset=1) return matrix[idx[0], idx[1]] def prompt_pairwise_stats(config: dict, z, metric: str) -> tuple[float, list[float]]: import torch import torch.nn.functional as F subset = config.get("prompt_subset", {}) num_prompts = int(subset.get("num_prompts", 200)) num_seeds = int(subset.get("num_seeds", 8)) per_prompt = [] for prompt_idx in range(num_prompts): group = z[prompt_idx * num_seeds : (prompt_idx + 1) * num_seeds] if metric == "cosine": group = F.normalize(group, dim=-1) distances = 1.0 - group @ group.T elif metric == "l2": distances = torch.cdist(group, group, p=2) else: raise ValueError(metric) per_prompt.append(float(upper_triangle_values(distances).mean().item())) return float(torch.tensor(per_prompt).mean().item()), per_prompt def embed_model(config: dict, model_cfg: dict, device, batch_size: int, num_workers: int, force: bool, model) -> Path: import torch from torch.utils.data import DataLoader from tqdm import tqdm from eval.javisbench.src.ImageBind.imagebind.models.imagebind_model import ModalityType out = embed_path(config, model_cfg) out.parent.mkdir(parents=True, exist_ok=True) if out.exists() and not force: return out samples = sample_paths(config, model_cfg) dataset_cls = make_dataset_class(config) dataloader = DataLoader( dataset_cls(samples), batch_size=batch_size, shuffle=False, num_workers=num_workers, collate_fn=collate, pin_memory=device.type == "cuda", ) video_embeds = [] audio_embeds = [] prompt_indices = [] seed_indices = [] video_paths = [] audio_paths = [] for batch in tqdm(dataloader, desc=f"embed {model_cfg['name']}"): inputs = { ModalityType.VISION: batch[ModalityType.VISION].to(device, non_blocking=True), ModalityType.AUDIO: batch[ModalityType.AUDIO].to(device, non_blocking=True), } with torch.inference_mode(): embeddings = model(inputs) video_embeds.append(embeddings[ModalityType.VISION].detach().cpu().float()) audio_embeds.append(embeddings[ModalityType.AUDIO].detach().cpu().float()) prompt_indices.append(batch["prompt_idx"].cpu()) seed_indices.append(batch["seed_idx"].cpu()) video_paths.extend(batch["video_path"]) audio_paths.extend(batch["audio_path"]) torch.save( { "model_name": model_cfg["name"], "video": torch.cat(video_embeds, dim=0), "audio": torch.cat(audio_embeds, dim=0), "prompt_idx": torch.cat(prompt_indices, dim=0), "seed_idx": torch.cat(seed_indices, dim=0), "video_path": video_paths, "audio_path": audio_paths, "num_prompts": int(config.get("prompt_subset", {}).get("num_prompts", 200)), "num_seeds": int(config.get("prompt_subset", {}).get("num_seeds", 8)), }, out, ) return out def compute_metrics(config: dict, path: Path) -> tuple[dict, dict]: import torch import torch.nn.functional as F subset = config.get("prompt_subset", {}) num_prompts = int(subset.get("num_prompts", 200)) num_seeds = int(subset.get("num_seeds", 8)) payload = torch.load(path, map_location="cpu", weights_only=True) count = num_prompts * num_seeds if (payload.get("num_prompts") != num_prompts or payload.get("num_seeds") != num_seeds or any(len(payload[key]) != count for key in ("video", "audio", "prompt_idx", "seed_idx"))): raise ValueError(f"Embedding count/config mismatch: {path}") order = torch.argsort(payload["prompt_idx"] * num_seeds + payload["seed_idx"]) expected = torch.arange(count) if not torch.equal((payload["prompt_idx"] * num_seeds + payload["seed_idx"])[order], expected): raise ValueError(f"Missing/duplicate prompt-seed embedding pairs: {path}") video = payload["video"][order].float() audio = payload["audio"][order].float() video_norm = F.normalize(video, dim=-1) audio_norm = F.normalize(audio, dim=-1) joint_concat = torch.cat([video_norm, audio_norm], dim=-1) joint_norm = F.normalize(joint_concat, dim=-1) summary = { "model_name": payload.get("model_name", payload.get("row_name")), "num_prompts": num_prompts, "num_seeds": num_seeds, "num_pairs_per_prompt": num_seeds * (num_seeds - 1) // 2, "embedding": "ImageBind-Huge", "joint_l2_definition": "concat(l2_normalize(video_embedding), l2_normalize(audio_embedding))", } per_prompt = {} for prefix, tensor in [("video", video_norm), ("audio", audio_norm), ("joint", joint_norm)]: mean_value, values = prompt_pairwise_stats(config, tensor, metric="cosine") summary[f"{prefix}_pairwise_cosine_distance"] = mean_value per_prompt[f"{prefix}_pairwise_cosine_distance"] = values for prefix, tensor in [("video", video_norm), ("audio", audio_norm), ("joint", joint_concat)]: mean_value, values = prompt_pairwise_stats(config, tensor, metric="l2") summary[f"{prefix}_pairwise_l2_distance"] = mean_value per_prompt[f"{prefix}_pairwise_l2_distance"] = values # Table IV's VA Avg. Div. is the arithmetic mean of video and audio. # With separately L2-normalized modalities, concatenating and normalizing # once more gives exactly the same pairwise cosine distance (up to rounding). summary["va_average_pairwise_cosine_distance"] = ( summary["video_pairwise_cosine_distance"] + summary["audio_pairwise_cosine_distance"] ) / 2 per_prompt["va_average_pairwise_cosine_distance"] = [ (v + a) / 2 for v, a in zip( per_prompt["video_pairwise_cosine_distance"], per_prompt["audio_pairwise_cosine_distance"], ) ] return summary, per_prompt def write_summary(config: dict, rows: list[dict]) -> None: root = cfg_path(config, "diversity_metric_root") write_json( root / "summary.json", { "updated_at_utc": utc_now(), "output_root": str(root), "num_rows": len(rows), "rows": rows, }, ) if not rows: return fieldnames = [ "model_name", "display_name", "video_pairwise_cosine_distance", "audio_pairwise_cosine_distance", "va_average_pairwise_cosine_distance", "joint_pairwise_cosine_distance", "video_pairwise_l2_distance", "audio_pairwise_l2_distance", "joint_pairwise_l2_distance", "num_prompts", "num_seeds", ] root.mkdir(parents=True, exist_ok=True) with (root / "summary.csv").open("w", newline="", encoding="utf-8") as f: writer = csv.DictWriter(f, fieldnames=fieldnames) writer.writeheader() for row in rows: writer.writerow({key: row.get(key) for key in fieldnames}) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--config", required=True) parser.add_argument("--gpu", default="0") parser.add_argument("--cpu", action="store_true", help="Run ImageBind embedding and statistics on CPU only") parser.add_argument("--only", default=None) parser.add_argument("--roles", default=None) parser.add_argument("--batch-size", type=int, default=4) parser.add_argument("--num-workers", type=int, default=4) parser.add_argument("--force-embeddings", action="store_true") parser.add_argument("--include-incomplete", action="store_true") return parser.parse_args() def main() -> int: args = parse_args() config = load_config(args.config) import os for key in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY", "all_proxy", "ALL_PROXY"): os.environ.pop(key, None) if args.cpu: os.environ["CUDA_VISIBLE_DEVICES"] = "" elif os.environ.get("CUDA_VISIBLE_DEVICES") == "": raise RuntimeError("CUDA_VISIBLE_DEVICES is empty; use --cpu to run without a GPU") else: os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu) os.environ.setdefault("HF_HUB_OFFLINE", "1") os.environ.setdefault("TRANSFORMERS_OFFLINE", "1") configure_javis_paths(config) import torch device = torch.device("cpu" if args.cpu else "cuda:0") if args.cpu: if torch.cuda.device_count(): raise RuntimeError("CPU mode must not expose CUDA devices") elif not torch.cuda.is_available(): raise RuntimeError("CUDA unavailable; use --cpu to run without a GPU") models = select_models(config, args.only, args.roles) if not args.include_incomplete: models = [model for model in models if complete_model(config, model)] else: incomplete = [model["name"] for model in models if not complete_model(config, model)] if incomplete: raise RuntimeError(f"Incomplete models: {incomplete}") if not models: print("No complete media pairs found; nothing to embed.", flush=True) return 0 imagebind = load_model(config, device) summaries = [] root = cfg_path(config, "diversity_metric_root") for model_cfg in models: started = time.time() print(f"model={model_cfg['name']} display={model_cfg.get('display_name')}", flush=True) emb = embed_model(config, model_cfg, device, args.batch_size, args.num_workers, args.force_embeddings, imagebind) metrics, per_prompt = compute_metrics(config, emb) metrics["display_name"] = model_cfg.get("display_name") metrics["embedding_path"] = str(emb) metrics["elapsed_seconds"] = round(time.time() - started, 1) per_prompt_path = root / "per_prompt" / f"{model_cfg['name']}.json" write_json(per_prompt_path, per_prompt) metrics["per_prompt_json"] = str(per_prompt_path) summaries.append(metrics) write_summary(config, summaries) print( f"done model={model_cfg['name']} " f"video_cos={metrics['video_pairwise_cosine_distance']:.4f} " f"audio_cos={metrics['audio_pairwise_cosine_distance']:.4f} " f"joint_cos={metrics['joint_pairwise_cosine_distance']:.4f}", flush=True, ) write_summary(config, summaries) return 0 if __name__ == "__main__": raise SystemExit(main())