Download code/evaluation/scripts/compute_diversity_imagebind_pairwise.py from luyu1021/turbot2va-ltx2: direct link, hf CLI and curl.
- Browser
- Download file 15 kB
-
https://huggingface.co/luyu1021/turbot2va-ltx2/resolve/main/code/evaluation/scripts/compute_diversity_imagebind_pairwise.py
- Command line
-
hf download hf://luyu1021/turbot2va-ltx2/code/evaluation/scripts/compute_diversity_imagebind_pairwise.py
-
curl -L -o compute_diversity_imagebind_pairwise.py https://huggingface.co/luyu1021/turbot2va-ltx2/resolve/main/code/evaluation/scripts/compute_diversity_imagebind_pairwise.py
15 kB
| #!/usr/bin/env python3 | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import sys | |
| import time | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| import json | |
| from common import cfg_path, load_config, media_dir_for_model, select_models, utc_now, write_json | |
| class Sample: | |
| model_name: str | |
| prompt_idx: int | |
| seed_idx: int | |
| video_path: str | |
| audio_path: str | |
| def configure_javis_paths(config: dict) -> None: | |
| javis_root = cfg_path(config, "javis_root") | |
| imagebind_root = javis_root / "eval/javisbench/src/ImageBind" | |
| for path in (javis_root, imagebind_root): | |
| path_str = str(path) | |
| if path_str not in sys.path: | |
| sys.path.insert(0, path_str) | |
| def sample_paths(config: dict, model: dict) -> list[Sample]: | |
| subset = config.get("prompt_subset", {}) | |
| num_prompts = int(subset.get("num_prompts", 200)) | |
| num_seeds = int(subset.get("num_seeds", 8)) | |
| out_dir = media_dir_for_model(config, model) | |
| samples = [] | |
| for prompt_idx in range(num_prompts): | |
| for seed_idx in range(num_seeds): | |
| stem = f"sample_{prompt_idx:04d}_seed{seed_idx:04d}" | |
| video_path = out_dir / f"{stem}.mp4" | |
| audio_path = out_dir / f"{stem}.wav" | |
| if (not video_path.is_file() or not audio_path.is_file() | |
| or video_path.stat().st_size == 0 or audio_path.stat().st_size == 0): | |
| raise FileNotFoundError(f"{model['name']}: missing {video_path} or {audio_path}") | |
| sidecar = out_dir / f"{stem}.json" | |
| if sidecar.is_file(): | |
| record = json.loads(sidecar.read_text(encoding="utf-8")) | |
| if record.get("index") != prompt_idx or record.get("seed_idx") != seed_idx: | |
| raise ValueError(f"Generated index/seed mismatch: {sidecar}") | |
| samples.append(Sample(model["name"], prompt_idx, seed_idx, str(video_path), str(audio_path))) | |
| return samples | |
| def complete_model(config: dict, model: dict) -> bool: | |
| try: | |
| sample_paths(config, model) | |
| return True | |
| except FileNotFoundError: | |
| return False | |
| def make_dataset_class(config: dict): | |
| import torch | |
| from torch.utils.data import Dataset | |
| class AVPathDataset(Dataset): | |
| def __init__(self, samples: list[Sample]): | |
| self.samples = samples | |
| def __len__(self): | |
| return len(self.samples) | |
| def __getitem__(self, index: int): | |
| configure_javis_paths(config) | |
| from eval.javisbench.src.ImageBind.imagebind import data as imagebind_data | |
| from eval.javisbench.src.ImageBind.imagebind.models.imagebind_model import ModalityType | |
| sample = self.samples[index] | |
| return { | |
| "prompt_idx": sample.prompt_idx, | |
| "seed_idx": sample.seed_idx, | |
| "video_path": sample.video_path, | |
| "audio_path": sample.audio_path, | |
| ModalityType.VISION: imagebind_data.load_and_transform_video_data([sample.video_path], "cpu")[0], | |
| ModalityType.AUDIO: imagebind_data.load_and_transform_audio_data([sample.audio_path], "cpu")[0], | |
| } | |
| return AVPathDataset | |
| def collate(batch): | |
| import torch | |
| from eval.javisbench.src.ImageBind.imagebind.models.imagebind_model import ModalityType | |
| return { | |
| "prompt_idx": torch.tensor([item["prompt_idx"] for item in batch], dtype=torch.long), | |
| "seed_idx": torch.tensor([item["seed_idx"] for item in batch], dtype=torch.long), | |
| "video_path": [item["video_path"] for item in batch], | |
| "audio_path": [item["audio_path"] for item in batch], | |
| ModalityType.VISION: torch.stack([item[ModalityType.VISION] for item in batch], dim=0), | |
| ModalityType.AUDIO: torch.stack([item[ModalityType.AUDIO] for item in batch], dim=0), | |
| } | |
| def load_model(config: dict, device): | |
| import os | |
| javis_root = cfg_path(config, "javis_root") | |
| os.chdir(javis_root) | |
| configure_javis_paths(config) | |
| from eval.javisbench.src.ImageBind.imagebind.models import imagebind_model | |
| model = imagebind_model.imagebind_huge(pretrained=True) | |
| model.eval().to(device) | |
| return model | |
| def embed_path(config: dict, model: dict) -> Path: | |
| return cfg_path(config, "diversity_metric_root") / "embeddings" / f"{model['name']}.pt" | |
| def upper_triangle_values(matrix): | |
| import torch | |
| n = matrix.shape[0] | |
| idx = torch.triu_indices(n, n, offset=1) | |
| return matrix[idx[0], idx[1]] | |
| def prompt_pairwise_stats(config: dict, z, metric: str) -> tuple[float, list[float]]: | |
| import torch | |
| import torch.nn.functional as F | |
| subset = config.get("prompt_subset", {}) | |
| num_prompts = int(subset.get("num_prompts", 200)) | |
| num_seeds = int(subset.get("num_seeds", 8)) | |
| per_prompt = [] | |
| for prompt_idx in range(num_prompts): | |
| group = z[prompt_idx * num_seeds : (prompt_idx + 1) * num_seeds] | |
| if metric == "cosine": | |
| group = F.normalize(group, dim=-1) | |
| distances = 1.0 - group @ group.T | |
| elif metric == "l2": | |
| distances = torch.cdist(group, group, p=2) | |
| else: | |
| raise ValueError(metric) | |
| per_prompt.append(float(upper_triangle_values(distances).mean().item())) | |
| return float(torch.tensor(per_prompt).mean().item()), per_prompt | |
| def embed_model(config: dict, model_cfg: dict, device, batch_size: int, num_workers: int, force: bool, model) -> Path: | |
| import torch | |
| from torch.utils.data import DataLoader | |
| from tqdm import tqdm | |
| from eval.javisbench.src.ImageBind.imagebind.models.imagebind_model import ModalityType | |
| out = embed_path(config, model_cfg) | |
| out.parent.mkdir(parents=True, exist_ok=True) | |
| if out.exists() and not force: | |
| return out | |
| samples = sample_paths(config, model_cfg) | |
| dataset_cls = make_dataset_class(config) | |
| dataloader = DataLoader( | |
| dataset_cls(samples), | |
| batch_size=batch_size, | |
| shuffle=False, | |
| num_workers=num_workers, | |
| collate_fn=collate, | |
| pin_memory=device.type == "cuda", | |
| ) | |
| video_embeds = [] | |
| audio_embeds = [] | |
| prompt_indices = [] | |
| seed_indices = [] | |
| video_paths = [] | |
| audio_paths = [] | |
| for batch in tqdm(dataloader, desc=f"embed {model_cfg['name']}"): | |
| inputs = { | |
| ModalityType.VISION: batch[ModalityType.VISION].to(device, non_blocking=True), | |
| ModalityType.AUDIO: batch[ModalityType.AUDIO].to(device, non_blocking=True), | |
| } | |
| with torch.inference_mode(): | |
| embeddings = model(inputs) | |
| video_embeds.append(embeddings[ModalityType.VISION].detach().cpu().float()) | |
| audio_embeds.append(embeddings[ModalityType.AUDIO].detach().cpu().float()) | |
| prompt_indices.append(batch["prompt_idx"].cpu()) | |
| seed_indices.append(batch["seed_idx"].cpu()) | |
| video_paths.extend(batch["video_path"]) | |
| audio_paths.extend(batch["audio_path"]) | |
| torch.save( | |
| { | |
| "model_name": model_cfg["name"], | |
| "video": torch.cat(video_embeds, dim=0), | |
| "audio": torch.cat(audio_embeds, dim=0), | |
| "prompt_idx": torch.cat(prompt_indices, dim=0), | |
| "seed_idx": torch.cat(seed_indices, dim=0), | |
| "video_path": video_paths, | |
| "audio_path": audio_paths, | |
| "num_prompts": int(config.get("prompt_subset", {}).get("num_prompts", 200)), | |
| "num_seeds": int(config.get("prompt_subset", {}).get("num_seeds", 8)), | |
| }, | |
| out, | |
| ) | |
| return out | |
| def compute_metrics(config: dict, path: Path) -> tuple[dict, dict]: | |
| import torch | |
| import torch.nn.functional as F | |
| subset = config.get("prompt_subset", {}) | |
| num_prompts = int(subset.get("num_prompts", 200)) | |
| num_seeds = int(subset.get("num_seeds", 8)) | |
| payload = torch.load(path, map_location="cpu", weights_only=True) | |
| count = num_prompts * num_seeds | |
| if (payload.get("num_prompts") != num_prompts or payload.get("num_seeds") != num_seeds | |
| or any(len(payload[key]) != count for key in ("video", "audio", "prompt_idx", "seed_idx"))): | |
| raise ValueError(f"Embedding count/config mismatch: {path}") | |
| order = torch.argsort(payload["prompt_idx"] * num_seeds + payload["seed_idx"]) | |
| expected = torch.arange(count) | |
| if not torch.equal((payload["prompt_idx"] * num_seeds + payload["seed_idx"])[order], expected): | |
| raise ValueError(f"Missing/duplicate prompt-seed embedding pairs: {path}") | |
| video = payload["video"][order].float() | |
| audio = payload["audio"][order].float() | |
| video_norm = F.normalize(video, dim=-1) | |
| audio_norm = F.normalize(audio, dim=-1) | |
| joint_concat = torch.cat([video_norm, audio_norm], dim=-1) | |
| joint_norm = F.normalize(joint_concat, dim=-1) | |
| summary = { | |
| "model_name": payload.get("model_name", payload.get("row_name")), | |
| "num_prompts": num_prompts, | |
| "num_seeds": num_seeds, | |
| "num_pairs_per_prompt": num_seeds * (num_seeds - 1) // 2, | |
| "embedding": "ImageBind-Huge", | |
| "joint_l2_definition": "concat(l2_normalize(video_embedding), l2_normalize(audio_embedding))", | |
| } | |
| per_prompt = {} | |
| for prefix, tensor in [("video", video_norm), ("audio", audio_norm), ("joint", joint_norm)]: | |
| mean_value, values = prompt_pairwise_stats(config, tensor, metric="cosine") | |
| summary[f"{prefix}_pairwise_cosine_distance"] = mean_value | |
| per_prompt[f"{prefix}_pairwise_cosine_distance"] = values | |
| for prefix, tensor in [("video", video_norm), ("audio", audio_norm), ("joint", joint_concat)]: | |
| mean_value, values = prompt_pairwise_stats(config, tensor, metric="l2") | |
| summary[f"{prefix}_pairwise_l2_distance"] = mean_value | |
| per_prompt[f"{prefix}_pairwise_l2_distance"] = values | |
| # Table IV's VA Avg. Div. is the arithmetic mean of video and audio. | |
| # With separately L2-normalized modalities, concatenating and normalizing | |
| # once more gives exactly the same pairwise cosine distance (up to rounding). | |
| summary["va_average_pairwise_cosine_distance"] = ( | |
| summary["video_pairwise_cosine_distance"] + summary["audio_pairwise_cosine_distance"] | |
| ) / 2 | |
| per_prompt["va_average_pairwise_cosine_distance"] = [ | |
| (v + a) / 2 for v, a in zip( | |
| per_prompt["video_pairwise_cosine_distance"], | |
| per_prompt["audio_pairwise_cosine_distance"], | |
| ) | |
| ] | |
| return summary, per_prompt | |
| def write_summary(config: dict, rows: list[dict]) -> None: | |
| root = cfg_path(config, "diversity_metric_root") | |
| write_json( | |
| root / "summary.json", | |
| { | |
| "updated_at_utc": utc_now(), | |
| "output_root": str(root), | |
| "num_rows": len(rows), | |
| "rows": rows, | |
| }, | |
| ) | |
| if not rows: | |
| return | |
| fieldnames = [ | |
| "model_name", | |
| "display_name", | |
| "video_pairwise_cosine_distance", | |
| "audio_pairwise_cosine_distance", | |
| "va_average_pairwise_cosine_distance", | |
| "joint_pairwise_cosine_distance", | |
| "video_pairwise_l2_distance", | |
| "audio_pairwise_l2_distance", | |
| "joint_pairwise_l2_distance", | |
| "num_prompts", | |
| "num_seeds", | |
| ] | |
| root.mkdir(parents=True, exist_ok=True) | |
| with (root / "summary.csv").open("w", newline="", encoding="utf-8") as f: | |
| writer = csv.DictWriter(f, fieldnames=fieldnames) | |
| writer.writeheader() | |
| for row in rows: | |
| writer.writerow({key: row.get(key) for key in fieldnames}) | |
| def parse_args() -> argparse.Namespace: | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--config", required=True) | |
| parser.add_argument("--gpu", default="0") | |
| parser.add_argument("--cpu", action="store_true", help="Run ImageBind embedding and statistics on CPU only") | |
| parser.add_argument("--only", default=None) | |
| parser.add_argument("--roles", default=None) | |
| parser.add_argument("--batch-size", type=int, default=4) | |
| parser.add_argument("--num-workers", type=int, default=4) | |
| parser.add_argument("--force-embeddings", action="store_true") | |
| parser.add_argument("--include-incomplete", action="store_true") | |
| return parser.parse_args() | |
| def main() -> int: | |
| args = parse_args() | |
| config = load_config(args.config) | |
| import os | |
| for key in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY", "all_proxy", "ALL_PROXY"): | |
| os.environ.pop(key, None) | |
| if args.cpu: | |
| os.environ["CUDA_VISIBLE_DEVICES"] = "" | |
| elif os.environ.get("CUDA_VISIBLE_DEVICES") == "": | |
| raise RuntimeError("CUDA_VISIBLE_DEVICES is empty; use --cpu to run without a GPU") | |
| else: | |
| os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu) | |
| os.environ.setdefault("HF_HUB_OFFLINE", "1") | |
| os.environ.setdefault("TRANSFORMERS_OFFLINE", "1") | |
| configure_javis_paths(config) | |
| import torch | |
| device = torch.device("cpu" if args.cpu else "cuda:0") | |
| if args.cpu: | |
| if torch.cuda.device_count(): | |
| raise RuntimeError("CPU mode must not expose CUDA devices") | |
| elif not torch.cuda.is_available(): | |
| raise RuntimeError("CUDA unavailable; use --cpu to run without a GPU") | |
| models = select_models(config, args.only, args.roles) | |
| if not args.include_incomplete: | |
| models = [model for model in models if complete_model(config, model)] | |
| else: | |
| incomplete = [model["name"] for model in models if not complete_model(config, model)] | |
| if incomplete: | |
| raise RuntimeError(f"Incomplete models: {incomplete}") | |
| if not models: | |
| print("No complete media pairs found; nothing to embed.", flush=True) | |
| return 0 | |
| imagebind = load_model(config, device) | |
| summaries = [] | |
| root = cfg_path(config, "diversity_metric_root") | |
| for model_cfg in models: | |
| started = time.time() | |
| print(f"model={model_cfg['name']} display={model_cfg.get('display_name')}", flush=True) | |
| emb = embed_model(config, model_cfg, device, args.batch_size, args.num_workers, args.force_embeddings, imagebind) | |
| metrics, per_prompt = compute_metrics(config, emb) | |
| metrics["display_name"] = model_cfg.get("display_name") | |
| metrics["embedding_path"] = str(emb) | |
| metrics["elapsed_seconds"] = round(time.time() - started, 1) | |
| per_prompt_path = root / "per_prompt" / f"{model_cfg['name']}.json" | |
| write_json(per_prompt_path, per_prompt) | |
| metrics["per_prompt_json"] = str(per_prompt_path) | |
| summaries.append(metrics) | |
| write_summary(config, summaries) | |
| print( | |
| f"done model={model_cfg['name']} " | |
| f"video_cos={metrics['video_pairwise_cosine_distance']:.4f} " | |
| f"audio_cos={metrics['audio_pairwise_cosine_distance']:.4f} " | |
| f"joint_cos={metrics['joint_pairwise_cosine_distance']:.4f}", | |
| flush=True, | |
| ) | |
| write_summary(config, summaries) | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |