turbot2va-ltx2 / code /evaluation /scripts /compute_diversity_imagebind_pairwise.py
luyu1021's picture
Add documented LTX-2 training source, recipes and checkpoint release plan
76b7b25 verified
Raw History Blame Contribute Delete
15 kB
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import csv
import sys
import time
from dataclasses import dataclass
from pathlib import Path
import json
from common import cfg_path, load_config, media_dir_for_model, select_models, utc_now, write_json
@dataclass(frozen=True)
class Sample:
model_name: str
prompt_idx: int
seed_idx: int
video_path: str
audio_path: str
def configure_javis_paths(config: dict) -> None:
javis_root = cfg_path(config, "javis_root")
imagebind_root = javis_root / "eval/javisbench/src/ImageBind"
for path in (javis_root, imagebind_root):
path_str = str(path)
if path_str not in sys.path:
sys.path.insert(0, path_str)
def sample_paths(config: dict, model: dict) -> list[Sample]:
subset = config.get("prompt_subset", {})
num_prompts = int(subset.get("num_prompts", 200))
num_seeds = int(subset.get("num_seeds", 8))
out_dir = media_dir_for_model(config, model)
samples = []
for prompt_idx in range(num_prompts):
for seed_idx in range(num_seeds):
stem = f"sample_{prompt_idx:04d}_seed{seed_idx:04d}"
video_path = out_dir / f"{stem}.mp4"
audio_path = out_dir / f"{stem}.wav"
if (not video_path.is_file() or not audio_path.is_file()
or video_path.stat().st_size == 0 or audio_path.stat().st_size == 0):
raise FileNotFoundError(f"{model['name']}: missing {video_path} or {audio_path}")
sidecar = out_dir / f"{stem}.json"
if sidecar.is_file():
record = json.loads(sidecar.read_text(encoding="utf-8"))
if record.get("index") != prompt_idx or record.get("seed_idx") != seed_idx:
raise ValueError(f"Generated index/seed mismatch: {sidecar}")
samples.append(Sample(model["name"], prompt_idx, seed_idx, str(video_path), str(audio_path)))
return samples
def complete_model(config: dict, model: dict) -> bool:
try:
sample_paths(config, model)
return True
except FileNotFoundError:
return False
def make_dataset_class(config: dict):
import torch
from torch.utils.data import Dataset
class AVPathDataset(Dataset):
def __init__(self, samples: list[Sample]):
self.samples = samples
def __len__(self):
return len(self.samples)
def __getitem__(self, index: int):
configure_javis_paths(config)
from eval.javisbench.src.ImageBind.imagebind import data as imagebind_data
from eval.javisbench.src.ImageBind.imagebind.models.imagebind_model import ModalityType
sample = self.samples[index]
return {
"prompt_idx": sample.prompt_idx,
"seed_idx": sample.seed_idx,
"video_path": sample.video_path,
"audio_path": sample.audio_path,
ModalityType.VISION: imagebind_data.load_and_transform_video_data([sample.video_path], "cpu")[0],
ModalityType.AUDIO: imagebind_data.load_and_transform_audio_data([sample.audio_path], "cpu")[0],
}
return AVPathDataset
def collate(batch):
import torch
from eval.javisbench.src.ImageBind.imagebind.models.imagebind_model import ModalityType
return {
"prompt_idx": torch.tensor([item["prompt_idx"] for item in batch], dtype=torch.long),
"seed_idx": torch.tensor([item["seed_idx"] for item in batch], dtype=torch.long),
"video_path": [item["video_path"] for item in batch],
"audio_path": [item["audio_path"] for item in batch],
ModalityType.VISION: torch.stack([item[ModalityType.VISION] for item in batch], dim=0),
ModalityType.AUDIO: torch.stack([item[ModalityType.AUDIO] for item in batch], dim=0),
}
def load_model(config: dict, device):
import os
javis_root = cfg_path(config, "javis_root")
os.chdir(javis_root)
configure_javis_paths(config)
from eval.javisbench.src.ImageBind.imagebind.models import imagebind_model
model = imagebind_model.imagebind_huge(pretrained=True)
model.eval().to(device)
return model
def embed_path(config: dict, model: dict) -> Path:
return cfg_path(config, "diversity_metric_root") / "embeddings" / f"{model['name']}.pt"
def upper_triangle_values(matrix):
import torch
n = matrix.shape[0]
idx = torch.triu_indices(n, n, offset=1)
return matrix[idx[0], idx[1]]
def prompt_pairwise_stats(config: dict, z, metric: str) -> tuple[float, list[float]]:
import torch
import torch.nn.functional as F
subset = config.get("prompt_subset", {})
num_prompts = int(subset.get("num_prompts", 200))
num_seeds = int(subset.get("num_seeds", 8))
per_prompt = []
for prompt_idx in range(num_prompts):
group = z[prompt_idx * num_seeds : (prompt_idx + 1) * num_seeds]
if metric == "cosine":
group = F.normalize(group, dim=-1)
distances = 1.0 - group @ group.T
elif metric == "l2":
distances = torch.cdist(group, group, p=2)
else:
raise ValueError(metric)
per_prompt.append(float(upper_triangle_values(distances).mean().item()))
return float(torch.tensor(per_prompt).mean().item()), per_prompt
def embed_model(config: dict, model_cfg: dict, device, batch_size: int, num_workers: int, force: bool, model) -> Path:
import torch
from torch.utils.data import DataLoader
from tqdm import tqdm
from eval.javisbench.src.ImageBind.imagebind.models.imagebind_model import ModalityType
out = embed_path(config, model_cfg)
out.parent.mkdir(parents=True, exist_ok=True)
if out.exists() and not force:
return out
samples = sample_paths(config, model_cfg)
dataset_cls = make_dataset_class(config)
dataloader = DataLoader(
dataset_cls(samples),
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
collate_fn=collate,
pin_memory=device.type == "cuda",
)
video_embeds = []
audio_embeds = []
prompt_indices = []
seed_indices = []
video_paths = []
audio_paths = []
for batch in tqdm(dataloader, desc=f"embed {model_cfg['name']}"):
inputs = {
ModalityType.VISION: batch[ModalityType.VISION].to(device, non_blocking=True),
ModalityType.AUDIO: batch[ModalityType.AUDIO].to(device, non_blocking=True),
}
with torch.inference_mode():
embeddings = model(inputs)
video_embeds.append(embeddings[ModalityType.VISION].detach().cpu().float())
audio_embeds.append(embeddings[ModalityType.AUDIO].detach().cpu().float())
prompt_indices.append(batch["prompt_idx"].cpu())
seed_indices.append(batch["seed_idx"].cpu())
video_paths.extend(batch["video_path"])
audio_paths.extend(batch["audio_path"])
torch.save(
{
"model_name": model_cfg["name"],
"video": torch.cat(video_embeds, dim=0),
"audio": torch.cat(audio_embeds, dim=0),
"prompt_idx": torch.cat(prompt_indices, dim=0),
"seed_idx": torch.cat(seed_indices, dim=0),
"video_path": video_paths,
"audio_path": audio_paths,
"num_prompts": int(config.get("prompt_subset", {}).get("num_prompts", 200)),
"num_seeds": int(config.get("prompt_subset", {}).get("num_seeds", 8)),
},
out,
)
return out
def compute_metrics(config: dict, path: Path) -> tuple[dict, dict]:
import torch
import torch.nn.functional as F
subset = config.get("prompt_subset", {})
num_prompts = int(subset.get("num_prompts", 200))
num_seeds = int(subset.get("num_seeds", 8))
payload = torch.load(path, map_location="cpu", weights_only=True)
count = num_prompts * num_seeds
if (payload.get("num_prompts") != num_prompts or payload.get("num_seeds") != num_seeds
or any(len(payload[key]) != count for key in ("video", "audio", "prompt_idx", "seed_idx"))):
raise ValueError(f"Embedding count/config mismatch: {path}")
order = torch.argsort(payload["prompt_idx"] * num_seeds + payload["seed_idx"])
expected = torch.arange(count)
if not torch.equal((payload["prompt_idx"] * num_seeds + payload["seed_idx"])[order], expected):
raise ValueError(f"Missing/duplicate prompt-seed embedding pairs: {path}")
video = payload["video"][order].float()
audio = payload["audio"][order].float()
video_norm = F.normalize(video, dim=-1)
audio_norm = F.normalize(audio, dim=-1)
joint_concat = torch.cat([video_norm, audio_norm], dim=-1)
joint_norm = F.normalize(joint_concat, dim=-1)
summary = {
"model_name": payload.get("model_name", payload.get("row_name")),
"num_prompts": num_prompts,
"num_seeds": num_seeds,
"num_pairs_per_prompt": num_seeds * (num_seeds - 1) // 2,
"embedding": "ImageBind-Huge",
"joint_l2_definition": "concat(l2_normalize(video_embedding), l2_normalize(audio_embedding))",
}
per_prompt = {}
for prefix, tensor in [("video", video_norm), ("audio", audio_norm), ("joint", joint_norm)]:
mean_value, values = prompt_pairwise_stats(config, tensor, metric="cosine")
summary[f"{prefix}_pairwise_cosine_distance"] = mean_value
per_prompt[f"{prefix}_pairwise_cosine_distance"] = values
for prefix, tensor in [("video", video_norm), ("audio", audio_norm), ("joint", joint_concat)]:
mean_value, values = prompt_pairwise_stats(config, tensor, metric="l2")
summary[f"{prefix}_pairwise_l2_distance"] = mean_value
per_prompt[f"{prefix}_pairwise_l2_distance"] = values
# Table IV's VA Avg. Div. is the arithmetic mean of video and audio.
# With separately L2-normalized modalities, concatenating and normalizing
# once more gives exactly the same pairwise cosine distance (up to rounding).
summary["va_average_pairwise_cosine_distance"] = (
summary["video_pairwise_cosine_distance"] + summary["audio_pairwise_cosine_distance"]
) / 2
per_prompt["va_average_pairwise_cosine_distance"] = [
(v + a) / 2 for v, a in zip(
per_prompt["video_pairwise_cosine_distance"],
per_prompt["audio_pairwise_cosine_distance"],
)
]
return summary, per_prompt
def write_summary(config: dict, rows: list[dict]) -> None:
root = cfg_path(config, "diversity_metric_root")
write_json(
root / "summary.json",
{
"updated_at_utc": utc_now(),
"output_root": str(root),
"num_rows": len(rows),
"rows": rows,
},
)
if not rows:
return
fieldnames = [
"model_name",
"display_name",
"video_pairwise_cosine_distance",
"audio_pairwise_cosine_distance",
"va_average_pairwise_cosine_distance",
"joint_pairwise_cosine_distance",
"video_pairwise_l2_distance",
"audio_pairwise_l2_distance",
"joint_pairwise_l2_distance",
"num_prompts",
"num_seeds",
]
root.mkdir(parents=True, exist_ok=True)
with (root / "summary.csv").open("w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=fieldnames)
writer.writeheader()
for row in rows:
writer.writerow({key: row.get(key) for key in fieldnames})
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--config", required=True)
parser.add_argument("--gpu", default="0")
parser.add_argument("--cpu", action="store_true", help="Run ImageBind embedding and statistics on CPU only")
parser.add_argument("--only", default=None)
parser.add_argument("--roles", default=None)
parser.add_argument("--batch-size", type=int, default=4)
parser.add_argument("--num-workers", type=int, default=4)
parser.add_argument("--force-embeddings", action="store_true")
parser.add_argument("--include-incomplete", action="store_true")
return parser.parse_args()
def main() -> int:
args = parse_args()
config = load_config(args.config)
import os
for key in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY", "all_proxy", "ALL_PROXY"):
os.environ.pop(key, None)
if args.cpu:
os.environ["CUDA_VISIBLE_DEVICES"] = ""
elif os.environ.get("CUDA_VISIBLE_DEVICES") == "":
raise RuntimeError("CUDA_VISIBLE_DEVICES is empty; use --cpu to run without a GPU")
else:
os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu)
os.environ.setdefault("HF_HUB_OFFLINE", "1")
os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
configure_javis_paths(config)
import torch
device = torch.device("cpu" if args.cpu else "cuda:0")
if args.cpu:
if torch.cuda.device_count():
raise RuntimeError("CPU mode must not expose CUDA devices")
elif not torch.cuda.is_available():
raise RuntimeError("CUDA unavailable; use --cpu to run without a GPU")
models = select_models(config, args.only, args.roles)
if not args.include_incomplete:
models = [model for model in models if complete_model(config, model)]
else:
incomplete = [model["name"] for model in models if not complete_model(config, model)]
if incomplete:
raise RuntimeError(f"Incomplete models: {incomplete}")
if not models:
print("No complete media pairs found; nothing to embed.", flush=True)
return 0
imagebind = load_model(config, device)
summaries = []
root = cfg_path(config, "diversity_metric_root")
for model_cfg in models:
started = time.time()
print(f"model={model_cfg['name']} display={model_cfg.get('display_name')}", flush=True)
emb = embed_model(config, model_cfg, device, args.batch_size, args.num_workers, args.force_embeddings, imagebind)
metrics, per_prompt = compute_metrics(config, emb)
metrics["display_name"] = model_cfg.get("display_name")
metrics["embedding_path"] = str(emb)
metrics["elapsed_seconds"] = round(time.time() - started, 1)
per_prompt_path = root / "per_prompt" / f"{model_cfg['name']}.json"
write_json(per_prompt_path, per_prompt)
metrics["per_prompt_json"] = str(per_prompt_path)
summaries.append(metrics)
write_summary(config, summaries)
print(
f"done model={model_cfg['name']} "
f"video_cos={metrics['video_pairwise_cosine_distance']:.4f} "
f"audio_cos={metrics['audio_pairwise_cosine_distance']:.4f} "
f"joint_cos={metrics['joint_pairwise_cosine_distance']:.4f}",
flush=True,
)
write_summary(config, summaries)
return 0
if __name__ == "__main__":
raise SystemExit(main())