#!/usr/bin/env python3 """Score complete prompt×seed groups already generated, without blocking generation.""" from __future__ import annotations import argparse import json import os import sys from pathlib import Path from common import utc_now, write_json from compute_diversity_imagebind_pairwise import ( Sample, collate, compute_metrics, configure_javis_paths, load_model, make_dataset_class, ) def ready_groups(media: Path, prompts: list[str], seeds: int) -> list[int]: result = [] for index, prompt in enumerate(prompts): for seed_idx in range(seeds): stem = f"sample_{index:04d}_seed{seed_idx:04d}" paths = [media / f"{stem}.{suffix}" for suffix in ("mp4", "wav", "json")] if any(not path.is_file() or path.stat().st_size == 0 for path in paths): break sidecar = json.loads(paths[-1].read_text(encoding="utf-8")) if sidecar.get("index") != index or sidecar.get("prompt") != prompt or sidecar.get("seed_idx") != seed_idx: raise ValueError(f"Generated media metadata mismatch: {paths[-1]}") else: result.append(index) return result def run(args: argparse.Namespace) -> int: import torch from torch.utils.data import DataLoader from tqdm import tqdm if args.batch_size < 1 or args.workers < 0 or args.seeds < 2: raise ValueError("batch-size must be positive, workers nonnegative, seeds at least two") prompts_file = args.prompts.resolve(strict=True) prompts = prompts_file.read_text(encoding="utf-8").splitlines() if len(prompts) != 200: raise ValueError(f"Expected 200 prompts: {prompts_file}") media = args.media.resolve(strict=True) root = args.javis_root.resolve(strict=True) output = args.output.resolve() group_indices = ready_groups(media, prompts, args.seeds) if args.limit_groups is not None: group_indices = group_indices[:args.limit_groups] if not group_indices: raise RuntimeError("No complete groups of audio-video media yet") print(f"complete prompts={len(group_indices)}/200, seeds={args.seeds}", flush=True) if output.exists() and any(output.iterdir()): raise FileExistsError(f"Use a new empty output directory: {output}") config = { "paths": {"javis_root": str(root), "diversity_metric_root": str(output)}, "prompt_subset": {"num_prompts": len(group_indices), "num_seeds": args.seeds}, } samples = [] for local_idx, prompt_idx in enumerate(group_indices): for seed_idx in range(args.seeds): stem = f"sample_{prompt_idx:04d}_seed{seed_idx:04d}" samples.append(Sample("published_turbot2av_main", local_idx, seed_idx, str(media / f"{stem}.mp4"), str(media / f"{stem}.wav"))) os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu) os.environ["HF_HUB_OFFLINE"] = "1" os.environ["TRANSFORMERS_OFFLINE"] = "1" configure_javis_paths(config) from eval.javisbench.src.ImageBind.imagebind.models.imagebind_model import ModalityType device = torch.device("cuda:0") if not torch.cuda.is_available(): raise RuntimeError("Assigned scoring GPU not available") model = load_model(config, device) loader = DataLoader(make_dataset_class(config)(samples), batch_size=args.batch_size, num_workers=args.workers, shuffle=False, collate_fn=collate, pin_memory=True) v_list, a_list, prompts_idx, seeds_idx = [], [], [], [] for batch in tqdm(loader, desc="diversity imagebind"): with torch.inference_mode(): emb = model({ ModalityType.VISION: batch[ModalityType.VISION].to(device), ModalityType.AUDIO: batch[ModalityType.AUDIO].to(device), }) v_list.append(emb[ModalityType.VISION].cpu().float()) a_list.append(emb[ModalityType.AUDIO].cpu().float()) prompts_idx.append(batch["prompt_idx"]) seeds_idx.append(batch["seed_idx"]) output.mkdir(parents=True, exist_ok=True) emb_path = output / "embeddings.pt" torch.save({ "model_name": "published_turbot2av_main", "num_prompts": len(group_indices), "num_seeds": args.seeds, "video": torch.cat(v_list), "audio": torch.cat(a_list), "prompt_idx": torch.cat(prompts_idx), "seed_idx": torch.cat(seeds_idx), }, emb_path) summary, per_prompt = compute_metrics(config, emb_path) write_json(output / "summary.json", { "created_at_utc": utc_now(), "prompt_indices": group_indices, "source": str(media), "seeds": args.seeds, "reference": {"video": 0.2322, "audio": 0.4197, "va_average": 0.3259}, "note": "Exploratory subset; not comparable to full 200-prompt Table IV unless all 200 groups are complete.", "metrics": summary, }) write_json(output / "per_prompt.json", per_prompt) print(json.dumps(summary, indent=2)) return 0 def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--media", type=Path, required=True) parser.add_argument("--prompts", type=Path, required=True) parser.add_argument("--javis-root", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--gpu", required=True) parser.add_argument("--seeds", type=int, default=8) parser.add_argument("--limit-groups", type=int) parser.add_argument("--batch-size", type=int, default=1) parser.add_argument("--workers", type=int, default=2) return run(parser.parse_args()) if __name__ == "__main__": sys.exit(main())