File size: 5,680 Bytes
76b7b25 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 | #!/usr/bin/env python3
"""Score complete prompt×seed groups already generated, without blocking generation."""
from __future__ import annotations
import argparse
import json
import os
import sys
from pathlib import Path
from common import utc_now, write_json
from compute_diversity_imagebind_pairwise import (
Sample, collate, compute_metrics, configure_javis_paths, load_model, make_dataset_class,
)
def ready_groups(media: Path, prompts: list[str], seeds: int) -> list[int]:
result = []
for index, prompt in enumerate(prompts):
for seed_idx in range(seeds):
stem = f"sample_{index:04d}_seed{seed_idx:04d}"
paths = [media / f"{stem}.{suffix}" for suffix in ("mp4", "wav", "json")]
if any(not path.is_file() or path.stat().st_size == 0 for path in paths):
break
sidecar = json.loads(paths[-1].read_text(encoding="utf-8"))
if sidecar.get("index") != index or sidecar.get("prompt") != prompt or sidecar.get("seed_idx") != seed_idx:
raise ValueError(f"Generated media metadata mismatch: {paths[-1]}")
else:
result.append(index)
return result
def run(args: argparse.Namespace) -> int:
import torch
from torch.utils.data import DataLoader
from tqdm import tqdm
if args.batch_size < 1 or args.workers < 0 or args.seeds < 2:
raise ValueError("batch-size must be positive, workers nonnegative, seeds at least two")
prompts_file = args.prompts.resolve(strict=True)
prompts = prompts_file.read_text(encoding="utf-8").splitlines()
if len(prompts) != 200:
raise ValueError(f"Expected 200 prompts: {prompts_file}")
media = args.media.resolve(strict=True)
root = args.javis_root.resolve(strict=True)
output = args.output.resolve()
group_indices = ready_groups(media, prompts, args.seeds)
if args.limit_groups is not None:
group_indices = group_indices[:args.limit_groups]
if not group_indices:
raise RuntimeError("No complete groups of audio-video media yet")
print(f"complete prompts={len(group_indices)}/200, seeds={args.seeds}", flush=True)
if output.exists() and any(output.iterdir()):
raise FileExistsError(f"Use a new empty output directory: {output}")
config = {
"paths": {"javis_root": str(root), "diversity_metric_root": str(output)},
"prompt_subset": {"num_prompts": len(group_indices), "num_seeds": args.seeds},
}
samples = []
for local_idx, prompt_idx in enumerate(group_indices):
for seed_idx in range(args.seeds):
stem = f"sample_{prompt_idx:04d}_seed{seed_idx:04d}"
samples.append(Sample("published_turbot2av_main", local_idx, seed_idx,
str(media / f"{stem}.mp4"), str(media / f"{stem}.wav")))
os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu)
os.environ["HF_HUB_OFFLINE"] = "1"
os.environ["TRANSFORMERS_OFFLINE"] = "1"
configure_javis_paths(config)
from eval.javisbench.src.ImageBind.imagebind.models.imagebind_model import ModalityType
device = torch.device("cuda:0")
if not torch.cuda.is_available():
raise RuntimeError("Assigned scoring GPU not available")
model = load_model(config, device)
loader = DataLoader(make_dataset_class(config)(samples), batch_size=args.batch_size,
num_workers=args.workers, shuffle=False, collate_fn=collate, pin_memory=True)
v_list, a_list, prompts_idx, seeds_idx = [], [], [], []
for batch in tqdm(loader, desc="diversity imagebind"):
with torch.inference_mode():
emb = model({
ModalityType.VISION: batch[ModalityType.VISION].to(device),
ModalityType.AUDIO: batch[ModalityType.AUDIO].to(device),
})
v_list.append(emb[ModalityType.VISION].cpu().float())
a_list.append(emb[ModalityType.AUDIO].cpu().float())
prompts_idx.append(batch["prompt_idx"])
seeds_idx.append(batch["seed_idx"])
output.mkdir(parents=True, exist_ok=True)
emb_path = output / "embeddings.pt"
torch.save({
"model_name": "published_turbot2av_main",
"num_prompts": len(group_indices), "num_seeds": args.seeds,
"video": torch.cat(v_list), "audio": torch.cat(a_list),
"prompt_idx": torch.cat(prompts_idx), "seed_idx": torch.cat(seeds_idx),
}, emb_path)
summary, per_prompt = compute_metrics(config, emb_path)
write_json(output / "summary.json", {
"created_at_utc": utc_now(),
"prompt_indices": group_indices,
"source": str(media), "seeds": args.seeds,
"reference": {"video": 0.2322, "audio": 0.4197, "va_average": 0.3259},
"note": "Exploratory subset; not comparable to full 200-prompt Table IV unless all 200 groups are complete.",
"metrics": summary,
})
write_json(output / "per_prompt.json", per_prompt)
print(json.dumps(summary, indent=2))
return 0
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--media", type=Path, required=True)
parser.add_argument("--prompts", type=Path, required=True)
parser.add_argument("--javis-root", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--gpu", required=True)
parser.add_argument("--seeds", type=int, default=8)
parser.add_argument("--limit-groups", type=int)
parser.add_argument("--batch-size", type=int, default=1)
parser.add_argument("--workers", type=int, default=2)
return run(parser.parse_args())
if __name__ == "__main__":
sys.exit(main())
|