File size: 5,680 Bytes
76b7b25
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
#!/usr/bin/env python3
"""Score complete prompt×seed groups already generated, without blocking generation."""
from __future__ import annotations

import argparse
import json
import os
import sys
from pathlib import Path

from common import utc_now, write_json
from compute_diversity_imagebind_pairwise import (
    Sample, collate, compute_metrics, configure_javis_paths, load_model, make_dataset_class,
)


def ready_groups(media: Path, prompts: list[str], seeds: int) -> list[int]:
    result = []
    for index, prompt in enumerate(prompts):
        for seed_idx in range(seeds):
            stem = f"sample_{index:04d}_seed{seed_idx:04d}"
            paths = [media / f"{stem}.{suffix}" for suffix in ("mp4", "wav", "json")]
            if any(not path.is_file() or path.stat().st_size == 0 for path in paths):
                break
            sidecar = json.loads(paths[-1].read_text(encoding="utf-8"))
            if sidecar.get("index") != index or sidecar.get("prompt") != prompt or sidecar.get("seed_idx") != seed_idx:
                raise ValueError(f"Generated media metadata mismatch: {paths[-1]}")
        else:
            result.append(index)
    return result


def run(args: argparse.Namespace) -> int:
    import torch
    from torch.utils.data import DataLoader
    from tqdm import tqdm

    if args.batch_size < 1 or args.workers < 0 or args.seeds < 2:
        raise ValueError("batch-size must be positive, workers nonnegative, seeds at least two")
    prompts_file = args.prompts.resolve(strict=True)
    prompts = prompts_file.read_text(encoding="utf-8").splitlines()
    if len(prompts) != 200:
        raise ValueError(f"Expected 200 prompts: {prompts_file}")
    media = args.media.resolve(strict=True)
    root = args.javis_root.resolve(strict=True)
    output = args.output.resolve()
    group_indices = ready_groups(media, prompts, args.seeds)
    if args.limit_groups is not None:
        group_indices = group_indices[:args.limit_groups]
    if not group_indices:
        raise RuntimeError("No complete groups of audio-video media yet")
    print(f"complete prompts={len(group_indices)}/200, seeds={args.seeds}", flush=True)
    if output.exists() and any(output.iterdir()):
        raise FileExistsError(f"Use a new empty output directory: {output}")
    config = {
        "paths": {"javis_root": str(root), "diversity_metric_root": str(output)},
        "prompt_subset": {"num_prompts": len(group_indices), "num_seeds": args.seeds},
    }
    samples = []
    for local_idx, prompt_idx in enumerate(group_indices):
        for seed_idx in range(args.seeds):
            stem = f"sample_{prompt_idx:04d}_seed{seed_idx:04d}"
            samples.append(Sample("published_turbot2av_main", local_idx, seed_idx,
                                  str(media / f"{stem}.mp4"), str(media / f"{stem}.wav")))
    os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu)
    os.environ["HF_HUB_OFFLINE"] = "1"
    os.environ["TRANSFORMERS_OFFLINE"] = "1"
    configure_javis_paths(config)
    from eval.javisbench.src.ImageBind.imagebind.models.imagebind_model import ModalityType

    device = torch.device("cuda:0")
    if not torch.cuda.is_available():
        raise RuntimeError("Assigned scoring GPU not available")
    model = load_model(config, device)
    loader = DataLoader(make_dataset_class(config)(samples), batch_size=args.batch_size,
                        num_workers=args.workers, shuffle=False, collate_fn=collate, pin_memory=True)
    v_list, a_list, prompts_idx, seeds_idx = [], [], [], []
    for batch in tqdm(loader, desc="diversity imagebind"):
        with torch.inference_mode():
            emb = model({
                ModalityType.VISION: batch[ModalityType.VISION].to(device),
                ModalityType.AUDIO: batch[ModalityType.AUDIO].to(device),
            })
        v_list.append(emb[ModalityType.VISION].cpu().float())
        a_list.append(emb[ModalityType.AUDIO].cpu().float())
        prompts_idx.append(batch["prompt_idx"])
        seeds_idx.append(batch["seed_idx"])
    output.mkdir(parents=True, exist_ok=True)
    emb_path = output / "embeddings.pt"
    torch.save({
        "model_name": "published_turbot2av_main",
        "num_prompts": len(group_indices), "num_seeds": args.seeds,
        "video": torch.cat(v_list), "audio": torch.cat(a_list),
        "prompt_idx": torch.cat(prompts_idx), "seed_idx": torch.cat(seeds_idx),
    }, emb_path)
    summary, per_prompt = compute_metrics(config, emb_path)
    write_json(output / "summary.json", {
        "created_at_utc": utc_now(),
        "prompt_indices": group_indices,
        "source": str(media), "seeds": args.seeds,
        "reference": {"video": 0.2322, "audio": 0.4197, "va_average": 0.3259},
        "note": "Exploratory subset; not comparable to full 200-prompt Table IV unless all 200 groups are complete.",
        "metrics": summary,
    })
    write_json(output / "per_prompt.json", per_prompt)
    print(json.dumps(summary, indent=2))
    return 0


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--media", type=Path, required=True)
    parser.add_argument("--prompts", type=Path, required=True)
    parser.add_argument("--javis-root", type=Path, required=True)
    parser.add_argument("--output", type=Path, required=True)
    parser.add_argument("--gpu", required=True)
    parser.add_argument("--seeds", type=int, default=8)
    parser.add_argument("--limit-groups", type=int)
    parser.add_argument("--batch-size", type=int, default=1)
    parser.add_argument("--workers", type=int, default=2)
    return run(parser.parse_args())


if __name__ == "__main__":
    sys.exit(main())