File size: 5,014 Bytes
76b7b25
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
#!/usr/bin/env python3
"""CPU-only inventory of a 200-prompt evaluation split and its generated media.

Only files labelled as model outputs count; GT videos are never used as predictions.
"""
from __future__ import annotations

import argparse
import csv
import hashlib
import json
from pathlib import Path

from common import utc_now, write_json


def digest_file(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as stream:
        for block in iter(lambda: stream.read(4 * 1024 * 1024), b""):
            digest.update(block)
    return digest.hexdigest()


def inspect_pair(media: Path, index: int, seed: int | None = None) -> dict:
    stem = f"sample_{index:04d}" + (f"_seed{seed:04d}" if seed is not None else "")
    video, audio = media / f"{stem}.mp4", media / f"{stem}.wav"
    return {
        "video": video.is_file() and video.stat().st_size > 0,
        "audio": audio.is_file() and audio.stat().st_size > 0,
    }


def inspect(prompts_file: Path, dataset_root: Path, student: Path,
            teacher: Path, diversity: Path, seeds: int = 8) -> dict:
    if seeds < 2:
        raise ValueError("Diversity requires at least two seeds")
    prompts_file = prompts_file.resolve(strict=True)
    dataset_root = dataset_root.resolve(strict=True)
    prompts = prompts_file.read_text(encoding="utf-8").splitlines()
    if len(prompts) != 200 or any(not prompt.strip() for prompt in prompts):
        raise ValueError("Expected exactly 200 non-empty prompts")
    with (dataset_root / "prompts_200.csv").open(newline="", encoding="utf-8") as stream:
        csv_rows = list(csv.DictReader(stream))
    if len(csv_rows) != 200 or any(
        row["id"] != f"sample_{i:04d}" or row["prompt"] != prompt
        for i, (row, prompt) in enumerate(zip(csv_rows, prompts))
    ):
        raise ValueError("prompts_200.csv does not match prompts_200.txt")
    with (dataset_root / "gt_manifest_200.csv").open(newline="", encoding="utf-8") as stream:
        gt_rows = list(csv.DictReader(stream))
    if len(gt_rows) != 200 or any(
        row["id"] != f"sample_{i:04d}" or row["prompt"] != prompt
        for i, (row, prompt) in enumerate(zip(gt_rows, prompts))
    ):
        raise ValueError("GT manifest is not aligned to the 200 prompts")

    gt_present = [i for i in range(200) if (dataset_root / "gt_video" / f"sample_{i:04d}.mp4").is_file()]
    models = {}
    for label, directory in (("student", student), ("teacher_40_step", teacher)):
        pairs = [i for i in range(200) if all(inspect_pair(directory, i).values())]
        models[label] = {
            "media_dir": str(directory.resolve()), "valid_pair_count": len(pairs),
            "missing_pair_indices": sorted(set(range(200)) - set(pairs)),
            "ready_for_full_javisbench": len(pairs) == 200,
        }
    pairs = {(i, j) for i in range(200) for j in range(seeds)
             if all(inspect_pair(diversity, i, j).values())}
    models["student_diversity"] = {
        "media_dir": str(diversity.resolve()), "num_seeds": seeds,
        "valid_pair_count": len(pairs), "expected_pair_count": 200 * seeds,
        "complete_prompt_count": sum(all((i, j) in pairs for j in range(seeds)) for i in range(200)),
        "ready_for_full_diversity": len(pairs) == 200 * seeds,
    }
    return {
        "checked_at_utc": utc_now(),
        "device": "cpu_inventory_only",
        "prompts_file": str(prompts_file),
        "prompts_sha256": digest_file(prompts_file),
        "num_prompts": len(prompts),
        "gt_video_present": len(gt_present),
        "gt_video_missing_indices": sorted(set(range(200)) - set(gt_present)),
        "gt_is_not_model_output": True,
        "models": models,
        "prompt_enhancement": "unverified: original author-side 200-prompt list/expansion metadata unavailable",
    }


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--dataset-root", type=Path, required=True)
    parser.add_argument("--student-media", type=Path, required=True)
    parser.add_argument("--teacher-media", type=Path, required=True)
    parser.add_argument("--diversity-media", type=Path, required=True)
    parser.add_argument("--output", type=Path, required=True)
    parser.add_argument("--num-seeds", type=int, default=8)
    args = parser.parse_args()
    result = inspect(
        args.dataset_root / "prompts_200.txt", args.dataset_root,
        args.student_media, args.teacher_media, args.diversity_media,
        args.num_seeds,
    )
    write_json(args.output, result)
    print(json.dumps({
        "prompts": result["num_prompts"],
        "gt_present": result["gt_video_present"],
        "student_pairs": result["models"]["student"]["valid_pair_count"],
        "teacher_pairs": result["models"]["teacher_40_step"]["valid_pair_count"],
        "diversity_pairs": result["models"]["student_diversity"]["valid_pair_count"],
        "output": str(args.output.resolve()),
    }, indent=2))


if __name__ == "__main__":
    main()