File size: 5,014 Bytes
76b7b25 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 | #!/usr/bin/env python3
"""CPU-only inventory of a 200-prompt evaluation split and its generated media.
Only files labelled as model outputs count; GT videos are never used as predictions.
"""
from __future__ import annotations
import argparse
import csv
import hashlib
import json
from pathlib import Path
from common import utc_now, write_json
def digest_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for block in iter(lambda: stream.read(4 * 1024 * 1024), b""):
digest.update(block)
return digest.hexdigest()
def inspect_pair(media: Path, index: int, seed: int | None = None) -> dict:
stem = f"sample_{index:04d}" + (f"_seed{seed:04d}" if seed is not None else "")
video, audio = media / f"{stem}.mp4", media / f"{stem}.wav"
return {
"video": video.is_file() and video.stat().st_size > 0,
"audio": audio.is_file() and audio.stat().st_size > 0,
}
def inspect(prompts_file: Path, dataset_root: Path, student: Path,
teacher: Path, diversity: Path, seeds: int = 8) -> dict:
if seeds < 2:
raise ValueError("Diversity requires at least two seeds")
prompts_file = prompts_file.resolve(strict=True)
dataset_root = dataset_root.resolve(strict=True)
prompts = prompts_file.read_text(encoding="utf-8").splitlines()
if len(prompts) != 200 or any(not prompt.strip() for prompt in prompts):
raise ValueError("Expected exactly 200 non-empty prompts")
with (dataset_root / "prompts_200.csv").open(newline="", encoding="utf-8") as stream:
csv_rows = list(csv.DictReader(stream))
if len(csv_rows) != 200 or any(
row["id"] != f"sample_{i:04d}" or row["prompt"] != prompt
for i, (row, prompt) in enumerate(zip(csv_rows, prompts))
):
raise ValueError("prompts_200.csv does not match prompts_200.txt")
with (dataset_root / "gt_manifest_200.csv").open(newline="", encoding="utf-8") as stream:
gt_rows = list(csv.DictReader(stream))
if len(gt_rows) != 200 or any(
row["id"] != f"sample_{i:04d}" or row["prompt"] != prompt
for i, (row, prompt) in enumerate(zip(gt_rows, prompts))
):
raise ValueError("GT manifest is not aligned to the 200 prompts")
gt_present = [i for i in range(200) if (dataset_root / "gt_video" / f"sample_{i:04d}.mp4").is_file()]
models = {}
for label, directory in (("student", student), ("teacher_40_step", teacher)):
pairs = [i for i in range(200) if all(inspect_pair(directory, i).values())]
models[label] = {
"media_dir": str(directory.resolve()), "valid_pair_count": len(pairs),
"missing_pair_indices": sorted(set(range(200)) - set(pairs)),
"ready_for_full_javisbench": len(pairs) == 200,
}
pairs = {(i, j) for i in range(200) for j in range(seeds)
if all(inspect_pair(diversity, i, j).values())}
models["student_diversity"] = {
"media_dir": str(diversity.resolve()), "num_seeds": seeds,
"valid_pair_count": len(pairs), "expected_pair_count": 200 * seeds,
"complete_prompt_count": sum(all((i, j) in pairs for j in range(seeds)) for i in range(200)),
"ready_for_full_diversity": len(pairs) == 200 * seeds,
}
return {
"checked_at_utc": utc_now(),
"device": "cpu_inventory_only",
"prompts_file": str(prompts_file),
"prompts_sha256": digest_file(prompts_file),
"num_prompts": len(prompts),
"gt_video_present": len(gt_present),
"gt_video_missing_indices": sorted(set(range(200)) - set(gt_present)),
"gt_is_not_model_output": True,
"models": models,
"prompt_enhancement": "unverified: original author-side 200-prompt list/expansion metadata unavailable",
}
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--dataset-root", type=Path, required=True)
parser.add_argument("--student-media", type=Path, required=True)
parser.add_argument("--teacher-media", type=Path, required=True)
parser.add_argument("--diversity-media", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--num-seeds", type=int, default=8)
args = parser.parse_args()
result = inspect(
args.dataset_root / "prompts_200.txt", args.dataset_root,
args.student_media, args.teacher_media, args.diversity_media,
args.num_seeds,
)
write_json(args.output, result)
print(json.dumps({
"prompts": result["num_prompts"],
"gt_present": result["gt_video_present"],
"student_pairs": result["models"]["student"]["valid_pair_count"],
"teacher_pairs": result["models"]["teacher_40_step"]["valid_pair_count"],
"diversity_pairs": result["models"]["student_diversity"]["valid_pair_count"],
"output": str(args.output.resolve()),
}, indent=2))
if __name__ == "__main__":
main()
|