#!/usr/bin/env python3 """CPU-only inventory of a 200-prompt evaluation split and its generated media. Only files labelled as model outputs count; GT videos are never used as predictions. """ from __future__ import annotations import argparse import csv import hashlib import json from pathlib import Path from common import utc_now, write_json def digest_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as stream: for block in iter(lambda: stream.read(4 * 1024 * 1024), b""): digest.update(block) return digest.hexdigest() def inspect_pair(media: Path, index: int, seed: int | None = None) -> dict: stem = f"sample_{index:04d}" + (f"_seed{seed:04d}" if seed is not None else "") video, audio = media / f"{stem}.mp4", media / f"{stem}.wav" return { "video": video.is_file() and video.stat().st_size > 0, "audio": audio.is_file() and audio.stat().st_size > 0, } def inspect(prompts_file: Path, dataset_root: Path, student: Path, teacher: Path, diversity: Path, seeds: int = 8) -> dict: if seeds < 2: raise ValueError("Diversity requires at least two seeds") prompts_file = prompts_file.resolve(strict=True) dataset_root = dataset_root.resolve(strict=True) prompts = prompts_file.read_text(encoding="utf-8").splitlines() if len(prompts) != 200 or any(not prompt.strip() for prompt in prompts): raise ValueError("Expected exactly 200 non-empty prompts") with (dataset_root / "prompts_200.csv").open(newline="", encoding="utf-8") as stream: csv_rows = list(csv.DictReader(stream)) if len(csv_rows) != 200 or any( row["id"] != f"sample_{i:04d}" or row["prompt"] != prompt for i, (row, prompt) in enumerate(zip(csv_rows, prompts)) ): raise ValueError("prompts_200.csv does not match prompts_200.txt") with (dataset_root / "gt_manifest_200.csv").open(newline="", encoding="utf-8") as stream: gt_rows = list(csv.DictReader(stream)) if len(gt_rows) != 200 or any( row["id"] != f"sample_{i:04d}" or row["prompt"] != prompt for i, (row, prompt) in enumerate(zip(gt_rows, prompts)) ): raise ValueError("GT manifest is not aligned to the 200 prompts") gt_present = [i for i in range(200) if (dataset_root / "gt_video" / f"sample_{i:04d}.mp4").is_file()] models = {} for label, directory in (("student", student), ("teacher_40_step", teacher)): pairs = [i for i in range(200) if all(inspect_pair(directory, i).values())] models[label] = { "media_dir": str(directory.resolve()), "valid_pair_count": len(pairs), "missing_pair_indices": sorted(set(range(200)) - set(pairs)), "ready_for_full_javisbench": len(pairs) == 200, } pairs = {(i, j) for i in range(200) for j in range(seeds) if all(inspect_pair(diversity, i, j).values())} models["student_diversity"] = { "media_dir": str(diversity.resolve()), "num_seeds": seeds, "valid_pair_count": len(pairs), "expected_pair_count": 200 * seeds, "complete_prompt_count": sum(all((i, j) in pairs for j in range(seeds)) for i in range(200)), "ready_for_full_diversity": len(pairs) == 200 * seeds, } return { "checked_at_utc": utc_now(), "device": "cpu_inventory_only", "prompts_file": str(prompts_file), "prompts_sha256": digest_file(prompts_file), "num_prompts": len(prompts), "gt_video_present": len(gt_present), "gt_video_missing_indices": sorted(set(range(200)) - set(gt_present)), "gt_is_not_model_output": True, "models": models, "prompt_enhancement": "unverified: original author-side 200-prompt list/expansion metadata unavailable", } def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--dataset-root", type=Path, required=True) parser.add_argument("--student-media", type=Path, required=True) parser.add_argument("--teacher-media", type=Path, required=True) parser.add_argument("--diversity-media", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--num-seeds", type=int, default=8) args = parser.parse_args() result = inspect( args.dataset_root / "prompts_200.txt", args.dataset_root, args.student_media, args.teacher_media, args.diversity_media, args.num_seeds, ) write_json(args.output, result) print(json.dumps({ "prompts": result["num_prompts"], "gt_present": result["gt_video_present"], "student_pairs": result["models"]["student"]["valid_pair_count"], "teacher_pairs": result["models"]["teacher_40_step"]["valid_pair_count"], "diversity_pairs": result["models"]["student_diversity"]["valid_pair_count"], "output": str(args.output.resolve()), }, indent=2)) if __name__ == "__main__": main()