Download code/evaluation/scripts/preflight_cpu_200.py from luyu1021/turbot2va-ltx2: direct link, hf CLI and curl.
- Browser
- Download file 5.01 kB
-
https://huggingface.co/luyu1021/turbot2va-ltx2/resolve/main/code/evaluation/scripts/preflight_cpu_200.py
- Command line
-
hf download hf://luyu1021/turbot2va-ltx2/code/evaluation/scripts/preflight_cpu_200.py
-
curl -L -o preflight_cpu_200.py https://huggingface.co/luyu1021/turbot2va-ltx2/resolve/main/code/evaluation/scripts/preflight_cpu_200.py
5.01 kB
| #!/usr/bin/env python3 | |
| """CPU-only inventory of a 200-prompt evaluation split and its generated media. | |
| Only files labelled as model outputs count; GT videos are never used as predictions. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| from common import utc_now, write_json | |
| def digest_file(path: Path) -> str: | |
| digest = hashlib.sha256() | |
| with path.open("rb") as stream: | |
| for block in iter(lambda: stream.read(4 * 1024 * 1024), b""): | |
| digest.update(block) | |
| return digest.hexdigest() | |
| def inspect_pair(media: Path, index: int, seed: int | None = None) -> dict: | |
| stem = f"sample_{index:04d}" + (f"_seed{seed:04d}" if seed is not None else "") | |
| video, audio = media / f"{stem}.mp4", media / f"{stem}.wav" | |
| return { | |
| "video": video.is_file() and video.stat().st_size > 0, | |
| "audio": audio.is_file() and audio.stat().st_size > 0, | |
| } | |
| def inspect(prompts_file: Path, dataset_root: Path, student: Path, | |
| teacher: Path, diversity: Path, seeds: int = 8) -> dict: | |
| if seeds < 2: | |
| raise ValueError("Diversity requires at least two seeds") | |
| prompts_file = prompts_file.resolve(strict=True) | |
| dataset_root = dataset_root.resolve(strict=True) | |
| prompts = prompts_file.read_text(encoding="utf-8").splitlines() | |
| if len(prompts) != 200 or any(not prompt.strip() for prompt in prompts): | |
| raise ValueError("Expected exactly 200 non-empty prompts") | |
| with (dataset_root / "prompts_200.csv").open(newline="", encoding="utf-8") as stream: | |
| csv_rows = list(csv.DictReader(stream)) | |
| if len(csv_rows) != 200 or any( | |
| row["id"] != f"sample_{i:04d}" or row["prompt"] != prompt | |
| for i, (row, prompt) in enumerate(zip(csv_rows, prompts)) | |
| ): | |
| raise ValueError("prompts_200.csv does not match prompts_200.txt") | |
| with (dataset_root / "gt_manifest_200.csv").open(newline="", encoding="utf-8") as stream: | |
| gt_rows = list(csv.DictReader(stream)) | |
| if len(gt_rows) != 200 or any( | |
| row["id"] != f"sample_{i:04d}" or row["prompt"] != prompt | |
| for i, (row, prompt) in enumerate(zip(gt_rows, prompts)) | |
| ): | |
| raise ValueError("GT manifest is not aligned to the 200 prompts") | |
| gt_present = [i for i in range(200) if (dataset_root / "gt_video" / f"sample_{i:04d}.mp4").is_file()] | |
| models = {} | |
| for label, directory in (("student", student), ("teacher_40_step", teacher)): | |
| pairs = [i for i in range(200) if all(inspect_pair(directory, i).values())] | |
| models[label] = { | |
| "media_dir": str(directory.resolve()), "valid_pair_count": len(pairs), | |
| "missing_pair_indices": sorted(set(range(200)) - set(pairs)), | |
| "ready_for_full_javisbench": len(pairs) == 200, | |
| } | |
| pairs = {(i, j) for i in range(200) for j in range(seeds) | |
| if all(inspect_pair(diversity, i, j).values())} | |
| models["student_diversity"] = { | |
| "media_dir": str(diversity.resolve()), "num_seeds": seeds, | |
| "valid_pair_count": len(pairs), "expected_pair_count": 200 * seeds, | |
| "complete_prompt_count": sum(all((i, j) in pairs for j in range(seeds)) for i in range(200)), | |
| "ready_for_full_diversity": len(pairs) == 200 * seeds, | |
| } | |
| return { | |
| "checked_at_utc": utc_now(), | |
| "device": "cpu_inventory_only", | |
| "prompts_file": str(prompts_file), | |
| "prompts_sha256": digest_file(prompts_file), | |
| "num_prompts": len(prompts), | |
| "gt_video_present": len(gt_present), | |
| "gt_video_missing_indices": sorted(set(range(200)) - set(gt_present)), | |
| "gt_is_not_model_output": True, | |
| "models": models, | |
| "prompt_enhancement": "unverified: original author-side 200-prompt list/expansion metadata unavailable", | |
| } | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--dataset-root", type=Path, required=True) | |
| parser.add_argument("--student-media", type=Path, required=True) | |
| parser.add_argument("--teacher-media", type=Path, required=True) | |
| parser.add_argument("--diversity-media", type=Path, required=True) | |
| parser.add_argument("--output", type=Path, required=True) | |
| parser.add_argument("--num-seeds", type=int, default=8) | |
| args = parser.parse_args() | |
| result = inspect( | |
| args.dataset_root / "prompts_200.txt", args.dataset_root, | |
| args.student_media, args.teacher_media, args.diversity_media, | |
| args.num_seeds, | |
| ) | |
| write_json(args.output, result) | |
| print(json.dumps({ | |
| "prompts": result["num_prompts"], | |
| "gt_present": result["gt_video_present"], | |
| "student_pairs": result["models"]["student"]["valid_pair_count"], | |
| "teacher_pairs": result["models"]["teacher_40_step"]["valid_pair_count"], | |
| "diversity_pairs": result["models"]["student_diversity"]["valid_pair_count"], | |
| "output": str(args.output.resolve()), | |
| }, indent=2)) | |
| if __name__ == "__main__": | |
| main() | |