Download code/evaluation/scripts/watch_evaluation.py from luyu1021/turbot2va-ltx2: direct link, hf CLI and curl.
- Browser
- Download file 7.21 kB
-
https://huggingface.co/luyu1021/turbot2va-ltx2/resolve/main/code/evaluation/scripts/watch_evaluation.py
- Command line
-
hf download hf://luyu1021/turbot2va-ltx2/code/evaluation/scripts/watch_evaluation.py
-
curl -L -o watch_evaluation.py https://huggingface.co/luyu1021/turbot2va-ltx2/resolve/main/code/evaluation/scripts/watch_evaluation.py
7.21 kB
| #!/usr/bin/env python3 | |
| """Watch in-progress AV media and score each completed experiment on a spare GPU.""" | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| import subprocess | |
| import sys | |
| import time | |
| from pathlib import Path | |
| from common import utc_now, write_json | |
| from run_checkpoint_curve import score_complete | |
| def media_count(directory: Path, indices: range, seeds: int | None = None) -> int: | |
| def complete(stem: str) -> bool: | |
| return all( | |
| (directory / f"{stem}.{suffix}").is_file() | |
| and (directory / f"{stem}.{suffix}").stat().st_size > 0 | |
| for suffix in ("mp4", "wav", "json") | |
| ) | |
| return sum( | |
| complete(f"sample_{i:04d}" + (f"_seed{j:04d}" if seeds is not None else "")) | |
| for i in indices for j in (range(seeds) if seeds is not None else (None,)) | |
| ) | |
| def run(args: argparse.Namespace) -> int: | |
| if args.poll_seconds < 2 or args.max_hours <= 0: | |
| raise ValueError("--poll-seconds >= 2 and --max-hours > 0 are required") | |
| script_root = Path(__file__).resolve().parent | |
| javis_python = args.javis_python.absolute() | |
| javis_root = args.javis_root.resolve(strict=True) | |
| media = {name: path.resolve() for name, path in ( | |
| ("student", args.student_media), ("teacher_40_step", args.teacher_media), | |
| ("diversity", args.diversity_media))} | |
| outputs = {name: args.output.resolve() / name for name in ("student", "teacher_40_step", "diversity")} | |
| status_file = args.output.resolve() / "watch_status.json" | |
| prompts = args.prompts.resolve(strict=True).read_text(encoding="utf-8").splitlines() | |
| if len(prompts) != 200: | |
| raise ValueError("Expected 200 prompts") | |
| started = time.monotonic() | |
| targets = {"student"} if args.student_only else {"student", "teacher_40_step", "diversity"} | |
| done: set[str] = { | |
| name for name in ("student", "teacher_40_step") | |
| if name in targets and score_complete(outputs[name] / "scores.json") | |
| } | |
| diversity_summary = outputs["diversity"] / "summary.json" | |
| if "diversity" in targets and diversity_summary.is_file(): | |
| payload = json.loads(diversity_summary.read_text(encoding="utf-8")) | |
| if len(payload.get("prompt_indices", [])) == 200 and payload.get("seeds") == 8: | |
| done.add("diversity") | |
| failures: dict[str, str] = {} | |
| while time.monotonic() - started < args.max_hours * 3600: | |
| counts = { | |
| "student": media_count(media["student"], range(200)), | |
| "teacher_40_step": media_count(media["teacher_40_step"], range(200)), | |
| "diversity": media_count(media["diversity"], range(200), seeds=8), | |
| } | |
| if "student" not in done and counts["student"] == 200: | |
| target = "student" | |
| elif "teacher_40_step" in targets and "teacher_40_step" not in done and counts["teacher_40_step"] == 200: | |
| target = "teacher_40_step" | |
| elif "diversity" in targets and "diversity" not in done and counts["diversity"] == 1600: | |
| target = "diversity" | |
| else: | |
| target = None | |
| write_json(status_file, { | |
| "updated_at_utc": utc_now(), "gpu": args.gpu, "counts": counts, | |
| "completed_scoring": sorted(done), "failures": failures, | |
| "active_scoring": target, | |
| }) | |
| print(f"{utc_now()} media {counts} scoring {target or 'waiting'}", flush=True) | |
| if targets <= done: | |
| return 0 | |
| if target is None: | |
| time.sleep(args.poll_seconds) | |
| continue | |
| env = dict(os.environ) | |
| env.update({ | |
| "CUDA_VISIBLE_DEVICES": str(args.gpu), "HF_HUB_OFFLINE": "1", | |
| "TRANSFORMERS_OFFLINE": "1", "PYTHONUNBUFFERED": "1", | |
| "OMP_NUM_THREADS": "4", "MKL_NUM_THREADS": "4", | |
| }) | |
| if target == "diversity": | |
| command = [ | |
| str(javis_python), str(script_root / "score_diversity_completed.py"), | |
| "--media", str(media[target]), "--prompts", str(args.prompts), | |
| "--javis-root", str(javis_root), "--output", str(outputs[target]), | |
| "--gpu", str(args.gpu), "--batch-size", "1", "--workers", "2", | |
| ] | |
| else: | |
| command = [ | |
| sys.executable, str(script_root / "score_existing_gpu.py"), | |
| "--model", target, "--media", str(media[target]), "--prompts", str(args.prompts), | |
| "--output", str(outputs[target]), "--javis-root", str(javis_root), | |
| "--javis-python", str(javis_python), "--gpu", str(args.gpu), "--workers", "4", | |
| ] | |
| print(f"start {target}: {' '.join(command)}", flush=True) | |
| result = subprocess.run(command, env=env, check=False) | |
| if result.returncode: | |
| failures[target] = f"exit code {result.returncode}" | |
| write_json(status_file, {"updated_at_utc": utc_now(), "counts": counts, | |
| "completed_scoring": sorted(done), "failures": failures}) | |
| raise RuntimeError(f"{target} scoring failed with {result.returncode}") | |
| if target == "diversity": | |
| summary = json.loads((outputs[target] / "summary.json").read_text(encoding="utf-8")) | |
| if len(summary["prompt_indices"]) != 200: | |
| raise RuntimeError("Diversity result does not cover all 200 prompts") | |
| reference = {"video_pairwise_cosine_distance": 0.2322, | |
| "audio_pairwise_cosine_distance": 0.4197, | |
| "va_average_pairwise_cosine_distance": 0.3259} | |
| metrics = summary["metrics"] | |
| write_json(outputs[target] / "paper_comparison.json", { | |
| "note": "Numeric comparison, pending prompt enhancement/checkpoint identity verification.", | |
| "metrics": {key: {"observed": metrics[key], "paper": value, | |
| "delta": metrics[key] - value} | |
| for key, value in reference.items()}, | |
| }) | |
| elif not score_complete(outputs[target] / "scores.json"): | |
| raise RuntimeError(f"{target} JavisBench output incomplete") | |
| done.add(target) | |
| raise TimeoutError(f"Evaluation watch timed out after {args.max_hours}h; status: {status_file}") | |
| def main() -> int: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--student-media", type=Path, required=True) | |
| parser.add_argument("--teacher-media", type=Path, required=True) | |
| parser.add_argument("--diversity-media", type=Path, required=True) | |
| parser.add_argument("--prompts", type=Path, required=True) | |
| parser.add_argument("--javis-root", type=Path, required=True) | |
| parser.add_argument("--javis-python", type=Path, required=True) | |
| parser.add_argument("--output", type=Path, required=True) | |
| parser.add_argument("--gpu", default="7") | |
| parser.add_argument("--poll-seconds", type=int, default=30) | |
| parser.add_argument("--max-hours", type=float, default=48) | |
| parser.add_argument("--student-only", action="store_true", help="Score student and exit without starting other evaluations") | |
| return run(parser.parse_args()) | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |