turbot2va-ltx2 / code /evaluation /scripts /watch_evaluation.py
luyu1021's picture
Add documented LTX-2 training source, recipes and checkpoint release plan
76b7b25 verified
Raw History Blame Contribute Delete
7.21 kB
#!/usr/bin/env python3
"""Watch in-progress AV media and score each completed experiment on a spare GPU."""
from __future__ import annotations
import argparse
import json
import os
import subprocess
import sys
import time
from pathlib import Path
from common import utc_now, write_json
from run_checkpoint_curve import score_complete
def media_count(directory: Path, indices: range, seeds: int | None = None) -> int:
def complete(stem: str) -> bool:
return all(
(directory / f"{stem}.{suffix}").is_file()
and (directory / f"{stem}.{suffix}").stat().st_size > 0
for suffix in ("mp4", "wav", "json")
)
return sum(
complete(f"sample_{i:04d}" + (f"_seed{j:04d}" if seeds is not None else ""))
for i in indices for j in (range(seeds) if seeds is not None else (None,))
)
def run(args: argparse.Namespace) -> int:
if args.poll_seconds < 2 or args.max_hours <= 0:
raise ValueError("--poll-seconds >= 2 and --max-hours > 0 are required")
script_root = Path(__file__).resolve().parent
javis_python = args.javis_python.absolute()
javis_root = args.javis_root.resolve(strict=True)
media = {name: path.resolve() for name, path in (
("student", args.student_media), ("teacher_40_step", args.teacher_media),
("diversity", args.diversity_media))}
outputs = {name: args.output.resolve() / name for name in ("student", "teacher_40_step", "diversity")}
status_file = args.output.resolve() / "watch_status.json"
prompts = args.prompts.resolve(strict=True).read_text(encoding="utf-8").splitlines()
if len(prompts) != 200:
raise ValueError("Expected 200 prompts")
started = time.monotonic()
targets = {"student"} if args.student_only else {"student", "teacher_40_step", "diversity"}
done: set[str] = {
name for name in ("student", "teacher_40_step")
if name in targets and score_complete(outputs[name] / "scores.json")
}
diversity_summary = outputs["diversity"] / "summary.json"
if "diversity" in targets and diversity_summary.is_file():
payload = json.loads(diversity_summary.read_text(encoding="utf-8"))
if len(payload.get("prompt_indices", [])) == 200 and payload.get("seeds") == 8:
done.add("diversity")
failures: dict[str, str] = {}
while time.monotonic() - started < args.max_hours * 3600:
counts = {
"student": media_count(media["student"], range(200)),
"teacher_40_step": media_count(media["teacher_40_step"], range(200)),
"diversity": media_count(media["diversity"], range(200), seeds=8),
}
if "student" not in done and counts["student"] == 200:
target = "student"
elif "teacher_40_step" in targets and "teacher_40_step" not in done and counts["teacher_40_step"] == 200:
target = "teacher_40_step"
elif "diversity" in targets and "diversity" not in done and counts["diversity"] == 1600:
target = "diversity"
else:
target = None
write_json(status_file, {
"updated_at_utc": utc_now(), "gpu": args.gpu, "counts": counts,
"completed_scoring": sorted(done), "failures": failures,
"active_scoring": target,
})
print(f"{utc_now()} media {counts} scoring {target or 'waiting'}", flush=True)
if targets <= done:
return 0
if target is None:
time.sleep(args.poll_seconds)
continue
env = dict(os.environ)
env.update({
"CUDA_VISIBLE_DEVICES": str(args.gpu), "HF_HUB_OFFLINE": "1",
"TRANSFORMERS_OFFLINE": "1", "PYTHONUNBUFFERED": "1",
"OMP_NUM_THREADS": "4", "MKL_NUM_THREADS": "4",
})
if target == "diversity":
command = [
str(javis_python), str(script_root / "score_diversity_completed.py"),
"--media", str(media[target]), "--prompts", str(args.prompts),
"--javis-root", str(javis_root), "--output", str(outputs[target]),
"--gpu", str(args.gpu), "--batch-size", "1", "--workers", "2",
]
else:
command = [
sys.executable, str(script_root / "score_existing_gpu.py"),
"--model", target, "--media", str(media[target]), "--prompts", str(args.prompts),
"--output", str(outputs[target]), "--javis-root", str(javis_root),
"--javis-python", str(javis_python), "--gpu", str(args.gpu), "--workers", "4",
]
print(f"start {target}: {' '.join(command)}", flush=True)
result = subprocess.run(command, env=env, check=False)
if result.returncode:
failures[target] = f"exit code {result.returncode}"
write_json(status_file, {"updated_at_utc": utc_now(), "counts": counts,
"completed_scoring": sorted(done), "failures": failures})
raise RuntimeError(f"{target} scoring failed with {result.returncode}")
if target == "diversity":
summary = json.loads((outputs[target] / "summary.json").read_text(encoding="utf-8"))
if len(summary["prompt_indices"]) != 200:
raise RuntimeError("Diversity result does not cover all 200 prompts")
reference = {"video_pairwise_cosine_distance": 0.2322,
"audio_pairwise_cosine_distance": 0.4197,
"va_average_pairwise_cosine_distance": 0.3259}
metrics = summary["metrics"]
write_json(outputs[target] / "paper_comparison.json", {
"note": "Numeric comparison, pending prompt enhancement/checkpoint identity verification.",
"metrics": {key: {"observed": metrics[key], "paper": value,
"delta": metrics[key] - value}
for key, value in reference.items()},
})
elif not score_complete(outputs[target] / "scores.json"):
raise RuntimeError(f"{target} JavisBench output incomplete")
done.add(target)
raise TimeoutError(f"Evaluation watch timed out after {args.max_hours}h; status: {status_file}")
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--student-media", type=Path, required=True)
parser.add_argument("--teacher-media", type=Path, required=True)
parser.add_argument("--diversity-media", type=Path, required=True)
parser.add_argument("--prompts", type=Path, required=True)
parser.add_argument("--javis-root", type=Path, required=True)
parser.add_argument("--javis-python", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--gpu", default="7")
parser.add_argument("--poll-seconds", type=int, default=30)
parser.add_argument("--max-hours", type=float, default=48)
parser.add_argument("--student-only", action="store_true", help="Score student and exit without starting other evaluations")
return run(parser.parse_args())
if __name__ == "__main__":
raise SystemExit(main())