File size: 5,862 Bytes
76b7b25 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 | #!/usr/bin/env python3
"""Generate and JavisBench-score one published full-state-dict checkpoint."""
from __future__ import annotations
import argparse
import hashlib
import json
import subprocess
import sys
from pathlib import Path
from common import load_config, unset_proxy_env
from run_checkpoint_curve import METRICS, SCORES, check_manifest, media_complete, score_complete, training_overlap, write_input
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--checkpoint", type=Path, required=True)
parser.add_argument("--checkpoint-info", type=Path, required=True)
parser.add_argument("--run-config", type=Path, required=True)
parser.add_argument("--prompts", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--javis-root", type=Path, required=True)
parser.add_argument("--javis-python", type=Path, required=True)
parser.add_argument("--ltx-root", type=Path, required=True)
parser.add_argument("--ltx-python", type=Path, required=True)
parser.add_argument("--gpu", default="0")
parser.add_argument("--seed", type=int, default=12345)
parser.add_argument("--limit", type=int, default=10)
parser.add_argument("--dry-run", action="store_true")
return parser.parse_args()
def run(args: argparse.Namespace) -> int:
checkpoint, info_path = args.checkpoint.resolve(), args.checkpoint_info.resolve()
config, prompts_path = args.run_config.resolve(), args.prompts.resolve()
output, javis_root, ltx_root = args.output.resolve(), args.javis_root.resolve(), args.ltx_root.resolve()
# A venv's python symlink must be executed through its own path.
javis_python, ltx_python = args.javis_python.absolute(), args.ltx_python.absolute()
infer = ltx_root / "packages/ltx-distillation/src/ltx_distillation/tools/run_av_inference_eval.py"
bench = javis_root / "eval/javisbench/main.py"
if args.limit < 1:
raise ValueError("--limit must be positive")
for path in (checkpoint, info_path, config, prompts_path, infer, bench, javis_python, ltx_python):
if not path.is_file():
raise FileNotFoundError(path)
info = json.loads(info_path.read_text(encoding="utf-8"))
if (checkpoint.stat().st_size != info["model_bytes"]
or info.get("format") != "pytorch_state_dict"
or info.get("checkpoint_name") != "turbot2av_main"):
raise ValueError(f"Published checkpoint size/format/name does not match {info_path}")
prompts = [line.strip() for line in prompts_path.read_text(encoding="utf-8").splitlines() if line.strip()]
if len(prompts) < args.limit:
raise ValueError(f"Expected {args.limit} prompts, found {len(prompts)}")
prompts = prompts[:args.limit]
cfg = load_config(config)
for key in ("checkpoint_path", "gemma_path"):
if not Path(cfg[key]).exists():
raise FileNotFoundError(f"{key}: {cfg[key]}")
mapping = Path(cfg["data_path"])
if not mapping.is_file():
raise FileNotFoundError(f"Training mapping missing: {mapping}")
overlap = training_overlap(mapping, prompts)
if overlap:
print(f"WARNING: {len(overlap)} evaluation prompts overlap training mapping {mapping}", flush=True)
manifest = {
"model_id": "luyu1021/TurboT2AV",
"checkpoint": str(checkpoint),
"checkpoint_sha256": info["model_sha256"],
"checkpoint_step": info["checkpoint_step"],
"config": str(config),
"config_sha256": hashlib.sha256(config.read_bytes()).hexdigest(),
"prompt_sha256": hashlib.sha256("\n".join(prompts).encode("utf-8")).hexdigest(),
"num_prompts": args.limit,
"train_overlap_count": len(overlap),
"train_mapping": str(mapping),
"seed": args.seed,
"metrics": METRICS,
"javis_root": str(javis_root),
"ltx_python": str(ltx_python),
"javis_python": str(javis_python),
}
check_manifest(output, manifest, args.dry_run)
media = output / "media"
score_file = output / "scores.json"
input_csv = output / "input.csv"
generate = [
str(ltx_python), str(infer), "--config_path", str(config),
"--prompts_file", str(prompts_path), "--output_dir", str(media),
"--model_kind", "student", "--student_param", "rcm_trig",
"--student_checkpoint", str(checkpoint), "--num_prompts", str(args.limit),
"--num_seeds", "1", "--seed", str(args.seed),
]
score = [
str(javis_python), "-m", "eval.javisbench.main",
"--input_file", str(input_csv), "--output_file", str(score_file),
"--metrics", *METRICS, "--max_audio_len_s", "4.0", "--num_workers", "8",
]
if args.dry_run:
print(f"published probe: generate={not media_complete(media, args.limit)} score={not score_complete(score_file)}")
print(f"generate: {generate}")
print(f"score: {score}")
return 0
env = unset_proxy_env()
env["CUDA_VISIBLE_DEVICES"] = args.gpu
env["PYTHONUNBUFFERED"] = "1"
if not score_complete(score_file):
if not media_complete(media, args.limit):
media.mkdir(parents=True, exist_ok=True)
subprocess.run(generate, cwd=ltx_root, env=env, check=True)
if not media_complete(media, args.limit):
raise RuntimeError(f"Incomplete media: {media}")
write_input(input_csv, prompts, media)
subprocess.run(score, cwd=javis_root, env=env, check=True)
if not score_complete(score_file):
raise RuntimeError(f"Incomplete JavisBench scores: {score_file}")
values = json.loads(score_file.read_text(encoding="utf-8"))
print(json.dumps({k: values[k] for k in SCORES}, indent=2))
return 0
if __name__ == "__main__":
sys.exit(run(parse_args()))
|