turbot2va-ltx2 / code /evaluation /scripts /run_diversity_generation.py
luyu1021's picture
Add documented LTX-2 training source, recipes and checkpoint release plan
76b7b25 verified
Raw History Blame Contribute Delete
12.7 kB
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
import random
import subprocess
import sys
import time
from pathlib import Path
from common import cfg_path, count_generated_media, load_config, model_output_dir, select_models, unset_proxy_env, utc_now, write_json
def subset_paths(config: dict) -> tuple[Path, Path, Path]:
subset = config.get("prompt_subset", {})
diversity_root = cfg_path(config, "diversity_root")
return (
Path(subset.get("prompts_file", diversity_root / "prompts_200.txt")).expanduser(),
Path(subset.get("prompt_ids_file", diversity_root / "prompt_ids_200.txt")).expanduser(),
Path(subset.get("prompt_manifest", diversity_root / "prompt_manifest.json")).expanduser(),
)
def ensure_prompt_subset(config: dict, force: bool = False) -> None:
source = cfg_path(config, "source_prompts")
prompts_file, prompt_ids_file, prompt_manifest = subset_paths(config)
subset = config.get("prompt_subset", {})
num_prompts = int(subset.get("num_prompts", 200))
num_seeds = int(subset.get("num_seeds", 8))
base_seed = int(subset.get("base_seed", 12345))
sample_seed = int(subset.get("prompt_sample_seed", 20260619))
anchors = [int(item) for item in subset.get("anchor_original_prompt_ids", [])]
if prompts_file.exists() and prompt_ids_file.exists() and prompt_manifest.exists() and not force:
return
prompts_file.parent.mkdir(parents=True, exist_ok=True)
prompts = [line.strip() for line in source.read_text(encoding="utf-8").splitlines() if line.strip()]
if len(prompts) < num_prompts:
raise RuntimeError(f"Need at least {num_prompts} prompts in {source}, found {len(prompts)}")
missing_anchors = [idx for idx in anchors if idx >= len(prompts)]
if missing_anchors:
raise RuntimeError(f"Anchor prompt ids out of range: {missing_anchors}")
rng = random.Random(sample_seed)
remaining = [idx for idx in range(len(prompts)) if idx not in anchors]
selected_ids = anchors + rng.sample(remaining, num_prompts - len(anchors))
prompts_file.write_text("".join(prompts[idx] + "\n" for idx in selected_ids), encoding="utf-8")
prompt_ids_file.write_text("".join(f"{idx}\n" for idx in selected_ids), encoding="utf-8")
write_json(
prompt_manifest,
{
"source_prompts": str(source),
"num_source_prompts": len(prompts),
"num_selected_prompts": num_prompts,
"num_seeds": num_seeds,
"base_seed": base_seed,
"prompt_sample_seed": sample_seed,
"anchor_original_prompt_ids": anchors,
"selected_original_prompt_ids": selected_ids,
"seed_rule": "seed = base_seed + local_prompt_index * num_seeds + seed_index",
},
)
def expected_count(config: dict) -> int:
subset = config.get("prompt_subset", {})
return int(subset.get("num_prompts", 200)) * int(subset.get("num_seeds", 8))
def model_complete(config: dict, model: dict) -> bool:
counts = count_generated_media(model_output_dir(config, model))
expected = expected_count(config)
return counts["mp4"] >= expected and counts["wav"] >= expected and counts["json"] >= expected
def validate_model(config: dict, model: dict) -> None:
paths = [Path(model["config_path"])]
if model.get("model_kind", "student") == "student":
if not model.get("student_checkpoint"):
raise RuntimeError(f"{model['name']}: missing student_checkpoint")
paths.append(Path(model["student_checkpoint"]))
missing = [str(path) for path in paths if not path.exists()]
if missing:
raise FileNotFoundError(f"{model['name']}: missing required paths:\n" + "\n".join(missing))
def build_cmd(config: dict, model: dict, num_shards: int, shard_id: int) -> list[str]:
subset = config.get("prompt_subset", {})
prompts_file, _, _ = subset_paths(config)
python = cfg_path(config, "ltx_python")
infer_script = cfg_path(config, "ltx_infer_script")
output_dir = model_output_dir(config, model)
model_kind = str(model.get("model_kind", "student"))
cmd = [
str(python),
str(infer_script),
"--config_path",
str(model["config_path"]),
"--prompts_file",
str(prompts_file),
"--output_dir",
str(output_dir),
"--model_kind",
model_kind,
"--num_prompts",
str(int(subset.get("num_prompts", 200))),
"--seed",
str(int(subset.get("base_seed", 12345))),
"--num_seeds",
str(int(subset.get("num_seeds", 8))),
"--num_shards",
str(num_shards),
"--shard_id",
str(shard_id),
]
if model_kind == "student":
cmd.extend(["--student_checkpoint", str(model["student_checkpoint"]), "--student_param", str(model.get("student_param", "auto"))])
else:
cmd.extend(["--teacher_mode", str(model.get("teacher_mode", "native_rf")), "--teacher_steps", str(model.get("teacher_steps", 40))])
return cmd
def launch_process(config: dict, model: dict, gpu: str, shard_id: int, num_shards: int, log_path: Path) -> dict:
ltx_root = cfg_path(config, "ltx_root")
python = cfg_path(config, "ltx_python")
infer_script = cfg_path(config, "ltx_infer_script")
for path in (python, infer_script):
if not path.exists():
raise FileNotFoundError(path)
output_dir = model_output_dir(config, model)
output_dir.mkdir(parents=True, exist_ok=True)
log_path.parent.mkdir(parents=True, exist_ok=True)
log_f = log_path.open("a", encoding="utf-8")
cmd = build_cmd(config, model, num_shards, shard_id)
env = unset_proxy_env()
env["CUDA_VISIBLE_DEVICES"] = str(gpu)
env.setdefault("PYTHONUNBUFFERED", "1")
log_f.write(f"\n\n===== launch diversity-generation {utc_now()} =====\n")
log_f.write("cmd: " + " ".join(cmd) + "\n")
log_f.write(f"CUDA_VISIBLE_DEVICES={gpu}\n")
log_f.flush()
proc = subprocess.Popen(cmd, cwd=str(ltx_root), env=env, stdout=log_f, stderr=subprocess.STDOUT, text=True)
print(f"launch generation gpu={gpu} model={model['name']} shard={shard_id}/{num_shards} pid={proc.pid} log={log_path}", flush=True)
return {
"model": model,
"gpu": gpu,
"shard_id": shard_id,
"num_shards": num_shards,
"proc": proc,
"log_f": log_f,
"log_path": log_path,
"started": time.time(),
}
def write_summary(config: dict, pending: list[dict], active: dict[str, dict], completed: list[str], failed: list[dict]) -> None:
diversity_root = cfg_path(config, "diversity_root")
payload = {
"updated_at_utc": utc_now(),
"strategy": "one_model_per_gpu_by_default",
"diversity_root": str(diversity_root),
"expected_per_model": expected_count(config),
"pending": [model["name"] for model in pending],
"active": {
key: {
"model": run["model"]["name"],
"gpu": run["gpu"],
"pid": run["proc"].pid,
"log": str(run["log_path"]),
"elapsed_seconds": round(time.time() - run["started"], 1),
"counts": count_generated_media(model_output_dir(config, run["model"])),
}
for key, run in active.items()
},
"completed": completed,
"failed": failed,
"models": [
{
"name": model["name"],
"display_name": model.get("display_name"),
"role": model.get("role"),
"output_dir": str(model_output_dir(config, model)),
"counts": count_generated_media(model_output_dir(config, model)),
"complete": model_complete(config, model),
}
for model in config.get("models", [])
],
}
write_json(diversity_root / "generation_summary.json", payload)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--config", required=True)
parser.add_argument("--gpus", default="0")
parser.add_argument("--only", default=None, help="comma-separated model names")
parser.add_argument("--roles", default=None, help="comma-separated role filters")
parser.add_argument("--force", action="store_true")
parser.add_argument("--force-prompt-subset", action="store_true")
parser.add_argument("--poll-seconds", type=int, default=30)
parser.add_argument("--shard-single-model", action="store_true", help="split one selected model across all GPUs")
parser.add_argument("--smoke", action="store_true", help="run one prompt x one seed for first selected model")
return parser.parse_args()
def main() -> int:
args = parse_args()
config = load_config(args.config)
gpus = [gpu.strip() for gpu in args.gpus.split(",") if gpu.strip()]
if not gpus:
raise RuntimeError("No GPUs specified")
ensure_prompt_subset(config, force=args.force_prompt_subset)
models = select_models(config, args.only, args.roles)
for model in models:
validate_model(config, model)
if args.smoke:
model = models[0]
log_path = cfg_path(config, "diversity_root") / "logs" / f"_smoke_{model['name']}.log"
# Smoke still calls the inference script. The inference script controls
# prompt count through CLI arguments, so build a temporary minimal config
# by overriding prompt_subset in memory.
config["prompt_subset"] = dict(config.get("prompt_subset", {}), num_prompts=1, num_seeds=1)
run = launch_process(config, model, gpus[0], shard_id=0, num_shards=1, log_path=log_path)
ret = run["proc"].wait()
run["log_f"].close()
print(f"smoke done ret={ret} counts={count_generated_media(model_output_dir(config, model))}", flush=True)
return ret
if args.shard_single_model:
if len(models) != 1:
raise RuntimeError("--shard-single-model requires exactly one selected model")
model = models[0]
active = {}
for shard_id, gpu in enumerate(gpus):
log_path = cfg_path(config, "diversity_root") / "logs" / model["name"] / f"shard_{shard_id:02d}.log"
active[f"{gpu}:{shard_id}"] = launch_process(config, model, gpu, shard_id, len(gpus), log_path)
failed = []
while active:
time.sleep(args.poll_seconds)
for key, run in list(active.items()):
ret = run["proc"].poll()
if ret is None:
continue
run["log_f"].close()
if ret != 0:
failed.append({"model": model["name"], "gpu": run["gpu"], "shard_id": run["shard_id"], "returncode": ret, "log": str(run["log_path"])})
del active[key]
write_summary(config, [], active, [model["name"]] if model_complete(config, model) else [], failed)
return 1 if failed else 0
pending = [model for model in models if args.force or not model_complete(config, model)]
completed = [model["name"] for model in models if model_complete(config, model) and not args.force]
failed: list[dict] = []
active: dict[str, dict] = {}
write_summary(config, pending, active, completed, failed)
while pending or active:
for gpu in gpus:
if gpu in active or not pending:
continue
model = pending.pop(0)
log_path = cfg_path(config, "diversity_root") / "logs" / model["name"] / "run.log"
active[gpu] = launch_process(config, model, gpu, shard_id=0, num_shards=1, log_path=log_path)
time.sleep(args.poll_seconds)
for gpu, run in list(active.items()):
ret = run["proc"].poll()
model = run["model"]
if ret is None:
print(f"progress generation gpu={gpu} model={model['name']} counts={count_generated_media(model_output_dir(config, model))}", flush=True)
continue
run["log_f"].close()
if ret == 0 and model_complete(config, model):
completed.append(model["name"])
print(f"done generation gpu={gpu} model={model['name']} wall={time.time() - run['started']:.1f}s", flush=True)
else:
failed.append({"model": model["name"], "gpu": gpu, "returncode": ret, "log": str(run["log_path"])})
print(f"failed generation gpu={gpu} model={model['name']} ret={ret}", flush=True)
del active[gpu]
write_summary(config, pending, active, completed, failed)
return 1 if failed else 0
if __name__ == "__main__":
sys.exit(main())