#!/usr/bin/env python3 from __future__ import annotations import argparse import json import random import subprocess import sys import time from pathlib import Path from common import cfg_path, count_generated_media, load_config, model_output_dir, select_models, unset_proxy_env, utc_now, write_json def subset_paths(config: dict) -> tuple[Path, Path, Path]: subset = config.get("prompt_subset", {}) diversity_root = cfg_path(config, "diversity_root") return ( Path(subset.get("prompts_file", diversity_root / "prompts_200.txt")).expanduser(), Path(subset.get("prompt_ids_file", diversity_root / "prompt_ids_200.txt")).expanduser(), Path(subset.get("prompt_manifest", diversity_root / "prompt_manifest.json")).expanduser(), ) def ensure_prompt_subset(config: dict, force: bool = False) -> None: source = cfg_path(config, "source_prompts") prompts_file, prompt_ids_file, prompt_manifest = subset_paths(config) subset = config.get("prompt_subset", {}) num_prompts = int(subset.get("num_prompts", 200)) num_seeds = int(subset.get("num_seeds", 8)) base_seed = int(subset.get("base_seed", 12345)) sample_seed = int(subset.get("prompt_sample_seed", 20260619)) anchors = [int(item) for item in subset.get("anchor_original_prompt_ids", [])] if prompts_file.exists() and prompt_ids_file.exists() and prompt_manifest.exists() and not force: return prompts_file.parent.mkdir(parents=True, exist_ok=True) prompts = [line.strip() for line in source.read_text(encoding="utf-8").splitlines() if line.strip()] if len(prompts) < num_prompts: raise RuntimeError(f"Need at least {num_prompts} prompts in {source}, found {len(prompts)}") missing_anchors = [idx for idx in anchors if idx >= len(prompts)] if missing_anchors: raise RuntimeError(f"Anchor prompt ids out of range: {missing_anchors}") rng = random.Random(sample_seed) remaining = [idx for idx in range(len(prompts)) if idx not in anchors] selected_ids = anchors + rng.sample(remaining, num_prompts - len(anchors)) prompts_file.write_text("".join(prompts[idx] + "\n" for idx in selected_ids), encoding="utf-8") prompt_ids_file.write_text("".join(f"{idx}\n" for idx in selected_ids), encoding="utf-8") write_json( prompt_manifest, { "source_prompts": str(source), "num_source_prompts": len(prompts), "num_selected_prompts": num_prompts, "num_seeds": num_seeds, "base_seed": base_seed, "prompt_sample_seed": sample_seed, "anchor_original_prompt_ids": anchors, "selected_original_prompt_ids": selected_ids, "seed_rule": "seed = base_seed + local_prompt_index * num_seeds + seed_index", }, ) def expected_count(config: dict) -> int: subset = config.get("prompt_subset", {}) return int(subset.get("num_prompts", 200)) * int(subset.get("num_seeds", 8)) def model_complete(config: dict, model: dict) -> bool: counts = count_generated_media(model_output_dir(config, model)) expected = expected_count(config) return counts["mp4"] >= expected and counts["wav"] >= expected and counts["json"] >= expected def validate_model(config: dict, model: dict) -> None: paths = [Path(model["config_path"])] if model.get("model_kind", "student") == "student": if not model.get("student_checkpoint"): raise RuntimeError(f"{model['name']}: missing student_checkpoint") paths.append(Path(model["student_checkpoint"])) missing = [str(path) for path in paths if not path.exists()] if missing: raise FileNotFoundError(f"{model['name']}: missing required paths:\n" + "\n".join(missing)) def build_cmd(config: dict, model: dict, num_shards: int, shard_id: int) -> list[str]: subset = config.get("prompt_subset", {}) prompts_file, _, _ = subset_paths(config) python = cfg_path(config, "ltx_python") infer_script = cfg_path(config, "ltx_infer_script") output_dir = model_output_dir(config, model) model_kind = str(model.get("model_kind", "student")) cmd = [ str(python), str(infer_script), "--config_path", str(model["config_path"]), "--prompts_file", str(prompts_file), "--output_dir", str(output_dir), "--model_kind", model_kind, "--num_prompts", str(int(subset.get("num_prompts", 200))), "--seed", str(int(subset.get("base_seed", 12345))), "--num_seeds", str(int(subset.get("num_seeds", 8))), "--num_shards", str(num_shards), "--shard_id", str(shard_id), ] if model_kind == "student": cmd.extend(["--student_checkpoint", str(model["student_checkpoint"]), "--student_param", str(model.get("student_param", "auto"))]) else: cmd.extend(["--teacher_mode", str(model.get("teacher_mode", "native_rf")), "--teacher_steps", str(model.get("teacher_steps", 40))]) return cmd def launch_process(config: dict, model: dict, gpu: str, shard_id: int, num_shards: int, log_path: Path) -> dict: ltx_root = cfg_path(config, "ltx_root") python = cfg_path(config, "ltx_python") infer_script = cfg_path(config, "ltx_infer_script") for path in (python, infer_script): if not path.exists(): raise FileNotFoundError(path) output_dir = model_output_dir(config, model) output_dir.mkdir(parents=True, exist_ok=True) log_path.parent.mkdir(parents=True, exist_ok=True) log_f = log_path.open("a", encoding="utf-8") cmd = build_cmd(config, model, num_shards, shard_id) env = unset_proxy_env() env["CUDA_VISIBLE_DEVICES"] = str(gpu) env.setdefault("PYTHONUNBUFFERED", "1") log_f.write(f"\n\n===== launch diversity-generation {utc_now()} =====\n") log_f.write("cmd: " + " ".join(cmd) + "\n") log_f.write(f"CUDA_VISIBLE_DEVICES={gpu}\n") log_f.flush() proc = subprocess.Popen(cmd, cwd=str(ltx_root), env=env, stdout=log_f, stderr=subprocess.STDOUT, text=True) print(f"launch generation gpu={gpu} model={model['name']} shard={shard_id}/{num_shards} pid={proc.pid} log={log_path}", flush=True) return { "model": model, "gpu": gpu, "shard_id": shard_id, "num_shards": num_shards, "proc": proc, "log_f": log_f, "log_path": log_path, "started": time.time(), } def write_summary(config: dict, pending: list[dict], active: dict[str, dict], completed: list[str], failed: list[dict]) -> None: diversity_root = cfg_path(config, "diversity_root") payload = { "updated_at_utc": utc_now(), "strategy": "one_model_per_gpu_by_default", "diversity_root": str(diversity_root), "expected_per_model": expected_count(config), "pending": [model["name"] for model in pending], "active": { key: { "model": run["model"]["name"], "gpu": run["gpu"], "pid": run["proc"].pid, "log": str(run["log_path"]), "elapsed_seconds": round(time.time() - run["started"], 1), "counts": count_generated_media(model_output_dir(config, run["model"])), } for key, run in active.items() }, "completed": completed, "failed": failed, "models": [ { "name": model["name"], "display_name": model.get("display_name"), "role": model.get("role"), "output_dir": str(model_output_dir(config, model)), "counts": count_generated_media(model_output_dir(config, model)), "complete": model_complete(config, model), } for model in config.get("models", []) ], } write_json(diversity_root / "generation_summary.json", payload) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--config", required=True) parser.add_argument("--gpus", default="0") parser.add_argument("--only", default=None, help="comma-separated model names") parser.add_argument("--roles", default=None, help="comma-separated role filters") parser.add_argument("--force", action="store_true") parser.add_argument("--force-prompt-subset", action="store_true") parser.add_argument("--poll-seconds", type=int, default=30) parser.add_argument("--shard-single-model", action="store_true", help="split one selected model across all GPUs") parser.add_argument("--smoke", action="store_true", help="run one prompt x one seed for first selected model") return parser.parse_args() def main() -> int: args = parse_args() config = load_config(args.config) gpus = [gpu.strip() for gpu in args.gpus.split(",") if gpu.strip()] if not gpus: raise RuntimeError("No GPUs specified") ensure_prompt_subset(config, force=args.force_prompt_subset) models = select_models(config, args.only, args.roles) for model in models: validate_model(config, model) if args.smoke: model = models[0] log_path = cfg_path(config, "diversity_root") / "logs" / f"_smoke_{model['name']}.log" # Smoke still calls the inference script. The inference script controls # prompt count through CLI arguments, so build a temporary minimal config # by overriding prompt_subset in memory. config["prompt_subset"] = dict(config.get("prompt_subset", {}), num_prompts=1, num_seeds=1) run = launch_process(config, model, gpus[0], shard_id=0, num_shards=1, log_path=log_path) ret = run["proc"].wait() run["log_f"].close() print(f"smoke done ret={ret} counts={count_generated_media(model_output_dir(config, model))}", flush=True) return ret if args.shard_single_model: if len(models) != 1: raise RuntimeError("--shard-single-model requires exactly one selected model") model = models[0] active = {} for shard_id, gpu in enumerate(gpus): log_path = cfg_path(config, "diversity_root") / "logs" / model["name"] / f"shard_{shard_id:02d}.log" active[f"{gpu}:{shard_id}"] = launch_process(config, model, gpu, shard_id, len(gpus), log_path) failed = [] while active: time.sleep(args.poll_seconds) for key, run in list(active.items()): ret = run["proc"].poll() if ret is None: continue run["log_f"].close() if ret != 0: failed.append({"model": model["name"], "gpu": run["gpu"], "shard_id": run["shard_id"], "returncode": ret, "log": str(run["log_path"])}) del active[key] write_summary(config, [], active, [model["name"]] if model_complete(config, model) else [], failed) return 1 if failed else 0 pending = [model for model in models if args.force or not model_complete(config, model)] completed = [model["name"] for model in models if model_complete(config, model) and not args.force] failed: list[dict] = [] active: dict[str, dict] = {} write_summary(config, pending, active, completed, failed) while pending or active: for gpu in gpus: if gpu in active or not pending: continue model = pending.pop(0) log_path = cfg_path(config, "diversity_root") / "logs" / model["name"] / "run.log" active[gpu] = launch_process(config, model, gpu, shard_id=0, num_shards=1, log_path=log_path) time.sleep(args.poll_seconds) for gpu, run in list(active.items()): ret = run["proc"].poll() model = run["model"] if ret is None: print(f"progress generation gpu={gpu} model={model['name']} counts={count_generated_media(model_output_dir(config, model))}", flush=True) continue run["log_f"].close() if ret == 0 and model_complete(config, model): completed.append(model["name"]) print(f"done generation gpu={gpu} model={model['name']} wall={time.time() - run['started']:.1f}s", flush=True) else: failed.append({"model": model["name"], "gpu": gpu, "returncode": ret, "log": str(run["log_path"])}) print(f"failed generation gpu={gpu} model={model['name']} ret={ret}", flush=True) del active[gpu] write_summary(config, pending, active, completed, failed) return 1 if failed else 0 if __name__ == "__main__": sys.exit(main())