#!/usr/bin/env python3 """Turn a named, documented recipe into an explicit torchrun command. All edits apply to a new effective config in the chosen output directory. Stage transitions are weights-only; --resume requires full training state. --dry-run creates nothing and starts no CUDA process. """ from __future__ import annotations import argparse import json import os from pathlib import Path import shlex import subprocess import sys import yaml ROOT = Path(__file__).resolve().parents[1] def main() -> None: recipes = json.loads((ROOT / 'configs/recipes.json').read_text()) parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('recipe', choices=sorted(recipes)) parser.add_argument('--base', default=os.environ.get('TURBO_CHECKPOINT_PATH')) parser.add_argument('--gemma', default=os.environ.get('TURBO_GEMMA_PATH')) parser.add_argument('--prompts', default=os.environ.get('TURBO_DATA_PATH')) parser.add_argument('--latents', default=os.environ.get('TURBO_SCM_DATA_PATH')) parser.add_argument('--output', type=Path, required=True) group = parser.add_mutually_exclusive_group() group.add_argument('--init', type=Path, help='Earlier-stage student weights; start new optimizer and step counter.') group.add_argument('--resume', type=Path, help='Same-stage checkpoint with optimizer, RNG and data cursor.') parser.add_argument('--max-steps', type=int) parser.add_argument('--gpus', type=int, default=8) parser.add_argument('--nodes', type=int, default=1) parser.add_argument('--node-rank', type=int, default=0) parser.add_argument('--master-addr', default='127.0.0.1') parser.add_argument('--master-port', type=int, default=29500) parser.add_argument('--benchmark-every', type=int) parser.add_argument('--keep-optimizer-states', type=int, help='0 keeps all; 2 keeps only the newest two.') parser.add_argument('--dry-run', action='store_true') args = parser.parse_args() recipe = recipes[args.recipe] cfg = yaml.safe_load((ROOT / recipe['config']).read_text()) missing = [name for name in ['base', 'gemma', 'prompts'] if not getattr(args, name)] if cfg.get('scm_enabled') or cfg.get('dcm_enabled'): if not args.latents: missing.append('latents') if missing: parser.error('Supply paths for: ' + ', '.join(missing)) if recipe['parent_artifact'] and not (args.init or args.resume): parser.error(f'This recipe requires its parent {recipe["parent_artifact"]!r}; specify --init or --resume.') if recipe['parent_requires_full_resume'] and args.init: parser.error('SCM continuation preserves the SCM optimizer; use --resume with the full prefix checkpoint.') if recipe['mode'] == 'dcm' and args.init: parser.error('These DCM A/B recipes start from the official base; --init is not an audited DCM fork.') if args.nodes < 1 or args.gpus < 1 or not 0 <= args.node_rank < args.nodes: parser.error('Invalid distributed topology.') if args.nodes > 1 and args.master_addr in ['127.0.0.1', 'localhost']: parser.error('Multiple nodes require a shared, reachable --master-addr.') output = args.output.resolve() cfg.update(checkpoint_path=str(Path(args.base).resolve()), gemma_path=str(Path(args.gemma).resolve()), data_path=str(Path(args.prompts).resolve()), output_path=str(output.parent), wandb_name=output.name, stop_file_path=str(output / 'STOP_REQUESTED')) if args.latents: cfg['scm_data_path'] = str(Path(args.latents).resolve()) if args.init or args.resume: cfg['resume_checkpoint'] = str((args.init or args.resume).resolve()) cfg['resume_training_state'] = bool(args.resume) cfg['require_full_resume_state'] = bool(args.resume) cfg['checkpoint_load_mode'] = 'serial' if args.resume else 'parallel' for key, value in [('max_steps', args.max_steps), ('benchmark_iters', args.benchmark_every), ('keep_optimizer_states', args.keep_optimizer_states)]: if value is not None: if value < 0: parser.error(f'{key} must be >= 0') cfg[key] = value env = os.environ.copy() # train_distillation.py respects these existing environment path overrides. for key, var in [('checkpoint_path','TURBO_CHECKPOINT_PATH'),('gemma_path','TURBO_GEMMA_PATH'), ('data_path','TURBO_DATA_PATH'),('scm_data_path','TURBO_SCM_DATA_PATH')]: env[var] = str(cfg[key]) env.update(LTX_RUN_DIR_NAME=output.name, WANDB_MODE=env.get('WANDB_MODE', 'offline'), PYTHONUNBUFFERED='1') if cfg.get('critic_enabled'): env.setdefault('TURBO_PARK_CRITIC_OPTIM', '1') env.setdefault('TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC', '3600') env.setdefault('TURBO_DIST_TIMEOUT_MINUTES', '120') if args.nodes == 1: env.setdefault('PYTORCH_CUDA_ALLOC_CONF', 'expandable_segments:True') packages = ROOT / 'LTX-2/packages' local_paths = [str(packages / p / 'src') for p in ['ltx-core','ltx-pipelines','ltx-causal','ltx-distillation']] env['PYTHONPATH'] = os.pathsep.join(local_paths + ([env['PYTHONPATH']] if env.get('PYTHONPATH') else [])) config_file = output / 'launch_config.yaml' cmd = [sys.executable, '-m', 'torch.distributed.run', f'--nproc_per_node={args.gpus}', f'--nnodes={args.nodes}', f'--node_rank={args.node_rank}', f'--master_addr={args.master_addr}', f'--master_port={args.master_port}', '-m', 'ltx_distillation.train_distillation', '--config_path', str(config_file)] print(yaml.safe_dump(cfg, allow_unicode=True, sort_keys=False)) print('Command:', shlex.join(cmd), flush=True) if args.dry_run: return for name in ['base', 'gemma', 'prompts', 'latents']: value = getattr(args, name) if value and not Path(value).exists(): parser.error(f'{name} does not exist: {value}') if args.init or args.resume: if not (args.init or args.resume).exists(): parser.error('Checkpoint does not exist.') if not args.resume and ((output/'train.log').exists() or (output/'checkpoints').exists()): parser.error('Output contains an existing run; choose a new output directory or use --resume.') if (output/'STOP_REQUESTED').exists(): parser.error('STOP_REQUESTED exists; remove it deliberately before continuation.') output.mkdir(parents=True, exist_ok=True) config_file.write_text(yaml.safe_dump(cfg, allow_unicode=True, sort_keys=False)) raise SystemExit(subprocess.call(cmd, cwd=ROOT, env=env)) if __name__ == '__main__': main()