File size: 6,664 Bytes
76b7b25 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 | #!/usr/bin/env python3
"""Turn a named, documented recipe into an explicit torchrun command.
All edits apply to a new effective config in the chosen output directory.
Stage transitions are weights-only; --resume requires full training state.
--dry-run creates nothing and starts no CUDA process.
"""
from __future__ import annotations
import argparse
import json
import os
from pathlib import Path
import shlex
import subprocess
import sys
import yaml
ROOT = Path(__file__).resolve().parents[1]
def main() -> None:
recipes = json.loads((ROOT / 'configs/recipes.json').read_text())
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('recipe', choices=sorted(recipes))
parser.add_argument('--base', default=os.environ.get('TURBO_CHECKPOINT_PATH'))
parser.add_argument('--gemma', default=os.environ.get('TURBO_GEMMA_PATH'))
parser.add_argument('--prompts', default=os.environ.get('TURBO_DATA_PATH'))
parser.add_argument('--latents', default=os.environ.get('TURBO_SCM_DATA_PATH'))
parser.add_argument('--output', type=Path, required=True)
group = parser.add_mutually_exclusive_group()
group.add_argument('--init', type=Path, help='Earlier-stage student weights; start new optimizer and step counter.')
group.add_argument('--resume', type=Path, help='Same-stage checkpoint with optimizer, RNG and data cursor.')
parser.add_argument('--max-steps', type=int)
parser.add_argument('--gpus', type=int, default=8)
parser.add_argument('--nodes', type=int, default=1)
parser.add_argument('--node-rank', type=int, default=0)
parser.add_argument('--master-addr', default='127.0.0.1')
parser.add_argument('--master-port', type=int, default=29500)
parser.add_argument('--benchmark-every', type=int)
parser.add_argument('--keep-optimizer-states', type=int, help='0 keeps all; 2 keeps only the newest two.')
parser.add_argument('--dry-run', action='store_true')
args = parser.parse_args()
recipe = recipes[args.recipe]
cfg = yaml.safe_load((ROOT / recipe['config']).read_text())
missing = [name for name in ['base', 'gemma', 'prompts'] if not getattr(args, name)]
if cfg.get('scm_enabled') or cfg.get('dcm_enabled'):
if not args.latents:
missing.append('latents')
if missing:
parser.error('Supply paths for: ' + ', '.join(missing))
if recipe['parent_artifact'] and not (args.init or args.resume):
parser.error(f'This recipe requires its parent {recipe["parent_artifact"]!r}; specify --init or --resume.')
if recipe['parent_requires_full_resume'] and args.init:
parser.error('SCM continuation preserves the SCM optimizer; use --resume with the full prefix checkpoint.')
if recipe['mode'] == 'dcm' and args.init:
parser.error('These DCM A/B recipes start from the official base; --init is not an audited DCM fork.')
if args.nodes < 1 or args.gpus < 1 or not 0 <= args.node_rank < args.nodes:
parser.error('Invalid distributed topology.')
if args.nodes > 1 and args.master_addr in ['127.0.0.1', 'localhost']:
parser.error('Multiple nodes require a shared, reachable --master-addr.')
output = args.output.resolve()
cfg.update(checkpoint_path=str(Path(args.base).resolve()), gemma_path=str(Path(args.gemma).resolve()),
data_path=str(Path(args.prompts).resolve()), output_path=str(output.parent),
wandb_name=output.name, stop_file_path=str(output / 'STOP_REQUESTED'))
if args.latents:
cfg['scm_data_path'] = str(Path(args.latents).resolve())
if args.init or args.resume:
cfg['resume_checkpoint'] = str((args.init or args.resume).resolve())
cfg['resume_training_state'] = bool(args.resume)
cfg['require_full_resume_state'] = bool(args.resume)
cfg['checkpoint_load_mode'] = 'serial' if args.resume else 'parallel'
for key, value in [('max_steps', args.max_steps), ('benchmark_iters', args.benchmark_every),
('keep_optimizer_states', args.keep_optimizer_states)]:
if value is not None:
if value < 0:
parser.error(f'{key} must be >= 0')
cfg[key] = value
env = os.environ.copy()
# train_distillation.py respects these existing environment path overrides.
for key, var in [('checkpoint_path','TURBO_CHECKPOINT_PATH'),('gemma_path','TURBO_GEMMA_PATH'),
('data_path','TURBO_DATA_PATH'),('scm_data_path','TURBO_SCM_DATA_PATH')]:
env[var] = str(cfg[key])
env.update(LTX_RUN_DIR_NAME=output.name, WANDB_MODE=env.get('WANDB_MODE', 'offline'), PYTHONUNBUFFERED='1')
if cfg.get('critic_enabled'):
env.setdefault('TURBO_PARK_CRITIC_OPTIM', '1')
env.setdefault('TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC', '3600')
env.setdefault('TURBO_DIST_TIMEOUT_MINUTES', '120')
if args.nodes == 1:
env.setdefault('PYTORCH_CUDA_ALLOC_CONF', 'expandable_segments:True')
packages = ROOT / 'LTX-2/packages'
local_paths = [str(packages / p / 'src') for p in ['ltx-core','ltx-pipelines','ltx-causal','ltx-distillation']]
env['PYTHONPATH'] = os.pathsep.join(local_paths + ([env['PYTHONPATH']] if env.get('PYTHONPATH') else []))
config_file = output / 'launch_config.yaml'
cmd = [sys.executable, '-m', 'torch.distributed.run', f'--nproc_per_node={args.gpus}',
f'--nnodes={args.nodes}', f'--node_rank={args.node_rank}', f'--master_addr={args.master_addr}',
f'--master_port={args.master_port}', '-m', 'ltx_distillation.train_distillation',
'--config_path', str(config_file)]
print(yaml.safe_dump(cfg, allow_unicode=True, sort_keys=False))
print('Command:', shlex.join(cmd), flush=True)
if args.dry_run:
return
for name in ['base', 'gemma', 'prompts', 'latents']:
value = getattr(args, name)
if value and not Path(value).exists():
parser.error(f'{name} does not exist: {value}')
if args.init or args.resume:
if not (args.init or args.resume).exists():
parser.error('Checkpoint does not exist.')
if not args.resume and ((output/'train.log').exists() or (output/'checkpoints').exists()):
parser.error('Output contains an existing run; choose a new output directory or use --resume.')
if (output/'STOP_REQUESTED').exists():
parser.error('STOP_REQUESTED exists; remove it deliberately before continuation.')
output.mkdir(parents=True, exist_ok=True)
config_file.write_text(yaml.safe_dump(cfg, allow_unicode=True, sort_keys=False))
raise SystemExit(subprocess.call(cmd, cwd=ROOT, env=env))
if __name__ == '__main__':
main()
|