File size: 6,664 Bytes
76b7b25
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
#!/usr/bin/env python3
"""Turn a named, documented recipe into an explicit torchrun command.

All edits apply to a new effective config in the chosen output directory.
Stage transitions are weights-only; --resume requires full training state.
--dry-run creates nothing and starts no CUDA process.
"""
from __future__ import annotations

import argparse
import json
import os
from pathlib import Path
import shlex
import subprocess
import sys

import yaml

ROOT = Path(__file__).resolve().parents[1]


def main() -> None:
    recipes = json.loads((ROOT / 'configs/recipes.json').read_text())
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('recipe', choices=sorted(recipes))
    parser.add_argument('--base', default=os.environ.get('TURBO_CHECKPOINT_PATH'))
    parser.add_argument('--gemma', default=os.environ.get('TURBO_GEMMA_PATH'))
    parser.add_argument('--prompts', default=os.environ.get('TURBO_DATA_PATH'))
    parser.add_argument('--latents', default=os.environ.get('TURBO_SCM_DATA_PATH'))
    parser.add_argument('--output', type=Path, required=True)
    group = parser.add_mutually_exclusive_group()
    group.add_argument('--init', type=Path, help='Earlier-stage student weights; start new optimizer and step counter.')
    group.add_argument('--resume', type=Path, help='Same-stage checkpoint with optimizer, RNG and data cursor.')
    parser.add_argument('--max-steps', type=int)
    parser.add_argument('--gpus', type=int, default=8)
    parser.add_argument('--nodes', type=int, default=1)
    parser.add_argument('--node-rank', type=int, default=0)
    parser.add_argument('--master-addr', default='127.0.0.1')
    parser.add_argument('--master-port', type=int, default=29500)
    parser.add_argument('--benchmark-every', type=int)
    parser.add_argument('--keep-optimizer-states', type=int, help='0 keeps all; 2 keeps only the newest two.')
    parser.add_argument('--dry-run', action='store_true')
    args = parser.parse_args()
    recipe = recipes[args.recipe]
    cfg = yaml.safe_load((ROOT / recipe['config']).read_text())
    missing = [name for name in ['base', 'gemma', 'prompts'] if not getattr(args, name)]
    if cfg.get('scm_enabled') or cfg.get('dcm_enabled'):
        if not args.latents:
            missing.append('latents')
    if missing:
        parser.error('Supply paths for: ' + ', '.join(missing))
    if recipe['parent_artifact'] and not (args.init or args.resume):
        parser.error(f'This recipe requires its parent {recipe["parent_artifact"]!r}; specify --init or --resume.')
    if recipe['parent_requires_full_resume'] and args.init:
        parser.error('SCM continuation preserves the SCM optimizer; use --resume with the full prefix checkpoint.')
    if recipe['mode'] == 'dcm' and args.init:
        parser.error('These DCM A/B recipes start from the official base; --init is not an audited DCM fork.')
    if args.nodes < 1 or args.gpus < 1 or not 0 <= args.node_rank < args.nodes:
        parser.error('Invalid distributed topology.')
    if args.nodes > 1 and args.master_addr in ['127.0.0.1', 'localhost']:
        parser.error('Multiple nodes require a shared, reachable --master-addr.')
    output = args.output.resolve()
    cfg.update(checkpoint_path=str(Path(args.base).resolve()), gemma_path=str(Path(args.gemma).resolve()),
               data_path=str(Path(args.prompts).resolve()), output_path=str(output.parent),
               wandb_name=output.name, stop_file_path=str(output / 'STOP_REQUESTED'))
    if args.latents:
        cfg['scm_data_path'] = str(Path(args.latents).resolve())
    if args.init or args.resume:
        cfg['resume_checkpoint'] = str((args.init or args.resume).resolve())
    cfg['resume_training_state'] = bool(args.resume)
    cfg['require_full_resume_state'] = bool(args.resume)
    cfg['checkpoint_load_mode'] = 'serial' if args.resume else 'parallel'
    for key, value in [('max_steps', args.max_steps), ('benchmark_iters', args.benchmark_every),
                       ('keep_optimizer_states', args.keep_optimizer_states)]:
        if value is not None:
            if value < 0:
                parser.error(f'{key} must be >= 0')
            cfg[key] = value
    env = os.environ.copy()
    # train_distillation.py respects these existing environment path overrides.
    for key, var in [('checkpoint_path','TURBO_CHECKPOINT_PATH'),('gemma_path','TURBO_GEMMA_PATH'),
                     ('data_path','TURBO_DATA_PATH'),('scm_data_path','TURBO_SCM_DATA_PATH')]:
        env[var] = str(cfg[key])
    env.update(LTX_RUN_DIR_NAME=output.name, WANDB_MODE=env.get('WANDB_MODE', 'offline'), PYTHONUNBUFFERED='1')
    if cfg.get('critic_enabled'):
        env.setdefault('TURBO_PARK_CRITIC_OPTIM', '1')
    env.setdefault('TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC', '3600')
    env.setdefault('TURBO_DIST_TIMEOUT_MINUTES', '120')
    if args.nodes == 1:
        env.setdefault('PYTORCH_CUDA_ALLOC_CONF', 'expandable_segments:True')
    packages = ROOT / 'LTX-2/packages'
    local_paths = [str(packages / p / 'src') for p in ['ltx-core','ltx-pipelines','ltx-causal','ltx-distillation']]
    env['PYTHONPATH'] = os.pathsep.join(local_paths + ([env['PYTHONPATH']] if env.get('PYTHONPATH') else []))
    config_file = output / 'launch_config.yaml'
    cmd = [sys.executable, '-m', 'torch.distributed.run', f'--nproc_per_node={args.gpus}',
           f'--nnodes={args.nodes}', f'--node_rank={args.node_rank}', f'--master_addr={args.master_addr}',
           f'--master_port={args.master_port}', '-m', 'ltx_distillation.train_distillation',
           '--config_path', str(config_file)]
    print(yaml.safe_dump(cfg, allow_unicode=True, sort_keys=False))
    print('Command:', shlex.join(cmd), flush=True)
    if args.dry_run:
        return
    for name in ['base', 'gemma', 'prompts', 'latents']:
        value = getattr(args, name)
        if value and not Path(value).exists():
            parser.error(f'{name} does not exist: {value}')
    if args.init or args.resume:
        if not (args.init or args.resume).exists():
            parser.error('Checkpoint does not exist.')
    if not args.resume and ((output/'train.log').exists() or (output/'checkpoints').exists()):
        parser.error('Output contains an existing run; choose a new output directory or use --resume.')
    if (output/'STOP_REQUESTED').exists():
        parser.error('STOP_REQUESTED exists; remove it deliberately before continuation.')
    output.mkdir(parents=True, exist_ok=True)
    config_file.write_text(yaml.safe_dump(cfg, allow_unicode=True, sort_keys=False))
    raise SystemExit(subprocess.call(cmd, cwd=ROOT, env=env))


if __name__ == '__main__':
    main()