diff --git a/LTX-2/packages/ltx-distillation/configs/bidirectional_rcm.yaml b/LTX-2/packages/ltx-distillation/configs/bidirectional_rcm.yaml index 336719d..f966250 100644 --- a/LTX-2/packages/ltx-distillation/configs/bidirectional_rcm.yaml +++ b/LTX-2/packages/ltx-distillation/configs/bidirectional_rcm.yaml @@ -140,20 +140,36 @@ text_encoder_fsdp_wrap_strategy: size # Gemma unshardable 22 GiB replica, frozen ~22 GiB # real_score and fake_score only pay a host-device copy per DMD forward; # bidirectional_dcm.yaml already offloads real_score for the same reason. -# Offload only the trainable critic. Measured on one node (8 GPUs, rcm): -# generator steps are the memory ceiling, because they hold the generator's -# activation graph while fake_score also runs. Leaving both scores resident -# dies at step 5 -- the first generator step -- with OutOfMemoryError on a -# 4.50 GiB allocation; so does offloading real_score alone, which still peaks -# at 90.5 GiB of 95 GiB there even though its critic steps peak at only -# 68-74 GiB. Offloading fake_score is what frees the generator step: it peaks -# at 68.2 GiB and survives, and critic steps still take 26-27s because the -# PCIe copy of the critic's weights overlaps its compute. Offloading both -# works too but buys nothing (27-33s critic, 63.8 GiB peak). Setting -# fsdp_backward_prefetch to "pre" changed neither step time nor peak memory, -# so it stays at "post". -real_score_cpu_offload: false -fake_score_cpu_offload: true +# Offload the frozen teacher and park the critic's optimizer state; see +# TURBO_PARK_CRITIC_OPTIM in train_distillation.py, which this pairing needs. +# +# Generator steps set the memory ceiling, because they hold the generator's +# activation graph while fake_score also runs. Measured on one node, 8 GPUs: +# +# real=false fake=false OOM at step 5, the first generator step +# real=true fake=false OOM too, 90.5 GiB of 95 GiB at the generator step, +# though its critic steps peak at only 68-74 GiB +# real=false fake=true survives, 68.2 GiB peak, critic steps 33s +# real=true fake=true survives but buys nothing, critic steps 27-33s +# +# The second row misses by a few GiB, and the critic's AdamW state is 8.79 GiB +# per rank that nothing reads during a generator step, so parking it on the +# host covers the gap. That combination keeps the critic's weights resident and +# cuts critic steps from 33s to 22s: 28.7s/step against 37.2s, measured over +# 711 steps of the slower setting and 11 of this one. Parking costs 2.6s per +# generator step, one step in five, so about 1.8% back. +# +# Parking cannot be expressed as fake_score_cpu_offload: FSDP fixes CPUOffload +# when the module is wrapped, so that flag also pays on the four-in-five critic +# steps that need the weights on the device. +# +# Verified with checkpoint_iters=2 and the benchmark enabled, i.e. a write of +# 70 GiB of optimizer shards immediately before a generator step, which is 250x +# more frequent than the 500 used in production and the case that fragments the +# allocator worst. Setting fsdp_backward_prefetch to "pre" changed neither step +# time nor peak memory, so it stays at "post". +real_score_cpu_offload: true +fake_score_cpu_offload: false text_encoder_cpu_offload: false # Gemma cannot be FSDP-sharded (it deadlocks every rank in diff --git a/LTX-2/packages/ltx-distillation/src/ltx_distillation/train_distillation.py b/LTX-2/packages/ltx-distillation/src/ltx_distillation/train_distillation.py index 1eed3dd..f6b9174 100644 --- a/LTX-2/packages/ltx-distillation/src/ltx_distillation/train_distillation.py +++ b/LTX-2/packages/ltx-distillation/src/ltx_distillation/train_distillation.py @@ -1879,6 +1879,15 @@ class Trainer: generator_grad_norm = torch.nn.utils.clip_grad_norm_( self.dmd.generator.parameters(), self.max_grad_norm ) + if getattr(config, "stop_on_nonfinite", False): + finite = ( + torch.isfinite(generator_loss.detach()).all() + & torch.isfinite(generator_grad_norm).all() + ).to(dtype=torch.int32) + if self.world_size > 1: + dist.all_reduce(finite, op=dist.ReduceOp.MIN) + if not finite.item(): + raise FloatingPointError(f"Non-finite generator loss/gradient at step {self.step}") self.generator_optimizer.step() # Update EMA (exponential moving average) of generator weights @@ -1938,6 +1947,15 @@ class Trainer: critic_grad_norm = torch.nn.utils.clip_grad_norm_( self.dmd.fake_score.parameters(), self.max_grad_norm ) + if getattr(config, "stop_on_nonfinite", False): + finite = ( + torch.isfinite(critic_loss.detach()).all() + & torch.isfinite(critic_grad_norm).all() + ).to(dtype=torch.int32) + if self.world_size > 1: + dist.all_reduce(finite, op=dist.ReduceOp.MIN) + if not finite.item(): + raise FloatingPointError(f"Non-finite critic loss/gradient at step {self.step}") self.critic_optimizer.step() else: critic_loss = None @@ -2839,6 +2857,20 @@ class Trainer: if self.critic_scheduler is not None: self.critic_scheduler.step(self.step) + stop_file = getattr(self.config, "stop_file_path", None) + if stop_file: + requested = torch.tensor( + [int(os.path.isfile(stop_file)) if self.is_main_process else 0], + dtype=torch.int32, + device=self.device, + ) + if self.world_size > 1: + dist.broadcast(requested, src=0) + if requested.item(): + if self.is_main_process: + print(f"[Stop] Requested at next_step={self.step}; saving full state", flush=True) + break + # Optional: max steps limit max_steps = getattr(self.config, "max_steps", None) if max_steps and self.step >= max_steps: