File size: 6,309 Bytes
76b7b25 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 | diff --git a/LTX-2/packages/ltx-distillation/configs/bidirectional_rcm.yaml b/LTX-2/packages/ltx-distillation/configs/bidirectional_rcm.yaml
index 336719d..f966250 100644
--- a/LTX-2/packages/ltx-distillation/configs/bidirectional_rcm.yaml
+++ b/LTX-2/packages/ltx-distillation/configs/bidirectional_rcm.yaml
@@ -140,20 +140,36 @@ text_encoder_fsdp_wrap_strategy: size
# Gemma unshardable 22 GiB replica, frozen ~22 GiB
# real_score and fake_score only pay a host-device copy per DMD forward;
# bidirectional_dcm.yaml already offloads real_score for the same reason.
-# Offload only the trainable critic. Measured on one node (8 GPUs, rcm):
-# generator steps are the memory ceiling, because they hold the generator's
-# activation graph while fake_score also runs. Leaving both scores resident
-# dies at step 5 -- the first generator step -- with OutOfMemoryError on a
-# 4.50 GiB allocation; so does offloading real_score alone, which still peaks
-# at 90.5 GiB of 95 GiB there even though its critic steps peak at only
-# 68-74 GiB. Offloading fake_score is what frees the generator step: it peaks
-# at 68.2 GiB and survives, and critic steps still take 26-27s because the
-# PCIe copy of the critic's weights overlaps its compute. Offloading both
-# works too but buys nothing (27-33s critic, 63.8 GiB peak). Setting
-# fsdp_backward_prefetch to "pre" changed neither step time nor peak memory,
-# so it stays at "post".
-real_score_cpu_offload: false
-fake_score_cpu_offload: true
+# Offload the frozen teacher and park the critic's optimizer state; see
+# TURBO_PARK_CRITIC_OPTIM in train_distillation.py, which this pairing needs.
+#
+# Generator steps set the memory ceiling, because they hold the generator's
+# activation graph while fake_score also runs. Measured on one node, 8 GPUs:
+#
+# real=false fake=false OOM at step 5, the first generator step
+# real=true fake=false OOM too, 90.5 GiB of 95 GiB at the generator step,
+# though its critic steps peak at only 68-74 GiB
+# real=false fake=true survives, 68.2 GiB peak, critic steps 33s
+# real=true fake=true survives but buys nothing, critic steps 27-33s
+#
+# The second row misses by a few GiB, and the critic's AdamW state is 8.79 GiB
+# per rank that nothing reads during a generator step, so parking it on the
+# host covers the gap. That combination keeps the critic's weights resident and
+# cuts critic steps from 33s to 22s: 28.7s/step against 37.2s, measured over
+# 711 steps of the slower setting and 11 of this one. Parking costs 2.6s per
+# generator step, one step in five, so about 1.8% back.
+#
+# Parking cannot be expressed as fake_score_cpu_offload: FSDP fixes CPUOffload
+# when the module is wrapped, so that flag also pays on the four-in-five critic
+# steps that need the weights on the device.
+#
+# Verified with checkpoint_iters=2 and the benchmark enabled, i.e. a write of
+# 70 GiB of optimizer shards immediately before a generator step, which is 250x
+# more frequent than the 500 used in production and the case that fragments the
+# allocator worst. Setting fsdp_backward_prefetch to "pre" changed neither step
+# time nor peak memory, so it stays at "post".
+real_score_cpu_offload: true
+fake_score_cpu_offload: false
text_encoder_cpu_offload: false
# Gemma cannot be FSDP-sharded (it deadlocks every rank in
diff --git a/LTX-2/packages/ltx-distillation/src/ltx_distillation/train_distillation.py b/LTX-2/packages/ltx-distillation/src/ltx_distillation/train_distillation.py
index 1eed3dd..f6b9174 100644
--- a/LTX-2/packages/ltx-distillation/src/ltx_distillation/train_distillation.py
+++ b/LTX-2/packages/ltx-distillation/src/ltx_distillation/train_distillation.py
@@ -1879,6 +1879,15 @@ class Trainer:
generator_grad_norm = torch.nn.utils.clip_grad_norm_(
self.dmd.generator.parameters(), self.max_grad_norm
)
+ if getattr(config, "stop_on_nonfinite", False):
+ finite = (
+ torch.isfinite(generator_loss.detach()).all()
+ & torch.isfinite(generator_grad_norm).all()
+ ).to(dtype=torch.int32)
+ if self.world_size > 1:
+ dist.all_reduce(finite, op=dist.ReduceOp.MIN)
+ if not finite.item():
+ raise FloatingPointError(f"Non-finite generator loss/gradient at step {self.step}")
self.generator_optimizer.step()
# Update EMA (exponential moving average) of generator weights
@@ -1938,6 +1947,15 @@ class Trainer:
critic_grad_norm = torch.nn.utils.clip_grad_norm_(
self.dmd.fake_score.parameters(), self.max_grad_norm
)
+ if getattr(config, "stop_on_nonfinite", False):
+ finite = (
+ torch.isfinite(critic_loss.detach()).all()
+ & torch.isfinite(critic_grad_norm).all()
+ ).to(dtype=torch.int32)
+ if self.world_size > 1:
+ dist.all_reduce(finite, op=dist.ReduceOp.MIN)
+ if not finite.item():
+ raise FloatingPointError(f"Non-finite critic loss/gradient at step {self.step}")
self.critic_optimizer.step()
else:
critic_loss = None
@@ -2839,6 +2857,20 @@ class Trainer:
if self.critic_scheduler is not None:
self.critic_scheduler.step(self.step)
+ stop_file = getattr(self.config, "stop_file_path", None)
+ if stop_file:
+ requested = torch.tensor(
+ [int(os.path.isfile(stop_file)) if self.is_main_process else 0],
+ dtype=torch.int32,
+ device=self.device,
+ )
+ if self.world_size > 1:
+ dist.broadcast(requested, src=0)
+ if requested.item():
+ if self.is_main_process:
+ print(f"[Stop] Requested at next_step={self.step}; saving full state", flush=True)
+ break
+
# Optional: max steps limit
max_steps = getattr(self.config, "max_steps", None)
if max_steps and self.step >= max_steps:
|