File size: 6,309 Bytes
76b7b25
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
diff --git a/LTX-2/packages/ltx-distillation/configs/bidirectional_rcm.yaml b/LTX-2/packages/ltx-distillation/configs/bidirectional_rcm.yaml
index 336719d..f966250 100644
--- a/LTX-2/packages/ltx-distillation/configs/bidirectional_rcm.yaml
+++ b/LTX-2/packages/ltx-distillation/configs/bidirectional_rcm.yaml
@@ -140,20 +140,36 @@ text_encoder_fsdp_wrap_strategy: size
 #   Gemma       unshardable 22 GiB replica, frozen  ~22 GiB
 # real_score and fake_score only pay a host-device copy per DMD forward;
 # bidirectional_dcm.yaml already offloads real_score for the same reason.
-# Offload only the trainable critic. Measured on one node (8 GPUs, rcm):
-# generator steps are the memory ceiling, because they hold the generator's
-# activation graph while fake_score also runs. Leaving both scores resident
-# dies at step 5 -- the first generator step -- with OutOfMemoryError on a
-# 4.50 GiB allocation; so does offloading real_score alone, which still peaks
-# at 90.5 GiB of 95 GiB there even though its critic steps peak at only
-# 68-74 GiB. Offloading fake_score is what frees the generator step: it peaks
-# at 68.2 GiB and survives, and critic steps still take 26-27s because the
-# PCIe copy of the critic's weights overlaps its compute. Offloading both
-# works too but buys nothing (27-33s critic, 63.8 GiB peak). Setting
-# fsdp_backward_prefetch to "pre" changed neither step time nor peak memory,
-# so it stays at "post".
-real_score_cpu_offload: false
-fake_score_cpu_offload: true
+# Offload the frozen teacher and park the critic's optimizer state; see
+# TURBO_PARK_CRITIC_OPTIM in train_distillation.py, which this pairing needs.
+#
+# Generator steps set the memory ceiling, because they hold the generator's
+# activation graph while fake_score also runs. Measured on one node, 8 GPUs:
+#
+#   real=false fake=false  OOM at step 5, the first generator step
+#   real=true  fake=false  OOM too, 90.5 GiB of 95 GiB at the generator step,
+#                          though its critic steps peak at only 68-74 GiB
+#   real=false fake=true   survives, 68.2 GiB peak, critic steps 33s
+#   real=true  fake=true   survives but buys nothing, critic steps 27-33s
+#
+# The second row misses by a few GiB, and the critic's AdamW state is 8.79 GiB
+# per rank that nothing reads during a generator step, so parking it on the
+# host covers the gap. That combination keeps the critic's weights resident and
+# cuts critic steps from 33s to 22s: 28.7s/step against 37.2s, measured over
+# 711 steps of the slower setting and 11 of this one. Parking costs 2.6s per
+# generator step, one step in five, so about 1.8% back.
+#
+# Parking cannot be expressed as fake_score_cpu_offload: FSDP fixes CPUOffload
+# when the module is wrapped, so that flag also pays on the four-in-five critic
+# steps that need the weights on the device.
+#
+# Verified with checkpoint_iters=2 and the benchmark enabled, i.e. a write of
+# 70 GiB of optimizer shards immediately before a generator step, which is 250x
+# more frequent than the 500 used in production and the case that fragments the
+# allocator worst. Setting fsdp_backward_prefetch to "pre" changed neither step
+# time nor peak memory, so it stays at "post".
+real_score_cpu_offload: true
+fake_score_cpu_offload: false
 text_encoder_cpu_offload: false
 
 # Gemma cannot be FSDP-sharded (it deadlocks every rank in
diff --git a/LTX-2/packages/ltx-distillation/src/ltx_distillation/train_distillation.py b/LTX-2/packages/ltx-distillation/src/ltx_distillation/train_distillation.py
index 1eed3dd..f6b9174 100644
--- a/LTX-2/packages/ltx-distillation/src/ltx_distillation/train_distillation.py
+++ b/LTX-2/packages/ltx-distillation/src/ltx_distillation/train_distillation.py
@@ -1879,6 +1879,15 @@ class Trainer:
                 generator_grad_norm = torch.nn.utils.clip_grad_norm_(
                     self.dmd.generator.parameters(), self.max_grad_norm
                 )
+            if getattr(config, "stop_on_nonfinite", False):
+                finite = (
+                    torch.isfinite(generator_loss.detach()).all()
+                    & torch.isfinite(generator_grad_norm).all()
+                ).to(dtype=torch.int32)
+                if self.world_size > 1:
+                    dist.all_reduce(finite, op=dist.ReduceOp.MIN)
+                if not finite.item():
+                    raise FloatingPointError(f"Non-finite generator loss/gradient at step {self.step}")
             self.generator_optimizer.step()
 
             # Update EMA (exponential moving average) of generator weights
@@ -1938,6 +1947,15 @@ class Trainer:
                 critic_grad_norm = torch.nn.utils.clip_grad_norm_(
                     self.dmd.fake_score.parameters(), self.max_grad_norm
                 )
+            if getattr(config, "stop_on_nonfinite", False):
+                finite = (
+                    torch.isfinite(critic_loss.detach()).all()
+                    & torch.isfinite(critic_grad_norm).all()
+                ).to(dtype=torch.int32)
+                if self.world_size > 1:
+                    dist.all_reduce(finite, op=dist.ReduceOp.MIN)
+                if not finite.item():
+                    raise FloatingPointError(f"Non-finite critic loss/gradient at step {self.step}")
             self.critic_optimizer.step()
         else:
             critic_loss = None
@@ -2839,6 +2857,20 @@ class Trainer:
             if self.critic_scheduler is not None:
                 self.critic_scheduler.step(self.step)
 
+            stop_file = getattr(self.config, "stop_file_path", None)
+            if stop_file:
+                requested = torch.tensor(
+                    [int(os.path.isfile(stop_file)) if self.is_main_process else 0],
+                    dtype=torch.int32,
+                    device=self.device,
+                )
+                if self.world_size > 1:
+                    dist.broadcast(requested, src=0)
+                if requested.item():
+                    if self.is_main_process:
+                        print(f"[Stop] Requested at next_step={self.step}; saving full state", flush=True)
+                    break
+
             # Optional: max steps limit
             max_steps = getattr(self.config, "max_steps", None)
             if max_steps and self.step >= max_steps: