Download code/docs/provenance/source_worktree.patch from luyu1021/turbot2va-ltx2: direct link, hf CLI and curl.
- Browser
- Download file 6.31 kB
-
https://huggingface.co/luyu1021/turbot2va-ltx2/resolve/main/code/docs/provenance/source_worktree.patch
- Command line
-
hf download hf://luyu1021/turbot2va-ltx2/code/docs/provenance/source_worktree.patch
-
curl -L -o source_worktree.patch https://huggingface.co/luyu1021/turbot2va-ltx2/resolve/main/code/docs/provenance/source_worktree.patch
6.31 kB
| diff --git a/LTX-2/packages/ltx-distillation/configs/bidirectional_rcm.yaml b/LTX-2/packages/ltx-distillation/configs/bidirectional_rcm.yaml | |
| index 336719d..f966250 100644 | |
| --- a/LTX-2/packages/ltx-distillation/configs/bidirectional_rcm.yaml | |
| +++ b/LTX-2/packages/ltx-distillation/configs/bidirectional_rcm.yaml | |
| text_encoder_fsdp_wrap_strategy: size | |
| # Gemma unshardable 22 GiB replica, frozen ~22 GiB | |
| # real_score and fake_score only pay a host-device copy per DMD forward; | |
| # bidirectional_dcm.yaml already offloads real_score for the same reason. | |
| -# Offload only the trainable critic. Measured on one node (8 GPUs, rcm): | |
| -# generator steps are the memory ceiling, because they hold the generator's | |
| -# activation graph while fake_score also runs. Leaving both scores resident | |
| -# dies at step 5 -- the first generator step -- with OutOfMemoryError on a | |
| -# 4.50 GiB allocation; so does offloading real_score alone, which still peaks | |
| -# at 90.5 GiB of 95 GiB there even though its critic steps peak at only | |
| -# 68-74 GiB. Offloading fake_score is what frees the generator step: it peaks | |
| -# at 68.2 GiB and survives, and critic steps still take 26-27s because the | |
| -# PCIe copy of the critic's weights overlaps its compute. Offloading both | |
| -# works too but buys nothing (27-33s critic, 63.8 GiB peak). Setting | |
| -# fsdp_backward_prefetch to "pre" changed neither step time nor peak memory, | |
| -# so it stays at "post". | |
| -real_score_cpu_offload: false | |
| -fake_score_cpu_offload: true | |
| +# Offload the frozen teacher and park the critic's optimizer state; see | |
| +# TURBO_PARK_CRITIC_OPTIM in train_distillation.py, which this pairing needs. | |
| +# | |
| +# Generator steps set the memory ceiling, because they hold the generator's | |
| +# activation graph while fake_score also runs. Measured on one node, 8 GPUs: | |
| +# | |
| +# real=false fake=false OOM at step 5, the first generator step | |
| +# real=true fake=false OOM too, 90.5 GiB of 95 GiB at the generator step, | |
| +# though its critic steps peak at only 68-74 GiB | |
| +# real=false fake=true survives, 68.2 GiB peak, critic steps 33s | |
| +# real=true fake=true survives but buys nothing, critic steps 27-33s | |
| +# | |
| +# The second row misses by a few GiB, and the critic's AdamW state is 8.79 GiB | |
| +# per rank that nothing reads during a generator step, so parking it on the | |
| +# host covers the gap. That combination keeps the critic's weights resident and | |
| +# cuts critic steps from 33s to 22s: 28.7s/step against 37.2s, measured over | |
| +# 711 steps of the slower setting and 11 of this one. Parking costs 2.6s per | |
| +# generator step, one step in five, so about 1.8% back. | |
| +# | |
| +# Parking cannot be expressed as fake_score_cpu_offload: FSDP fixes CPUOffload | |
| +# when the module is wrapped, so that flag also pays on the four-in-five critic | |
| +# steps that need the weights on the device. | |
| +# | |
| +# Verified with checkpoint_iters=2 and the benchmark enabled, i.e. a write of | |
| +# 70 GiB of optimizer shards immediately before a generator step, which is 250x | |
| +# more frequent than the 500 used in production and the case that fragments the | |
| +# allocator worst. Setting fsdp_backward_prefetch to "pre" changed neither step | |
| +# time nor peak memory, so it stays at "post". | |
| +real_score_cpu_offload: true | |
| +fake_score_cpu_offload: false | |
| text_encoder_cpu_offload: false | |
| # Gemma cannot be FSDP-sharded (it deadlocks every rank in | |
| diff --git a/LTX-2/packages/ltx-distillation/src/ltx_distillation/train_distillation.py b/LTX-2/packages/ltx-distillation/src/ltx_distillation/train_distillation.py | |
| index 1eed3dd..f6b9174 100644 | |
| --- a/LTX-2/packages/ltx-distillation/src/ltx_distillation/train_distillation.py | |
| +++ b/LTX-2/packages/ltx-distillation/src/ltx_distillation/train_distillation.py | |
| class Trainer: | |
| generator_grad_norm = torch.nn.utils.clip_grad_norm_( | |
| self.dmd.generator.parameters(), self.max_grad_norm | |
| ) | |
| + if getattr(config, "stop_on_nonfinite", False): | |
| + finite = ( | |
| + torch.isfinite(generator_loss.detach()).all() | |
| + & torch.isfinite(generator_grad_norm).all() | |
| + ).to(dtype=torch.int32) | |
| + if self.world_size > 1: | |
| + dist.all_reduce(finite, op=dist.ReduceOp.MIN) | |
| + if not finite.item(): | |
| + raise FloatingPointError(f"Non-finite generator loss/gradient at step {self.step}") | |
| self.generator_optimizer.step() | |
| # Update EMA (exponential moving average) of generator weights | |
| class Trainer: | |
| critic_grad_norm = torch.nn.utils.clip_grad_norm_( | |
| self.dmd.fake_score.parameters(), self.max_grad_norm | |
| ) | |
| + if getattr(config, "stop_on_nonfinite", False): | |
| + finite = ( | |
| + torch.isfinite(critic_loss.detach()).all() | |
| + & torch.isfinite(critic_grad_norm).all() | |
| + ).to(dtype=torch.int32) | |
| + if self.world_size > 1: | |
| + dist.all_reduce(finite, op=dist.ReduceOp.MIN) | |
| + if not finite.item(): | |
| + raise FloatingPointError(f"Non-finite critic loss/gradient at step {self.step}") | |
| self.critic_optimizer.step() | |
| else: | |
| critic_loss = None | |
| class Trainer: | |
| if self.critic_scheduler is not None: | |
| self.critic_scheduler.step(self.step) | |
| + stop_file = getattr(self.config, "stop_file_path", None) | |
| + if stop_file: | |
| + requested = torch.tensor( | |
| + [int(os.path.isfile(stop_file)) if self.is_main_process else 0], | |
| + dtype=torch.int32, | |
| + device=self.device, | |
| + ) | |
| + if self.world_size > 1: | |
| + dist.broadcast(requested, src=0) | |
| + if requested.item(): | |
| + if self.is_main_process: | |
| + print(f"[Stop] Requested at next_step={self.step}; saving full state", flush=True) | |
| + break | |
| + | |
| # Optional: max steps limit | |
| max_steps = getattr(self.config, "max_steps", None) | |
| if max_steps and self.step >= max_steps: | |