moose Claude Opus 4.5 commited on
Commit
bbac3b5
·
1 Parent(s): 73a98d6

Reduce memory during AOT compilation by offloading text encoder

Browse files

Move text encoder (~16GB) to CPU before torch.export to free memory
during the memory-intensive AOT compilation step. Move it back to
GPU after compilation completes.

This should help stay under the 96GB build memory limit.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Files changed (1) hide show
  1. app.py +9 -0
app.py CHANGED
@@ -1,3 +1,4 @@
 
1
  import gradio as gr
2
  import numpy as np
3
  import random
@@ -112,8 +113,16 @@ pipe.transformer.__class__ = QwenImageTransformer2DModel
112
  pipe.transformer.set_attn_processor(QwenDoubleStreamAttnProcessorFA3())
113
 
114
  # --- Ahead-of-time compilation ---
 
 
 
 
 
115
  optimize_pipeline_(pipe, image=[Image.new("RGB", (1024, 1024)), Image.new("RGB", (1024, 1024))], prompt="prompt")
116
 
 
 
 
117
  # --- UI Constants and Helpers ---
118
  MAX_SEED = np.iinfo(np.int32).max
119
 
 
1
+ import gc
2
  import gradio as gr
3
  import numpy as np
4
  import random
 
113
  pipe.transformer.set_attn_processor(QwenDoubleStreamAttnProcessorFA3())
114
 
115
  # --- Ahead-of-time compilation ---
116
+ # Temporarily offload text encoder to CPU to free ~16GB during memory-intensive torch.export
117
+ pipe.text_encoder.to('cpu')
118
+ gc.collect()
119
+ torch.cuda.empty_cache()
120
+
121
  optimize_pipeline_(pipe, image=[Image.new("RGB", (1024, 1024)), Image.new("RGB", (1024, 1024))], prompt="prompt")
122
 
123
+ # Move text encoder back to GPU for inference
124
+ pipe.text_encoder.to(device)
125
+
126
  # --- UI Constants and Helpers ---
127
  MAX_SEED = np.iinfo(np.int32).max
128