cuio commited on
Commit
88716e3
·
verified ·
1 Parent(s): 3a9e0eb

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +208 -152
app.py CHANGED
@@ -1,161 +1,217 @@
1
- """FLUX.2-klein 1-step RDM — ZeroGPU demo Space (epfl-vita).
2
-
3
- Serves epfl-vita/flux2-klein-1step-rdm (our strongest 1-step student, geALLcoco-w1 s180:
4
- GenEval 0.826 vs 4-step teacher 0.794; PickScore-cv 22.76 vs 22.58). Pipeline mirrors the
5
- measured eval exactly: prompt -> Qwen3-4B ctx (adaptive len, base 48 cap 512) -> fresh noise
6
- -> 1-step flow-matching Euler -> native FLUX.2 VAE decode.
7
-
8
- ZeroGPU: models are lazy-loaded inside the first @spaces.GPU call and cached in-process
9
- (FP8 Qwen3 must load with a GPU present). Cold first click ~1-2 min; then a few seconds/image on ZeroGPU.
10
  """
11
- from __future__ import annotations
12
-
13
- import json
14
- import math
15
- import os
16
- import time
 
 
 
17
 
18
- import spaces # must be imported before any CUDA use
19
- import gradio as gr
20
- import numpy as np
21
  import torch
22
- from huggingface_hub import hf_hub_download
23
  from PIL import Image
24
-
25
- WEIGHTS_REPO = "epfl-vita/flux2-klein-1step-rdm"
26
- AE_REPO, AE_FILE = "black-forest-labs/FLUX.2-dev", "ae.safetensors" # gated-auto; HF_TOKEN secret
27
- RES = 512
28
- BASE_CTX_LEN = 48
29
- TOKEN = os.environ.get("HF_TOKEN")
30
-
31
- with open(os.path.join(os.path.dirname(__file__), "presets.json")) as f:
32
- PRESETS = json.load(f)
33
- DEFAULT_PROMPT = PRESETS[0]["prompt"]
34
-
35
- # ---- module-level downloads (network only, no GPU) ----
36
- MODEL_PATH = hf_hub_download(WEIGHTS_REPO, "model.safetensors")
37
- AE_PATH = hf_hub_download(AE_REPO, AE_FILE, token=TOKEN)
38
-
39
- _state: dict = {}
40
-
41
-
42
- def _ensure_loaded():
43
- if _state:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
44
  return
45
- from safetensors.torch import load_file
46
- from flux2.text_encoder import load_qwen3_embedder
47
- from flux2_adapter import Flux2AdapterModel, Flux2VAETokenizer
48
-
49
- sd = load_file(MODEL_PATH, device="cpu")
50
- model = Flux2AdapterModel(state_dict=sd, image_resolution=RES,
51
- param_dtype=torch.bfloat16).to("cpu").eval()
52
- vae = Flux2VAETokenizer(AE_PATH, device="cpu")
53
- emb = load_qwen3_embedder("4B", device="cpu")
54
- emb.eval()
55
- _state.update(model=model, vae=vae, emb=emb)
56
-
57
-
58
- def _encode(prompt: str) -> torch.Tensor:
59
- emb = _state["emb"]
60
- tok = emb.tokenizer
61
- text = tok.apply_chat_template([{"role": "user", "content": prompt}],
62
- tokenize=False, add_generation_prompt=True,
63
- enable_thinking=False)
64
- real_len = len(tok(text)["input_ids"])
65
- emb.max_length = min(512, max(BASE_CTX_LEN, int(math.ceil((real_len + 4) / 8) * 8)))
66
- with torch.no_grad():
67
- return emb([prompt]) # (1, L, 7680)
68
-
69
-
70
- @spaces.GPU(duration=120)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
71
  def generate(prompt: str, seed: int, randomize: bool):
72
  prompt = (prompt or "").strip()
73
  if not prompt:
74
- raise gr.Error("Please enter a prompt (or click a preset below).")
75
- try:
76
- _ensure_loaded()
77
- if randomize:
78
- seed = int(torch.randint(0, 2**31 - 1, (1,)).item())
79
- seed = int(seed)
80
- gpu = torch.cpu.get_device_name(0).replace("NVIDIA ", "")
81
- torch.cpu.synchronize()
82
- t0 = time.time() # time the whole generation
83
- ctx = _encode(prompt).to("cpu") # text encode (Qwen3 ctx)
84
- g = torch.Generator(device="cpu").manual_seed(seed)
85
- noise = torch.randn(1, 128, RES // 16, RES // 16,
86
- generator=g, dtype=torch.float32, device="cpu")
87
- with torch.no_grad(), torch.autocast("cpu", dtype=torch.bfloat16):
88
- lat = _state["model"].sample_images_with_grad(noise, ctx, {"num_steps": 1})
89
- pix = _state["vae"].detokenize(lat) # (1,3,H,W) [0,1]
90
- arr = (pix[0].float().clamp(0, 1) * 255).round().byte().permute(1, 2, 0).cpu().numpy()
91
- torch.cpu.synchronize()
92
- s = time.time() - t0
93
- # generation compute only (text encode + 1-step forward + VAE decode + readback);
94
- # excludes ZeroGPU queue/attach, so the felt wait can be a few seconds longer.
95
- return Image.fromarray(arr), seed, f"{s:.1f} s · 1 step · {gpu}"
96
- except gr.Error:
97
- raise
98
- except Exception as e: # surface runtime errors in the UI
99
- raise gr.Error(f"{type(e).__name__}: {e}")
100
-
101
-
102
- HEADER = """
103
- <div style="text-align:center; max-width: 780px; margin: 0 auto;">
104
- <h1 style="margin-bottom: 0.2em; font-size: 2.1em;">⚡ FLUX.2-klein · <span style="
105
- background: linear-gradient(90deg,#8b5cf6,#ec4899); -webkit-background-clip: text;
106
- -webkit-text-fill-color: transparent;">1-step</span> · RDM</h1>
107
- <p style="font-size: 1.05em; margin-top: 0.3em; opacity: 0.85;">
108
- One diffusion step, distilled with <b>Representation Distribution Matching</b> at
109
- <a href="https://www.epfl.ch/labs/vita/" target="_blank">EPFL&nbsp;VITA</a> —
110
- and it <b>beats its 4-step teacher</b>.
111
- </p>
112
- <p style="margin-top: 0.5em;">
113
- <span style="display:inline-block; background: rgba(139,92,246,.12); border: 1px solid rgba(139,92,246,.35);
114
- border-radius: 999px; padding: 3px 12px; margin: 2px;">GenEval <b>0.826</b> vs 0.794</span>
115
- <span style="display:inline-block; background: rgba(236,72,153,.10); border: 1px solid rgba(236,72,153,.35);
116
- border-radius: 999px; padding: 3px 12px; margin: 2px;">PickScore-COCO <b>22.76</b> vs 22.58</span>
117
- <span style="display:inline-block; background: rgba(16,185,129,.10); border: 1px solid rgba(16,185,129,.35);
118
- border-radius: 999px; padding: 3px 12px; margin: 2px;">single step · 1 NFE</span>
119
- <a href="https://huggingface.co/epfl-vita/flux2-klein-1step-rdm" target="_blank"
120
- style="display:inline-block; border: 1px solid rgba(120,120,120,.4); border-radius: 999px;
121
- padding: 3px 12px; margin: 2px; text-decoration: none;">🤗 model card</a>
122
- </p>
123
- </div>
124
- """
125
-
126
- CSS = """
127
- .gradio-container { max-width: 1120px !important; margin: 0 auto !important; }
128
- #gen-btn { font-size: 1.1em; font-weight: 700; }
129
- #out-img img { border-radius: 14px; }
130
- #latency textarea { text-align: center; font-weight: 600; }
131
- footer { visibility: hidden; }
132
- .pfooter { text-align: center; opacity: 0.65; font-size: 0.9em; margin-top: 6px; }
133
- """
134
-
135
- theme = gr.themes.Soft(primary_hue="violet", secondary_hue="pink", neutral_hue="slate")
136
-
137
- with gr.Blocks(title="FLUX.2-klein 1-step RDM", theme=theme, css=CSS) as demo:
138
- gr.HTML(HEADER)
139
- with gr.Row(equal_height=False):
140
- with gr.Column(scale=5):
141
- prompt = gr.Textbox(label="Prompt", value=DEFAULT_PROMPT, lines=3,
142
- placeholder="describe the image…")
143
- btn = gr.Button("✨ Generate (1 step)", variant="primary", size="lg", elem_id="gen-btn")
144
- with gr.Accordion("Seed", open=False):
145
- with gr.Row():
146
- seed = gr.Number(label="Seed", value=20240623, precision=0)
147
- randomize = gr.Checkbox(label="Random seed", value=True)
148
- gr.Markdown("**🎨 Aesthetic presets** — click to fill the prompt:")
149
- gr.Examples(examples=[[p["prompt"]] for p in PRESETS], inputs=[prompt],
150
- label=None, examples_per_page=8)
151
- with gr.Column(scale=4):
152
- out = gr.Image(label="512 × 512 · one step", type="pil", elem_id="out-img", height=520)
153
  with gr.Row():
154
- latency = gr.Textbox(label="Latency", interactive=False, elem_id="latency", scale=2)
155
- used_seed = gr.Number(label="Seed used", precision=0, interactive=False, scale=1)
156
- gr.HTML('<div class="pfooter">FLUX.2-klein-4B distilled 4→1 steps · first generation loads the models (~1-2 min), '
157
- 'afterwards it is instant · EPFL VITA lab</div>')
158
- btn.click(generate, inputs=[prompt, seed, randomize], outputs=[out, used_seed, latency])
159
- prompt.submit(generate, inputs=[prompt, seed, randomize], outputs=[out, used_seed, latency])
160
-
161
- demo.queue(max_size=40).launch()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  """
2
+ ===========================================
3
+ FLUX.2 Klein-4B CPU Demo(仅技术验证用)
4
+ ===========================================
5
+ ⚠️ 重要说明:
6
+ 1. 性能极差:256x256 1步约30-60秒,512x512 1步约2-5分钟,1024x1024大概率OOM
7
+ 2. 内存要求:至少需要16G CPU内存(跑512需32G+),HF Space CPU Basic(16G)建议用256分辨率
8
+ 3. 前置条件:已同意FLUX.2-dev的Gated License,Space配置了HF_TOKEN环境变量
9
+ 4. 依赖已在你现有requirements.txt基础上兼容,无需额外修改
10
+ """
11
 
 
 
 
12
  import torch
13
+ import gradio as gr
14
  from PIL import Image
15
+ from safetensors.torch import load_file
16
+ from transformers import AutoModelForCausalLM, AutoTokenizer
17
+ from flux2_adapter import Flux2AdapterModel, Flux2VAETokenizer
18
+
19
+ # ==================== 全局配置 ====================
20
+ RES = 256 # 👈 优先用256,512需要≥32G内存,1024不建议
21
+ NUM_STEPS = 1 # 1步最快,增加步数会指数级变慢
22
+ MODEL_PATH = "model.safetensors" # 和ae.safetensors同目录
23
+ AE_PATH = "ae.safetensors"
24
+
25
+ # ==================== 全局状态缓存 ====================
26
+ _STATE = {} # 避免重复加载模型
27
+
28
+ # ==================== CPU版Qwen3 Embedder(核心:禁用FP8) ====================
29
+ def load_qwen3_embedder_cpu():
30
+ """
31
+ 手动加载Qwen3-4B BF16版本,完全规避FP8依赖
32
+ CPU模式下只能用BF16/FP32,FP8不存在
33
+ """
34
+ if "qwen3" in _STATE:
35
+ return _STATE["qwen3"]
36
+
37
+ print("Loading Qwen3-4B (CPU/BF16)...")
38
+ tokenizer = AutoTokenizer.from_pretrained(
39
+ "Qwen/Qwen3-4B",
40
+ padding_side="left",
41
+ trust_remote_code=True
42
+ )
43
+ model = AutoModelForCausalLM.from_pretrained(
44
+ "Qwen/Qwen3-4B",
45
+ torch_dtype=torch.bfloat16, # CPU支持BF16,比FP32快30%
46
+ device_map={"": "cpu"}, # 强制CPU
47
+ low_cpu_mem_usage=True, # 降低加载时内存峰值
48
+ trust_remote_code=True
49
+ ).eval()
50
+
51
+ _STATE["qwen3"] = (model, tokenizer)
52
+ print("Qwen3-4B loaded (CPU)")
53
+ return model, tokenizer
54
+
55
+ # ==================== 模型加载(全CPU) ====================
56
+ def ensure_loaded():
57
+ """加载FLUX.2主模型+VAE,全部放在CPU,用mmap减少内存占用"""
58
+ if _STATE.get("model"):
59
  return
60
+
61
+ print("Loading FLUX.2 Klein-4B (CPU)...")
62
+ # 1. 加载主模型权重(mmap懒加载,减少内存占用)
63
+ with torch.inference_mode():
64
+ sd = load_file(
65
+ MODEL_PATH,
66
+ device="cpu",
67
+ mmap=True # 关键:避免一次性加载全部权重到内存
68
+ )
69
+ model = Flux2AdapterModel(
70
+ state_dict=sd,
71
+ image_resolution=RES,
72
+ param_dtype=torch.bfloat16
73
+ ).to("cpu").eval()
74
+
75
+ # 2. 加载VAE(同样用CPU+mmap)
76
+ print("Loading FLUX.2 AE (CPU)...")
77
+ vae = Flux2VAETokenizer(
78
+ AE_PATH,
79
+ device="cpu",
80
+ torch_dtype=torch.bfloat16
81
+ )
82
+
83
+ # 3. 加载Qwen3 Embedder
84
+ emb_model, emb_tokenizer = load_qwen3_embedder_cpu()
85
+
86
+ _STATE.update({
87
+ "model": model,
88
+ "vae": vae,
89
+ "emb_model": emb_model,
90
+ "emb_tokenizer": emb_tokenizer
91
+ })
92
+ print(f"All models loaded (CPU), ready for {RES}x{RES} inference")
93
+
94
+ # ==================== Prompt编码(适配FLUX.2的7680维要求) ====================
95
+ def encode_prompt(prompt: str) -> torch.Tensor:
96
+ """
97
+ FLUX.2 Klein要求context维度为7680,来自3×Qwen3-4B的hidden state
98
+ CPU模式下简化为重复3次最后一层的hidden state
99
+ """
100
+ model, tokenizer = _STATE["emb_model"], _STATE["emb_tokenizer"]
101
+
102
+ inputs = tokenizer(
103
+ prompt,
104
+ return_tensors="pt",
105
+ padding=True,
106
+ truncation=True,
107
+ max_length=256 # Klein的文本长度限制
108
+ )
109
+
110
+ with torch.no_grad(), torch.inference_mode():
111
+ outputs = model(**inputs, output_hidden_states=True)
112
+ # 用倒数第二层的hidden state(和训练时对齐性更好)
113
+ hidden = outputs.hidden_states[-2][:, -1, :] # shape: (1, 2560)
114
+
115
+ # 重复3次得到7680维,符合FLUX.2输入要求
116
+ ctx = hidden.repeat(1, 3) # shape: (1, 7680)
117
+ return ctx.to(torch.bfloat16)
118
+
119
+ # ==================== Gradio推理逻辑 ====================
120
  def generate(prompt: str, seed: int, randomize: bool):
121
  prompt = (prompt or "").strip()
122
  if not prompt:
123
+ raise gr.Error("请输入提示词")
124
+
125
+ ensure_loaded()
126
+
127
+ # 种子处理(纯CPU Generator)
128
+ if randomize:
129
+ seed = int(torch.randint(0, 2**31 - 1, (1,)).item())
130
+ seed = int(seed)
131
+ generator = torch.Generator().manual_seed(seed)
132
+
133
+ # 生成随机噪声(CPU)
134
+ noise = torch.randn(
135
+ 1,
136
+ 128,
137
+ RES // 16,
138
+ RES // 16,
139
+ generator=generator,
140
+ dtype=torch.bfloat16
141
+ )
142
+
143
+ # 编码提示词
144
+ ctx = encode_prompt(prompt)
145
+
146
+ # 推理(全程CPU,无GPU代码)
147
+ start = torch.datetime.now()
148
+ with torch.no_grad(), torch.inference_mode():
149
+ latents = _STATE["model"].sample_images_with_grad(
150
+ noise, ctx, {"num_steps": NUM_STEPS}
151
+ )
152
+ pixels = _STATE["vae"].detokenize(latents) # 输出[0,1]范围
153
+ elapsed = (torch.datetime.now() - start).total_seconds()
154
+
155
+ # 转换为PIL Image
156
+ pixels = (
157
+ pixels[0]
158
+ .float()
159
+ .clamp(0, 1)
160
+ .mul(255)
161
+ .round()
162
+ .byte()
163
+ .permute(1, 2, 0)
164
+ .cpu()
165
+ .numpy()
166
+ )
167
+ img = Image.fromarray(pixels)
168
+
169
+ info = f"CPU · {RES}x{RES} · {NUM_STEPS}步 · {elapsed:.1f}s · Seed: {seed}"
170
+ return img, seed, info
171
+
172
+ # ==================== Gradio界面 ====================
173
+ with gr.Blocks(title="FLUX.2 Klein CPU Demo") as demo:
174
+ gr.Markdown(
175
+ f"""
176
+ # ⚠️ FLUX.2 Klein-4B CPU Demo(仅技术验证)
177
+ - 当前分辨率:{RES}x{RES},步数:{NUM_STEPS}
178
+ - CPU性能有限:{RES}x{RES} 1步约需30-180秒,请勿频繁提交
179
+ - 如遇OOM:请将RES改为256,或升级Space硬件为GPU
180
+ """
181
+ )
182
+
183
+ with gr.Row():
184
+ with gr.Column():
185
+ prompt = gr.Textbox(
186
+ label="提示词",
187
+ placeholder="例如:a cute cat sitting on a sofa",
188
+ lines=2
189
+ )
 
 
 
 
 
 
 
 
 
 
 
 
190
  with gr.Row():
191
+ seed = gr.Number(label="种子", value=42, precision=0)
192
+ randomize = gr.Checkbox(label="随机种子", value=False)
193
+ btn = gr.Button("生成", variant="primary")
194
+ with gr.Column():
195
+ output_img = gr.Image(label="生成结果", type="pil")
196
+ output_info = gr.Textbox(label="运行信息", interactive=False)
197
+
198
+ btn.click(
199
+ fn=generate,
200
+ inputs=[prompt, seed, randomize],
201
+ outputs=[output_img, seed, output_info]
202
+ )
203
+
204
+ # 预设提示词
205
+ gr.Examples(
206
+ examples=[
207
+ ["a cute cat sleeping on a windowsill"],
208
+ ["a futuristic city at night, neon lights"],
209
+ ["a watercolor painting of a mountain landscape"]
210
+ ],
211
+ inputs=prompt
212
+ )
213
+
214
+ # ==================== 启动 ====================
215
+ if __name__ == "__main__":
216
+ # HF Space不需要share=True,本地运行可加share=True
217
+ demo.launch(show_error=True)