Update app.py
Browse files
app.py
CHANGED
|
@@ -1,281 +1,82 @@
|
|
| 1 |
"""
|
| 2 |
-
===========================================
|
| 3 |
-
FLUX.2 Klein-4B CPU Demo
|
| 4 |
-
===========================================
|
| 5 |
-
⚠️
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
-
|
| 9 |
-
|
| 10 |
-
- Hardware: 16GB CPU (HuggingFace Space CPU Basic)
|
| 11 |
-
|
| 12 |
-
Expected performance: 40-90s per image after cold start
|
| 13 |
-
=============================================================================
|
| 14 |
"""
|
| 15 |
|
| 16 |
import torch
|
| 17 |
-
import torch.nn as nn
|
| 18 |
import gradio as gr
|
| 19 |
from PIL import Image
|
| 20 |
from safetensors.torch import load_file
|
| 21 |
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 22 |
-
import
|
| 23 |
-
|
| 24 |
-
# =============================================================================
|
| 25 |
-
# EMBEDDED FLUX.2 ADAPTER (from your flux2_adapter.py)
|
| 26 |
-
# =============================================================================
|
| 27 |
-
FLUX2_VAE_DOWNSAMPLE = 16
|
| 28 |
-
FLUX2_LATENT_CHANNELS = 128
|
| 29 |
-
|
| 30 |
-
class Flux2AdapterModel(nn.Module):
|
| 31 |
-
def __init__(self, state_dict: dict, image_resolution: int = 256,
|
| 32 |
-
param_dtype: torch.dtype = torch.float32, guidance: float = 1.0):
|
| 33 |
-
super().__init__()
|
| 34 |
-
import flux2.model as _fm
|
| 35 |
-
from flux2 import sampling as _sampling
|
| 36 |
-
|
| 37 |
-
self.image_resolution = int(image_resolution)
|
| 38 |
-
self.guidance = float(guidance)
|
| 39 |
-
assert self.image_resolution % FLUX2_VAE_DOWNSAMPLE == 0
|
| 40 |
-
|
| 41 |
-
params = _fm.Klein4BParams()
|
| 42 |
-
assert params.in_channels == FLUX2_LATENT_CHANNELS
|
| 43 |
-
|
| 44 |
-
# Use meta device to avoid memory allocation during init
|
| 45 |
-
with torch.device("meta"):
|
| 46 |
-
model = _fm.Flux2(params).to(torch.float32)
|
| 47 |
-
|
| 48 |
-
# Remove "model." prefix if present
|
| 49 |
-
if all(k.startswith("model.") for k in list(state_dict.keys())[:8]):
|
| 50 |
-
state_dict = {k[len("model."):]: v for k, v in state_dict.items()}
|
| 51 |
-
|
| 52 |
-
model.load_state_dict(state_dict, strict=True, assign=True)
|
| 53 |
-
self.model = model.to(dtype=param_dtype, device="cpu")
|
| 54 |
-
|
| 55 |
-
self._batched_prc_img = _sampling.batched_prc_img
|
| 56 |
-
self._batched_prc_txt = _sampling.batched_prc_txt
|
| 57 |
-
self._get_schedule = _sampling.get_schedule
|
| 58 |
-
self._timestep_embedding = _fm.timestep_embedding
|
| 59 |
-
|
| 60 |
-
self.in_channels = FLUX2_LATENT_CHANNELS
|
| 61 |
-
self.input_size = self.image_resolution // FLUX2_VAE_DOWNSAMPLE
|
| 62 |
-
|
| 63 |
-
@property
|
| 64 |
-
def device(self) -> torch.device:
|
| 65 |
-
return next(self.model.parameters()).device
|
| 66 |
-
|
| 67 |
-
def _run_dit(self, x, x_ids, ctx, ctx_ids, t_vec):
|
| 68 |
-
m = self.model
|
| 69 |
-
num_txt_tokens = ctx.shape[1]
|
| 70 |
-
vec = m.time_in(self._timestep_embedding(t_vec, 256))
|
| 71 |
-
if getattr(m, "use_guidance_embed", False):
|
| 72 |
-
guid = torch.full((x.shape[0],), self.guidance,
|
| 73 |
-
dtype=x.dtype, device=x.device)
|
| 74 |
-
vec = vec + m.guidance_in(self._timestep_embedding(guid, 256))
|
| 75 |
-
mod_img = m.double_stream_modulation_img(vec)
|
| 76 |
-
mod_txt = m.double_stream_modulation_txt(vec)
|
| 77 |
-
single_mod, _ = m.single_stream_modulation(vec)
|
| 78 |
-
img = m.img_in(x)
|
| 79 |
-
txt = m.txt_in(ctx)
|
| 80 |
-
pe_x = m.pe_embedder(x_ids)
|
| 81 |
-
pe_ctx = m.pe_embedder(ctx_ids)
|
| 82 |
-
|
| 83 |
-
for block in m.double_blocks:
|
| 84 |
-
img, txt, _ = block.forward_kv_extract(img, txt, pe_x, pe_ctx,
|
| 85 |
-
mod_img, mod_txt, 0)
|
| 86 |
-
img = torch.cat((txt, img), dim=1)
|
| 87 |
-
pe = torch.cat((pe_ctx, pe_x), dim=2)
|
| 88 |
-
for block in m.single_blocks:
|
| 89 |
-
img, _ = block.forward_kv_extract(img, pe, single_mod, num_txt_tokens, 0)
|
| 90 |
-
img = img[:, num_txt_tokens:, ...]
|
| 91 |
-
return m.final_layer(img, vec)
|
| 92 |
-
|
| 93 |
-
def sample_images_with_grad(self, noise: torch.Tensor, condition: torch.Tensor,
|
| 94 |
-
sampling_args: dict) -> torch.Tensor:
|
| 95 |
-
B = noise.shape[0]
|
| 96 |
-
ctx = condition.to(device=noise.device, dtype=noise.dtype)
|
| 97 |
-
x, x_ids = self._batched_prc_img(noise)
|
| 98 |
-
ctx, ctx_ids = self._batched_prc_txt(ctx)
|
| 99 |
-
H, W = noise.shape[-2], noise.shape[-1]
|
| 100 |
-
num_steps = int(sampling_args.get("num_steps", 1))
|
| 101 |
-
timesteps = self._get_schedule(num_steps, x.shape[1])
|
| 102 |
-
|
| 103 |
-
for t_curr, t_prev in zip(timesteps[:-1], timesteps[1:]):
|
| 104 |
-
t_vec = torch.full((B,), t_curr, dtype=x.dtype, device=x.device)
|
| 105 |
-
pred = self._run_dit(x, x_ids, ctx, ctx_ids, t_vec)
|
| 106 |
-
x = x + (t_prev - t_curr) * pred
|
| 107 |
-
|
| 108 |
-
from einops import rearrange
|
| 109 |
-
return rearrange(x, "b (h w) c -> b c h w", h=H, w=W)
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
class Flux2VAETokenizer(nn.Module):
|
| 113 |
-
def __init__(self, ae_path: str, device="cpu",
|
| 114 |
-
torch_dtype: torch.dtype = torch.float32):
|
| 115 |
-
super().__init__()
|
| 116 |
-
from flux2.autoencoder import AutoEncoder, AutoEncoderParams
|
| 117 |
-
from safetensors.torch import load_file as load_sft
|
| 118 |
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
ae.load_state_dict(sd, strict=True, assign=True)
|
| 124 |
-
ae = ae.to(device=device, dtype=torch_dtype)
|
| 125 |
-
for p in ae.parameters():
|
| 126 |
-
p.requires_grad = False
|
| 127 |
-
ae.eval()
|
| 128 |
-
self.vae = ae
|
| 129 |
-
|
| 130 |
-
def denormalize_z(self, z):
|
| 131 |
-
return z
|
| 132 |
-
|
| 133 |
-
def decode(self, z):
|
| 134 |
-
z = z.to(dtype=next(self.vae.parameters()).dtype)
|
| 135 |
-
return self.vae.decode(z)
|
| 136 |
-
|
| 137 |
-
@torch.inference_mode()
|
| 138 |
-
def detokenize(self, z):
|
| 139 |
-
return torch.clamp(self.decode(self.denormalize_z(z)) * 0.5 + 0.5, 0.0, 1.0)
|
| 140 |
-
|
| 141 |
-
# =============================================================================
|
| 142 |
-
# CONFIGURATION (16GB CPU OPTIMIZED)
|
| 143 |
-
# =============================================================================
|
| 144 |
-
RES = 256 # NEVER CHANGE - 512 WILL OOM ON 16GB
|
| 145 |
-
NUM_STEPS = 1 # 1 step only for CPU
|
| 146 |
-
MAX_PROMPT_LEN = 128 # Short prompts save memory
|
| 147 |
-
MODEL_PATH = "model.safetensors"
|
| 148 |
AE_PATH = "ae.safetensors"
|
| 149 |
|
| 150 |
-
#
|
| 151 |
-
_STATE = {}
|
| 152 |
|
| 153 |
-
# ========================================
|
| 154 |
-
# MEMORY-EFFICIENT QWEN3-0.6B LOADER (4B VERSION WILL OOM)
|
| 155 |
-
# =============================================================================
|
| 156 |
def load_qwen3_embedder_cpu():
|
| 157 |
-
"""
|
|
|
|
|
|
|
|
|
|
| 158 |
if "qwen3" in _STATE:
|
| 159 |
return _STATE["qwen3"]
|
| 160 |
-
|
| 161 |
-
print("
|
| 162 |
-
|
| 163 |
tokenizer = AutoTokenizer.from_pretrained(
|
| 164 |
-
"Qwen/Qwen3-
|
| 165 |
padding_side="left",
|
| 166 |
trust_remote_code=True
|
| 167 |
)
|
| 168 |
-
|
| 169 |
-
# Critical memory optimizations for 16GB
|
| 170 |
model = AutoModelForCausalLM.from_pretrained(
|
| 171 |
-
"Qwen/Qwen3-
|
| 172 |
-
torch_dtype=torch.
|
| 173 |
-
device_map={"": "cpu"},
|
| 174 |
-
low_cpu_mem_usage=True,
|
| 175 |
-
use_safetensors=True,
|
| 176 |
trust_remote_code=True
|
| 177 |
).eval()
|
| 178 |
-
|
| 179 |
-
# Freeze parameters to prevent gradient computation
|
| 180 |
-
for param in model.parameters():
|
| 181 |
-
param.requires_grad = False
|
| 182 |
-
|
| 183 |
_STATE["qwen3"] = (model, tokenizer)
|
| 184 |
-
print("
|
| 185 |
return model, tokenizer
|
| 186 |
|
| 187 |
-
# ========================================
|
| 188 |
-
# PROMPT ENCODER (ADAPTED FOR QWEN3-0.6B → FLUX.2 7680D)
|
| 189 |
-
# =============================================================================
|
| 190 |
-
def encode_prompt(prompt: str) -> torch.Tensor:
|
| 191 |
-
"""Encode prompt with Qwen3-0.6B and project to FLUX.2's expected 7680D"""
|
| 192 |
-
model, tokenizer = _STATE["qwen3"]
|
| 193 |
-
|
| 194 |
-
# Truncate to save memory
|
| 195 |
-
inputs = tokenizer(
|
| 196 |
-
prompt,
|
| 197 |
-
return_tensors="pt",
|
| 198 |
-
padding=True,
|
| 199 |
-
truncation=True,
|
| 200 |
-
max_length=MAX_PROMPT_LEN
|
| 201 |
-
)
|
| 202 |
-
|
| 203 |
-
with torch.no_grad(), torch.inference_mode():
|
| 204 |
-
outputs = model(**inputs, output_hidden_states=True)
|
| 205 |
-
# Use second-to-last layer (better semantic representation)
|
| 206 |
-
hidden = outputs.hidden_states[-2][:, -1, :] # Shape: (1, 1024)
|
| 207 |
-
|
| 208 |
-
# Project 1024D → 7680D (required by FLUX.2)
|
| 209 |
-
# Using zero-padding (compromise for memory constraints)
|
| 210 |
-
hidden = torch.nn.functional.pad(hidden, (0, 7680 - 1024))
|
| 211 |
-
return hidden.to(torch.float32)
|
| 212 |
-
|
| 213 |
-
# =============================================================================
|
| 214 |
-
# MODEL LOADING WITH MAXIMUM MEMORY OPTIMIZATION
|
| 215 |
-
# =============================================================================
|
| 216 |
-
from huggingface_hub import hf_hub_download, login
|
| 217 |
-
import os
|
| 218 |
-
|
| 219 |
def ensure_loaded():
|
| 220 |
if _STATE.get("model"):
|
| 221 |
return
|
| 222 |
|
| 223 |
-
|
| 224 |
-
hf_token = os.environ.get("HF_TOKEN")
|
| 225 |
-
if not hf_token:
|
| 226 |
-
raise RuntimeError(
|
| 227 |
-
"HF_TOKEN not found. "
|
| 228 |
-
"Go to Space Settings → Secrets → add HF_TOKEN"
|
| 229 |
-
)
|
| 230 |
-
login(token=hf_token)
|
| 231 |
-
print("[Setup] Authenticated to HuggingFace")
|
| 232 |
-
|
| 233 |
-
# ---------- 2. 下载模型文件(若不存在) ----------
|
| 234 |
-
print("[Setup] Ensuring model.safetensors exists...")
|
| 235 |
-
global MODEL_PATH, AE_PATH
|
| 236 |
-
MODEL_PATH = hf_hub_download(
|
| 237 |
-
repo_id="black-forest-labs/FLUX.2-dev",
|
| 238 |
-
filename="model.safetensors",
|
| 239 |
-
cache_dir="./models", # 本地缓存,避免权限问题
|
| 240 |
-
local_files_only=False,
|
| 241 |
-
token=hf_token
|
| 242 |
-
)
|
| 243 |
|
| 244 |
-
|
| 245 |
-
AE_PATH = hf_hub_download(
|
| 246 |
-
repo_id="black-forest-labs/FLUX.2-dev",
|
| 247 |
-
filename="ae.safetensors",
|
| 248 |
-
cache_dir="./models",
|
| 249 |
-
local_files_only=False,
|
| 250 |
-
token=hf_token
|
| 251 |
-
)
|
| 252 |
-
|
| 253 |
-
print(f"[Setup] Model: {MODEL_PATH}")
|
| 254 |
-
print(f"[Setup] AE: {AE_PATH}")
|
| 255 |
-
|
| 256 |
-
# ---------- 3. 加载 FLUX.2 主模型 ----------
|
| 257 |
-
print("[Memory] Loading FLUX.2 Klein-4B (16GB optimized)...")
|
| 258 |
sd = load_file(MODEL_PATH, device="cpu")
|
| 259 |
|
| 260 |
model = Flux2AdapterModel(
|
| 261 |
state_dict=sd,
|
| 262 |
image_resolution=RES,
|
| 263 |
-
param_dtype=torch.
|
| 264 |
).to("cpu").eval()
|
| 265 |
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
# ---------- 4. 加载 VAE ----------
|
| 270 |
-
print("[Memory] Loading VAE...")
|
| 271 |
vae = Flux2VAETokenizer(
|
| 272 |
AE_PATH,
|
| 273 |
device="cpu",
|
| 274 |
-
torch_dtype=torch.
|
| 275 |
)
|
| 276 |
|
| 277 |
-
#
|
| 278 |
-
print("[Memory] Loading Qwen3-0.6B...")
|
| 279 |
emb_model, emb_tokenizer = load_qwen3_embedder_cpu()
|
| 280 |
|
| 281 |
_STATE.update({
|
|
@@ -285,47 +86,72 @@ def ensure_loaded():
|
|
| 285 |
"emb_tokenizer": emb_tokenizer
|
| 286 |
})
|
| 287 |
|
| 288 |
-
print(f"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 289 |
|
| 290 |
-
# ========================================
|
| 291 |
-
# GRADIO INFERENCE FUNCTION
|
| 292 |
-
# =============================================================================
|
| 293 |
def generate(prompt: str, seed: int, randomize: bool):
|
| 294 |
prompt = (prompt or "").strip()
|
| 295 |
if not prompt:
|
| 296 |
-
raise gr.Error("
|
| 297 |
-
|
| 298 |
ensure_loaded()
|
| 299 |
-
|
| 300 |
-
#
|
| 301 |
if randomize:
|
| 302 |
seed = int(torch.randint(0, 2**31 - 1, (1,)).item())
|
| 303 |
seed = int(seed)
|
| 304 |
generator = torch.Generator().manual_seed(seed)
|
| 305 |
-
|
| 306 |
-
#
|
| 307 |
noise = torch.randn(
|
| 308 |
-
1,
|
|
|
|
|
|
|
|
|
|
| 309 |
generator=generator,
|
| 310 |
-
dtype=torch.
|
| 311 |
)
|
| 312 |
-
|
| 313 |
-
#
|
| 314 |
ctx = encode_prompt(prompt)
|
| 315 |
-
|
| 316 |
-
#
|
| 317 |
-
|
| 318 |
with torch.no_grad(), torch.inference_mode():
|
| 319 |
latents = _STATE["model"].sample_images_with_grad(
|
| 320 |
noise, ctx, {"num_steps": NUM_STEPS}
|
| 321 |
)
|
| 322 |
-
pixels = _STATE["vae"].detokenize(latents)
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
|
| 326 |
-
# Convert to PIL Image
|
| 327 |
pixels = (
|
| 328 |
pixels[0]
|
|
|
|
| 329 |
.clamp(0, 1)
|
| 330 |
.mul(255)
|
| 331 |
.round()
|
|
@@ -334,47 +160,36 @@ def generate(prompt: str, seed: int, randomize: bool):
|
|
| 334 |
.cpu()
|
| 335 |
.numpy()
|
| 336 |
)
|
| 337 |
-
|
| 338 |
img = Image.fromarray(pixels)
|
| 339 |
-
|
|
|
|
| 340 |
return img, seed, info
|
| 341 |
|
| 342 |
-
# ========================================
|
| 343 |
-
|
| 344 |
-
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
|
| 352 |
-
- Single-step sampling only
|
| 353 |
-
- Cold start: ~2-3 minutes
|
| 354 |
-
- Generation time: **40-90 seconds per image**
|
| 355 |
-
|
| 356 |
-
**Tips for stability:**
|
| 357 |
-
- Keep prompts short (<{MAX_PROMPT_LEN} tokens)
|
| 358 |
-
- Wait for generation to complete before submitting new requests
|
| 359 |
-
- If OOM occurs, restart the Space
|
| 360 |
-
""")
|
| 361 |
|
| 362 |
with gr.Row():
|
| 363 |
-
with gr.Column(
|
| 364 |
prompt = gr.Textbox(
|
| 365 |
-
label="
|
| 366 |
-
placeholder=
|
| 367 |
-
lines=2
|
| 368 |
-
max_lines=3
|
| 369 |
)
|
| 370 |
with gr.Row():
|
| 371 |
-
seed = gr.Number(label="
|
| 372 |
-
randomize = gr.Checkbox(label="
|
| 373 |
-
btn = gr.Button("
|
| 374 |
-
|
| 375 |
-
|
| 376 |
-
|
| 377 |
-
output_info = gr.Textbox(label="Status", interactive=False)
|
| 378 |
|
| 379 |
btn.click(
|
| 380 |
fn=generate,
|
|
@@ -382,40 +197,17 @@ with gr.Blocks(title="FLUX.2 Klein-4B CPU (16GB Optimized)") as demo:
|
|
| 382 |
outputs=[output_img, seed, output_info]
|
| 383 |
)
|
| 384 |
|
| 385 |
-
#
|
| 386 |
gr.Examples(
|
| 387 |
examples=[
|
| 388 |
-
["a cute cat"],
|
| 389 |
-
["
|
| 390 |
-
["
|
| 391 |
],
|
| 392 |
-
inputs=prompt
|
| 393 |
-
label="Click to try (short prompts only)"
|
| 394 |
)
|
| 395 |
-
|
| 396 |
-
gr.Markdown("""
|
| 397 |
-
---
|
| 398 |
-
**Technical Notes:**
|
| 399 |
-
- Models loaded with `low_cpu_mem_usage=True`
|
| 400 |
-
- Using float32 (bfloat16 causes memory spikes on CPU)
|
| 401 |
-
- Text encoder projected from 1024D → 7680D via padding
|
| 402 |
-
- Safe for 16GB CPU environments
|
| 403 |
-
""")
|
| 404 |
|
| 405 |
-
# ========================================
|
| 406 |
-
# LAUNCH
|
| 407 |
-
# =============================================================================
|
| 408 |
if __name__ == "__main__":
|
| 409 |
-
#
|
| 410 |
-
|
| 411 |
-
|
| 412 |
-
print("Starting FLUX.2 Klein-4B CPU Demo (16GB Optimized)")
|
| 413 |
-
print(f"Resolution: {RES}x{RES}")
|
| 414 |
-
print(f"Steps: {NUM_STEPS}")
|
| 415 |
-
print("Models will load on first request (~2-3min)")
|
| 416 |
-
|
| 417 |
-
demo.launch(
|
| 418 |
-
show_error=True,
|
| 419 |
-
quiet=False,
|
| 420 |
-
share=False # HF Spaces sets this automatically
|
| 421 |
-
)
|
|
|
|
| 1 |
"""
|
| 2 |
+
===========================================
|
| 3 |
+
FLUX.2 Klein-4B CPU Demo(仅技术验证用)
|
| 4 |
+
===========================================
|
| 5 |
+
⚠️ 重要说明:
|
| 6 |
+
1. 性能极差:256x256 1步约30-60秒,512x512 1步约2-5分钟,1024x1024大概率OOM
|
| 7 |
+
2. 内存要求:至少需要16G CPU内存(跑512需32G+),HF Space CPU Basic(16G)建议用256分辨率
|
| 8 |
+
3. 前置条件:已同意FLUX.2-dev的Gated License,Space配置了HF_TOKEN环境变量
|
| 9 |
+
4. 依赖已在你现有requirements.txt基础上兼容,无需额外修改
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
"""
|
| 11 |
|
| 12 |
import torch
|
|
|
|
| 13 |
import gradio as gr
|
| 14 |
from PIL import Image
|
| 15 |
from safetensors.torch import load_file
|
| 16 |
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 17 |
+
from flux2_adapter import Flux2AdapterModel, Flux2VAETokenizer
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
|
| 19 |
+
# ==================== 全局配置 ====================
|
| 20 |
+
RES = 256 # 👈 优先用256,512需要≥32G内存,1024不建议
|
| 21 |
+
NUM_STEPS = 1 # 1步最快,增加步数会指数级变慢
|
| 22 |
+
MODEL_PATH = "model.safetensors" # 和ae.safetensors同目录
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
AE_PATH = "ae.safetensors"
|
| 24 |
|
| 25 |
+
# ==================== 全局状态缓存 ====================
|
| 26 |
+
_STATE = {} # 避免重复加载模型
|
| 27 |
|
| 28 |
+
# ==================== CPU版Qwen3 Embedder(核心:禁用FP8) ====================
|
|
|
|
|
|
|
| 29 |
def load_qwen3_embedder_cpu():
|
| 30 |
+
"""
|
| 31 |
+
手动加载Qwen3-4B BF16版本,完全规避FP8依赖
|
| 32 |
+
CPU模式下只能用BF16/FP32,FP8不存在
|
| 33 |
+
"""
|
| 34 |
if "qwen3" in _STATE:
|
| 35 |
return _STATE["qwen3"]
|
| 36 |
+
|
| 37 |
+
print("Loading Qwen3-4B (CPU/BF16)...")
|
|
|
|
| 38 |
tokenizer = AutoTokenizer.from_pretrained(
|
| 39 |
+
"Qwen/Qwen3-4B",
|
| 40 |
padding_side="left",
|
| 41 |
trust_remote_code=True
|
| 42 |
)
|
|
|
|
|
|
|
| 43 |
model = AutoModelForCausalLM.from_pretrained(
|
| 44 |
+
"Qwen/Qwen3-4B",
|
| 45 |
+
torch_dtype=torch.bfloat16, # CPU支持BF16,比FP32快30%
|
| 46 |
+
device_map={"": "cpu"}, # 强制CPU
|
| 47 |
+
low_cpu_mem_usage=True, # 降低加载时内存峰值
|
|
|
|
| 48 |
trust_remote_code=True
|
| 49 |
).eval()
|
| 50 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
| 51 |
_STATE["qwen3"] = (model, tokenizer)
|
| 52 |
+
print("Qwen3-4B loaded (CPU)")
|
| 53 |
return model, tokenizer
|
| 54 |
|
| 55 |
+
# ==================== 模型加载(全CPU) ====================
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
def ensure_loaded():
|
| 57 |
if _STATE.get("model"):
|
| 58 |
return
|
| 59 |
|
| 60 |
+
print("Loading FLUX.2 Klein-4B (CPU)...")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
|
| 62 |
+
# 1. 加载主模型权重(CPU,lazy load 自动生效)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
sd = load_file(MODEL_PATH, device="cpu")
|
| 64 |
|
| 65 |
model = Flux2AdapterModel(
|
| 66 |
state_dict=sd,
|
| 67 |
image_resolution=RES,
|
| 68 |
+
param_dtype=torch.bfloat16
|
| 69 |
).to("cpu").eval()
|
| 70 |
|
| 71 |
+
# 2. 加载 VAE
|
| 72 |
+
print("Loading FLUX.2 AE (CPU)...")
|
|
|
|
|
|
|
|
|
|
| 73 |
vae = Flux2VAETokenizer(
|
| 74 |
AE_PATH,
|
| 75 |
device="cpu",
|
| 76 |
+
torch_dtype=torch.bfloat16
|
| 77 |
)
|
| 78 |
|
| 79 |
+
# 3. 加载 Qwen3 Embedder
|
|
|
|
| 80 |
emb_model, emb_tokenizer = load_qwen3_embedder_cpu()
|
| 81 |
|
| 82 |
_STATE.update({
|
|
|
|
| 86 |
"emb_tokenizer": emb_tokenizer
|
| 87 |
})
|
| 88 |
|
| 89 |
+
print(f"All models loaded (CPU), ready for {RES}x{RES} inference")
|
| 90 |
+
# ==================== Prompt编码(适配FLUX.2的7680维要求) ====================
|
| 91 |
+
def encode_prompt(prompt: str) -> torch.Tensor:
|
| 92 |
+
"""
|
| 93 |
+
FLUX.2 Klein要求context维度为7680,来自3×Qwen3-4B的hidden state
|
| 94 |
+
CPU模式下简化为重复3次最后一层的hidden state
|
| 95 |
+
"""
|
| 96 |
+
model, tokenizer = _STATE["emb_model"], _STATE["emb_tokenizer"]
|
| 97 |
+
|
| 98 |
+
inputs = tokenizer(
|
| 99 |
+
prompt,
|
| 100 |
+
return_tensors="pt",
|
| 101 |
+
padding=True,
|
| 102 |
+
truncation=True,
|
| 103 |
+
max_length=256 # Klein的文本长度限制
|
| 104 |
+
)
|
| 105 |
+
|
| 106 |
+
with torch.no_grad(), torch.inference_mode():
|
| 107 |
+
outputs = model(**inputs, output_hidden_states=True)
|
| 108 |
+
# 用倒数第二层的hidden state(和训练时对齐性更好)
|
| 109 |
+
hidden = outputs.hidden_states[-2][:, -1, :] # shape: (1, 2560)
|
| 110 |
+
|
| 111 |
+
# 重复3次得到7680维,符合FLUX.2输入要求
|
| 112 |
+
ctx = hidden.repeat(1, 3) # shape: (1, 7680)
|
| 113 |
+
return ctx.to(torch.bfloat16)
|
| 114 |
|
| 115 |
+
# ==================== Gradio推理逻辑 ====================
|
|
|
|
|
|
|
| 116 |
def generate(prompt: str, seed: int, randomize: bool):
|
| 117 |
prompt = (prompt or "").strip()
|
| 118 |
if not prompt:
|
| 119 |
+
raise gr.Error("请输入提示词")
|
| 120 |
+
|
| 121 |
ensure_loaded()
|
| 122 |
+
|
| 123 |
+
# 种子处理(纯CPU Generator)
|
| 124 |
if randomize:
|
| 125 |
seed = int(torch.randint(0, 2**31 - 1, (1,)).item())
|
| 126 |
seed = int(seed)
|
| 127 |
generator = torch.Generator().manual_seed(seed)
|
| 128 |
+
|
| 129 |
+
# 生成随机噪声(CPU)
|
| 130 |
noise = torch.randn(
|
| 131 |
+
1,
|
| 132 |
+
128,
|
| 133 |
+
RES // 16,
|
| 134 |
+
RES // 16,
|
| 135 |
generator=generator,
|
| 136 |
+
dtype=torch.bfloat16
|
| 137 |
)
|
| 138 |
+
|
| 139 |
+
# 编码提示词
|
| 140 |
ctx = encode_prompt(prompt)
|
| 141 |
+
|
| 142 |
+
# 推理(全程CPU,无GPU代码)
|
| 143 |
+
start = torch.datetime.now()
|
| 144 |
with torch.no_grad(), torch.inference_mode():
|
| 145 |
latents = _STATE["model"].sample_images_with_grad(
|
| 146 |
noise, ctx, {"num_steps": NUM_STEPS}
|
| 147 |
)
|
| 148 |
+
pixels = _STATE["vae"].detokenize(latents) # 输出[0,1]范围
|
| 149 |
+
elapsed = (torch.datetime.now() - start).total_seconds()
|
| 150 |
+
|
| 151 |
+
# 转换为PIL Image
|
|
|
|
| 152 |
pixels = (
|
| 153 |
pixels[0]
|
| 154 |
+
.float()
|
| 155 |
.clamp(0, 1)
|
| 156 |
.mul(255)
|
| 157 |
.round()
|
|
|
|
| 160 |
.cpu()
|
| 161 |
.numpy()
|
| 162 |
)
|
|
|
|
| 163 |
img = Image.fromarray(pixels)
|
| 164 |
+
|
| 165 |
+
info = f"CPU · {RES}x{RES} · {NUM_STEPS}步 · {elapsed:.1f}s · Seed: {seed}"
|
| 166 |
return img, seed, info
|
| 167 |
|
| 168 |
+
# ==================== Gradio界面 ====================
|
| 169 |
+
with gr.Blocks(title="FLUX.2 Klein CPU Demo") as demo:
|
| 170 |
+
gr.Markdown(
|
| 171 |
+
f"""
|
| 172 |
+
# ⚠️ FLUX.2 Klein-4B CPU Demo(仅技术验证)
|
| 173 |
+
- 当前分辨率:{RES}x{RES},步数:{NUM_STEPS}
|
| 174 |
+
- CPU性能有限:{RES}x{RES} 1步约需30-180秒,请勿频繁提交
|
| 175 |
+
- 如遇OOM:请将RES改为256,或升级Space硬件为GPU
|
| 176 |
+
"""
|
| 177 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 178 |
|
| 179 |
with gr.Row():
|
| 180 |
+
with gr.Column():
|
| 181 |
prompt = gr.Textbox(
|
| 182 |
+
label="提示词",
|
| 183 |
+
placeholder="例如:a cute cat sitting on a sofa",
|
| 184 |
+
lines=2
|
|
|
|
| 185 |
)
|
| 186 |
with gr.Row():
|
| 187 |
+
seed = gr.Number(label="种子", value=42, precision=0)
|
| 188 |
+
randomize = gr.Checkbox(label="随机种子", value=False)
|
| 189 |
+
btn = gr.Button("生成", variant="primary")
|
| 190 |
+
with gr.Column():
|
| 191 |
+
output_img = gr.Image(label="生成结果", type="pil")
|
| 192 |
+
output_info = gr.Textbox(label="运行信息", interactive=False)
|
|
|
|
| 193 |
|
| 194 |
btn.click(
|
| 195 |
fn=generate,
|
|
|
|
| 197 |
outputs=[output_img, seed, output_info]
|
| 198 |
)
|
| 199 |
|
| 200 |
+
# 预设提示词
|
| 201 |
gr.Examples(
|
| 202 |
examples=[
|
| 203 |
+
["a cute cat sleeping on a windowsill"],
|
| 204 |
+
["a futuristic city at night, neon lights"],
|
| 205 |
+
["a watercolor painting of a mountain landscape"]
|
| 206 |
],
|
| 207 |
+
inputs=prompt
|
|
|
|
| 208 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 209 |
|
| 210 |
+
# ==================== 启动 ====================
|
|
|
|
|
|
|
| 211 |
if __name__ == "__main__":
|
| 212 |
+
# HF Space不需要share=True,本地运行可加share=True
|
| 213 |
+
demo.launch(show_error=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|