akhaliq HF Staff commited on
Commit
7e95a12
·
verified ·
1 Parent(s): e0272e2

merge/: native (non-distilled) image DiT loader (bit-exact remap) + manual N-step sampler + control test

Browse files
Files changed (1) hide show
  1. merge/expD_native_dit.py +357 -0
merge/expD_native_dit.py ADDED
@@ -0,0 +1,357 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Experiment D: attach ControlNet-Union-2.0 to the NATIVE (non-distilled) MiniMax-H3-Image
2
+ DiT and run 32 flow-matching steps — the trajectory closest to the branch's training.
3
+
4
+ The native transformer/ checkpoint uses VideoX-Fun key layout (fused qkv, video_patch_proj,
5
+ final_layer.video_out), so we remap its keys onto diffusers' MiniMaxH3ImageTransformer2DModel
6
+ (num_pdd_heads=1), then reuse the proven attach/wrap/control-row code from the merge repo.
7
+
8
+ Runs (1024x1024, seed 904231, cabin prompt, canny of reference1.png):
9
+ native_base.png (no control), native_scale0.0/0.5/1.0.png
10
+ Success criterion: base is coherent (remap correct) AND scale1.0 structure follows edges.
11
+
12
+ Self-contained job script. Uploads frames to assets/validation6/.
13
+ """
14
+ import os
15
+ import subprocess
16
+ import sys
17
+ import time
18
+
19
+ os.environ.setdefault("HF_XET_HIGH_PERFORMANCE", "1")
20
+ WORK = "/work"
21
+ OUT = f"{WORK}/out"
22
+ os.makedirs(OUT, exist_ok=True)
23
+
24
+ BASE = "MiniMaxAI/MiniMax-H3-Image"
25
+ CONTROLNET_REPO = "alibaba-pai/MiniMax-H3-Fun-Controlnet-Union-2.0"
26
+ CONTROLNET_FILE = "MiniMax-H3-Fun-Controlnet-Union-2.0.safetensors"
27
+ MERGE_REPO = "akhaliq/MiniMax-H3-Image-Fun-Controlnet-Union-2.0"
28
+ ENCODER_SPACE = "CozyByte/minimax-h3-image-encoder"
29
+ SEED = 904231
30
+ NUM_STEPS = 32
31
+ PROMPT = "A cozy cabin in a snowy pine forest at dusk, warm light in the windows, photorealistic."
32
+
33
+
34
+ def log(msg):
35
+ print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
36
+
37
+
38
+ def sh(cmd):
39
+ log(f"$ {' '.join(cmd)}")
40
+ subprocess.run(cmd, check=True)
41
+
42
+
43
+ # 0. deps
44
+ sh([sys.executable, "-m", "pip", "install", "--no-cache-dir", "torch", "torchvision",
45
+ "--index-url", "https://download.pytorch.org/whl/cu126"])
46
+ sh([sys.executable, "-m", "pip", "install", "--no-cache-dir",
47
+ "numpy", "diffusers>=0.43", "transformers", "accelerate", "safetensors",
48
+ "gradio_client", "opencv-python-headless", "hf_xet", "Pillow"])
49
+
50
+ import json
51
+ import re
52
+
53
+ import cv2
54
+ import numpy as np
55
+ import torch
56
+ from PIL import Image
57
+ from huggingface_hub import hf_hub_download, snapshot_download
58
+ from safetensors.torch import load_file
59
+
60
+ # 1. downloads
61
+ log("downloading native transformer + vae + scheduler + control ckpt")
62
+ snap = snapshot_download(BASE, allow_patterns=[
63
+ "transformer/*", "vae_diffusers/*", "scheduler/*", "transformer_diffusers/config.json",
64
+ ])
65
+ ckpt = hf_hub_download(CONTROLNET_REPO, CONTROLNET_FILE)
66
+ attach_src = hf_hub_download(MERGE_REPO, "merge/validate_controlnet.py")
67
+ sys.path.insert(0, os.path.dirname(attach_src))
68
+ from validate_controlnet import ( # noqa: E402
69
+ attach_control_branch, wrap_forward_with_control, build_control_rows, psnr,
70
+ )
71
+
72
+ # 2. build diffusers-class model with 1 pdd head, on meta device
73
+ from accelerate import init_empty_weights # noqa: E402
74
+ from diffusers import AutoencoderKLMiniMaxH3Image, MiniMaxH3Scheduler # noqa: E402
75
+ from diffusers.models.transformers.transformer_minimax_h3_image import ( # noqa: E402
76
+ MiniMaxH3ImageTransformer2DModel,
77
+ )
78
+
79
+ cfg = json.load(open(f"{snap}/transformer_diffusers/config.json"))
80
+ cfg["num_pdd_heads"] = 1
81
+ with init_empty_weights():
82
+ model = MiniMaxH3ImageTransformer2DModel.from_config(cfg)
83
+ target_keys = set(model.state_dict().keys())
84
+ log(f"model skeleton: {len(target_keys)} keys")
85
+
86
+ # 3. load native checkpoint (12 shards) and remap
87
+ from glob import glob # noqa: E402
88
+
89
+ native_sd = {}
90
+ for shard in sorted(glob(f"{snap}/transformer/*.safetensors")):
91
+ native_sd.update(load_file(shard))
92
+ log(f"native checkpoint: {len(native_sd)} tensors")
93
+
94
+
95
+ def _sub_map(sub):
96
+ """Map a VideoX-Fun block submodule name to the diffusers one (shared by main
97
+ transformer blocks and token_refiner refiner blocks)."""
98
+ if sub.startswith("attn.out_proj."):
99
+ return "attn.to_out.0." + sub[len("attn.out_proj."):]
100
+ if sub.startswith("attn.q_norm."):
101
+ return "attn.norm_q." + sub[len("attn.q_norm."):]
102
+ if sub.startswith("attn.k_norm."):
103
+ return "attn.norm_k." + sub[len("attn.k_norm."):]
104
+ if sub.startswith("mlp.fc1."):
105
+ return "ff.net.0.proj." + sub[len("mlp.fc1."):]
106
+ if sub.startswith("mlp.fc2."):
107
+ return "ff.net.2." + sub[len("mlp.fc2."):]
108
+ if sub.startswith("adaln_proj."):
109
+ suffix = sub[len("adaln_proj."):]
110
+ if suffix.startswith("linear."):
111
+ return sub # already diffusers-style
112
+ return "adaln_proj.linear." + suffix
113
+ return sub # norm1/norm2, attn.norm_q/norm_k already match
114
+
115
+
116
+ def remap_key(k):
117
+ if k.startswith("blocks."):
118
+ rest = k[len("blocks."):]
119
+ n, sub = rest.split(".", 1)
120
+ return f"transformer_blocks.{n}." + _sub_map(sub)
121
+ if k.startswith("token_refiner.blocks."):
122
+ rest = k[len("token_refiner.blocks."):]
123
+ n, sub = rest.split(".", 1)
124
+ return f"token_refiner.refiner_blocks.{n}." + _sub_map(sub)
125
+ if k.startswith("video_patch_proj."):
126
+ return "proj_in." + k[len("video_patch_proj."):]
127
+ if k.startswith("condition_proj."):
128
+ return "context_embedder." + k[len("condition_proj."):]
129
+ if k.startswith("final_layer.video_out."):
130
+ return "pdd_proj_out.0." + k[len("final_layer.video_out."):]
131
+ if k.startswith("final_layer.adaln_proj."):
132
+ return "norm_out." + k[len("final_layer.adaln_proj."):]
133
+ if k.startswith("final_layer.norm."):
134
+ return "norm_out.norm." + k[len("final_layer.norm."):]
135
+ if k.startswith("time_embedder.proj_in."):
136
+ return "time_embedder.linear_1." + k[len("time_embedder.proj_in."):]
137
+ if k == "time_embedder.proj_in.weight":
138
+ return "time_embedder.linear_1.weight"
139
+ if k.startswith("time_embedder.proj_out."):
140
+ return "time_embedder.linear_2." + k[len("time_embedder.proj_out."):]
141
+ return k # token_refiner.final_norm, rope buffers, etc.
142
+
143
+
144
+ def qkv_prefix(k):
145
+ if k.startswith("blocks."):
146
+ n = re.match(r"blocks\.(\d+)\.", k).group(1)
147
+ return f"transformer_blocks.{n}.attn."
148
+ n = re.match(r"token_refiner\.blocks\.(\d+)\.", k).group(1)
149
+ return f"token_refiner.refiner_blocks.{n}.attn."
150
+
151
+
152
+ remapped = {}
153
+ for k, v in native_sd.items():
154
+ if k.endswith("attn.qkv_proj.weight"):
155
+ # raw shards store QKV per-head interleaved [h0: q|k|v, h1: q|k|v, ...];
156
+ # de-interleave to [q_all; k_all; v_all] first (official converter semantics)
157
+ assert v.shape[0] == 21504, (k, v.shape)
158
+ grouped = v.reshape(56, 3 * 128, *v.shape[1:])
159
+ q, kk, vv = grouped.split(128, dim=1)
160
+ pre = qkv_prefix(k)
161
+ remapped[pre + "to_q.weight"] = q.reshape(7168, *v.shape[1:]).contiguous()
162
+ remapped[pre + "to_k.weight"] = kk.reshape(7168, *v.shape[1:]).contiguous()
163
+ remapped[pre + "to_v.weight"] = vv.reshape(7168, *v.shape[1:]).contiguous()
164
+ continue
165
+ if k.endswith("mlp.fc1.weight"):
166
+ # native stores [gate; value]; diffusers SwiGLU reads [value; gate] — swap halves
167
+ half = v.shape[0] // 2
168
+ v = torch.cat([v[half:], v[:half]], dim=0)
169
+ remapped[remap_key(k)] = v
170
+
171
+ # the native time_embedder linear_1 has no bias; zero bias is the exact equivalent
172
+ skel = model.state_dict()
173
+ if "time_embedder.linear_1.bias" in skel and "time_embedder.linear_1.bias" not in remapped:
174
+ remapped["time_embedder.linear_1.bias"] = torch.zeros_like(skel["time_embedder.linear_1.bias"])
175
+
176
+ # fuzzy fallback for unmapped targets (e.g. norm_out internals with different names)
177
+ mapped = set(remapped)
178
+ unmapped_targets = target_keys - mapped
179
+ used_sources = {"__qkv__"}
180
+ src_by_shape = {}
181
+ for k, v in native_sd.items():
182
+ src_by_shape.setdefault(tuple(v.shape), []).append(k)
183
+ fuzzy = {}
184
+ for tk in sorted(unmapped_targets):
185
+ tshape = tuple(model.state_dict()[tk].shape)
186
+ cands = [c for c in src_by_shape.get(tshape, []) if c not in remapped.values()]
187
+ if len(cands) == 1:
188
+ fuzzy[tk] = cands[0]
189
+ for tk, sk in fuzzy.items():
190
+ log(f"fuzzy map: {sk} -> {tk}")
191
+ remapped[tk] = native_sd[sk]
192
+
193
+ still_missing = sorted(target_keys - set(remapped))
194
+ unexpected = sorted(set(remapped) - target_keys)
195
+ log(f"remap: {len(remapped)} mapped, missing={len(still_missing)}, unexpected={len(unexpected)}")
196
+ if still_missing:
197
+ log(f"MISSING: {still_missing[:20]}")
198
+ if unexpected:
199
+ log(f"UNEXPECTED: {unexpected[:20]}")
200
+
201
+ log("materializing model")
202
+ missing, unexp = model.load_state_dict(remapped, strict=False, assign=True)
203
+ model = model.to(torch.bfloat16)
204
+ # diffusers keeps these in fp32; restore after the blanket cast
205
+ for fp32_mod in ("proj_in", "time_embedder", "pdd_proj_out", "rope"):
206
+ getattr(model, fp32_mod).to(torch.float32)
207
+ del remapped, native_sd
208
+
209
+ # 4. attach control branch + wrap
210
+ attach_control_branch(model, ckpt)
211
+ wrap_forward_with_control(model)
212
+ meta_left = [n for n, p in model.named_parameters() if p.is_meta]
213
+ assert not meta_left, f"still on meta device: {meta_left[:10]}"
214
+ model = model.cuda()
215
+ vae = AutoencoderKLMiniMaxH3Image.from_pretrained(BASE, subfolder="vae_diffusers", torch_dtype=torch.bfloat16).cuda()
216
+ scheduler = MiniMaxH3Scheduler.from_pretrained(BASE, subfolder="scheduler")
217
+ log("model + control + vae on cuda")
218
+
219
+ # 5. prompt embeds from the encoder Space
220
+ from gradio_client import Client # noqa: E402
221
+ from safetensors import safe_open # noqa: E402
222
+
223
+ client = Client(ENCODER_SPACE, token=os.environ.get("HF_TOKEN"), httpx_kwargs={"timeout": 600})
224
+ try:
225
+ embeds_path = client.predict(PROMPT, [], api_name="/encode")
226
+ except Exception as e:
227
+ log(f"empty image list rejected ({e}); white png fallback")
228
+ from gradio_client import handle_file
229
+ white = "/tmp/white.png"
230
+ Image.new("RGB", (64, 64), (255, 255, 255)).save(white)
231
+ embeds_path = client.predict(PROMPT, [handle_file(white)], api_name="/encode")
232
+ if isinstance(embeds_path, (tuple, list)):
233
+ embeds_path = embeds_path[0]
234
+ if isinstance(embeds_path, dict):
235
+ embeds_path = embeds_path["path"]
236
+ with safe_open(embeds_path, framework="pt") as h:
237
+ prompt_embeds = h.get_tensor("prompt_embeds")
238
+ text_token_tags = h.get_tensor("text_token_tags")
239
+ log(f"prompt embeds {tuple(prompt_embeds.shape)}")
240
+
241
+ # 6. packed layout + initial latents
242
+ from diffusers.modular_pipelines.minimax_h3_image.before_denoise import ( # noqa: E402
243
+ build_image_packed_sequence, patchify_video_latents,
244
+ )
245
+ from diffusers.utils.torch_utils import randn_tensor # noqa: E402
246
+
247
+ device = "cuda"
248
+ LAT = 64
249
+ position_ids, token_tags, image_indices, text_indices, num_ref = build_image_packed_sequence(
250
+ text_token_tags, [], LAT, LAT, (1, 2, 2), 0,
251
+ )
252
+ position_ids = position_ids.to(device, torch.float32)
253
+ token_tags, image_indices, text_indices = token_tags.to(device), image_indices.to(device), text_indices.to(device)
254
+ seq_len = token_tags.shape[0]
255
+ log(f"packed: seq_len={seq_len}, image rows={image_indices.numel()}")
256
+
257
+ ref = hf_hub_download(BASE, "examples/reference1.png")
258
+ img = np.asarray(Image.open(ref).convert("RGB"))
259
+ edges = cv2.Canny(cv2.cvtColor(img, cv2.COLOR_RGB2GRAY), 100, 200)
260
+ canny_rgb = np.repeat(edges[..., None], 3, axis=-1)
261
+ Image.fromarray(canny_rgb).save(f"{OUT}/control_canny.png")
262
+
263
+ fake_pipe = type("P", (), {})()
264
+ fake_pipe.vae = vae
265
+ fake_pipe._execution_device = device
266
+ fake_pipe.transformer = model
267
+ fake_pipe.vae_spatial_compression_ratio = 16
268
+ control_rows = build_control_rows(fake_pipe, canny_rgb, LAT, LAT)
269
+ log(f"control rows {tuple(control_rows.shape)}")
270
+
271
+ # GRID_MODE pdd6: exact PDD 6-step grid (isolates head-vs-grid question);
272
+ # anything else: uniform linspace grid
273
+ GRID_MODE = os.environ.get("GRID_MODE", "linspace")
274
+ if GRID_MODE == "pdd6":
275
+ sigmas = torch.tensor([1.0, 0.9675236940383911, 0.8875485062599182, 0.75,
276
+ 0.5327739119529724, 0.23200887441635132, 0.0])
277
+ elif GRID_MODE == "shift":
278
+ s = float(os.environ.get("SHIFT", "12.0"))
279
+ u = torch.linspace(1.0, 0.0, NUM_STEPS + 1)
280
+ sigmas = s * u / (1 + (s - 1) * u)
281
+ sigmas[-1] = 0.0
282
+ log(f"shifted grid s={s}: sigmas {[round(float(x), 4) for x in sigmas[:5]]}...")
283
+ else:
284
+ sigmas = torch.linspace(1.0, 0.0, NUM_STEPS + 1)
285
+
286
+
287
+ def unpatchify(rows, h=LAT, w=LAT, ch=24, patch=(1, 2, 2)):
288
+ pt, ph, pw = patch
289
+ x = rows.reshape(1, h // ph, w // pw, ch, pt, ph, pw)
290
+ x = x.permute(0, 3, 4, 1, 5, 2, 6)
291
+ return x.reshape(1, ch, pt, h, w)
292
+
293
+
294
+ def decode(rows):
295
+ z = unpatchify(rows).squeeze(2).float() # (1, 24, 64, 64)
296
+ mean = torch.tensor(vae.config.latents_mean, device=device).view(1, -1, 1, 1)
297
+ std = torch.tensor(vae.config.latents_std, device=device).view(1, -1, 1, 1)
298
+ z = z * std + mean
299
+ with torch.no_grad():
300
+ x = vae.decode(z.to(torch.bfloat16), return_dict=False)[0]
301
+ x = (x.float() / 2 + 0.5).clamp(0, 1)[0].permute(1, 2, 0).cpu().numpy()
302
+ return Image.fromarray((x * 255).round().astype(np.uint8))
303
+
304
+
305
+ @torch.no_grad()
306
+ def generate(rows_ctrl, scale, tag):
307
+ # the scheduler is stateful: reset its step index for every generation
308
+ scheduler.set_timesteps(sigmas=sigmas, device=device)
309
+ ts = scheduler.timesteps
310
+ latents = patchify_video_latents(
311
+ randn_tensor((1, 24, 1, LAT, LAT), generator=torch.Generator("cpu").manual_seed(SEED),
312
+ device=device, dtype=torch.float32), (1, 2, 2))
313
+ for i, t in enumerate(ts):
314
+ unique = torch.full((1,), float(t), dtype=torch.float32, device=device)
315
+ tidx = torch.zeros(seq_len, dtype=torch.long, device=device)
316
+ vel = model(
317
+ hidden_states=latents[None],
318
+ encoder_hidden_states=prompt_embeds.to(device, torch.bfloat16),
319
+ timestep=unique,
320
+ timestep_indices=tidx,
321
+ token_tags=token_tags,
322
+ position_ids=position_ids,
323
+ image_indices=image_indices,
324
+ text_indices=text_indices,
325
+ pdd_step_index=0,
326
+ return_dict=False,
327
+ control_rows=rows_ctrl,
328
+ control_context_scale=scale,
329
+ )[0]
330
+ latents = scheduler.step(vel[0].float(), t, latents, return_dict=False)[0]
331
+ img = decode(latents)
332
+ img.save(f"{OUT}/{tag}.png")
333
+ peak = torch.cuda.max_memory_allocated() / 2**30
334
+ log(f"saved {OUT}/{tag}.png (peak GPU {peak:.1f} GiB)")
335
+ return img
336
+
337
+
338
+ tag_prefix = f"native_{GRID_MODE}{os.environ.get('SHIFT', '') if GRID_MODE == 'shift' else ''}_"
339
+ base = generate(None, 1.0, tag_prefix + "base")
340
+ if GRID_MODE == "shift":
341
+ out = generate(control_rows, 1.0, tag_prefix + "scale1.0")
342
+ log(f"PSNR(scale1.0 vs base) = {psnr(base, out):.2f} dB")
343
+ elif GRID_MODE != "pdd6":
344
+ gated = generate(control_rows, 0.0, tag_prefix + "scale0.0")
345
+ log(f"PSNR(scale0.0 vs base) = {psnr(base, gated):.2f} dB (expect > 40)")
346
+ for s in (0.5, 1.0):
347
+ out = generate(control_rows, s, tag_prefix + f"scale{s}")
348
+ log(f"PSNR(scale{s} vs base) = {psnr(base, out):.2f} dB")
349
+
350
+ from huggingface_hub import HfApi # noqa: E402
351
+
352
+ HfApi().upload_folder(
353
+ folder_path=OUT, path_in_repo="assets/validation6", repo_id=MERGE_REPO, repo_type="model",
354
+ commit_message=f"Native (non-distilled) image DiT + control branch, grid={os.environ.get('GRID_MODE', 'linspace')}",
355
+ )
356
+ log("UPLOAD DONE")
357
+ log("ALL DONE")