akhaliq HF Staff commited on
Commit
05f34d5
·
verified ·
1 Parent(s): e7045eb

merge/: hint spatial-structure probe (offline control-branch replay)

Browse files
Files changed (1) hide show
  1. merge/diag_hints.py +152 -0
merge/diag_hints.py ADDED
@@ -0,0 +1,152 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Probe: do the control hints carry the control image's spatial structure?
2
+
3
+ Captures real intermediates (packed seq, temb, adaln_indices, rotary_emb,
4
+ image_indices) from one denoise forward, then replays the control branch
5
+ offline with circle-left vs circle-right control rows and reports per-row
6
+ hint norms as 32x32 maps. Decides between:
7
+ A. control rows/hints are spatially flat -> wiring/prep bug (fixable)
8
+ B. hints carry structure but DiT ignores -> training-trajectory mismatch
9
+ """
10
+ import os
11
+
12
+ os.environ.setdefault("HF_XET_HIGH_PERFORMANCE", "1")
13
+
14
+ import cv2
15
+ import numpy as np
16
+ import torch
17
+ from PIL import Image
18
+ from huggingface_hub import hf_hub_download
19
+ from safetensors import safe_open
20
+ from gradio_client import Client
21
+
22
+ from validate_controlnet import (
23
+ attach_control_branch,
24
+ wrap_forward_with_control,
25
+ build_control_rows,
26
+ log,
27
+ ENCODER_SPACE,
28
+ BASE,
29
+ CONTROLNET_REPO,
30
+ CONTROLNET_FILE,
31
+ SEED,
32
+ )
33
+
34
+ PROMPT = "A rough weathered concrete wall texture filling the entire frame, soft even light, photorealistic."
35
+ OUT = "/work/diag2"
36
+ os.makedirs(OUT, exist_ok=True)
37
+
38
+
39
+ def circle_control(cx):
40
+ img = np.zeros((1024, 1024), np.uint8)
41
+ cv2.circle(img, (cx, 512), 200, 255, 8)
42
+ return np.repeat(img[..., None], 3, axis=-1)
43
+
44
+
45
+ def norm_map(rows_2d, tag):
46
+ """rows_2d: (1024, C) -> 32x32 L2-norm map + left/right stats + heatmap png."""
47
+ norms = rows_2d.float().norm(dim=-1).reshape(32, 32).cpu().numpy()
48
+ l, r = norms[:, :16].mean(), norms[:, 16:].mean()
49
+ log(f"{tag}: row-norm left={l:.4f} right={r:.4f} ratio L/R={l/max(r,1e-9):.3f} "
50
+ f"min={norms.min():.4f} max={norms.max():.4f}")
51
+ heat = ((norms - norms.min()) / max(np.ptp(norms), 1e-9) * 255).astype(np.uint8)
52
+ Image.fromarray(heat).resize((256, 256), Image.NEAREST).save(f"{OUT}/{tag}_heatmap.png")
53
+ return norms
54
+
55
+
56
+ def main():
57
+ log("loading pipeline (cached)")
58
+ ckpt = hf_hub_download(CONTROLNET_REPO, CONTROLNET_FILE)
59
+ from diffusers import ModularPipeline
60
+
61
+ blocks = ModularPipeline.from_pretrained(BASE).blocks
62
+ blocks.sub_blocks.pop("text_encoder")
63
+ pipe = blocks.init_pipeline(BASE)
64
+ pipe.load_components(dtype=torch.bfloat16)
65
+ model = pipe.transformer
66
+ attach_control_branch(model, ckpt)
67
+ wrap_forward_with_control(model)
68
+ pipe.to("cuda")
69
+
70
+ # capture: forward kwargs (image_indices...) + block-0 inputs (packed, temb, adaln, rope)
71
+ cap = {}
72
+ orig_forward = model.forward
73
+ def rec(hidden_states, encoder_hidden_states, timestep, timestep_indices,
74
+ token_tags, position_ids, image_indices, text_indices, pdd_step_index,
75
+ return_dict=True, control_rows=None, control_context_scale=1.0):
76
+ cap.setdefault("image_indices", image_indices)
77
+ cap.setdefault("text_indices", text_indices)
78
+ cap.setdefault("position_ids", position_ids)
79
+ return orig_forward(hidden_states, encoder_hidden_states, timestep,
80
+ timestep_indices, token_tags, position_ids, image_indices,
81
+ text_indices, pdd_step_index, return_dict=return_dict,
82
+ control_rows=control_rows, control_context_scale=control_context_scale)
83
+ model.forward = rec
84
+
85
+ block0 = {}
86
+ def pre_hook(m, args):
87
+ if "packed" not in block0:
88
+ block0["packed"], block0["temb"], block0["adaln"], block0["rope"] = (
89
+ args[0].detach(), args[1].detach(), args[2].detach(),
90
+ (args[3][0].detach(), args[3][1].detach()) if isinstance(args[3], tuple) else args[3].detach(),
91
+ )
92
+ h = model.transformer_blocks[0].register_forward_pre_hook(pre_hook)
93
+
94
+ client = Client(ENCODER_SPACE, token=os.environ.get("HF_TOKEN"), httpx_kwargs={"timeout": 600})
95
+ embeds_path = client.predict(PROMPT, [], api_name="/encode")
96
+ if isinstance(embeds_path, (tuple, list)):
97
+ embeds_path = embeds_path[0]
98
+ if isinstance(embeds_path, dict):
99
+ embeds_path = embeds_path["path"]
100
+ with safe_open(embeds_path, framework="pt") as f:
101
+ prompt_embeds = f.get_tensor("prompt_embeds")
102
+ text_token_tags = f.get_tensor("text_token_tags")
103
+
104
+ ratio = getattr(pipe, "vae_spatial_compression_ratio", 16)
105
+ rows_left = build_control_rows(pipe, circle_control(256), 1024 // ratio, 1024 // ratio)
106
+
107
+ log("one generation to capture intermediates")
108
+ pipe(
109
+ prompt_embeds=prompt_embeds.to("cuda", torch.bfloat16),
110
+ text_token_tags=text_token_tags,
111
+ width=1024, height=1024, min_generation_area=1024,
112
+ generator=torch.Generator("cpu").manual_seed(SEED),
113
+ output="images",
114
+ denoiser_input_fields={"control_rows": rows_left, "control_context_scale": 1.0},
115
+ )
116
+ h.remove()
117
+ model.forward = orig_forward
118
+ assert "packed" in block0 and "image_indices" in cap, (list(block0), list(cap))
119
+ log(f"captured: packed {tuple(block0['packed'].shape)}, image rows {cap['image_indices'].shape[0]}")
120
+
121
+ rows_right = build_control_rows(pipe, circle_control(768), 1024 // ratio, 1024 // ratio)
122
+
123
+ with torch.no_grad():
124
+ for tag, rows in (("left", rows_left), ("right", rows_right)):
125
+ # stage 1: control_proj_in output
126
+ ce = model.control_proj_in(rows.to(next(model.control_proj_in.parameters()).dtype))
127
+ norm_map(ce[0], f"projin_{tag}")
128
+ # stage 2: replay control blocks on the captured intermediates
129
+ c = block0["packed"].index_copy(1, cap["image_indices"], ce.to(block0["packed"].dtype))
130
+ hints = []
131
+ for n, blk in enumerate(model.control_blocks):
132
+ if n == 0:
133
+ c = blk.before_proj(c) + block0["packed"]
134
+ c = blk(c, block0["temb"], block0["adaln"], block0["rope"])
135
+ hints.append(blk.after_proj(c))
136
+ img_rows = cap["image_indices"]
137
+ norm_map(hints[0][0, img_rows], f"hint0_{tag}")
138
+ norm_map(hints[9][0, img_rows], f"hint9_{tag}")
139
+ del hints
140
+ torch.cuda.empty_cache()
141
+
142
+ from huggingface_hub import HfApi
143
+ HfApi().upload_folder(
144
+ folder_path=OUT, path_in_repo="assets/diag2",
145
+ repo_id="akhaliq/MiniMax-H3-Image-Fun-Controlnet-Union-2.0", repo_type="model",
146
+ commit_message="Hint spatial-structure probe (control branch replay)",
147
+ )
148
+ log("DONE")
149
+
150
+
151
+ if __name__ == "__main__":
152
+ main()