akhaliq HF Staff commited on
Commit
5bbb7e1
·
verified ·
1 Parent(s): 05f34d5

merge/: spatial routing diagnostic (circle left vs right)

Browse files
Files changed (1) hide show
  1. merge/diag_spatial.py +123 -0
merge/diag_spatial.py ADDED
@@ -0,0 +1,123 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Diagnostic: does the control branch route SPATIAL structure into the H3-Image DiT?
2
+
3
+ Generates a texture wall with (a) no control, (b) a circle-edge control on the
4
+ LEFT half, (c) the same circle on the RIGHT half, same seed. If the control
5
+ signal is spatially routed, output edge density should be higher on the circle
6
+ side. If outputs show no side-correlated structure, the control rows are not
7
+ landing on the right positions (wiring/order bug), not a training mismatch.
8
+ """
9
+ import os
10
+
11
+ os.environ.setdefault("HF_XET_HIGH_PERFORMANCE", "1")
12
+
13
+ import cv2
14
+ import numpy as np
15
+ import torch
16
+ from PIL import Image
17
+ from huggingface_hub import hf_hub_download
18
+ from safetensors import safe_open
19
+ from gradio_client import Client
20
+
21
+ from validate_controlnet import (
22
+ attach_control_branch,
23
+ wrap_forward_with_control,
24
+ build_control_rows,
25
+ log,
26
+ ENCODER_SPACE,
27
+ BASE,
28
+ CONTROLNET_REPO,
29
+ CONTROLNET_FILE,
30
+ SEED,
31
+ )
32
+
33
+ PROMPT = "A rough weathered concrete wall texture filling the entire frame, soft even light, photorealistic."
34
+ OUT = "/work/diag"
35
+ os.makedirs(OUT, exist_ok=True)
36
+
37
+
38
+ def circle_control(cx):
39
+ img = np.zeros((1024, 1024), np.uint8)
40
+ cv2.circle(img, (cx, 512), 200, 255, 8)
41
+ return np.repeat(img[..., None], 3, axis=-1)
42
+
43
+
44
+ def edge_density(pil_img):
45
+ gray = cv2.cvtColor(np.asarray(pil_img), cv2.COLOR_RGB2GRAY)
46
+ edges = cv2.Canny(gray, 100, 200) > 0
47
+ return edges[:, :512].mean(), edges[:, 512:].mean()
48
+
49
+
50
+ def main():
51
+ log("loading pipeline (cached)")
52
+ ckpt = hf_hub_download(CONTROLNET_REPO, CONTROLNET_FILE)
53
+ from diffusers import ModularPipeline
54
+
55
+ blocks = ModularPipeline.from_pretrained(BASE).blocks
56
+ blocks.sub_blocks.pop("text_encoder")
57
+ pipe = blocks.init_pipeline(BASE)
58
+ pipe.load_components(dtype=torch.bfloat16)
59
+ attach_control_branch(pipe.transformer, ckpt)
60
+ wrap_forward_with_control(pipe.transformer)
61
+ pipe.to("cuda")
62
+ log("pipeline ready")
63
+
64
+ client = Client(ENCODER_SPACE, token=os.environ.get("HF_TOKEN"), httpx_kwargs={"timeout": 600})
65
+ try:
66
+ embeds_path = client.predict(PROMPT, [], api_name="/encode")
67
+ except Exception:
68
+ from gradio_client import handle_file
69
+
70
+ white = "/tmp/white.png"
71
+ Image.new("RGB", (64, 64), (255, 255, 255)).save(white)
72
+ embeds_path = client.predict(PROMPT, [handle_file(white)], api_name="/encode")
73
+ if isinstance(embeds_path, (tuple, list)):
74
+ embeds_path = embeds_path[0]
75
+ if isinstance(embeds_path, dict):
76
+ embeds_path = embeds_path["path"]
77
+ with safe_open(embeds_path, framework="pt") as h:
78
+ prompt_embeds = h.get_tensor("prompt_embeds")
79
+ text_token_tags = h.get_tensor("text_token_tags")
80
+
81
+ ratio = getattr(pipe, "vae_spatial_compression_ratio", 16)
82
+
83
+ def gen(ctrl_rgb, scale, tag):
84
+ rows = None
85
+ if ctrl_rgb is not None:
86
+ rows = build_control_rows(pipe, ctrl_rgb, 1024 // ratio, 1024 // ratio)
87
+ result = pipe(
88
+ prompt_embeds=prompt_embeds.to("cuda", torch.bfloat16),
89
+ text_token_tags=text_token_tags,
90
+ width=1024,
91
+ height=1024,
92
+ min_generation_area=1024,
93
+ generator=torch.Generator("cpu").manual_seed(SEED),
94
+ output="images",
95
+ denoiser_input_fields={"control_rows": rows, "control_context_scale": scale},
96
+ )
97
+ img = result[0]
98
+ img.save(f"{OUT}/{tag}.png")
99
+ l, r = edge_density(img)
100
+ log(f"{tag}: edge density left={l:.4f} right={r:.4f} ratio L/R={l/max(r,1e-9):.3f}")
101
+ return img
102
+
103
+ Image.fromarray(circle_control(256)).save(f"{OUT}/control_circle_left.png")
104
+ Image.fromarray(circle_control(768)).save(f"{OUT}/control_circle_right.png")
105
+
106
+ gen(None, 1.0, "base")
107
+ gen(circle_control(256), 1.0, "circle_left")
108
+ gen(circle_control(768), 1.0, "circle_right")
109
+
110
+ from huggingface_hub import HfApi
111
+
112
+ HfApi().upload_folder(
113
+ folder_path=OUT,
114
+ path_in_repo="assets/diag",
115
+ repo_id="akhaliq/MiniMax-H3-Image-Fun-Controlnet-Union-2.0",
116
+ repo_type="model",
117
+ commit_message="Spatial routing diagnostic: circle control left vs right",
118
+ )
119
+ log("DONE")
120
+
121
+
122
+ if __name__ == "__main__":
123
+ main()