jennyxue commited on
Commit
bd26bf3
·
verified ·
1 Parent(s): 9cdf574

6bit build of HiDream-O1-Image

Browse files
6bit/chat_template.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {
2
+ "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {%- if messages[0].content is string %}\n {{- messages[0].content }}\n {%- else %}\n {%- for content in messages[0].content %}\n {%- if 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].content is string %}\n {{- messages[0].content }}\n {%- else %}\n {%- for content in messages[0].content %}\n {%- if 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set image_count = namespace(value=0) %}\n{%- set video_count = namespace(value=0) %}\n{%- for message in messages %}\n {%- if message.role == \"user\" %}\n {{- '<|im_start|>' + message.role + '\\n' }}\n {%- if message.content is string %}\n {{- message.content }}\n {%- else %}\n {%- for content in message.content %}\n {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}\n <|vision_start|><|image_pad|><|vision_end|>\n {%- elif content.type == 'video' or 'video' in content %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}\n <|vision_start|><|video_pad|><|vision_end|>\n {%- elif 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' }}\n {%- if message.content is string %}\n {{- message.content }}\n {%- else %}\n {%- for content_item in message.content %}\n {%- if 'text' in content_item %}\n {{- content_item.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and message.content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {%- if message.content is string %}\n {{- message.content }}\n {%- else %}\n {%- for content in message.content %}\n {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}\n <|vision_start|><|image_pad|><|vision_end|>\n {%- elif content.type == 'video' or 'video' in content %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}\n <|vision_start|><|video_pad|><|vision_end|>\n {%- elif 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n"
3
+ }
6bit/config.json ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_type": "qwen3_vl",
3
+ "architecture": "hidream_o1_image",
4
+ "text_config": {
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 151643,
8
+ "dtype": "bfloat16",
9
+ "eos_token_id": 151645,
10
+ "head_dim": 128,
11
+ "hidden_act": "silu",
12
+ "hidden_size": 4096,
13
+ "initializer_range": 0.02,
14
+ "intermediate_size": 12288,
15
+ "max_position_embeddings": 262144,
16
+ "model_type": "qwen3_vl_text",
17
+ "num_attention_heads": 32,
18
+ "num_hidden_layers": 36,
19
+ "num_key_value_heads": 8,
20
+ "rms_norm_eps": 1e-06,
21
+ "rope_scaling": {
22
+ "mrope_interleaved": true,
23
+ "mrope_section": [
24
+ 24,
25
+ 20,
26
+ 20
27
+ ],
28
+ "rope_type": "default"
29
+ },
30
+ "rope_theta": 5000000,
31
+ "use_cache": true,
32
+ "vocab_size": 151936
33
+ },
34
+ "vision_config": {
35
+ "deepstack_visual_indexes": [
36
+ 8,
37
+ 16,
38
+ 24
39
+ ],
40
+ "depth": 27,
41
+ "hidden_act": "gelu_pytorch_tanh",
42
+ "hidden_size": 1152,
43
+ "in_channels": 3,
44
+ "initializer_range": 0.02,
45
+ "intermediate_size": 4304,
46
+ "model_type": "qwen3_vl",
47
+ "num_heads": 16,
48
+ "num_position_embeddings": 2304,
49
+ "out_hidden_size": 4096,
50
+ "patch_size": 16,
51
+ "spatial_merge_size": 2,
52
+ "temporal_patch_size": 2
53
+ },
54
+ "image_token_id": 151655,
55
+ "video_token_id": 151656,
56
+ "vision_start_token_id": 151652,
57
+ "tms_token_id": 151673,
58
+ "patch_size": 32,
59
+ "quantization": {
60
+ "group_size": 64,
61
+ "bits": 6,
62
+ "mode": "affine"
63
+ },
64
+ "precision": "6bit"
65
+ }
6bit/generation_config.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 151643,
3
+ "pad_token_id": 151643,
4
+ "do_sample": true,
5
+ "eos_token_id": [
6
+ 151645,
7
+ 151643
8
+ ],
9
+ "top_k": 20,
10
+ "top_p": 0.8,
11
+ "repetition_penalty": 1.0,
12
+ "temperature": 0.7,
13
+ "transformers_version": "4.56.0"
14
+ }
6bit/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
6bit/mlx_model.py ADDED
@@ -0,0 +1,596 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """HiDream-O1-Image (HiDream-ai, MIT) in MLX: text-to-image and instruction editing in pixel space.
2
+
3
+ The model is a Qwen3-VL-8B whose sequence ends with generation tokens: a timestep token (the `<|tms_token|>` slot,
4
+ filled with a timestep embedding) and the picture itself as 32x32x3 pixel patches (x_embedder in, final_layer2
5
+ out; no VAE). Text tokens attend causally among themselves; generation tokens attend to everything. So the text
6
+ prefix (with the Qwen3-VL vision tower's features of reference pictures, DeepStack included) is run once and its
7
+ keys / values are reused at every step: each step runs only the timestep token and the patches. The sampler is the
8
+ original's: FlowUniPCMultistepScheduler (bh2, order 2), 50 steps, shift 3, classifier-free guidance 5 against the
9
+ prompt " ", the starting noise scaled by 8 (models/pipeline.py, full model).
10
+
11
+ import mlx_model
12
+ model, processor = mlx_model.load("8bit")
13
+ img = mlx_model.generate(model, processor, "a red fox in the snow") # PIL image, 2048x2048
14
+ img = mlx_model.generate(model, processor, "make it night", ref_images=[Image.open("in.png")])
15
+
16
+ Needs `transformers` (tokenizer, chat template and Qwen3-VL image processor; the processor files sit in each folder).
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import json
22
+ import math
23
+ import os
24
+ from pathlib import Path
25
+
26
+ # The vision tower runs in float32 (see UNQUANTIZED); with MLX's default TF32 matmuls it drifts 2.5% from true float32
27
+ # over its 27 blocks. Read by MLX at the first float32 matmul, so set before any: only float32 matmuls are affected.
28
+ os.environ.setdefault("MLX_ENABLE_TF32", "0")
29
+
30
+ import mlx.core as mx
31
+ import mlx.nn as nn
32
+ import numpy as np
33
+ from mlx.utils import tree_map
34
+
35
+ GROUP = 64
36
+ PATCH = 32
37
+ TIMESTEP_TOKEN_NUM = 1
38
+ NOISE_SCALE = 8.0
39
+ T_EPS = 0.001
40
+ CONDITION_IMAGE_SIZE = 384
41
+ PREDEFINED_RESOLUTIONS = [(2048, 2048), (2304, 1728), (1728, 2304), (2560, 1440), (1440, 2560), (2496, 1664),
42
+ (1664, 2496), (3104, 1312), (1312, 3104), (2304, 1792), (1792, 2304)]
43
+ # Kept in 16-bit in every build: the pixel-space input / output layers and the timestep embedding (small, and every
44
+ # pixel goes through them), and the vision tower, which load() casts to float32: in bf16 its 27 blocks drift 12-15%
45
+ # from float32 (the original's bf16 run as much as this port's), and it runs once per reference picture.
46
+ UNQUANTIZED = ("x_embedder", "final_layer2", "t_embedder1", "visual")
47
+
48
+
49
+ def quantizable(path: str, module) -> bool:
50
+ if any(path.startswith(p) for p in UNQUANTIZED):
51
+ return False
52
+ if isinstance(module, nn.Embedding):
53
+ return True
54
+ return isinstance(module, nn.Linear) and module.weight.shape[-1] % GROUP == 0
55
+
56
+
57
+ def rotate_half(x):
58
+ h = x.shape[-1] // 2
59
+ return mx.concatenate([-x[..., h:], x[..., :h]], axis=-1)
60
+
61
+
62
+ # ------------------------------------------------------------------------------------------------ text model
63
+
64
+ class RMSNorm(nn.Module):
65
+ def __init__(self, dim, eps=1e-6):
66
+ super().__init__()
67
+ self.weight = mx.ones((dim,))
68
+ self.eps = eps
69
+
70
+ def __call__(self, x):
71
+ y = x.astype(mx.float32)
72
+ y = y * mx.rsqrt(mx.mean(y * y, axis=-1, keepdims=True) + self.eps)
73
+ return self.weight * y.astype(x.dtype)
74
+
75
+
76
+ class Attention(nn.Module):
77
+ def __init__(self, c):
78
+ super().__init__()
79
+ d, self.h, self.kv, self.hd = c["hidden_size"], c["num_attention_heads"], c["num_key_value_heads"], c["head_dim"]
80
+ self.q_proj = nn.Linear(d, self.h * self.hd, bias=False)
81
+ self.k_proj = nn.Linear(d, self.kv * self.hd, bias=False)
82
+ self.v_proj = nn.Linear(d, self.kv * self.hd, bias=False)
83
+ self.o_proj = nn.Linear(self.h * self.hd, d, bias=False)
84
+ self.q_norm = RMSNorm(self.hd, c["rms_norm_eps"])
85
+ self.k_norm = RMSNorm(self.hd, c["rms_norm_eps"])
86
+
87
+ def __call__(self, x, cos, sin, cache=None, causal=False):
88
+ b, n, _ = x.shape
89
+ q = self.q_norm(self.q_proj(x).reshape(b, n, self.h, self.hd)).transpose(0, 2, 1, 3)
90
+ k = self.k_norm(self.k_proj(x).reshape(b, n, self.kv, self.hd)).transpose(0, 2, 1, 3)
91
+ v = self.v_proj(x).reshape(b, n, self.kv, self.hd).transpose(0, 2, 1, 3)
92
+ q = q * cos + rotate_half(q) * sin
93
+ k = k * cos + rotate_half(k) * sin
94
+ kv = (k, v)
95
+ if cache is not None:
96
+ k, v = mx.concatenate([cache[0], k], axis=2), mx.concatenate([cache[1], v], axis=2)
97
+ y = mx.fast.scaled_dot_product_attention(q, k, v, scale=self.hd ** -0.5, mask="causal" if causal else None)
98
+ return self.o_proj(y.transpose(0, 2, 1, 3).reshape(b, n, -1)), kv
99
+
100
+
101
+ class MLP(nn.Module):
102
+ def __init__(self, d, f):
103
+ super().__init__()
104
+ self.gate_proj = nn.Linear(d, f, bias=False)
105
+ self.up_proj = nn.Linear(d, f, bias=False)
106
+ self.down_proj = nn.Linear(f, d, bias=False)
107
+
108
+ def __call__(self, x):
109
+ return self.down_proj(nn.silu(self.gate_proj(x)) * self.up_proj(x))
110
+
111
+
112
+ class DecoderLayer(nn.Module):
113
+ def __init__(self, c):
114
+ super().__init__()
115
+ self.self_attn = Attention(c)
116
+ self.mlp = MLP(c["hidden_size"], c["intermediate_size"])
117
+ self.input_layernorm = RMSNorm(c["hidden_size"], c["rms_norm_eps"])
118
+ self.post_attention_layernorm = RMSNorm(c["hidden_size"], c["rms_norm_eps"])
119
+
120
+ def __call__(self, x, cos, sin, cache=None, causal=False):
121
+ a, kv = self.self_attn(self.input_layernorm(x), cos, sin, cache, causal)
122
+ x = x + a
123
+ return x + self.mlp(self.post_attention_layernorm(x)), kv
124
+
125
+
126
+ class TextModel(nn.Module):
127
+ def __init__(self, c):
128
+ super().__init__()
129
+ self.embed_tokens = nn.Embedding(c["vocab_size"], c["hidden_size"])
130
+ self.layers = [DecoderLayer(c) for _ in range(c["num_hidden_layers"])]
131
+ self.norm = RMSNorm(c["hidden_size"], c["rms_norm_eps"])
132
+
133
+
134
+ def mrope_cos_sin(pos: np.ndarray, c: dict, dtype):
135
+ """pos: (3, n) T/H/W positions -> cos, sin (n, head_dim): interleaved MRoPE (Qwen3VLTextRotaryEmbedding)."""
136
+ hd, theta = c["head_dim"], c["rope_theta"]
137
+ sec = c["rope_scaling"]["mrope_section"]
138
+ inv = 1.0 / (theta ** (np.arange(0, hd, 2, dtype=np.int64).astype(np.float32) / hd))
139
+ f = pos.astype(np.float32)[:, :, None] * inv[None, None] # (3, n, hd/2), float32 like the original
140
+ ft = f[0].copy()
141
+ for dim, off in ((1, 1), (2, 2)):
142
+ idx = slice(off, sec[dim] * 3, 3)
143
+ ft[:, idx] = f[dim][:, idx]
144
+ emb = np.concatenate([ft, ft], axis=-1)
145
+ return mx.array(np.cos(emb)).astype(dtype), mx.array(np.sin(emb)).astype(dtype)
146
+
147
+
148
+ # ------------------------------------------------------------------------------------------------ vision tower
149
+
150
+ class VisionBlock(nn.Module):
151
+ def __init__(self, v):
152
+ super().__init__()
153
+ d = v["hidden_size"]
154
+ self.heads = v["num_heads"]
155
+ self.norm1 = nn.LayerNorm(d, eps=1e-6)
156
+ self.norm2 = nn.LayerNorm(d, eps=1e-6)
157
+ self.qkv = nn.Linear(d, 3 * d)
158
+ self.proj = nn.Linear(d, d)
159
+ self.linear_fc1 = nn.Linear(d, v["intermediate_size"])
160
+ self.linear_fc2 = nn.Linear(v["intermediate_size"], d)
161
+
162
+ def __call__(self, x, cos, sin, lengths):
163
+ n, d = x.shape
164
+ hd = d // self.heads
165
+ qkv = self.qkv(self.norm1(x)).reshape(n, 3, self.heads, hd)
166
+ q, k, v = qkv[:, 0], qkv[:, 1], qkv[:, 2]
167
+ qf, kf = q.astype(mx.float32), k.astype(mx.float32)
168
+ q = (qf * cos[:, None] + rotate_half(qf) * sin[:, None]).astype(x.dtype)
169
+ k = (kf * cos[:, None] + rotate_half(kf) * sin[:, None]).astype(x.dtype)
170
+ outs, s = [], 0
171
+ for L in lengths: # attention within each picture (cu_seqlens)
172
+ qi, ki, vi = (t[s:s + L].transpose(1, 0, 2)[None] for t in (q, k, v))
173
+ outs.append(mx.fast.scaled_dot_product_attention(qi, ki, vi, scale=hd ** -0.5)[0].transpose(1, 0, 2))
174
+ s += L
175
+ x = x + self.proj(mx.concatenate(outs, axis=0).reshape(n, d))
176
+ return x + self.linear_fc2(nn.gelu_approx(self.linear_fc1(self.norm2(x))))
177
+
178
+
179
+ class Merger(nn.Module):
180
+ def __init__(self, v, postshuffle):
181
+ super().__init__()
182
+ m = v["spatial_merge_size"] ** 2
183
+ self.width = v["hidden_size"] * m
184
+ self.postshuffle = postshuffle
185
+ self.norm = nn.LayerNorm(self.width if postshuffle else v["hidden_size"], eps=1e-6)
186
+ self.linear_fc1 = nn.Linear(self.width, self.width)
187
+ self.linear_fc2 = nn.Linear(self.width, v["out_hidden_size"])
188
+
189
+ def __call__(self, x):
190
+ x = self.norm(x.reshape(-1, self.width)) if self.postshuffle else self.norm(x).reshape(-1, self.width)
191
+ return self.linear_fc2(nn.gelu(self.linear_fc1(x)))
192
+
193
+
194
+ class VisionModel(nn.Module):
195
+ def __init__(self, v):
196
+ super().__init__()
197
+ self.v = v
198
+ p, t = v["patch_size"], v["temporal_patch_size"]
199
+ self.patch_embed = nn.Linear(v["in_channels"] * t * p * p, v["hidden_size"]) # the Conv3d, flattened
200
+ self.pos_embed = nn.Embedding(v["num_position_embeddings"], v["hidden_size"])
201
+ self.blocks = [VisionBlock(v) for _ in range(v["depth"])]
202
+ self.merger = Merger(v, False)
203
+ self.deepstack_merger_list = [Merger(v, True) for _ in v["deepstack_visual_indexes"]]
204
+
205
+ def _pos(self, grid):
206
+ side, m = int(self.v["num_position_embeddings"] ** 0.5), self.v["spatial_merge_size"]
207
+ idx, wts = [[] for _ in range(4)], [[] for _ in range(4)]
208
+ for t, h, w in grid:
209
+ hi = np.linspace(0, side - 1, h, dtype=np.float32)
210
+ wi = np.linspace(0, side - 1, w, dtype=np.float32)
211
+ hf, wf = hi.astype(np.int64), wi.astype(np.int64)
212
+ hc, wc = np.minimum(hf + 1, side - 1), np.minimum(wf + 1, side - 1)
213
+ dh, dw = hi - hf, wi - wf
214
+ for j, (a, b, wa, wb) in enumerate([(hf, wf, 1 - dh, 1 - dw), (hf, wc, 1 - dh, dw),
215
+ (hc, wf, dh, 1 - dw), (hc, wc, dh, dw)]):
216
+ ii = (a[:, None] * side + b[None]).reshape(-1)
217
+ ww = (wa[:, None] * wb[None]).reshape(-1)
218
+ order = np.arange(h * w).reshape(h // m, m, w // m, m).transpose(0, 2, 1, 3).reshape(-1)
219
+ idx[j].append(np.tile(ii[order], t))
220
+ wts[j].append(np.tile(ww[order], t))
221
+ dt = self.pos_embed.weight.dtype
222
+ out = 0
223
+ for j in range(4):
224
+ e = self.pos_embed(mx.array(np.concatenate(idx[j])))
225
+ out = out + e * mx.array(np.concatenate(wts[j]))[:, None].astype(dt)
226
+ return out
227
+
228
+ def _rope(self, grid):
229
+ hd = self.v["hidden_size"] // self.v["num_heads"]
230
+ dim = hd // 2
231
+ inv = 1.0 / (10000.0 ** (np.arange(0, dim, 2, dtype=np.float32) / dim))
232
+ m = self.v["spatial_merge_size"]
233
+ coords = []
234
+ for t, h, w in grid:
235
+ r = (np.arange(h // m)[:, None, None, None] * m + np.arange(m)[None, None, :, None])
236
+ c = (np.arange(w // m)[None, :, None, None] * m + np.arange(m)[None, None, None, :])
237
+ r = np.broadcast_to(r, (h // m, w // m, m, m)).reshape(-1)
238
+ c = np.broadcast_to(c, (h // m, w // m, m, m)).reshape(-1)
239
+ coords.append(np.tile(np.stack([r, c], -1), (t, 1)))
240
+ coords = np.concatenate(coords)
241
+ f = np.concatenate([coords[:, :1] * inv[None], coords[:, 1:] * inv[None]], axis=-1).astype(np.float32)
242
+ emb = np.concatenate([f, f], axis=-1)
243
+ return mx.array(np.cos(emb)), mx.array(np.sin(emb))
244
+
245
+ def __call__(self, pixels, grid):
246
+ """pixels: (N, 1536) from the Qwen3-VL image processor, grid: [(t, h, w)] -> (merged tokens, deepstack)."""
247
+ if self.patch_embed.weight.dtype != mx.float32: # float32 from the first use on (UNQUANTIZED)
248
+ self.update(tree_map(lambda a: a.astype(mx.float32), self.parameters()))
249
+ x = self.patch_embed(pixels.astype(self.patch_embed.weight.dtype)) + self._pos(grid)
250
+ cos, sin = self._rope(grid)
251
+ lengths = [t * h * w for t, h, w in grid]
252
+ ds = []
253
+ for i, blk in enumerate(self.blocks):
254
+ x = blk(x, cos, sin, lengths)
255
+ if i in self.v["deepstack_visual_indexes"]:
256
+ ds.append(self.deepstack_merger_list[self.v["deepstack_visual_indexes"].index(i)](x))
257
+ return self.merger(x), ds
258
+
259
+
260
+ # ------------------------------------------------------------------------------------------------ the generator
261
+
262
+ class TimestepEmbedder(nn.Module):
263
+ def __init__(self, d, freq=256):
264
+ super().__init__()
265
+ self.freq = freq
266
+ self.fc1 = nn.Linear(freq, d)
267
+ self.fc2 = nn.Linear(d, d)
268
+
269
+ def __call__(self, t):
270
+ half = self.freq // 2
271
+ freqs = mx.exp(-math.log(10000) * mx.arange(half, dtype=mx.float32) / half)
272
+ a = (t.astype(mx.float32) * 1000)[:, None] * freqs[None]
273
+ e = mx.concatenate([mx.cos(a), mx.sin(a)], axis=-1).astype(self.fc1.weight.dtype)
274
+ return self.fc2(nn.silu(self.fc1(e)))
275
+
276
+
277
+ class Model(nn.Module):
278
+ def __init__(self, config):
279
+ super().__init__()
280
+ self.config = config
281
+ c, v = config["text_config"], config["vision_config"]
282
+ d = c["hidden_size"]
283
+ self.visual = VisionModel(v)
284
+ self.language_model = TextModel(c)
285
+ self.t_embedder1 = TimestepEmbedder(d)
286
+ self.x_embedder_proj1 = nn.Linear(PATCH * PATCH * 3, d // 4, bias=False)
287
+ self.x_embedder_proj2 = nn.Linear(d // 4, d)
288
+ self.final_layer2 = nn.Linear(d, PATCH * PATCH * 3)
289
+
290
+ def prefix(self, ids, pos, image_embeds=None, deepstack=None):
291
+ """The causal text part (everything before the timestep token) -> per-layer keys / values."""
292
+ lm, c = self.language_model, self.config["text_config"]
293
+ x = lm.embed_tokens(mx.array(np.asarray(ids, dtype=np.int32))[None])
294
+ vis = np.nonzero(np.asarray(ids) == self.config["image_token_id"])[0]
295
+ if image_embeds is not None:
296
+ x[:, mx.array(vis)] = image_embeds.astype(x.dtype)[None]
297
+ cos, sin = mrope_cos_sin(pos, c, x.dtype)
298
+ caches = []
299
+ for i, layer in enumerate(lm.layers):
300
+ x, kv = layer(x, cos, sin, causal=True)
301
+ caches.append(kv)
302
+ if deepstack is not None and i < len(deepstack):
303
+ x[:, mx.array(vis)] = x[:, mx.array(vis)] + deepstack[i].astype(x.dtype)[None]
304
+ mx.eval(x, kv)
305
+ return caches
306
+
307
+ def step(self, caches, pos, t, patches):
308
+ """One pass of the generation tokens: [timestep token] + pixel patches (target, then references) ->
309
+ predicted clean patches for every patch token, float32."""
310
+ lm, c = self.language_model, self.config["text_config"]
311
+ dt = self.x_embedder_proj2.weight.dtype
312
+ te = self.t_embedder1(mx.array([t], dtype=mx.float32))[:, None]
313
+ pe = self.x_embedder_proj2(self.x_embedder_proj1(patches.astype(dt)))
314
+ x = mx.concatenate([te.astype(dt), pe], axis=1)
315
+ cos, sin = mrope_cos_sin(pos, c, dt)
316
+ for layer, kv in zip(lm.layers, caches):
317
+ x, _ = layer(x, cos, sin, cache=kv)
318
+ return self.final_layer2(lm.norm(x[:, 1:])).astype(mx.float32)
319
+
320
+
321
+ def load(folder):
322
+ """-> (model, processor)."""
323
+ from transformers import AutoProcessor
324
+ folder = Path(folder)
325
+ config = json.loads((folder / "config.json").read_text())
326
+ model = Model(config)
327
+ q = config.get("quantization")
328
+ if q:
329
+ nn.quantize(model, group_size=q["group_size"], bits=q["bits"], class_predicate=quantizable)
330
+ model.load_weights(str(folder / "model.safetensors"), strict=True)
331
+ mx.eval(model.parameters())
332
+ return model, AutoProcessor.from_pretrained(str(folder))
333
+
334
+
335
+ # ------------------------------------------------------------------------------------------------ positions
336
+
337
+ def rope_index(spatial_merge_size, image_token_id, video_token_id, vision_start_token_id, input_ids, image_grid_thw,
338
+ skip_vision_start_token, fix_point=4096):
339
+ """models/utils.py get_rope_index_fix_point for one sequence (numpy) -> (3, n) positions."""
340
+ ids = list(int(i) for i in input_ids)
341
+ arr = np.asarray(ids)
342
+ starts = np.nonzero(arr == vision_start_token_id)[0]
343
+ image_nums = int(np.sum(arr[starts + 1] == image_token_id))
344
+ out, st, image_index = [], 0, 0
345
+ for _ in range(image_nums):
346
+ ed = ids.index(image_token_id, st)
347
+ t, h, w = (int(x) for x in image_grid_thw[image_index])
348
+ image_index += 1
349
+ gt, gh, gw = t, h // spatial_merge_size, w // spatial_merge_size
350
+ text_len = max(0, ed - st - skip_vision_start_token[image_index - 1])
351
+ st_idx = out[-1].max() + 1 if out else 0
352
+ out.append(np.broadcast_to(np.arange(text_len)[None], (3, text_len)) + st_idx)
353
+ ti = np.repeat(np.arange(gt), gh * gw)
354
+ hi = np.tile(np.repeat(np.arange(gh), gw), gt)
355
+ wi = np.tile(np.arange(gw), gt * gh)
356
+ grid = np.stack([ti, hi, wi])
357
+ if skip_vision_start_token[image_index - 1]:
358
+ if fix_point > 0:
359
+ fix_point = fix_point - st_idx
360
+ out.append(grid + fix_point + st_idx)
361
+ fix_point = 0
362
+ else:
363
+ out.append(grid + text_len + st_idx)
364
+ st = ed + gt * gh * gw
365
+ if st < len(ids):
366
+ st_idx = out[-1].max() + 1 if out else 0
367
+ n = len(ids) - st
368
+ out.append(np.broadcast_to(np.arange(n)[None], (3, n)) + st_idx)
369
+ return np.concatenate(out, axis=1)
370
+
371
+
372
+ # ------------------------------------------------------------------------------------------------ sampler
373
+
374
+ class UniPC:
375
+ """FlowUniPCMultistepScheduler (models/fm_solvers_unipc.py) with the pipeline's settings: solver_order 2,
376
+ flow_prediction, predict_x0, bh2, lower_order_final, final sigma 0. numpy float32."""
377
+
378
+ def __init__(self, steps, shift):
379
+ alphas = np.linspace(1, 1 / 1000, 1000)[::-1].copy()
380
+ s = (1.0 - alphas).astype(np.float32)
381
+ s = shift * s / (1 + (shift - 1) * s)
382
+ smax, smin = float(s[0]), float(s[-1])
383
+ sig = np.linspace(smax, smin, steps + 1).copy()[:-1]
384
+ sig = shift * sig / (1 + (shift - 1) * sig)
385
+ self.timesteps = (sig * 1000).astype(np.int64)
386
+ self.sigmas = np.concatenate([sig, [0]]).astype(np.float32)
387
+ self.n = len(self.timesteps)
388
+ self.outputs, self.lower, self.last, self.i, self.order = [None, None], 0, None, 0, 1
389
+
390
+ def _lam(self, s):
391
+ s = np.float32(s)
392
+ return np.log(np.float32(1) - s) - np.log(s)
393
+
394
+ def _coeffs(self, h, order, rks):
395
+ hh = -h
396
+ h_phi_1 = np.expm1(hh)
397
+ h_phi_k = h_phi_1 / hh - 1
398
+ B_h = np.expm1(hh)
399
+ R, b, fact = [], [], 1
400
+ for i in range(1, order + 1):
401
+ R.append(np.power(rks, i - 1))
402
+ b.append(h_phi_k * fact / B_h)
403
+ fact *= i + 1
404
+ h_phi_k = h_phi_k / hh - 1 / fact
405
+ return h_phi_1, B_h, np.stack(R).astype(np.float32), np.array(b, np.float32)
406
+
407
+ def step(self, model_output, sample):
408
+ sig = self.sigmas
409
+ x0 = sample - sig[self.i] * model_output
410
+ if self.i > 0 and self.last is not None: # UniC corrector
411
+ m0, x = self.outputs[-1], self.last
412
+ st, s0 = sig[self.i], sig[self.i - 1]
413
+ h = self._lam(st) - self._lam(s0)
414
+ rks, D1s = [], []
415
+ for k in range(1, self.order):
416
+ rk = (self._lam(sig[self.i - (k + 1)]) - self._lam(s0)) / h
417
+ rks.append(rk); D1s.append((self.outputs[-(k + 1)] - m0) / rk)
418
+ rks.append(1.0)
419
+ h_phi_1, B_h, R, b = self._coeffs(h, self.order, np.array(rks, np.float32))
420
+ rhos = np.array([0.5], np.float32) if self.order == 1 else np.linalg.solve(R, b).astype(np.float32)
421
+ xt_ = st / s0 * x - (1 - st) * h_phi_1 * m0
422
+ corr = sum(r * d for r, d in zip(rhos[:-1], D1s)) if D1s else 0
423
+ sample = (xt_ - (1 - st) * B_h * (corr + rhos[-1] * (x0 - m0))).astype(np.float32)
424
+ self.outputs = [self.outputs[1], x0]
425
+ order = min(2, self.n - self.i)
426
+ self.order = min(order, self.lower + 1)
427
+ self.last = sample
428
+ m0 = self.outputs[-1] # UniP predictor
429
+ st, s0 = sig[self.i + 1], sig[self.i]
430
+ if st == 0:
431
+ prev = m0.copy() # the original's limit at sigma 0: h = inf, h_phi_1 = B_h = -1, order 1
432
+ else:
433
+ h = self._lam(st) - self._lam(s0)
434
+ rks, D1s = [], []
435
+ for k in range(1, self.order):
436
+ rk = (self._lam(sig[self.i - k]) - self._lam(s0)) / h
437
+ rks.append(rk); D1s.append((self.outputs[-(k + 1)] - m0) / rk)
438
+ rks.append(1.0)
439
+ h_phi_1, B_h, R, b = self._coeffs(h, self.order, np.array(rks, np.float32))
440
+ xt_ = st / s0 * sample - (1 - st) * h_phi_1 * m0
441
+ pred = 0.5 * D1s[0] if D1s else 0
442
+ prev = xt_ - (1 - st) * B_h * pred
443
+ self.lower = min(self.lower + 1, 2)
444
+ self.i += 1
445
+ return prev.astype(np.float32)
446
+
447
+
448
+ # ------------------------------------------------------------------------------------------------ pipeline
449
+
450
+ def _bf16(x: np.ndarray) -> np.ndarray:
451
+ return np.array(mx.array(x).astype(mx.bfloat16).astype(mx.float32))
452
+
453
+
454
+ def to_patches(img: np.ndarray) -> np.ndarray:
455
+ """(H, W, 3) float in [-1, 1] -> (H/32 * W/32, 3*32*32) in the original's (C p1 p2) order."""
456
+ h, w, _ = img.shape
457
+ x = img.reshape(h // PATCH, PATCH, w // PATCH, PATCH, 3).transpose(0, 2, 4, 1, 3)
458
+ return x.reshape(-1, 3 * PATCH * PATCH)
459
+
460
+
461
+ def from_patches(x: np.ndarray, h: int, w: int) -> np.ndarray:
462
+ x = x.reshape(h // PATCH, w // PATCH, 3, PATCH, PATCH).transpose(0, 3, 1, 4, 2)
463
+ return x.reshape(h, w, 3)
464
+
465
+
466
+ def resize_pilimage(pil, image_size, patch_size=16):
467
+ from PIL import Image
468
+ while min(*pil.size) >= 2 * image_size:
469
+ pil = pil.resize(tuple(x // 2 for x in pil.size), resample=Image.BOX)
470
+ m, (width, height) = patch_size, pil.size
471
+ S_max = image_size * image_size
472
+ scale = math.sqrt(S_max / (width * height))
473
+ sizes = sorted([(round(width * scale) // m * m, round(height * scale) // m * m),
474
+ (round(width * scale) // m * m, math.floor(height * scale) // m * m),
475
+ (math.floor(width * scale) // m * m, round(height * scale) // m * m),
476
+ (math.floor(width * scale) // m * m, math.floor(height * scale) // m * m)],
477
+ key=lambda x: x[0] * x[1], reverse=True)
478
+ for new in sizes:
479
+ if new[0] * new[1] <= S_max:
480
+ break
481
+ s1, s2 = width / new[0], height / new[1]
482
+ if s1 < s2:
483
+ pil = pil.resize([new[0], round(height / s1)], resample=Image.BICUBIC)
484
+ top = (round(height / s1) - new[1]) // 2
485
+ return pil.crop((0, top, new[0], top + new[1]))
486
+ pil = pil.resize([round(width / s2), new[1]], resample=Image.BICUBIC)
487
+ left = (round(width / s2) - new[0]) // 2
488
+ return pil.crop((left, 0, left + new[0], new[1]))
489
+
490
+
491
+ def _samples(model, processor, prompt, refs, height, width, guidance):
492
+ """The pipeline's cond / uncond samples: (prefix ids, prefix positions, generation positions, vision inputs)."""
493
+ from PIL import Image
494
+ cfg = model.config
495
+ tok = processor.tokenizer
496
+ img_id, vid_id, vs_id = cfg["image_token_id"], cfg["video_token_id"], cfg["vision_start_token_id"]
497
+ tgt_len = (height // PATCH) * (width // PATCH)
498
+ tail = "<|boi_token|>" + "<|tms_token|>" * TIMESTEP_TOKEN_NUM
499
+ captions = [prompt] + ([" "] if guidance > 1.0 else [])
500
+ out, ref_patches, vlm = [], None, None
501
+ if refs:
502
+ K = len(refs)
503
+ mx_ = max(height, width)
504
+ max_size = (mx_ if K == 1 else mx_ * 48 // 64 if K == 2 else mx_ // 2 if K <= 4 else mx_ * 24 // 64 if K <= 8
505
+ else mx_ // 4)
506
+ rs = [resize_pilimage(p.convert("RGB"), max_size, PATCH) for p in refs]
507
+ ref_patches = np.concatenate([to_patches(np.asarray(r, np.float32) / 255 * 2 - 1) for r in rs])
508
+ ref_lens = [(r.height // PATCH) * (r.width // PATCH) for r in rs]
509
+ cs = CONDITION_IMAGE_SIZE if K <= 4 else CONDITION_IMAGE_SIZE * 48 // 64 if K <= 8 else CONDITION_IMAGE_SIZE // 2
510
+ vlm = []
511
+ for r in rs:
512
+ wq = math.sqrt(cs * cs * r.width / r.height); hq = wq / (r.width / r.height)
513
+ vlm.append(r.resize((int(wq / 32) * 32, int(hq / 32) * 32), resample=Image.LANCZOS))
514
+ tgt_grid = [[1, height // PATCH, width // PATCH]]
515
+ ref_grid = [[1, r.height // PATCH, r.width // PATCH] for r in rs]
516
+ for cap in captions:
517
+ if not refs:
518
+ text = processor.apply_chat_template([{"role": "user", "content": cap}], tokenize=False,
519
+ add_generation_prompt=True) + tail
520
+ ids = tok.encode(text, add_special_tokens=False)
521
+ pad = ids + [vs_id] + [img_id] * (tgt_len - 1)
522
+ pos = rope_index(1, img_id, vid_id, vs_id, pad, [[1, height // PATCH, width // PATCH]], [1])
523
+ vision = None
524
+ else:
525
+ content = [{"type": "image"} for _ in refs] + [{"type": "text", "text": cap}]
526
+ text = processor.apply_chat_template([{"role": "user", "content": content}], tokenize=False,
527
+ add_generation_prompt=True)
528
+ proc = processor(text=[text], images=vlm, padding="longest", return_tensors="np")
529
+ ids = list(proc["input_ids"][0]) + tok.encode(tail, add_special_tokens=False)
530
+ g = np.array(proc["image_grid_thw"]).copy()
531
+ gcond = g.copy(); gcond[:, 1:] //= cfg["vision_config"]["spatial_merge_size"]
532
+ pad = list(ids)
533
+ for L in [tgt_len] + ref_lens:
534
+ pad += [vs_id] + [img_id] * (L - 1)
535
+ pos = rope_index(1, img_id, vid_id, vs_id, pad, list(gcond) + tgt_grid + ref_grid,
536
+ [0] * len(refs) + [1] + [1] * len(refs))
537
+ vision = (np.array(proc["pixel_values"], np.float32), [tuple(int(v) for v in r) for r in g])
538
+ n = len(ids) - TIMESTEP_TOKEN_NUM
539
+ out.append((ids[:n], pos[:, :n], pos[:, n:], vision))
540
+ return out, ref_patches, tgt_len
541
+
542
+
543
+ def generate(model, processor, prompt: str, ref_images=(), width: int = 2048, height: int = 2048, steps: int = 50,
544
+ guidance: float = 5.0, shift: float = 3.0, seed: int = 32, noise_scale: float = NOISE_SCALE,
545
+ noise: np.ndarray | None = None, snap: bool = True, callback=None):
546
+ """-> PIL image. `noise`: (height, width, 3) standard normal to fix the start (default: MLX's, from `seed`).
547
+ `snap` keeps the original's resolution snapping (to its 2048-class sizes) for text-to-image."""
548
+ from PIL import Image
549
+ if snap and not ref_images:
550
+ width, height = min(PREDEFINED_RESOLUTIONS, key=lambda r: abs(r[0] / r[1] - width / height))
551
+ width, height = width // PATCH * PATCH, height // PATCH * PATCH
552
+ samples, ref_patches, tgt_len = _samples(model, processor, prompt, list(ref_images), height, width, guidance)
553
+ caches, gen_pos = [], []
554
+ for ids, ppos, gpos, vision in samples:
555
+ emb = ds = None
556
+ if vision is not None:
557
+ emb, ds = model.visual(mx.array(vision[0]), vision[1])
558
+ caches.append(model.prefix(ids, ppos, emb, ds))
559
+ gen_pos.append(gpos)
560
+ if noise is None:
561
+ noise = np.array(mx.random.normal((height, width, 3), key=mx.random.key(seed + 1)))
562
+ z = _bf16(to_patches(noise_scale * noise.astype(np.float32)))
563
+ refp = None if ref_patches is None else mx.array(ref_patches).astype(mx.bfloat16)
564
+ sched = UniPC(steps, shift)
565
+ for k, step_t in enumerate(sched.timesteps):
566
+ t = 1.0 - float(step_t) / 1000.0
567
+ sigma = max(np.float32(step_t) / np.float32(1000.0), np.float32(T_EPS))
568
+ zin = mx.array(z)[None].astype(mx.bfloat16)
569
+ if refp is not None:
570
+ zin = mx.concatenate([zin, refp[None]], axis=1)
571
+ vs = []
572
+ for cache, gpos in zip(caches, gen_pos):
573
+ xp = model.step(cache, gpos, t, zin)[0, :tgt_len]
574
+ vs.append((np.array(xp) - z) / sigma)
575
+ v = vs[0] if len(vs) == 1 else vs[1] + guidance * (vs[0] - vs[1])
576
+ z = _bf16(sched.step(-v, z))
577
+ if callback is not None:
578
+ callback(k, sched.n)
579
+ img = from_patches((z + 1) / 2, height, width)
580
+ return Image.fromarray(np.round(np.clip(img * 255, 0, 255)).astype(np.uint8))
581
+
582
+
583
+ # ------------------------------------------------------------------------------------------------ conversion
584
+
585
+ def rename(k: str) -> str | None:
586
+ """Hub name -> this file's name (None: not used)."""
587
+ if k == "lm_head.weight":
588
+ return None
589
+ k = k.removeprefix("model.")
590
+ k = k.replace("visual.blocks.", "visual.blocks.").replace(".attn.qkv.", ".qkv.").replace(".attn.proj.", ".proj.")
591
+ k = k.replace(".mlp.linear_fc1.", ".linear_fc1.").replace(".mlp.linear_fc2.", ".linear_fc2.")
592
+ k = k.replace("visual.patch_embed.proj.", "visual.patch_embed.")
593
+ k = k.replace("t_embedder1.mlp.0.", "t_embedder1.fc1.").replace("t_embedder1.mlp.2.", "t_embedder1.fc2.")
594
+ k = k.replace("x_embedder.proj1.", "x_embedder_proj1.").replace("x_embedder.proj2.", "x_embedder_proj2.")
595
+ k = k.replace("final_layer2.linear.", "final_layer2.")
596
+ return k
6bit/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:907a9251345de34f9de791c34250f35b820ef26ce408d8450327c1dc533a845f
3
+ size 7378149426
6bit/preprocessor_config.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "size": {
3
+ "longest_edge": 16777216,
4
+ "shortest_edge": 65536
5
+ },
6
+ "patch_size": 16,
7
+ "temporal_patch_size": 2,
8
+ "merge_size": 2,
9
+ "image_mean": [
10
+ 0.5,
11
+ 0.5,
12
+ 0.5
13
+ ],
14
+ "image_std": [
15
+ 0.5,
16
+ 0.5,
17
+ 0.5
18
+ ],
19
+ "processor_class": "Qwen3VLProcessor",
20
+ "image_processor_type": "Qwen2VLImageProcessorFast"
21
+ }
6bit/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
6bit/tokenizer_config.json ADDED
@@ -0,0 +1,244 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_prefix_space": false,
4
+ "added_tokens_decoder": {
5
+ "151643": {
6
+ "content": "<|endoftext|>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "151644": {
14
+ "content": "<|im_start|>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "151645": {
22
+ "content": "<|im_end|>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ },
29
+ "151646": {
30
+ "content": "<|object_ref_start|>",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
+ },
37
+ "151647": {
38
+ "content": "<|object_ref_end|>",
39
+ "lstrip": false,
40
+ "normalized": false,
41
+ "rstrip": false,
42
+ "single_word": false,
43
+ "special": true
44
+ },
45
+ "151648": {
46
+ "content": "<|box_start|>",
47
+ "lstrip": false,
48
+ "normalized": false,
49
+ "rstrip": false,
50
+ "single_word": false,
51
+ "special": true
52
+ },
53
+ "151649": {
54
+ "content": "<|box_end|>",
55
+ "lstrip": false,
56
+ "normalized": false,
57
+ "rstrip": false,
58
+ "single_word": false,
59
+ "special": true
60
+ },
61
+ "151650": {
62
+ "content": "<|quad_start|>",
63
+ "lstrip": false,
64
+ "normalized": false,
65
+ "rstrip": false,
66
+ "single_word": false,
67
+ "special": true
68
+ },
69
+ "151651": {
70
+ "content": "<|quad_end|>",
71
+ "lstrip": false,
72
+ "normalized": false,
73
+ "rstrip": false,
74
+ "single_word": false,
75
+ "special": true
76
+ },
77
+ "151652": {
78
+ "content": "<|vision_start|>",
79
+ "lstrip": false,
80
+ "normalized": false,
81
+ "rstrip": false,
82
+ "single_word": false,
83
+ "special": true
84
+ },
85
+ "151653": {
86
+ "content": "<|vision_end|>",
87
+ "lstrip": false,
88
+ "normalized": false,
89
+ "rstrip": false,
90
+ "single_word": false,
91
+ "special": true
92
+ },
93
+ "151654": {
94
+ "content": "<|vision_pad|>",
95
+ "lstrip": false,
96
+ "normalized": false,
97
+ "rstrip": false,
98
+ "single_word": false,
99
+ "special": true
100
+ },
101
+ "151655": {
102
+ "content": "<|image_pad|>",
103
+ "lstrip": false,
104
+ "normalized": false,
105
+ "rstrip": false,
106
+ "single_word": false,
107
+ "special": true
108
+ },
109
+ "151656": {
110
+ "content": "<|video_pad|>",
111
+ "lstrip": false,
112
+ "normalized": false,
113
+ "rstrip": false,
114
+ "single_word": false,
115
+ "special": true
116
+ },
117
+ "151657": {
118
+ "content": "<tool_call>",
119
+ "lstrip": false,
120
+ "normalized": false,
121
+ "rstrip": false,
122
+ "single_word": false,
123
+ "special": false
124
+ },
125
+ "151658": {
126
+ "content": "</tool_call>",
127
+ "lstrip": false,
128
+ "normalized": false,
129
+ "rstrip": false,
130
+ "single_word": false,
131
+ "special": false
132
+ },
133
+ "151659": {
134
+ "content": "<|fim_prefix|>",
135
+ "lstrip": false,
136
+ "normalized": false,
137
+ "rstrip": false,
138
+ "single_word": false,
139
+ "special": false
140
+ },
141
+ "151660": {
142
+ "content": "<|fim_middle|>",
143
+ "lstrip": false,
144
+ "normalized": false,
145
+ "rstrip": false,
146
+ "single_word": false,
147
+ "special": false
148
+ },
149
+ "151661": {
150
+ "content": "<|fim_suffix|>",
151
+ "lstrip": false,
152
+ "normalized": false,
153
+ "rstrip": false,
154
+ "single_word": false,
155
+ "special": false
156
+ },
157
+ "151662": {
158
+ "content": "<|fim_pad|>",
159
+ "lstrip": false,
160
+ "normalized": false,
161
+ "rstrip": false,
162
+ "single_word": false,
163
+ "special": false
164
+ },
165
+ "151663": {
166
+ "content": "<|repo_name|>",
167
+ "lstrip": false,
168
+ "normalized": false,
169
+ "rstrip": false,
170
+ "single_word": false,
171
+ "special": false
172
+ },
173
+ "151664": {
174
+ "content": "<|file_sep|>",
175
+ "lstrip": false,
176
+ "normalized": false,
177
+ "rstrip": false,
178
+ "single_word": false,
179
+ "special": false
180
+ },
181
+ "151665": {
182
+ "content": "<tool_response>",
183
+ "lstrip": false,
184
+ "normalized": false,
185
+ "rstrip": false,
186
+ "single_word": false,
187
+ "special": false
188
+ },
189
+ "151666": {
190
+ "content": "</tool_response>",
191
+ "lstrip": false,
192
+ "normalized": false,
193
+ "rstrip": false,
194
+ "single_word": false,
195
+ "special": false
196
+ },
197
+ "151667": {
198
+ "content": "<think>",
199
+ "lstrip": false,
200
+ "normalized": false,
201
+ "rstrip": false,
202
+ "single_word": false,
203
+ "special": false
204
+ },
205
+ "151668": {
206
+ "content": "</think>",
207
+ "lstrip": false,
208
+ "normalized": false,
209
+ "rstrip": false,
210
+ "single_word": false,
211
+ "special": false
212
+ }
213
+ },
214
+ "additional_special_tokens": [
215
+ "<|im_start|>",
216
+ "<|im_end|>",
217
+ "<|object_ref_start|>",
218
+ "<|object_ref_end|>",
219
+ "<|box_start|>",
220
+ "<|box_end|>",
221
+ "<|quad_start|>",
222
+ "<|quad_end|>",
223
+ "<|vision_start|>",
224
+ "<|vision_end|>",
225
+ "<|vision_pad|>",
226
+ "<|image_pad|>",
227
+ "<|video_pad|>",
228
+ "<|boi_token|>",
229
+ "<|bor_token|>",
230
+ "<|eor_token|>",
231
+ "<|bot_token|>",
232
+ "<|tms_token|>"
233
+ ],
234
+ "bos_token": null,
235
+ "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {%- if messages[0].content is string %}\n {{- messages[0].content }}\n {%- else %}\n {%- for content in messages[0].content %}\n {%- if 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].content is string %}\n {{- messages[0].content }}\n {%- else %}\n {%- for content in messages[0].content %}\n {%- if 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set image_count = namespace(value=0) %}\n{%- set video_count = namespace(value=0) %}\n{%- for message in messages %}\n {%- if message.role == \"user\" %}\n {{- '<|im_start|>' + message.role + '\\n' }}\n {%- if message.content is string %}\n {{- message.content }}\n {%- else %}\n {%- for content in message.content %}\n {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}\n <|vision_start|><|image_pad|><|vision_end|>\n {%- elif content.type == 'video' or 'video' in content %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}\n <|vision_start|><|video_pad|><|vision_end|>\n {%- elif 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' }}\n {%- if message.content is string %}\n {{- message.content }}\n {%- else %}\n {%- for content_item in message.content %}\n {%- if 'text' in content_item %}\n {{- content_item.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and message.content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {%- if message.content is string %}\n {{- message.content }}\n {%- else %}\n {%- for content in message.content %}\n {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}\n <|vision_start|><|image_pad|><|vision_end|>\n {%- elif content.type == 'video' or 'video' in content %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}\n <|vision_start|><|video_pad|><|vision_end|>\n {%- elif 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
236
+ "clean_up_tokenization_spaces": false,
237
+ "eos_token": "<|im_end|>",
238
+ "errors": "replace",
239
+ "model_max_length": 262144,
240
+ "pad_token": "<|endoftext|>",
241
+ "split_special_tokens": false,
242
+ "tokenizer_class": "Qwen2Tokenizer",
243
+ "unk_token": null
244
+ }
6bit/video_preprocessor_config.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "size": {
3
+ "longest_edge": 25165824,
4
+ "shortest_edge": 4096
5
+ },
6
+ "patch_size": 16,
7
+ "temporal_patch_size": 2,
8
+ "merge_size": 2,
9
+ "image_mean": [
10
+ 0.5,
11
+ 0.5,
12
+ 0.5
13
+ ],
14
+ "image_std": [
15
+ 0.5,
16
+ 0.5,
17
+ 0.5
18
+ ],
19
+ "processor_class": "Qwen3VLProcessor",
20
+ "video_processor_type": "Qwen3VLVideoProcessor"
21
+ }
6bit/vocab.json ADDED
The diff for this file is too large to render. See raw diff