Hugging Face's logo

baseten
/
GLM-5.2-Vision-FP8

Image-Text-to-Text
Safetensors
sglang
glm5v
multimodal
vision-language
glm
conversational
custom_code
fp8
Model card Files
xet
 Community
3
harrypart commited on
Commit
7ca59e2
·
verified ·
1 Parent(s): 697ef33

Add files using upload-large-folder tool

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. ._configuration_glm5v.py +0 -0
  2. ._kimi_k25_processor.py +0 -0
  3. .gitattributes +2 -0
  4. chat_template.jinja +121 -0
  5. config.json +1370 -0
  6. configuration_glm5v.py +152 -0
  7. generation_config.json +12 -0
  8. kimi_k25_processor.py +208 -0
  9. kimi_k25_vision_processing.py +251 -0
  10. media_utils.py +368 -0
  11. mm_projector.safetensors +3 -0
  12. model-00001-of-00141.safetensors +3 -0
  13. model-00002-of-00141.safetensors +3 -0
  14. model-00003-of-00141.safetensors +3 -0
  15. model-00004-of-00141.safetensors +3 -0
  16. model-00005-of-00141.safetensors +3 -0
  17. model-00006-of-00141.safetensors +3 -0
  18. model-00007-of-00141.safetensors +3 -0
  19. model-00008-of-00141.safetensors +3 -0
  20. model-00009-of-00141.safetensors +3 -0
  21. model-00010-of-00141.safetensors +3 -0
  22. model-00011-of-00141.safetensors +3 -0
  23. model-00012-of-00141.safetensors +3 -0
  24. model-00013-of-00141.safetensors +3 -0
  25. model-00014-of-00141.safetensors +3 -0
  26. model-00015-of-00141.safetensors +3 -0
  27. model-00016-of-00141.safetensors +3 -0
  28. model-00017-of-00141.safetensors +3 -0
  29. model-00018-of-00141.safetensors +3 -0
  30. model-00019-of-00141.safetensors +3 -0
  31. model-00020-of-00141.safetensors +3 -0
  32. model-00021-of-00141.safetensors +3 -0
  33. model-00022-of-00141.safetensors +3 -0
  34. model-00023-of-00141.safetensors +3 -0
  35. model-00024-of-00141.safetensors +3 -0
  36. model-00025-of-00141.safetensors +3 -0
  37. model-00026-of-00141.safetensors +3 -0
  38. model-00027-of-00141.safetensors +3 -0
  39. model-00028-of-00141.safetensors +3 -0
  40. model-00029-of-00141.safetensors +3 -0
  41. model-00030-of-00141.safetensors +3 -0
  42. model-00031-of-00141.safetensors +3 -0
  43. model-00032-of-00141.safetensors +3 -0
  44. model-00033-of-00141.safetensors +3 -0
  45. model-00034-of-00141.safetensors +3 -0
  46. model-00141-of-00141.safetensors +3 -0
  47. model.safetensors.index.json +3 -0
  48. preprocessor_config.json +30 -0
  49. tokenizer.json +3 -0
  50. tokenizer_config.json +33 -0
._configuration_glm5v.py ADDED
Binary file (163 Bytes). View file
 
._kimi_k25_processor.py ADDED
Binary file (163 Bytes). View file
 
.gitattributes CHANGED
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ model.safetensors.index.json filter=lfs diff=lfs merge=lfs -text
37
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
chat_template.jinja ADDED
@@ -0,0 +1,121 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [gMASK]<sop>
2
+ {%- set effective_reasoning_effort = 'high' if reasoning_effort is defined and reasoning_effort == 'high' else 'max' -%}
3
+ {%- if (enable_thinking is not defined or enable_thinking) and effective_reasoning_effort is not none -%}<|system|>Reasoning Effort: {{ effective_reasoning_effort | capitalize }}{%- endif -%}
4
+ {%- if tools -%}
5
+ {%- macro tool_to_json(tool) -%}
6
+ {%- set ns_tool = namespace(first=true) -%}
7
+ {{ '{' -}}
8
+ {%- for k, v in tool.items() -%}
9
+ {%- if k != 'defer_loading' and k != 'strict' -%}
10
+ {%- if not ns_tool.first -%}{{- ', ' -}}{%- endif -%}
11
+ {%- set ns_tool.first = false -%}
12
+ "{{ k }}": {{ v | tojson(ensure_ascii=False) }}
13
+ {%- endif -%}
14
+ {%- endfor -%}
15
+ {{- '}' -}}
16
+ {%- endmacro -%}
17
+ <|system|>
18
+ # Tools
19
+
20
+ You may call one or more functions to assist with the user query.
21
+
22
+ You are provided with function signatures within <tools></tools> XML tags:
23
+ <tools>
24
+ {% for tool in tools %}
25
+ {%- if 'function' in tool -%}
26
+ {%- set tool = tool['function'] -%}
27
+ {%- endif -%}
28
+ {% if tool.defer_loading is not defined or not tool.defer_loading %}
29
+ {{ tool_to_json(tool) }}
30
+ {% endif %}
31
+ {% endfor %}
32
+ </tools>
33
+
34
+ For each function call, output the function name and arguments within the following XML format:
35
+ <tool_call>{function-name}<arg_key>{arg-key-1}</arg_key><arg_value>{arg-value-1}</arg_value><arg_key>{arg-key-2}</arg_key><arg_value>{arg-value-2}</arg_value>...</tool_call>{%- endif -%}
36
+ {%- macro visible_text(content) -%}
37
+ {%- if content is string -%}
38
+ {{- content }}
39
+ {%- elif content is iterable and content is not mapping -%}
40
+ {%- for item in content -%}
41
+ {%- if item is mapping and item.type == 'text' -%}
42
+ {{- item.text }}
43
+ {%- elif item is string -%}
44
+ {{- item }}
45
+ {%- elif item is mapping and item.type in ['image', 'image_url'] -%}
46
+ {{- '<|begin_of_image|><|image|><|end_of_image|>' }}
47
+ {%- elif item is mapping and item.type in ['video', 'video_url', 'audio', 'audio_url', 'input_audio'] -%}
48
+ {%- set media_type = item.type | replace('_url', '') | replace('input_', '') -%}
49
+ {{- "<reminder>You are unable to process this " ~ media_type ~ " because you don't have multi-modal input ability. Try different methods.</reminder>" }}
50
+ {%- endif -%}
51
+ {%- endfor -%}
52
+ {%- else -%}
53
+ {{- content }}
54
+ {%- endif -%}
55
+ {%- endmacro -%}
56
+ {%- set ns = namespace(last_user_index=-1) -%}
57
+ {%- for m in messages %}
58
+ {%- if m.role == 'user' %}
59
+ {%- set ns.last_user_index = loop.index0 -%}
60
+ {%- endif %}
61
+ {%- endfor %}
62
+ {%- for m in messages -%}
63
+ {%- if m.role == 'user' -%}<|user|>{{ visible_text(m.content) }}
64
+ {%- elif m.role == 'assistant' -%}
65
+ <|assistant|>
66
+ {%- set content = visible_text(m.content) %}
67
+ {%- if m.reasoning_content is string %}
68
+ {%- set reasoning_content = m.reasoning_content %}
69
+ {%- elif '</think>' in content %}
70
+ {%- set reasoning_content = content.split('</think>')[0].split('<think>')[-1] %}
71
+ {%- set content = content.split('</think>')[-1] %}
72
+ {%- endif %}
73
+ {%- if ((clear_thinking is defined and not clear_thinking) or loop.index0 > ns.last_user_index) and reasoning_content is defined -%}
74
+ {{ '<think>' + reasoning_content + '</think>'}}
75
+ {%- else -%}
76
+ {{ '<think></think>' }}
77
+ {%- endif -%}
78
+ {%- if content.strip() -%}
79
+ {{ content.strip() }}
80
+ {%- endif -%}
81
+ {% if m.tool_calls %}
82
+ {% for tc in m.tool_calls %}
83
+ {%- if tc.function %}
84
+ {%- set tc = tc.function %}
85
+ {%- endif %}
86
+ {{- '<tool_call>' + tc.name -}}
87
+ {% set _args = tc.arguments %}{% for k, v in _args.items() %}<arg_key>{{ k }}</arg_key><arg_value>{{ v | tojson(ensure_ascii=False) if v is not string else v }}</arg_value>{% endfor %}</tool_call>{% endfor %}
88
+ {% endif %}
89
+ {%- elif m.role == 'tool' -%}
90
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
91
+ {{- '<|observation|>' -}}
92
+ {%- endif %}
93
+ {%- if m.content is string -%}
94
+ {{- '<tool_response>' + m.content + '</tool_response>' -}}
95
+ {%- elif m.content is iterable and m.content is not mapping and m.content and m.content.0.type == "tool_reference" -%}
96
+ {{- '<tool_response><tools>\n' -}}
97
+ {% for tr in m.content %}
98
+ {%- for tool in tools -%}
99
+ {%- if 'function' in tool -%}
100
+ {%- set tool = tool['function'] -%}
101
+ {%- endif -%}
102
+ {%- if tool.name == tr.name -%}
103
+ {{- tool_to_json(tool) + '\n' -}}
104
+ {%- endif -%}
105
+ {%- endfor -%}
106
+ {%- endfor -%}
107
+ {{- '</tools></tool_response>' -}}
108
+ {%- elif m.content is iterable and m.content is not mapping and m.content and m.content.0 is mapping and m.content.0.output is defined -%}
109
+ {%- for tr in m.content -%}
110
+ {{- '<tool_response>' + tr.output + '</tool_response>' -}}
111
+ {%- endfor -%}
112
+ {%- else -%}
113
+ {{- '<tool_response>' + visible_text(m.content) + '</tool_response>' -}}
114
+ {% endif -%}
115
+ {%- elif m.role == 'system' -%}
116
+ <|system|>{{ visible_text(m.content) }}
117
+ {%- endif -%}
118
+ {%- endfor -%}
119
+ {%- if add_generation_prompt -%}
120
+ <|assistant|>{{- '<think></think>' if (enable_thinking is defined and not enable_thinking) else '<think>' -}}
121
+ {%- endif -%}
config.json ADDED
@@ -0,0 +1,1370 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Glm5vForConditionalGeneration"
4
+ ],
5
+ "model_type": "glm5v",
6
+ "auto_map": {
7
+ "AutoConfig": "configuration_glm5v.Glm5vConfig"
8
+ },
9
+ "text_config": {
10
+ "architectures": [
11
+ "GlmMoeDsaForCausalLM"
12
+ ],
13
+ "attention_bias": false,
14
+ "attention_dropout": 0.0,
15
+ "dtype": "bfloat16",
16
+ "eos_token_id": [
17
+ 154820,
18
+ 154827,
19
+ 154829
20
+ ],
21
+ "ep_size": 1,
22
+ "first_k_dense_replace": 3,
23
+ "head_dim": 192,
24
+ "hidden_act": "silu",
25
+ "hidden_size": 6144,
26
+ "index_head_dim": 128,
27
+ "index_n_heads": 32,
28
+ "index_share_for_mtp_iteration": true,
29
+ "index_skip_topk_offset": 3,
30
+ "index_topk": 2048,
31
+ "index_topk_freq": 4,
32
+ "index_topk_pattern": null,
33
+ "indexer_rope_interleave": true,
34
+ "indexer_types": [
35
+ "full",
36
+ "full",
37
+ "full",
38
+ "shared",
39
+ "shared",
40
+ "shared",
41
+ "full",
42
+ "shared",
43
+ "shared",
44
+ "shared",
45
+ "full",
46
+ "shared",
47
+ "shared",
48
+ "shared",
49
+ "full",
50
+ "shared",
51
+ "shared",
52
+ "shared",
53
+ "full",
54
+ "shared",
55
+ "shared",
56
+ "shared",
57
+ "full",
58
+ "shared",
59
+ "shared",
60
+ "shared",
61
+ "full",
62
+ "shared",
63
+ "shared",
64
+ "shared",
65
+ "full",
66
+ "shared",
67
+ "shared",
68
+ "shared",
69
+ "full",
70
+ "shared",
71
+ "shared",
72
+ "shared",
73
+ "full",
74
+ "shared",
75
+ "shared",
76
+ "shared",
77
+ "full",
78
+ "shared",
79
+ "shared",
80
+ "shared",
81
+ "full",
82
+ "shared",
83
+ "shared",
84
+ "shared",
85
+ "full",
86
+ "shared",
87
+ "shared",
88
+ "shared",
89
+ "full",
90
+ "shared",
91
+ "shared",
92
+ "shared",
93
+ "full",
94
+ "shared",
95
+ "shared",
96
+ "shared",
97
+ "full",
98
+ "shared",
99
+ "shared",
100
+ "shared",
101
+ "full",
102
+ "shared",
103
+ "shared",
104
+ "shared",
105
+ "full",
106
+ "shared",
107
+ "shared",
108
+ "shared",
109
+ "full",
110
+ "shared",
111
+ "shared",
112
+ "shared"
113
+ ],
114
+ "initializer_range": 0.02,
115
+ "intermediate_size": 12288,
116
+ "kv_lora_rank": 512,
117
+ "max_position_embeddings": 1048576,
118
+ "mlp_layer_types": [
119
+ "dense",
120
+ "dense",
121
+ "dense",
122
+ "sparse",
123
+ "sparse",
124
+ "sparse",
125
+ "sparse",
126
+ "sparse",
127
+ "sparse",
128
+ "sparse",
129
+ "sparse",
130
+ "sparse",
131
+ "sparse",
132
+ "sparse",
133
+ "sparse",
134
+ "sparse",
135
+ "sparse",
136
+ "sparse",
137
+ "sparse",
138
+ "sparse",
139
+ "sparse",
140
+ "sparse",
141
+ "sparse",
142
+ "sparse",
143
+ "sparse",
144
+ "sparse",
145
+ "sparse",
146
+ "sparse",
147
+ "sparse",
148
+ "sparse",
149
+ "sparse",
150
+ "sparse",
151
+ "sparse",
152
+ "sparse",
153
+ "sparse",
154
+ "sparse",
155
+ "sparse",
156
+ "sparse",
157
+ "sparse",
158
+ "sparse",
159
+ "sparse",
160
+ "sparse",
161
+ "sparse",
162
+ "sparse",
163
+ "sparse",
164
+ "sparse",
165
+ "sparse",
166
+ "sparse",
167
+ "sparse",
168
+ "sparse",
169
+ "sparse",
170
+ "sparse",
171
+ "sparse",
172
+ "sparse",
173
+ "sparse",
174
+ "sparse",
175
+ "sparse",
176
+ "sparse",
177
+ "sparse",
178
+ "sparse",
179
+ "sparse",
180
+ "sparse",
181
+ "sparse",
182
+ "sparse",
183
+ "sparse",
184
+ "sparse",
185
+ "sparse",
186
+ "sparse",
187
+ "sparse",
188
+ "sparse",
189
+ "sparse",
190
+ "sparse",
191
+ "sparse",
192
+ "sparse",
193
+ "sparse",
194
+ "sparse",
195
+ "sparse",
196
+ "sparse"
197
+ ],
198
+ "model_type": "glm_moe_dsa",
199
+ "moe_intermediate_size": 2048,
200
+ "moe_layer_freq": 1,
201
+ "n_group": 1,
202
+ "n_routed_experts": 256,
203
+ "n_shared_experts": 1,
204
+ "norm_topk_prob": true,
205
+ "num_attention_heads": 64,
206
+ "num_experts_per_tok": 8,
207
+ "num_hidden_layers": 78,
208
+ "num_key_value_heads": 64,
209
+ "num_nextn_predict_layers": 1,
210
+ "pad_token_id": 154820,
211
+ "pretraining_tp": 1,
212
+ "q_lora_rank": 2048,
213
+ "qk_head_dim": 256,
214
+ "qk_nope_head_dim": 192,
215
+ "qk_rope_head_dim": 64,
216
+ "rms_norm_eps": 1e-05,
217
+ "rope_interleave": true,
218
+ "rope_parameters": {
219
+ "rope_theta": 8000000,
220
+ "rope_type": "default"
221
+ },
222
+ "routed_scaling_factor": 2.5,
223
+ "scoring_func": "sigmoid",
224
+ "tie_word_embeddings": false,
225
+ "topk_group": 1,
226
+ "topk_method": "noaux_tc",
227
+ "transformers_version": "5.12.0",
228
+ "use_cache": true,
229
+ "v_head_dim": 256,
230
+ "vocab_size": 154880,
231
+ "quantization_config": {
232
+ "activation_scheme": "dynamic",
233
+ "fmt": "e4m3",
234
+ "quant_method": "fp8",
235
+ "weight_block_size": [
236
+ 128,
237
+ 128
238
+ ],
239
+ "modules_to_not_convert": [
240
+ "model.layers.59.post_attention_layernorm",
241
+ "model.layers.47.mlp.gate.e_score_correction_bias",
242
+ "model.layers.71.self_attn.kv_a_layernorm",
243
+ "model.layers.46.mlp.gate.e_score_correction_bias",
244
+ "model.layers.30.self_attn.kv_a_layernorm",
245
+ "model.layers.23.self_attn.kv_a_layernorm",
246
+ "model.layers.73.mlp.gate.e_score_correction_bias",
247
+ "model.layers.36.mlp.gate",
248
+ "model.layers.12.self_attn.q_a_layernorm",
249
+ "model.layers.51.mlp.gate",
250
+ "model.layers.47.self_attn.kv_a_layernorm",
251
+ "model.layers.45.self_attn.q_a_layernorm",
252
+ "model.layers.15.self_attn.q_a_layernorm",
253
+ "model.layers.39.input_layernorm",
254
+ "model.layers.50.self_attn.kv_a_layernorm",
255
+ "model.layers.19.post_attention_layernorm",
256
+ "model.layers.71.input_layernorm",
257
+ "model.layers.72.self_attn.kv_a_layernorm",
258
+ "model.layers.70.mlp.gate",
259
+ "model.layers.52.input_layernorm",
260
+ "model.layers.78.self_attn.kv_a_layernorm",
261
+ "model.layers.48.post_attention_layernorm",
262
+ "model.layers.4.self_attn.q_a_layernorm",
263
+ "model.layers.62.self_attn.q_a_layernorm",
264
+ "model.layers.4.post_attention_layernorm",
265
+ "model.layers.38.self_attn.indexers_proj",
266
+ "model.layers.61.mlp.gate.e_score_correction_bias",
267
+ "model.layers.14.mlp.gate.e_score_correction_bias",
268
+ "model.layers.36.self_attn.kv_a_layernorm",
269
+ "model.layers.65.mlp.gate",
270
+ "model.layers.13.self_attn.kv_a_layernorm",
271
+ "model.layers.57.mlp.gate",
272
+ "model.layers.65.self_attn.kv_a_layernorm",
273
+ "model.layers.68.input_layernorm",
274
+ "model.layers.70.self_attn.kv_a_layernorm",
275
+ "model.layers.24.mlp.gate.e_score_correction_bias",
276
+ "model.layers.4.input_layernorm",
277
+ "model.layers.20.input_layernorm",
278
+ "model.layers.65.self_attn.q_a_layernorm",
279
+ "model.layers.38.mlp.gate",
280
+ "model.layers.76.mlp.gate.e_score_correction_bias",
281
+ "model.layers.62.input_layernorm",
282
+ "model.layers.45.input_layernorm",
283
+ "model.layers.68.post_attention_layernorm",
284
+ "model.layers.72.input_layernorm",
285
+ "model.layers.32.mlp.gate.e_score_correction_bias",
286
+ "model.layers.27.input_layernorm",
287
+ "model.layers.16.post_attention_layernorm",
288
+ "model.layers.35.self_attn.q_a_layernorm",
289
+ "model.layers.58.self_attn.indexers_proj",
290
+ "model.layers.3.mlp.gate.e_score_correction_bias",
291
+ "model.layers.49.self_attn.q_a_layernorm",
292
+ "model.layers.66.mlp.gate.e_score_correction_bias",
293
+ "model.layers.10.self_attn.q_a_layernorm",
294
+ "model.layers.51.mlp.gate.e_score_correction_bias",
295
+ "model.layers.64.self_attn.q_a_layernorm",
296
+ "model.layers.70.input_layernorm",
297
+ "model.layers.71.post_attention_layernorm",
298
+ "model.layers.67.post_attention_layernorm",
299
+ "model.layers.6.self_attn.indexer.k_norm",
300
+ "model.layers.3.mlp.gate",
301
+ "model.layers.11.mlp.gate",
302
+ "model.layers.17.input_layernorm",
303
+ "model.layers.13.mlp.gate",
304
+ "model.layers.15.mlp.gate",
305
+ "model.layers.71.self_attn.q_a_layernorm",
306
+ "model.layers.33.mlp.gate.e_score_correction_bias",
307
+ "model.layers.72.mlp.gate.e_score_correction_bias",
308
+ "model.layers.65.post_attention_layernorm",
309
+ "model.layers.78.hnorm",
310
+ "model.layers.40.post_attention_layernorm",
311
+ "model.layers.12.mlp.gate.e_score_correction_bias",
312
+ "model.layers.8.input_layernorm",
313
+ "model.layers.73.mlp.gate",
314
+ "model.layers.11.post_attention_layernorm",
315
+ "model.layers.61.input_layernorm",
316
+ "model.layers.14.self_attn.kv_a_layernorm",
317
+ "model.layers.39.self_attn.kv_a_layernorm",
318
+ "model.layers.70.self_attn.indexer.k_norm",
319
+ "model.layers.45.mlp.gate",
320
+ "model.layers.51.self_attn.q_a_layernorm",
321
+ "model.layers.64.post_attention_layernorm",
322
+ "model.layers.20.post_attention_layernorm",
323
+ "model.layers.66.self_attn.kv_a_layernorm",
324
+ "model.layers.10.self_attn.indexer.k_norm.bias",
325
+ "model.layers.23.input_layernorm",
326
+ "model.layers.38.self_attn.indexer.k_norm",
327
+ "model.layers.19.self_attn.kv_a_layernorm",
328
+ "model.layers.40.self_attn.q_a_layernorm",
329
+ "model.layers.10.self_attn.indexer.k_norm",
330
+ "model.layers.44.post_attention_layernorm",
331
+ "model.layers.70.post_attention_layernorm",
332
+ "model.layers.41.mlp.gate.e_score_correction_bias",
333
+ "model.layers.36.post_attention_layernorm",
334
+ "model.layers.30.self_attn.indexer.k_norm.bias",
335
+ "model.layers.54.self_attn.q_a_layernorm",
336
+ "model.layers.59.mlp.gate",
337
+ "model.layers.69.post_attention_layernorm",
338
+ "lm_head",
339
+ "model.layers.77.self_attn.kv_a_layernorm",
340
+ "model.layers.75.self_attn.q_a_layernorm",
341
+ "model.layers.56.self_attn.q_a_layernorm",
342
+ "model.layers.50.self_attn.indexer.k_norm.bias",
343
+ "model.layers.54.mlp.gate",
344
+ "model.layers.6.self_attn.indexer.k_norm.bias",
345
+ "model.layers.15.input_layernorm",
346
+ "model.layers.29.self_attn.q_a_layernorm",
347
+ "model.layers.54.self_attn.indexers_proj",
348
+ "model.layers.73.self_attn.kv_a_layernorm",
349
+ "model.layers.0.self_attn.indexer.k_norm",
350
+ "model.layers.58.self_attn.indexer.k_norm",
351
+ "model.layers.36.input_layernorm",
352
+ "model.layers.47.self_attn.q_a_layernorm",
353
+ "model.layers.26.mlp.gate.e_score_correction_bias",
354
+ "model.layers.50.input_layernorm",
355
+ "model.layers.16.self_attn.q_a_layernorm",
356
+ "model.layers.19.mlp.gate.e_score_correction_bias",
357
+ "model.layers.78.input_layernorm",
358
+ "model.layers.27.mlp.gate",
359
+ "model.layers.19.self_attn.q_a_layernorm",
360
+ "model.layers.25.input_layernorm",
361
+ "model.layers.33.self_attn.kv_a_layernorm",
362
+ "model.layers.32.mlp.gate",
363
+ "model.layers.61.self_attn.kv_a_layernorm",
364
+ "model.layers.50.self_attn.indexers_proj",
365
+ "model.layers.34.self_attn.indexers_proj",
366
+ "model.layers.4.mlp.gate.e_score_correction_bias",
367
+ "model.layers.76.post_attention_layernorm",
368
+ "model.layers.57.post_attention_layernorm",
369
+ "model.layers.73.post_attention_layernorm",
370
+ "model.layers.68.mlp.gate.e_score_correction_bias",
371
+ "model.layers.34.self_attn.kv_a_layernorm",
372
+ "model.layers.78.enorm",
373
+ "model.layers.30.mlp.gate",
374
+ "model.layers.10.self_attn.indexers_proj",
375
+ "model.layers.34.input_layernorm",
376
+ "model.layers.34.post_attention_layernorm",
377
+ "model.layers.20.mlp.gate.e_score_correction_bias",
378
+ "model.layers.54.self_attn.indexer.k_norm.bias",
379
+ "model.layers.44.mlp.gate",
380
+ "model.layers.25.mlp.gate.e_score_correction_bias",
381
+ "model.layers.26.self_attn.indexers_proj",
382
+ "model.layers.66.self_attn.indexer.k_norm.bias",
383
+ "model.layers.41.post_attention_layernorm",
384
+ "model.layers.51.post_attention_layernorm",
385
+ "model.layers.22.self_attn.indexer.k_norm.bias",
386
+ "model.layers.21.mlp.gate.e_score_correction_bias",
387
+ "model.layers.15.self_attn.kv_a_layernorm",
388
+ "model.layers.25.self_attn.kv_a_layernorm",
389
+ "model.layers.55.mlp.gate",
390
+ "model.layers.11.self_attn.q_a_layernorm",
391
+ "model.layers.72.mlp.gate",
392
+ "model.layers.21.self_attn.kv_a_layernorm",
393
+ "model.layers.59.mlp.gate.e_score_correction_bias",
394
+ "model.layers.2.input_layernorm",
395
+ "model.layers.5.mlp.gate",
396
+ "model.layers.54.self_attn.kv_a_layernorm",
397
+ "model.layers.0.self_attn.indexers_proj",
398
+ "model.layers.71.mlp.gate.e_score_correction_bias",
399
+ "model.layers.6.self_attn.q_a_layernorm",
400
+ "model.layers.14.mlp.gate",
401
+ "model.layers.44.self_attn.kv_a_layernorm",
402
+ "model.layers.61.post_attention_layernorm",
403
+ "model.layers.64.self_attn.kv_a_layernorm",
404
+ "model.layers.1.post_attention_layernorm",
405
+ "model.layers.4.mlp.gate",
406
+ "model.layers.22.post_attention_layernorm",
407
+ "model.layers.21.self_attn.q_a_layernorm",
408
+ "model.layers.74.self_attn.q_a_layernorm",
409
+ "model.layers.18.self_attn.indexers_proj",
410
+ "model.layers.75.input_layernorm",
411
+ "model.layers.70.self_attn.q_a_layernorm",
412
+ "model.layers.22.self_attn.indexer.k_norm",
413
+ "model.layers.42.mlp.gate.e_score_correction_bias",
414
+ "model.layers.5.self_attn.kv_a_layernorm",
415
+ "model.layers.31.mlp.gate.e_score_correction_bias",
416
+ "model.layers.62.self_attn.indexer.k_norm.bias",
417
+ "model.layers.29.post_attention_layernorm",
418
+ "model.layers.6.self_attn.kv_a_layernorm",
419
+ "model.layers.48.input_layernorm",
420
+ "model.layers.74.self_attn.indexer.k_norm.bias",
421
+ "model.layers.7.input_layernorm",
422
+ "model.layers.39.self_attn.q_a_layernorm",
423
+ "model.layers.41.input_layernorm",
424
+ "model.layers.54.self_attn.indexer.k_norm",
425
+ "model.layers.32.self_attn.q_a_layernorm",
426
+ "model.layers.39.mlp.gate.e_score_correction_bias",
427
+ "model.layers.42.input_layernorm",
428
+ "model.layers.43.mlp.gate",
429
+ "model.layers.72.self_attn.q_a_layernorm",
430
+ "model.layers.37.mlp.gate.e_score_correction_bias",
431
+ "model.layers.5.post_attention_layernorm",
432
+ "model.layers.28.mlp.gate",
433
+ "model.layers.2.post_attention_layernorm",
434
+ "model.layers.49.post_attention_layernorm",
435
+ "model.layers.43.self_attn.kv_a_layernorm",
436
+ "model.layers.12.input_layernorm",
437
+ "model.layers.56.mlp.gate",
438
+ "model.layers.5.input_layernorm",
439
+ "model.layers.29.mlp.gate.e_score_correction_bias",
440
+ "model.layers.4.self_attn.kv_a_layernorm",
441
+ "model.layers.20.self_attn.q_a_layernorm",
442
+ "model.layers.59.self_attn.q_a_layernorm",
443
+ "model.layers.68.mlp.gate",
444
+ "model.layers.21.post_attention_layernorm",
445
+ "model.layers.26.self_attn.q_a_layernorm",
446
+ "model.layers.52.self_attn.q_a_layernorm",
447
+ "model.layers.29.self_attn.kv_a_layernorm",
448
+ "model.layers.24.input_layernorm",
449
+ "model.layers.34.self_attn.indexer.k_norm.bias",
450
+ "model.layers.43.input_layernorm",
451
+ "model.layers.27.self_attn.kv_a_layernorm",
452
+ "model.layers.31.input_layernorm",
453
+ "model.layers.69.mlp.gate",
454
+ "model.layers.26.post_attention_layernorm",
455
+ "model.layers.3.self_attn.q_a_layernorm",
456
+ "model.layers.37.post_attention_layernorm",
457
+ "model.layers.52.mlp.gate",
458
+ "model.layers.73.input_layernorm",
459
+ "model.layers.19.input_layernorm",
460
+ "model.layers.13.self_attn.q_a_layernorm",
461
+ "model.layers.76.input_layernorm",
462
+ "model.layers.8.mlp.gate",
463
+ "model.layers.63.post_attention_layernorm",
464
+ "model.layers.58.mlp.gate",
465
+ "model.layers.31.post_attention_layernorm",
466
+ "model.layers.23.mlp.gate.e_score_correction_bias",
467
+ "model.layers.34.self_attn.q_a_layernorm",
468
+ "model.layers.78.self_attn.indexer.k_norm.bias",
469
+ "model.layers.13.mlp.gate.e_score_correction_bias",
470
+ "model.layers.71.mlp.gate",
471
+ "model.layers.14.input_layernorm",
472
+ "model.layers.17.mlp.gate.e_score_correction_bias",
473
+ "model.layers.8.post_attention_layernorm",
474
+ "model.layers.45.self_attn.kv_a_layernorm",
475
+ "model.layers.46.self_attn.indexer.k_norm",
476
+ "model.layers.20.self_attn.kv_a_layernorm",
477
+ "model.layers.78.self_attn.indexer.k_norm",
478
+ "model.layers.37.mlp.gate",
479
+ "model.layers.7.mlp.gate",
480
+ "model.layers.30.mlp.gate.e_score_correction_bias",
481
+ "model.layers.67.mlp.gate",
482
+ "model.layers.73.self_attn.q_a_layernorm",
483
+ "model.layers.10.self_attn.kv_a_layernorm",
484
+ "model.layers.61.mlp.gate",
485
+ "model.layers.22.mlp.gate.e_score_correction_bias",
486
+ "model.layers.58.input_layernorm",
487
+ "model.layers.8.self_attn.kv_a_layernorm",
488
+ "model.layers.33.self_attn.q_a_layernorm",
489
+ "model.layers.55.self_attn.q_a_layernorm",
490
+ "model.layers.32.input_layernorm",
491
+ "model.layers.17.self_attn.kv_a_layernorm",
492
+ "model.layers.46.self_attn.indexer.k_norm.bias",
493
+ "model.layers.11.mlp.gate.e_score_correction_bias",
494
+ "model.layers.9.input_layernorm",
495
+ "model.layers.77.input_layernorm",
496
+ "model.layers.16.self_attn.kv_a_layernorm",
497
+ "model.layers.45.post_attention_layernorm",
498
+ "model.layers.74.mlp.gate",
499
+ "model.layers.68.self_attn.q_a_layernorm",
500
+ "model.layers.55.self_attn.kv_a_layernorm",
501
+ "model.layers.44.input_layernorm",
502
+ "model.layers.60.self_attn.kv_a_layernorm",
503
+ "model.layers.14.post_attention_layernorm",
504
+ "model.layers.62.self_attn.indexers_proj",
505
+ "model.layers.0.self_attn.q_a_layernorm",
506
+ "model.layers.72.post_attention_layernorm",
507
+ "model.layers.78.post_attention_layernorm",
508
+ "model.layers.60.mlp.gate.e_score_correction_bias",
509
+ "model.layers.26.self_attn.indexer.k_norm.bias",
510
+ "model.layers.38.post_attention_layernorm",
511
+ "model.layers.50.post_attention_layernorm",
512
+ "model.layers.28.input_layernorm",
513
+ "model.layers.30.self_attn.indexer.k_norm",
514
+ "model.layers.37.self_attn.q_a_layernorm",
515
+ "model.layers.56.input_layernorm",
516
+ "model.layers.27.post_attention_layernorm",
517
+ "model.layers.74.input_layernorm",
518
+ "model.layers.78.self_attn.q_a_layernorm",
519
+ "model.layers.42.self_attn.q_a_layernorm",
520
+ "model.layers.26.self_attn.indexer.k_norm",
521
+ "model.layers.42.self_attn.kv_a_layernorm",
522
+ "model.layers.1.self_attn.indexer.k_norm",
523
+ "model.layers.39.post_attention_layernorm",
524
+ "model.layers.48.mlp.gate.e_score_correction_bias",
525
+ "model.layers.74.post_attention_layernorm",
526
+ "model.layers.69.self_attn.q_a_layernorm",
527
+ "model.layers.13.input_layernorm",
528
+ "model.layers.74.self_attn.indexer.k_norm",
529
+ "model.layers.29.mlp.gate",
530
+ "model.layers.7.mlp.gate.e_score_correction_bias",
531
+ "model.layers.3.self_attn.kv_a_layernorm",
532
+ "model.layers.35.self_attn.kv_a_layernorm",
533
+ "model.layers.46.self_attn.indexers_proj",
534
+ "model.layers.0.self_attn.kv_a_layernorm",
535
+ "model.layers.43.self_attn.q_a_layernorm",
536
+ "model.norm",
537
+ "model.layers.1.self_attn.kv_a_layernorm",
538
+ "model.layers.38.self_attn.indexer.k_norm.bias",
539
+ "model.layers.30.self_attn.q_a_layernorm",
540
+ "model.layers.0.input_layernorm",
541
+ "model.layers.47.input_layernorm",
542
+ "model.layers.8.self_attn.q_a_layernorm",
543
+ "model.layers.61.self_attn.q_a_layernorm",
544
+ "model.layers.1.self_attn.indexer.k_norm.bias",
545
+ "model.layers.62.mlp.gate.e_score_correction_bias",
546
+ "model.layers.66.self_attn.q_a_layernorm",
547
+ "model.layers.46.self_attn.kv_a_layernorm",
548
+ "model.layers.66.mlp.gate",
549
+ "model.layers.69.input_layernorm",
550
+ "model.layers.69.self_attn.kv_a_layernorm",
551
+ "model.layers.22.input_layernorm",
552
+ "model.layers.30.post_attention_layernorm",
553
+ "model.layers.24.self_attn.q_a_layernorm",
554
+ "model.layers.48.mlp.gate",
555
+ "model.layers.7.self_attn.kv_a_layernorm",
556
+ "model.layers.67.input_layernorm",
557
+ "model.layers.27.self_attn.q_a_layernorm",
558
+ "model.layers.23.mlp.gate",
559
+ "model.layers.45.mlp.gate.e_score_correction_bias",
560
+ "model.layers.76.self_attn.kv_a_layernorm",
561
+ "model.layers.22.self_attn.q_a_layernorm",
562
+ "model.layers.78.mlp.gate",
563
+ "model.layers.9.mlp.gate",
564
+ "model.layers.57.self_attn.q_a_layernorm",
565
+ "model.layers.78.mlp.gate.e_score_correction_bias",
566
+ "model.layers.23.post_attention_layernorm",
567
+ "model.layers.38.self_attn.q_a_layernorm",
568
+ "model.layers.53.mlp.gate",
569
+ "model.layers.77.self_attn.q_a_layernorm",
570
+ "model.layers.62.self_attn.kv_a_layernorm",
571
+ "model.layers.12.mlp.gate",
572
+ "model.layers.34.self_attn.indexer.k_norm",
573
+ "model.layers.46.self_attn.q_a_layernorm",
574
+ "model.layers.36.self_attn.q_a_layernorm",
575
+ "model.layers.58.self_attn.indexer.k_norm.bias",
576
+ "model.layers.60.post_attention_layernorm",
577
+ "model.layers.28.post_attention_layernorm",
578
+ "model.layers.2.self_attn.kv_a_layernorm",
579
+ "model.layers.70.self_attn.indexers_proj",
580
+ "model.layers.16.mlp.gate.e_score_correction_bias",
581
+ "model.layers.2.self_attn.indexer.k_norm",
582
+ "model.layers.31.self_attn.kv_a_layernorm",
583
+ "model.layers.64.input_layernorm",
584
+ "model.layers.6.mlp.gate.e_score_correction_bias",
585
+ "model.layers.50.self_attn.indexer.k_norm",
586
+ "model.layers.58.self_attn.q_a_layernorm",
587
+ "model.layers.57.mlp.gate.e_score_correction_bias",
588
+ "model.layers.13.post_attention_layernorm",
589
+ "model.layers.28.self_attn.kv_a_layernorm",
590
+ "model.layers.31.self_attn.q_a_layernorm",
591
+ "model.layers.1.self_attn.q_a_layernorm",
592
+ "model.layers.17.mlp.gate",
593
+ "model.layers.15.post_attention_layernorm",
594
+ "model.layers.18.self_attn.indexer.k_norm.bias",
595
+ "model.layers.18.self_attn.indexer.k_norm",
596
+ "model.layers.43.mlp.gate.e_score_correction_bias",
597
+ "model.layers.11.input_layernorm",
598
+ "model.layers.9.post_attention_layernorm",
599
+ "model.layers.33.input_layernorm",
600
+ "model.layers.66.input_layernorm",
601
+ "model.layers.78.self_attn.indexers_proj",
602
+ "model.layers.54.post_attention_layernorm",
603
+ "model.layers.53.mlp.gate.e_score_correction_bias",
604
+ "model.layers.5.mlp.gate.e_score_correction_bias",
605
+ "model.layers.19.mlp.gate",
606
+ "model.layers.49.mlp.gate",
607
+ "model.layers.66.self_attn.indexers_proj",
608
+ "model.layers.14.self_attn.indexer.k_norm.bias",
609
+ "model.layers.41.mlp.gate",
610
+ "model.layers.42.self_attn.indexer.k_norm.bias",
611
+ "model.layers.65.input_layernorm",
612
+ "model.layers.75.mlp.gate",
613
+ "model.layers.12.self_attn.kv_a_layernorm",
614
+ "model.layers.53.input_layernorm",
615
+ "model.layers.58.self_attn.kv_a_layernorm",
616
+ "model.layers.60.self_attn.q_a_layernorm",
617
+ "model.layers.6.mlp.gate",
618
+ "model.layers.49.self_attn.kv_a_layernorm",
619
+ "model.layers.16.mlp.gate",
620
+ "model.layers.22.mlp.gate",
621
+ "model.layers.58.post_attention_layernorm",
622
+ "model.layers.15.mlp.gate.e_score_correction_bias",
623
+ "model.layers.26.input_layernorm",
624
+ "model.layers.3.post_attention_layernorm",
625
+ "model.layers.6.input_layernorm",
626
+ "model.layers.22.self_attn.kv_a_layernorm",
627
+ "model.layers.16.input_layernorm",
628
+ "model.layers.33.post_attention_layernorm",
629
+ "model.layers.17.post_attention_layernorm",
630
+ "model.layers.35.mlp.gate",
631
+ "model.layers.56.post_attention_layernorm",
632
+ "model.layers.7.self_attn.q_a_layernorm",
633
+ "model.layers.24.mlp.gate",
634
+ "model.layers.24.post_attention_layernorm",
635
+ "model.layers.36.mlp.gate.e_score_correction_bias",
636
+ "model.layers.56.mlp.gate.e_score_correction_bias",
637
+ "model.layers.32.post_attention_layernorm",
638
+ "model.layers.47.post_attention_layernorm",
639
+ "model.layers.40.mlp.gate",
640
+ "model.layers.41.self_attn.kv_a_layernorm",
641
+ "model.layers.52.self_attn.kv_a_layernorm",
642
+ "model.layers.9.self_attn.kv_a_layernorm",
643
+ "model.layers.77.mlp.gate.e_score_correction_bias",
644
+ "model.layers.78.shared_head.norm",
645
+ "model.layers.53.post_attention_layernorm",
646
+ "model.layers.63.self_attn.kv_a_layernorm",
647
+ "model.layers.48.self_attn.kv_a_layernorm",
648
+ "model.layers.28.mlp.gate.e_score_correction_bias",
649
+ "model.layers.8.mlp.gate.e_score_correction_bias",
650
+ "model.layers.52.mlp.gate.e_score_correction_bias",
651
+ "model.layers.52.post_attention_layernorm",
652
+ "model.layers.18.self_attn.q_a_layernorm",
653
+ "model.layers.50.mlp.gate.e_score_correction_bias",
654
+ "model.layers.64.mlp.gate.e_score_correction_bias",
655
+ "model.layers.38.mlp.gate.e_score_correction_bias",
656
+ "model.layers.59.input_layernorm",
657
+ "model.layers.21.input_layernorm",
658
+ "model.layers.39.mlp.gate",
659
+ "model.layers.77.post_attention_layernorm",
660
+ "model.layers.76.self_attn.q_a_layernorm",
661
+ "model.layers.12.post_attention_layernorm",
662
+ "model.layers.43.post_attention_layernorm",
663
+ "model.layers.35.input_layernorm",
664
+ "model.layers.62.mlp.gate",
665
+ "model.layers.38.self_attn.kv_a_layernorm",
666
+ "model.layers.63.mlp.gate",
667
+ "model.layers.27.mlp.gate.e_score_correction_bias",
668
+ "model.layers.18.mlp.gate",
669
+ "model.layers.25.post_attention_layernorm",
670
+ "model.layers.55.mlp.gate.e_score_correction_bias",
671
+ "model.layers.66.self_attn.indexer.k_norm",
672
+ "model.layers.48.self_attn.q_a_layernorm",
673
+ "model.layers.78.eh_proj",
674
+ "model.layers.77.mlp.gate",
675
+ "model.layers.6.self_attn.indexers_proj",
676
+ "model.layers.14.self_attn.q_a_layernorm",
677
+ "model.layers.2.self_attn.q_a_layernorm",
678
+ "model.layers.42.mlp.gate",
679
+ "model.layers.54.mlp.gate.e_score_correction_bias",
680
+ "model.layers.60.input_layernorm",
681
+ "model.layers.40.self_attn.kv_a_layernorm",
682
+ "model.layers.51.self_attn.kv_a_layernorm",
683
+ "model.layers.18.input_layernorm",
684
+ "model.layers.30.self_attn.indexers_proj",
685
+ "model.layers.53.self_attn.kv_a_layernorm",
686
+ "model.layers.6.post_attention_layernorm",
687
+ "model.layers.70.self_attn.indexer.k_norm.bias",
688
+ "model.layers.54.input_layernorm",
689
+ "model.layers.41.self_attn.q_a_layernorm",
690
+ "model.layers.18.mlp.gate.e_score_correction_bias",
691
+ "model.layers.34.mlp.gate.e_score_correction_bias",
692
+ "model.layers.47.mlp.gate",
693
+ "model.layers.46.mlp.gate",
694
+ "model.layers.25.self_attn.q_a_layernorm",
695
+ "model.layers.63.mlp.gate.e_score_correction_bias",
696
+ "model.layers.66.post_attention_layernorm",
697
+ "model.layers.76.mlp.gate",
698
+ "model.embed_tokens",
699
+ "model.layers.67.self_attn.kv_a_layernorm",
700
+ "model.layers.67.mlp.gate.e_score_correction_bias",
701
+ "model.layers.26.self_attn.kv_a_layernorm",
702
+ "model.layers.31.mlp.gate",
703
+ "model.layers.18.self_attn.kv_a_layernorm",
704
+ "model.layers.34.mlp.gate",
705
+ "model.layers.55.post_attention_layernorm",
706
+ "model.layers.30.input_layernorm",
707
+ "model.layers.50.mlp.gate",
708
+ "model.layers.63.input_layernorm",
709
+ "model.layers.11.self_attn.kv_a_layernorm",
710
+ "model.layers.24.self_attn.kv_a_layernorm",
711
+ "model.layers.75.mlp.gate.e_score_correction_bias",
712
+ "model.layers.10.post_attention_layernorm",
713
+ "model.layers.1.self_attn.indexers_proj",
714
+ "model.layers.42.self_attn.indexers_proj",
715
+ "model.layers.46.input_layernorm",
716
+ "model.layers.23.self_attn.q_a_layernorm",
717
+ "model.layers.63.self_attn.q_a_layernorm",
718
+ "model.layers.14.self_attn.indexer.k_norm",
719
+ "model.layers.14.self_attn.indexers_proj",
720
+ "model.layers.60.mlp.gate",
721
+ "model.layers.70.mlp.gate.e_score_correction_bias",
722
+ "model.layers.53.self_attn.q_a_layernorm",
723
+ "model.layers.0.post_attention_layernorm",
724
+ "model.layers.35.mlp.gate.e_score_correction_bias",
725
+ "model.layers.75.self_attn.kv_a_layernorm",
726
+ "model.layers.1.input_layernorm",
727
+ "model.layers.42.post_attention_layernorm",
728
+ "model.layers.37.input_layernorm",
729
+ "model.layers.9.mlp.gate.e_score_correction_bias",
730
+ "model.layers.46.post_attention_layernorm",
731
+ "model.layers.26.mlp.gate",
732
+ "model.layers.65.mlp.gate.e_score_correction_bias",
733
+ "model.layers.67.self_attn.q_a_layernorm",
734
+ "model.layers.44.mlp.gate.e_score_correction_bias",
735
+ "model.layers.33.mlp.gate",
736
+ "model.layers.29.input_layernorm",
737
+ "model.layers.18.post_attention_layernorm",
738
+ "model.layers.75.post_attention_layernorm",
739
+ "model.layers.44.self_attn.q_a_layernorm",
740
+ "model.layers.10.mlp.gate",
741
+ "model.layers.21.mlp.gate",
742
+ "model.layers.58.mlp.gate.e_score_correction_bias",
743
+ "model.layers.7.post_attention_layernorm",
744
+ "model.layers.50.self_attn.q_a_layernorm",
745
+ "model.layers.9.self_attn.q_a_layernorm",
746
+ "model.layers.2.self_attn.indexers_proj",
747
+ "model.layers.0.self_attn.indexer.k_norm.bias",
748
+ "model.layers.74.self_attn.kv_a_layernorm",
749
+ "model.layers.74.mlp.gate.e_score_correction_bias",
750
+ "model.layers.49.mlp.gate.e_score_correction_bias",
751
+ "model.layers.62.post_attention_layernorm",
752
+ "model.layers.57.self_attn.kv_a_layernorm",
753
+ "model.layers.40.input_layernorm",
754
+ "model.layers.69.mlp.gate.e_score_correction_bias",
755
+ "model.layers.35.post_attention_layernorm",
756
+ "model.layers.74.self_attn.indexers_proj",
757
+ "model.layers.55.input_layernorm",
758
+ "model.layers.49.input_layernorm",
759
+ "model.layers.40.mlp.gate.e_score_correction_bias",
760
+ "model.layers.25.mlp.gate",
761
+ "model.layers.2.self_attn.indexer.k_norm.bias",
762
+ "model.layers.68.self_attn.kv_a_layernorm",
763
+ "model.layers.51.input_layernorm",
764
+ "model.layers.22.self_attn.indexers_proj",
765
+ "model.layers.10.input_layernorm",
766
+ "model.layers.42.self_attn.indexer.k_norm",
767
+ "model.layers.56.self_attn.kv_a_layernorm",
768
+ "model.layers.17.self_attn.q_a_layernorm",
769
+ "model.layers.59.self_attn.kv_a_layernorm",
770
+ "model.layers.3.input_layernorm",
771
+ "model.layers.57.input_layernorm",
772
+ "model.layers.38.input_layernorm",
773
+ "model.layers.62.self_attn.indexer.k_norm",
774
+ "model.layers.20.mlp.gate",
775
+ "model.layers.64.mlp.gate",
776
+ "model.layers.28.self_attn.q_a_layernorm",
777
+ "model.layers.32.self_attn.kv_a_layernorm",
778
+ "model.layers.10.mlp.gate.e_score_correction_bias",
779
+ "model.layers.5.self_attn.q_a_layernorm",
780
+ "model.layers.37.self_attn.kv_a_layernorm"
781
+ ]
782
+ }
783
+ },
784
+ "vision_config": {
785
+ "patch_size": 14,
786
+ "init_pos_emb_height": 64,
787
+ "init_pos_emb_width": 64,
788
+ "init_pos_emb_time": 4,
789
+ "pos_emb_type": "divided_fixed",
790
+ "num_attention_heads": 16,
791
+ "num_hidden_layers": 27,
792
+ "hidden_size": 1152,
793
+ "intermediate_size": 4304,
794
+ "vt_num_attention_heads": 16,
795
+ "vt_num_hidden_layers": 27,
796
+ "vt_hidden_size": 1152,
797
+ "vt_intermediate_size": 4304,
798
+ "merge_kernel_size": [
799
+ 2,
800
+ 2
801
+ ],
802
+ "video_attn_type": "spatial_temporal",
803
+ "merge_type": "sd2_tpool",
804
+ "mm_projector_type": "patchmerger",
805
+ "mm_hidden_size": 1152,
806
+ "projector_hidden_act": "gelu",
807
+ "projector_ln_eps": 1e-05,
808
+ "text_hidden_size": 6144
809
+ },
810
+ "ignore_index": -100,
811
+ "media_placeholder_token_id": 154854,
812
+ "pad_token_id": 154820,
813
+ "use_unified_vision_chunk": true,
814
+ "video_placeholder": "<|glm5v_video_placeholder|>",
815
+ "encoder_only": false,
816
+ "language_only": false,
817
+ "tie_word_embeddings": false,
818
+ "quantization_config": {
819
+ "activation_scheme": "dynamic",
820
+ "fmt": "e4m3",
821
+ "quant_method": "fp8",
822
+ "weight_block_size": [
823
+ 128,
824
+ 128
825
+ ],
826
+ "modules_to_not_convert": [
827
+ "model.layers.59.post_attention_layernorm",
828
+ "model.layers.47.mlp.gate.e_score_correction_bias",
829
+ "model.layers.71.self_attn.kv_a_layernorm",
830
+ "model.layers.46.mlp.gate.e_score_correction_bias",
831
+ "model.layers.30.self_attn.kv_a_layernorm",
832
+ "model.layers.23.self_attn.kv_a_layernorm",
833
+ "model.layers.73.mlp.gate.e_score_correction_bias",
834
+ "model.layers.36.mlp.gate",
835
+ "model.layers.12.self_attn.q_a_layernorm",
836
+ "model.layers.51.mlp.gate",
837
+ "model.layers.47.self_attn.kv_a_layernorm",
838
+ "model.layers.45.self_attn.q_a_layernorm",
839
+ "model.layers.15.self_attn.q_a_layernorm",
840
+ "model.layers.39.input_layernorm",
841
+ "model.layers.50.self_attn.kv_a_layernorm",
842
+ "model.layers.19.post_attention_layernorm",
843
+ "model.layers.71.input_layernorm",
844
+ "model.layers.72.self_attn.kv_a_layernorm",
845
+ "model.layers.70.mlp.gate",
846
+ "model.layers.52.input_layernorm",
847
+ "model.layers.78.self_attn.kv_a_layernorm",
848
+ "model.layers.48.post_attention_layernorm",
849
+ "model.layers.4.self_attn.q_a_layernorm",
850
+ "model.layers.62.self_attn.q_a_layernorm",
851
+ "model.layers.4.post_attention_layernorm",
852
+ "model.layers.38.self_attn.indexers_proj",
853
+ "model.layers.61.mlp.gate.e_score_correction_bias",
854
+ "model.layers.14.mlp.gate.e_score_correction_bias",
855
+ "model.layers.36.self_attn.kv_a_layernorm",
856
+ "model.layers.65.mlp.gate",
857
+ "model.layers.13.self_attn.kv_a_layernorm",
858
+ "model.layers.57.mlp.gate",
859
+ "model.layers.65.self_attn.kv_a_layernorm",
860
+ "model.layers.68.input_layernorm",
861
+ "model.layers.70.self_attn.kv_a_layernorm",
862
+ "model.layers.24.mlp.gate.e_score_correction_bias",
863
+ "model.layers.4.input_layernorm",
864
+ "model.layers.20.input_layernorm",
865
+ "model.layers.65.self_attn.q_a_layernorm",
866
+ "model.layers.38.mlp.gate",
867
+ "model.layers.76.mlp.gate.e_score_correction_bias",
868
+ "model.layers.62.input_layernorm",
869
+ "model.layers.45.input_layernorm",
870
+ "model.layers.68.post_attention_layernorm",
871
+ "model.layers.72.input_layernorm",
872
+ "model.layers.32.mlp.gate.e_score_correction_bias",
873
+ "model.layers.27.input_layernorm",
874
+ "model.layers.16.post_attention_layernorm",
875
+ "model.layers.35.self_attn.q_a_layernorm",
876
+ "model.layers.58.self_attn.indexers_proj",
877
+ "model.layers.3.mlp.gate.e_score_correction_bias",
878
+ "model.layers.49.self_attn.q_a_layernorm",
879
+ "model.layers.66.mlp.gate.e_score_correction_bias",
880
+ "model.layers.10.self_attn.q_a_layernorm",
881
+ "model.layers.51.mlp.gate.e_score_correction_bias",
882
+ "model.layers.64.self_attn.q_a_layernorm",
883
+ "model.layers.70.input_layernorm",
884
+ "model.layers.71.post_attention_layernorm",
885
+ "model.layers.67.post_attention_layernorm",
886
+ "model.layers.6.self_attn.indexer.k_norm",
887
+ "model.layers.3.mlp.gate",
888
+ "model.layers.11.mlp.gate",
889
+ "model.layers.17.input_layernorm",
890
+ "model.layers.13.mlp.gate",
891
+ "model.layers.15.mlp.gate",
892
+ "model.layers.71.self_attn.q_a_layernorm",
893
+ "model.layers.33.mlp.gate.e_score_correction_bias",
894
+ "model.layers.72.mlp.gate.e_score_correction_bias",
895
+ "model.layers.65.post_attention_layernorm",
896
+ "model.layers.78.hnorm",
897
+ "model.layers.40.post_attention_layernorm",
898
+ "model.layers.12.mlp.gate.e_score_correction_bias",
899
+ "model.layers.8.input_layernorm",
900
+ "model.layers.73.mlp.gate",
901
+ "model.layers.11.post_attention_layernorm",
902
+ "model.layers.61.input_layernorm",
903
+ "model.layers.14.self_attn.kv_a_layernorm",
904
+ "model.layers.39.self_attn.kv_a_layernorm",
905
+ "model.layers.70.self_attn.indexer.k_norm",
906
+ "model.layers.45.mlp.gate",
907
+ "model.layers.51.self_attn.q_a_layernorm",
908
+ "model.layers.64.post_attention_layernorm",
909
+ "model.layers.20.post_attention_layernorm",
910
+ "model.layers.66.self_attn.kv_a_layernorm",
911
+ "model.layers.10.self_attn.indexer.k_norm.bias",
912
+ "model.layers.23.input_layernorm",
913
+ "model.layers.38.self_attn.indexer.k_norm",
914
+ "model.layers.19.self_attn.kv_a_layernorm",
915
+ "model.layers.40.self_attn.q_a_layernorm",
916
+ "model.layers.10.self_attn.indexer.k_norm",
917
+ "model.layers.44.post_attention_layernorm",
918
+ "model.layers.70.post_attention_layernorm",
919
+ "model.layers.41.mlp.gate.e_score_correction_bias",
920
+ "model.layers.36.post_attention_layernorm",
921
+ "model.layers.30.self_attn.indexer.k_norm.bias",
922
+ "model.layers.54.self_attn.q_a_layernorm",
923
+ "model.layers.59.mlp.gate",
924
+ "model.layers.69.post_attention_layernorm",
925
+ "lm_head",
926
+ "model.layers.77.self_attn.kv_a_layernorm",
927
+ "model.layers.75.self_attn.q_a_layernorm",
928
+ "model.layers.56.self_attn.q_a_layernorm",
929
+ "model.layers.50.self_attn.indexer.k_norm.bias",
930
+ "model.layers.54.mlp.gate",
931
+ "model.layers.6.self_attn.indexer.k_norm.bias",
932
+ "model.layers.15.input_layernorm",
933
+ "model.layers.29.self_attn.q_a_layernorm",
934
+ "model.layers.54.self_attn.indexers_proj",
935
+ "model.layers.73.self_attn.kv_a_layernorm",
936
+ "model.layers.0.self_attn.indexer.k_norm",
937
+ "model.layers.58.self_attn.indexer.k_norm",
938
+ "model.layers.36.input_layernorm",
939
+ "model.layers.47.self_attn.q_a_layernorm",
940
+ "model.layers.26.mlp.gate.e_score_correction_bias",
941
+ "model.layers.50.input_layernorm",
942
+ "model.layers.16.self_attn.q_a_layernorm",
943
+ "model.layers.19.mlp.gate.e_score_correction_bias",
944
+ "model.layers.78.input_layernorm",
945
+ "model.layers.27.mlp.gate",
946
+ "model.layers.19.self_attn.q_a_layernorm",
947
+ "model.layers.25.input_layernorm",
948
+ "model.layers.33.self_attn.kv_a_layernorm",
949
+ "model.layers.32.mlp.gate",
950
+ "model.layers.61.self_attn.kv_a_layernorm",
951
+ "model.layers.50.self_attn.indexers_proj",
952
+ "model.layers.34.self_attn.indexers_proj",
953
+ "model.layers.4.mlp.gate.e_score_correction_bias",
954
+ "model.layers.76.post_attention_layernorm",
955
+ "model.layers.57.post_attention_layernorm",
956
+ "model.layers.73.post_attention_layernorm",
957
+ "model.layers.68.mlp.gate.e_score_correction_bias",
958
+ "model.layers.34.self_attn.kv_a_layernorm",
959
+ "model.layers.78.enorm",
960
+ "model.layers.30.mlp.gate",
961
+ "model.layers.10.self_attn.indexers_proj",
962
+ "model.layers.34.input_layernorm",
963
+ "model.layers.34.post_attention_layernorm",
964
+ "model.layers.20.mlp.gate.e_score_correction_bias",
965
+ "model.layers.54.self_attn.indexer.k_norm.bias",
966
+ "model.layers.44.mlp.gate",
967
+ "model.layers.25.mlp.gate.e_score_correction_bias",
968
+ "model.layers.26.self_attn.indexers_proj",
969
+ "model.layers.66.self_attn.indexer.k_norm.bias",
970
+ "model.layers.41.post_attention_layernorm",
971
+ "model.layers.51.post_attention_layernorm",
972
+ "model.layers.22.self_attn.indexer.k_norm.bias",
973
+ "model.layers.21.mlp.gate.e_score_correction_bias",
974
+ "model.layers.15.self_attn.kv_a_layernorm",
975
+ "model.layers.25.self_attn.kv_a_layernorm",
976
+ "model.layers.55.mlp.gate",
977
+ "model.layers.11.self_attn.q_a_layernorm",
978
+ "model.layers.72.mlp.gate",
979
+ "model.layers.21.self_attn.kv_a_layernorm",
980
+ "model.layers.59.mlp.gate.e_score_correction_bias",
981
+ "model.layers.2.input_layernorm",
982
+ "model.layers.5.mlp.gate",
983
+ "model.layers.54.self_attn.kv_a_layernorm",
984
+ "model.layers.0.self_attn.indexers_proj",
985
+ "model.layers.71.mlp.gate.e_score_correction_bias",
986
+ "model.layers.6.self_attn.q_a_layernorm",
987
+ "model.layers.14.mlp.gate",
988
+ "model.layers.44.self_attn.kv_a_layernorm",
989
+ "model.layers.61.post_attention_layernorm",
990
+ "model.layers.64.self_attn.kv_a_layernorm",
991
+ "model.layers.1.post_attention_layernorm",
992
+ "model.layers.4.mlp.gate",
993
+ "model.layers.22.post_attention_layernorm",
994
+ "model.layers.21.self_attn.q_a_layernorm",
995
+ "model.layers.74.self_attn.q_a_layernorm",
996
+ "model.layers.18.self_attn.indexers_proj",
997
+ "model.layers.75.input_layernorm",
998
+ "model.layers.70.self_attn.q_a_layernorm",
999
+ "model.layers.22.self_attn.indexer.k_norm",
1000
+ "model.layers.42.mlp.gate.e_score_correction_bias",
1001
+ "model.layers.5.self_attn.kv_a_layernorm",
1002
+ "model.layers.31.mlp.gate.e_score_correction_bias",
1003
+ "model.layers.62.self_attn.indexer.k_norm.bias",
1004
+ "model.layers.29.post_attention_layernorm",
1005
+ "model.layers.6.self_attn.kv_a_layernorm",
1006
+ "model.layers.48.input_layernorm",
1007
+ "model.layers.74.self_attn.indexer.k_norm.bias",
1008
+ "model.layers.7.input_layernorm",
1009
+ "model.layers.39.self_attn.q_a_layernorm",
1010
+ "model.layers.41.input_layernorm",
1011
+ "model.layers.54.self_attn.indexer.k_norm",
1012
+ "model.layers.32.self_attn.q_a_layernorm",
1013
+ "model.layers.39.mlp.gate.e_score_correction_bias",
1014
+ "model.layers.42.input_layernorm",
1015
+ "model.layers.43.mlp.gate",
1016
+ "model.layers.72.self_attn.q_a_layernorm",
1017
+ "model.layers.37.mlp.gate.e_score_correction_bias",
1018
+ "model.layers.5.post_attention_layernorm",
1019
+ "model.layers.28.mlp.gate",
1020
+ "model.layers.2.post_attention_layernorm",
1021
+ "model.layers.49.post_attention_layernorm",
1022
+ "model.layers.43.self_attn.kv_a_layernorm",
1023
+ "model.layers.12.input_layernorm",
1024
+ "model.layers.56.mlp.gate",
1025
+ "model.layers.5.input_layernorm",
1026
+ "model.layers.29.mlp.gate.e_score_correction_bias",
1027
+ "model.layers.4.self_attn.kv_a_layernorm",
1028
+ "model.layers.20.self_attn.q_a_layernorm",
1029
+ "model.layers.59.self_attn.q_a_layernorm",
1030
+ "model.layers.68.mlp.gate",
1031
+ "model.layers.21.post_attention_layernorm",
1032
+ "model.layers.26.self_attn.q_a_layernorm",
1033
+ "model.layers.52.self_attn.q_a_layernorm",
1034
+ "model.layers.29.self_attn.kv_a_layernorm",
1035
+ "model.layers.24.input_layernorm",
1036
+ "model.layers.34.self_attn.indexer.k_norm.bias",
1037
+ "model.layers.43.input_layernorm",
1038
+ "model.layers.27.self_attn.kv_a_layernorm",
1039
+ "model.layers.31.input_layernorm",
1040
+ "model.layers.69.mlp.gate",
1041
+ "model.layers.26.post_attention_layernorm",
1042
+ "model.layers.3.self_attn.q_a_layernorm",
1043
+ "model.layers.37.post_attention_layernorm",
1044
+ "model.layers.52.mlp.gate",
1045
+ "model.layers.73.input_layernorm",
1046
+ "model.layers.19.input_layernorm",
1047
+ "model.layers.13.self_attn.q_a_layernorm",
1048
+ "model.layers.76.input_layernorm",
1049
+ "model.layers.8.mlp.gate",
1050
+ "model.layers.63.post_attention_layernorm",
1051
+ "model.layers.58.mlp.gate",
1052
+ "model.layers.31.post_attention_layernorm",
1053
+ "model.layers.23.mlp.gate.e_score_correction_bias",
1054
+ "model.layers.34.self_attn.q_a_layernorm",
1055
+ "model.layers.78.self_attn.indexer.k_norm.bias",
1056
+ "model.layers.13.mlp.gate.e_score_correction_bias",
1057
+ "model.layers.71.mlp.gate",
1058
+ "model.layers.14.input_layernorm",
1059
+ "model.layers.17.mlp.gate.e_score_correction_bias",
1060
+ "model.layers.8.post_attention_layernorm",
1061
+ "model.layers.45.self_attn.kv_a_layernorm",
1062
+ "model.layers.46.self_attn.indexer.k_norm",
1063
+ "model.layers.20.self_attn.kv_a_layernorm",
1064
+ "model.layers.78.self_attn.indexer.k_norm",
1065
+ "model.layers.37.mlp.gate",
1066
+ "model.layers.7.mlp.gate",
1067
+ "model.layers.30.mlp.gate.e_score_correction_bias",
1068
+ "model.layers.67.mlp.gate",
1069
+ "model.layers.73.self_attn.q_a_layernorm",
1070
+ "model.layers.10.self_attn.kv_a_layernorm",
1071
+ "model.layers.61.mlp.gate",
1072
+ "model.layers.22.mlp.gate.e_score_correction_bias",
1073
+ "model.layers.58.input_layernorm",
1074
+ "model.layers.8.self_attn.kv_a_layernorm",
1075
+ "model.layers.33.self_attn.q_a_layernorm",
1076
+ "model.layers.55.self_attn.q_a_layernorm",
1077
+ "model.layers.32.input_layernorm",
1078
+ "model.layers.17.self_attn.kv_a_layernorm",
1079
+ "model.layers.46.self_attn.indexer.k_norm.bias",
1080
+ "model.layers.11.mlp.gate.e_score_correction_bias",
1081
+ "model.layers.9.input_layernorm",
1082
+ "model.layers.77.input_layernorm",
1083
+ "model.layers.16.self_attn.kv_a_layernorm",
1084
+ "model.layers.45.post_attention_layernorm",
1085
+ "model.layers.74.mlp.gate",
1086
+ "model.layers.68.self_attn.q_a_layernorm",
1087
+ "model.layers.55.self_attn.kv_a_layernorm",
1088
+ "model.layers.44.input_layernorm",
1089
+ "model.layers.60.self_attn.kv_a_layernorm",
1090
+ "model.layers.14.post_attention_layernorm",
1091
+ "model.layers.62.self_attn.indexers_proj",
1092
+ "model.layers.0.self_attn.q_a_layernorm",
1093
+ "model.layers.72.post_attention_layernorm",
1094
+ "model.layers.78.post_attention_layernorm",
1095
+ "model.layers.60.mlp.gate.e_score_correction_bias",
1096
+ "model.layers.26.self_attn.indexer.k_norm.bias",
1097
+ "model.layers.38.post_attention_layernorm",
1098
+ "model.layers.50.post_attention_layernorm",
1099
+ "model.layers.28.input_layernorm",
1100
+ "model.layers.30.self_attn.indexer.k_norm",
1101
+ "model.layers.37.self_attn.q_a_layernorm",
1102
+ "model.layers.56.input_layernorm",
1103
+ "model.layers.27.post_attention_layernorm",
1104
+ "model.layers.74.input_layernorm",
1105
+ "model.layers.78.self_attn.q_a_layernorm",
1106
+ "model.layers.42.self_attn.q_a_layernorm",
1107
+ "model.layers.26.self_attn.indexer.k_norm",
1108
+ "model.layers.42.self_attn.kv_a_layernorm",
1109
+ "model.layers.1.self_attn.indexer.k_norm",
1110
+ "model.layers.39.post_attention_layernorm",
1111
+ "model.layers.48.mlp.gate.e_score_correction_bias",
1112
+ "model.layers.74.post_attention_layernorm",
1113
+ "model.layers.69.self_attn.q_a_layernorm",
1114
+ "model.layers.13.input_layernorm",
1115
+ "model.layers.74.self_attn.indexer.k_norm",
1116
+ "model.layers.29.mlp.gate",
1117
+ "model.layers.7.mlp.gate.e_score_correction_bias",
1118
+ "model.layers.3.self_attn.kv_a_layernorm",
1119
+ "model.layers.35.self_attn.kv_a_layernorm",
1120
+ "model.layers.46.self_attn.indexers_proj",
1121
+ "model.layers.0.self_attn.kv_a_layernorm",
1122
+ "model.layers.43.self_attn.q_a_layernorm",
1123
+ "model.norm",
1124
+ "model.layers.1.self_attn.kv_a_layernorm",
1125
+ "model.layers.38.self_attn.indexer.k_norm.bias",
1126
+ "model.layers.30.self_attn.q_a_layernorm",
1127
+ "model.layers.0.input_layernorm",
1128
+ "model.layers.47.input_layernorm",
1129
+ "model.layers.8.self_attn.q_a_layernorm",
1130
+ "model.layers.61.self_attn.q_a_layernorm",
1131
+ "model.layers.1.self_attn.indexer.k_norm.bias",
1132
+ "model.layers.62.mlp.gate.e_score_correction_bias",
1133
+ "model.layers.66.self_attn.q_a_layernorm",
1134
+ "model.layers.46.self_attn.kv_a_layernorm",
1135
+ "model.layers.66.mlp.gate",
1136
+ "model.layers.69.input_layernorm",
1137
+ "model.layers.69.self_attn.kv_a_layernorm",
1138
+ "model.layers.22.input_layernorm",
1139
+ "model.layers.30.post_attention_layernorm",
1140
+ "model.layers.24.self_attn.q_a_layernorm",
1141
+ "model.layers.48.mlp.gate",
1142
+ "model.layers.7.self_attn.kv_a_layernorm",
1143
+ "model.layers.67.input_layernorm",
1144
+ "model.layers.27.self_attn.q_a_layernorm",
1145
+ "model.layers.23.mlp.gate",
1146
+ "model.layers.45.mlp.gate.e_score_correction_bias",
1147
+ "model.layers.76.self_attn.kv_a_layernorm",
1148
+ "model.layers.22.self_attn.q_a_layernorm",
1149
+ "model.layers.78.mlp.gate",
1150
+ "model.layers.9.mlp.gate",
1151
+ "model.layers.57.self_attn.q_a_layernorm",
1152
+ "model.layers.78.mlp.gate.e_score_correction_bias",
1153
+ "model.layers.23.post_attention_layernorm",
1154
+ "model.layers.38.self_attn.q_a_layernorm",
1155
+ "model.layers.53.mlp.gate",
1156
+ "model.layers.77.self_attn.q_a_layernorm",
1157
+ "model.layers.62.self_attn.kv_a_layernorm",
1158
+ "model.layers.12.mlp.gate",
1159
+ "model.layers.34.self_attn.indexer.k_norm",
1160
+ "model.layers.46.self_attn.q_a_layernorm",
1161
+ "model.layers.36.self_attn.q_a_layernorm",
1162
+ "model.layers.58.self_attn.indexer.k_norm.bias",
1163
+ "model.layers.60.post_attention_layernorm",
1164
+ "model.layers.28.post_attention_layernorm",
1165
+ "model.layers.2.self_attn.kv_a_layernorm",
1166
+ "model.layers.70.self_attn.indexers_proj",
1167
+ "model.layers.16.mlp.gate.e_score_correction_bias",
1168
+ "model.layers.2.self_attn.indexer.k_norm",
1169
+ "model.layers.31.self_attn.kv_a_layernorm",
1170
+ "model.layers.64.input_layernorm",
1171
+ "model.layers.6.mlp.gate.e_score_correction_bias",
1172
+ "model.layers.50.self_attn.indexer.k_norm",
1173
+ "model.layers.58.self_attn.q_a_layernorm",
1174
+ "model.layers.57.mlp.gate.e_score_correction_bias",
1175
+ "model.layers.13.post_attention_layernorm",
1176
+ "model.layers.28.self_attn.kv_a_layernorm",
1177
+ "model.layers.31.self_attn.q_a_layernorm",
1178
+ "model.layers.1.self_attn.q_a_layernorm",
1179
+ "model.layers.17.mlp.gate",
1180
+ "model.layers.15.post_attention_layernorm",
1181
+ "model.layers.18.self_attn.indexer.k_norm.bias",
1182
+ "model.layers.18.self_attn.indexer.k_norm",
1183
+ "model.layers.43.mlp.gate.e_score_correction_bias",
1184
+ "model.layers.11.input_layernorm",
1185
+ "model.layers.9.post_attention_layernorm",
1186
+ "model.layers.33.input_layernorm",
1187
+ "model.layers.66.input_layernorm",
1188
+ "model.layers.78.self_attn.indexers_proj",
1189
+ "model.layers.54.post_attention_layernorm",
1190
+ "model.layers.53.mlp.gate.e_score_correction_bias",
1191
+ "model.layers.5.mlp.gate.e_score_correction_bias",
1192
+ "model.layers.19.mlp.gate",
1193
+ "model.layers.49.mlp.gate",
1194
+ "model.layers.66.self_attn.indexers_proj",
1195
+ "model.layers.14.self_attn.indexer.k_norm.bias",
1196
+ "model.layers.41.mlp.gate",
1197
+ "model.layers.42.self_attn.indexer.k_norm.bias",
1198
+ "model.layers.65.input_layernorm",
1199
+ "model.layers.75.mlp.gate",
1200
+ "model.layers.12.self_attn.kv_a_layernorm",
1201
+ "model.layers.53.input_layernorm",
1202
+ "model.layers.58.self_attn.kv_a_layernorm",
1203
+ "model.layers.60.self_attn.q_a_layernorm",
1204
+ "model.layers.6.mlp.gate",
1205
+ "model.layers.49.self_attn.kv_a_layernorm",
1206
+ "model.layers.16.mlp.gate",
1207
+ "model.layers.22.mlp.gate",
1208
+ "model.layers.58.post_attention_layernorm",
1209
+ "model.layers.15.mlp.gate.e_score_correction_bias",
1210
+ "model.layers.26.input_layernorm",
1211
+ "model.layers.3.post_attention_layernorm",
1212
+ "model.layers.6.input_layernorm",
1213
+ "model.layers.22.self_attn.kv_a_layernorm",
1214
+ "model.layers.16.input_layernorm",
1215
+ "model.layers.33.post_attention_layernorm",
1216
+ "model.layers.17.post_attention_layernorm",
1217
+ "model.layers.35.mlp.gate",
1218
+ "model.layers.56.post_attention_layernorm",
1219
+ "model.layers.7.self_attn.q_a_layernorm",
1220
+ "model.layers.24.mlp.gate",
1221
+ "model.layers.24.post_attention_layernorm",
1222
+ "model.layers.36.mlp.gate.e_score_correction_bias",
1223
+ "model.layers.56.mlp.gate.e_score_correction_bias",
1224
+ "model.layers.32.post_attention_layernorm",
1225
+ "model.layers.47.post_attention_layernorm",
1226
+ "model.layers.40.mlp.gate",
1227
+ "model.layers.41.self_attn.kv_a_layernorm",
1228
+ "model.layers.52.self_attn.kv_a_layernorm",
1229
+ "model.layers.9.self_attn.kv_a_layernorm",
1230
+ "model.layers.77.mlp.gate.e_score_correction_bias",
1231
+ "model.layers.78.shared_head.norm",
1232
+ "model.layers.53.post_attention_layernorm",
1233
+ "model.layers.63.self_attn.kv_a_layernorm",
1234
+ "model.layers.48.self_attn.kv_a_layernorm",
1235
+ "model.layers.28.mlp.gate.e_score_correction_bias",
1236
+ "model.layers.8.mlp.gate.e_score_correction_bias",
1237
+ "model.layers.52.mlp.gate.e_score_correction_bias",
1238
+ "model.layers.52.post_attention_layernorm",
1239
+ "model.layers.18.self_attn.q_a_layernorm",
1240
+ "model.layers.50.mlp.gate.e_score_correction_bias",
1241
+ "model.layers.64.mlp.gate.e_score_correction_bias",
1242
+ "model.layers.38.mlp.gate.e_score_correction_bias",
1243
+ "model.layers.59.input_layernorm",
1244
+ "model.layers.21.input_layernorm",
1245
+ "model.layers.39.mlp.gate",
1246
+ "model.layers.77.post_attention_layernorm",
1247
+ "model.layers.76.self_attn.q_a_layernorm",
1248
+ "model.layers.12.post_attention_layernorm",
1249
+ "model.layers.43.post_attention_layernorm",
1250
+ "model.layers.35.input_layernorm",
1251
+ "model.layers.62.mlp.gate",
1252
+ "model.layers.38.self_attn.kv_a_layernorm",
1253
+ "model.layers.63.mlp.gate",
1254
+ "model.layers.27.mlp.gate.e_score_correction_bias",
1255
+ "model.layers.18.mlp.gate",
1256
+ "model.layers.25.post_attention_layernorm",
1257
+ "model.layers.55.mlp.gate.e_score_correction_bias",
1258
+ "model.layers.66.self_attn.indexer.k_norm",
1259
+ "model.layers.48.self_attn.q_a_layernorm",
1260
+ "model.layers.78.eh_proj",
1261
+ "model.layers.77.mlp.gate",
1262
+ "model.layers.6.self_attn.indexers_proj",
1263
+ "model.layers.14.self_attn.q_a_layernorm",
1264
+ "model.layers.2.self_attn.q_a_layernorm",
1265
+ "model.layers.42.mlp.gate",
1266
+ "model.layers.54.mlp.gate.e_score_correction_bias",
1267
+ "model.layers.60.input_layernorm",
1268
+ "model.layers.40.self_attn.kv_a_layernorm",
1269
+ "model.layers.51.self_attn.kv_a_layernorm",
1270
+ "model.layers.18.input_layernorm",
1271
+ "model.layers.30.self_attn.indexers_proj",
1272
+ "model.layers.53.self_attn.kv_a_layernorm",
1273
+ "model.layers.6.post_attention_layernorm",
1274
+ "model.layers.70.self_attn.indexer.k_norm.bias",
1275
+ "model.layers.54.input_layernorm",
1276
+ "model.layers.41.self_attn.q_a_layernorm",
1277
+ "model.layers.18.mlp.gate.e_score_correction_bias",
1278
+ "model.layers.34.mlp.gate.e_score_correction_bias",
1279
+ "model.layers.47.mlp.gate",
1280
+ "model.layers.46.mlp.gate",
1281
+ "model.layers.25.self_attn.q_a_layernorm",
1282
+ "model.layers.63.mlp.gate.e_score_correction_bias",
1283
+ "model.layers.66.post_attention_layernorm",
1284
+ "model.layers.76.mlp.gate",
1285
+ "model.embed_tokens",
1286
+ "model.layers.67.self_attn.kv_a_layernorm",
1287
+ "model.layers.67.mlp.gate.e_score_correction_bias",
1288
+ "model.layers.26.self_attn.kv_a_layernorm",
1289
+ "model.layers.31.mlp.gate",
1290
+ "model.layers.18.self_attn.kv_a_layernorm",
1291
+ "model.layers.34.mlp.gate",
1292
+ "model.layers.55.post_attention_layernorm",
1293
+ "model.layers.30.input_layernorm",
1294
+ "model.layers.50.mlp.gate",
1295
+ "model.layers.63.input_layernorm",
1296
+ "model.layers.11.self_attn.kv_a_layernorm",
1297
+ "model.layers.24.self_attn.kv_a_layernorm",
1298
+ "model.layers.75.mlp.gate.e_score_correction_bias",
1299
+ "model.layers.10.post_attention_layernorm",
1300
+ "model.layers.1.self_attn.indexers_proj",
1301
+ "model.layers.42.self_attn.indexers_proj",
1302
+ "model.layers.46.input_layernorm",
1303
+ "model.layers.23.self_attn.q_a_layernorm",
1304
+ "model.layers.63.self_attn.q_a_layernorm",
1305
+ "model.layers.14.self_attn.indexer.k_norm",
1306
+ "model.layers.14.self_attn.indexers_proj",
1307
+ "model.layers.60.mlp.gate",
1308
+ "model.layers.70.mlp.gate.e_score_correction_bias",
1309
+ "model.layers.53.self_attn.q_a_layernorm",
1310
+ "model.layers.0.post_attention_layernorm",
1311
+ "model.layers.35.mlp.gate.e_score_correction_bias",
1312
+ "model.layers.75.self_attn.kv_a_layernorm",
1313
+ "model.layers.1.input_layernorm",
1314
+ "model.layers.42.post_attention_layernorm",
1315
+ "model.layers.37.input_layernorm",
1316
+ "model.layers.9.mlp.gate.e_score_correction_bias",
1317
+ "model.layers.46.post_attention_layernorm",
1318
+ "model.layers.26.mlp.gate",
1319
+ "model.layers.65.mlp.gate.e_score_correction_bias",
1320
+ "model.layers.67.self_attn.q_a_layernorm",
1321
+ "model.layers.44.mlp.gate.e_score_correction_bias",
1322
+ "model.layers.33.mlp.gate",
1323
+ "model.layers.29.input_layernorm",
1324
+ "model.layers.18.post_attention_layernorm",
1325
+ "model.layers.75.post_attention_layernorm",
1326
+ "model.layers.44.self_attn.q_a_layernorm",
1327
+ "model.layers.10.mlp.gate",
1328
+ "model.layers.21.mlp.gate",
1329
+ "model.layers.58.mlp.gate.e_score_correction_bias",
1330
+ "model.layers.7.post_attention_layernorm",
1331
+ "model.layers.50.self_attn.q_a_layernorm",
1332
+ "model.layers.9.self_attn.q_a_layernorm",
1333
+ "model.layers.2.self_attn.indexers_proj",
1334
+ "model.layers.0.self_attn.indexer.k_norm.bias",
1335
+ "model.layers.74.self_attn.kv_a_layernorm",
1336
+ "model.layers.74.mlp.gate.e_score_correction_bias",
1337
+ "model.layers.49.mlp.gate.e_score_correction_bias",
1338
+ "model.layers.62.post_attention_layernorm",
1339
+ "model.layers.57.self_attn.kv_a_layernorm",
1340
+ "model.layers.40.input_layernorm",
1341
+ "model.layers.69.mlp.gate.e_score_correction_bias",
1342
+ "model.layers.35.post_attention_layernorm",
1343
+ "model.layers.74.self_attn.indexers_proj",
1344
+ "model.layers.55.input_layernorm",
1345
+ "model.layers.49.input_layernorm",
1346
+ "model.layers.40.mlp.gate.e_score_correction_bias",
1347
+ "model.layers.25.mlp.gate",
1348
+ "model.layers.2.self_attn.indexer.k_norm.bias",
1349
+ "model.layers.68.self_attn.kv_a_layernorm",
1350
+ "model.layers.51.input_layernorm",
1351
+ "model.layers.22.self_attn.indexers_proj",
1352
+ "model.layers.10.input_layernorm",
1353
+ "model.layers.42.self_attn.indexer.k_norm",
1354
+ "model.layers.56.self_attn.kv_a_layernorm",
1355
+ "model.layers.17.self_attn.q_a_layernorm",
1356
+ "model.layers.59.self_attn.kv_a_layernorm",
1357
+ "model.layers.3.input_layernorm",
1358
+ "model.layers.57.input_layernorm",
1359
+ "model.layers.38.input_layernorm",
1360
+ "model.layers.62.self_attn.indexer.k_norm",
1361
+ "model.layers.20.mlp.gate",
1362
+ "model.layers.64.mlp.gate",
1363
+ "model.layers.28.self_attn.q_a_layernorm",
1364
+ "model.layers.32.self_attn.kv_a_layernorm",
1365
+ "model.layers.10.mlp.gate.e_score_correction_bias",
1366
+ "model.layers.5.self_attn.q_a_layernorm",
1367
+ "model.layers.37.self_attn.kv_a_layernorm"
1368
+ ]
1369
+ }
1370
+ }
configuration_glm5v.py ADDED
@@ -0,0 +1,152 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Glm5vConfig — remote-code config carried inside the assembled GLM5V SGLang
3
+ checkpoint (referenced by config.json ``auto_map``; loaded with
4
+ ``--trust-remote-code``, which the checkpoint already requires for the Kimi
5
+ image-processor remote code).
6
+
7
+ Self-contained: depends only on ``transformers``. Mirrors SGLang's in-tree
8
+ ``KimiK25Config`` structure (``vision_config`` + ``text_config`` + media
9
+ placeholder fields) with GLM-5.2 as the text model:
10
+
11
+ * ``text_config`` -> ``GlmMoeDsaConfig`` (transformers-native ``glm_moe_dsa``).
12
+ * ``vision_config``-> MoonViT fields; ``text_hidden_size`` (projector output
13
+ dim) retargeted to GLM hidden 6144.
14
+ * ``media_placeholder_token_id`` -> GLM ``<|image|>`` = 154854.
15
+ """
16
+
17
+ from transformers import AutoConfig
18
+ from transformers.configuration_utils import PretrainedConfig
19
+
20
+
21
+ class Glm5vVisionConfig(PretrainedConfig):
22
+ """MoonViT vision tower + PatchMerger projector config.
23
+
24
+ Field names/defaults mirror SGLang's ``KimiK25VisionConfig`` (declared
25
+ names like ``hidden_size``) while the official Kimi checkpoint's ``vt_*``
26
+ names arrive via **kwargs and are stored as attributes — SGLang's model
27
+ code reads both families (tower: ``hidden_size``; projector:
28
+ ``vt_hidden_size``/``text_hidden_size``).
29
+ """
30
+
31
+ model_type = "glm5v_vision"
32
+
33
+ def __init__(
34
+ self,
35
+ # Vision tower
36
+ patch_size: int = 14,
37
+ init_pos_emb_height: int = 64,
38
+ init_pos_emb_width: int = 64,
39
+ init_pos_emb_time: int = 4,
40
+ pos_emb_type: str = "divided_fixed",
41
+ num_attention_heads: int = 16,
42
+ num_hidden_layers: int = 27,
43
+ hidden_size: int = 1152,
44
+ intermediate_size: int = 4304,
45
+ merge_kernel_size=(2, 2),
46
+ video_attn_type: str = "spatial_temporal",
47
+ merge_type: str = "sd2_tpool",
48
+ # MM projector
49
+ mm_projector_type: str = "patchmerger",
50
+ mm_hidden_size: int | None = None,
51
+ projector_hidden_act: str = "gelu",
52
+ projector_ln_eps: float = 1e-5,
53
+ text_hidden_size: int = 6144, # GLM-5.2 hidden (Kimi default is 7168)
54
+ **kwargs,
55
+ ):
56
+ super().__init__(**kwargs)
57
+ self.patch_size = patch_size
58
+ self.init_pos_emb_height = init_pos_emb_height
59
+ self.init_pos_emb_width = init_pos_emb_width
60
+ self.init_pos_emb_time = init_pos_emb_time
61
+ self.pos_emb_type = pos_emb_type
62
+ self.num_attention_heads = num_attention_heads
63
+ self.num_hidden_layers = num_hidden_layers
64
+ self.hidden_size = hidden_size
65
+ self.intermediate_size = intermediate_size
66
+ self.merge_kernel_size = merge_kernel_size
67
+ self.video_attn_type = video_attn_type
68
+ self.merge_type = merge_type
69
+ self.mm_projector_type = mm_projector_type
70
+ self.mm_hidden_size = mm_hidden_size if mm_hidden_size is not None else hidden_size
71
+ self.projector_hidden_act = projector_hidden_act
72
+ self.projector_ln_eps = projector_ln_eps
73
+ self.text_hidden_size = text_hidden_size
74
+
75
+
76
+ class Glm5vConfig(PretrainedConfig):
77
+ """glm5v top-level config: MoonViT ``vision_config`` + GLM-5.2 ``text_config``."""
78
+
79
+ model_type = "glm5v"
80
+
81
+ def __init__(
82
+ self,
83
+ text_config=None,
84
+ vision_config=None,
85
+ ignore_index: int = -100,
86
+ media_placeholder_token_id: int = 154854, # GLM <|image|>
87
+ pad_token_id: int = 154820,
88
+ use_unified_vision_chunk: bool = True,
89
+ video_placeholder: str = "<|glm5v_video_placeholder|>",
90
+ encoder_only: bool = False,
91
+ language_only: bool = False,
92
+ **kwargs,
93
+ ):
94
+ # Vision config (MoonViT).
95
+ if vision_config is None:
96
+ self.vision_config = Glm5vVisionConfig()
97
+ elif isinstance(vision_config, dict):
98
+ self.vision_config = Glm5vVisionConfig(**vision_config)
99
+ else:
100
+ self.vision_config = vision_config
101
+
102
+ # Text config (GLM-5.2 / glm_moe_dsa), built via AutoConfig so the
103
+ # transformers-native GlmMoeDsaConfig class is used.
104
+ raw_text = dict(text_config) if isinstance(text_config, dict) else None
105
+ if text_config is None:
106
+ self.text_config = AutoConfig.for_model("glm_moe_dsa")
107
+ elif isinstance(text_config, dict):
108
+ tc = dict(text_config)
109
+ tc.setdefault("model_type", "glm_moe_dsa")
110
+ self.text_config = AutoConfig.for_model(**tc)
111
+ else:
112
+ self.text_config = text_config
113
+
114
+ # transformers 5.8.x GlmMoeDsaConfig drops/clobbers raw DSA fields the
115
+ # sparse-attention path needs. SGLang applies this same restore for
116
+ # bare GlmMoeDsaForCausalLM checkpoints (see its HfModelConfigParser;
117
+ # fixed upstream by transformers PR #46338, gone once >= 5.10); our
118
+ # top-level arch is Glm5v so we replicate it here.
119
+ if raw_text is not None:
120
+ for key in ("qk_rope_head_dim", "index_topk_freq"):
121
+ if key in raw_text:
122
+ setattr(self.text_config, key, raw_text[key])
123
+ if hasattr(self.text_config, "qk_nope_head_dim") and hasattr(
124
+ self.text_config, "qk_rope_head_dim"
125
+ ):
126
+ self.text_config.qk_head_dim = (
127
+ self.text_config.qk_nope_head_dim
128
+ + self.text_config.qk_rope_head_dim
129
+ )
130
+
131
+ self.ignore_index = ignore_index
132
+ self.media_placeholder_token_id = media_placeholder_token_id
133
+ self.use_unified_vision_chunk = use_unified_vision_chunk
134
+ self.video_placeholder = video_placeholder
135
+ self.encoder_only = encoder_only
136
+ self.language_only = language_only
137
+
138
+ # Propagate quantization config from the text model (Kimi pattern):
139
+ # only the GLM text Linears are FP8; vision/projector stay bf16 by
140
+ # construction in the model code.
141
+ if getattr(self.text_config, "quantization_config", None) is not None:
142
+ self.quantization_config = self.text_config.quantization_config
143
+
144
+ super().__init__(pad_token_id=pad_token_id, **kwargs)
145
+
146
+ @property
147
+ def hidden_size(self) -> int:
148
+ return self.text_config.hidden_size
149
+
150
+ @property
151
+ def vocab_size(self) -> int:
152
+ return self.text_config.vocab_size
generation_config.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "eos_token_id": [
4
+ 154820,
5
+ 154827,
6
+ 154829
7
+ ],
8
+ "pad_token_id": 154820,
9
+ "temperature": 1.0,
10
+ "top_p": 0.95,
11
+ "transformers_version": "5.12.0"
12
+ }
kimi_k25_processor.py ADDED
@@ -0,0 +1,208 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers.feature_extraction_utils import BatchFeature
2
+ from transformers.processing_utils import ProcessorMixin
3
+ from transformers.utils import logging
4
+
5
+ logger = logging.get_logger(__name__)
6
+
7
+
8
+ class KimiK25Processor(ProcessorMixin):
9
+ r"""
10
+ Constructs a KimiK25 processor which wraps a KimiK25 image processor and a tokenizer into a single processor.
11
+
12
+ [`KimiK25Processor`] offers all the functionalities of [`KimiK25ImageProcessor`] and [`TikTokenTokenizer`]. See the
13
+ [`~KimiK25Processor.__call__`] and [`~KimiK25Processor.decode`] for more information.
14
+
15
+ Args:
16
+ image_processor ([`KimiK25ImageProcessor`], *optional*):
17
+ The image processor is a required input.
18
+ tokenizer ([`TikTokenTokenizer`], *optional*):
19
+ The tokenizer is a required input.
20
+ chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages
21
+ in a chat into a tokenizable string.
22
+ """
23
+
24
+ attributes = ["image_processor", "tokenizer"]
25
+ valid_kwargs = ["chat_template"]
26
+ image_processor_class = "AutoImageProcessor"
27
+ tokenizer_class = "AutoTokenizer"
28
+
29
+ def __init__(
30
+ self,
31
+ image_processor=None,
32
+ tokenizer=None,
33
+ chat_template=None,
34
+ **kwargs,
35
+ ):
36
+ super().__init__(image_processor,
37
+ tokenizer,
38
+ chat_template=chat_template)
39
+ self.media_processor = image_processor
40
+ # A special temporal placeholder to be replaced by actual video placeholders
41
+ self.video_placeholder = "<|kimi_k25_video_placeholder|>"
42
+
43
+ def update_raw_text(self, text: str, video_prompts: list[str]) -> str:
44
+ # replace video prompt in text with video chunk prompts
45
+ video_count = text.count(self.video_placeholder)
46
+ if video_count == 0:
47
+ return text
48
+ assert video_count == len(video_prompts)
49
+ text_parts = text.split(self.video_placeholder)
50
+ assert len(text_parts) == len(video_prompts) + 1
51
+ text = "".join([
52
+ text_parts[i] + video_prompts[i] for i in range(len(video_prompts))
53
+ ])
54
+ text += text_parts[-1]
55
+ return text
56
+
57
+ def preprocess_medias(self, medias: list[dict]) -> list[dict]:
58
+ updated_medias = []
59
+ video_prompts = []
60
+ for media in medias:
61
+ if media['type'] == 'image':
62
+ updated_medias.append(media)
63
+ elif media['type'] == 'video':
64
+ video_chunks = self.media_processor.split_video_chunks(
65
+ media['video'])
66
+ updated_medias.extend(video_chunks)
67
+ video_prompts.append("".join(
68
+ [vc['prompt'] for vc in video_chunks]))
69
+ else:
70
+ raise ValueError(f"unsupported media type: {media['type']}")
71
+ return updated_medias, video_prompts
72
+
73
+ # glm5v: the image placeholder expanded per patch (GLM <|image|> = 154854).
74
+ # The chat template wraps it as <|begin_of_image|><|image|><|end_of_image|>.
75
+ GLM5V_IMAGE_TOKEN = "<|image|>"
76
+
77
+ def __call__(self,
78
+ messages: list[dict] = None,
79
+ medias: list[dict] = None,
80
+ text: str = None,
81
+ images: list = None,
82
+ return_tensors: str = "pt",
83
+ **kwargs) -> BatchFeature:
84
+ """
85
+ Process multimodal inputs for Kimi-K2.5 model.
86
+
87
+ This processor accepts ordered messages and extracts both media and text in a single pass.
88
+ text will be automatically updated if video input detected in messages
89
+
90
+ Args:
91
+ messages: List of message dicts with 'role' and 'content' fields.
92
+ If provided, medias and text will be extracted automatically.
93
+ medias: Pre-extracted list of media dicts. If None, extracted from messages.
94
+ text: Pre-formatted text string. If None, generated via apply_chat_template.
95
+ images: Standard HF VLM API (``processor(text=..., images=[...])``), as
96
+ called by generic drivers (e.g. slime's rollout prompt prep).
97
+ Converted to ``medias`` and each ``<|image|>`` placeholder in
98
+ ``text`` is expanded to that image's per-patch token count, so
99
+ the returned ``input_ids`` align with ``pixel_values`` (same
100
+ semantics as Qwen-family processors and the serving-layer
101
+ wrappers in vLLM/SGLang).
102
+ return_tensors: Format of returned tensors ('pt', 'np', 'tf'). Default: 'pt'.
103
+ **kwargs: Additional arguments passed to tokenizer.apply_chat_template.
104
+
105
+ Returns:
106
+ BatchFeature with fields: input_ids, attention_mask, pixel_values, grid_thws.
107
+ """
108
+ if images is not None and medias is None and text is not None:
109
+ # Standard HF call: expand placeholders, run the media preprocess, and
110
+ # return with STANDARD-HF dtypes: input_ids/attention_mask as python
111
+ # lists (callers like slime's rollout do `sample.tokens += tokens`),
112
+ # media tensors as `return_tensors` (default pt) for the train side.
113
+ if not isinstance(images, (list, tuple)):
114
+ images = [images]
115
+ medias = [{"type": "image", "image": img} for img in images]
116
+ parts = text.split(self.GLM5V_IMAGE_TOKEN)
117
+ if len(parts) - 1 != len(images):
118
+ raise ValueError(
119
+ f"got {len(images)} images but {len(parts) - 1} "
120
+ f"{self.GLM5V_IMAGE_TOKEN!r} placeholders in text")
121
+ expanded = [parts[0]]
122
+ for media, part in zip(medias, parts[1:]):
123
+ num_tokens = self.media_processor.media_tokens_calculator(media)
124
+ expanded.append(self.GLM5V_IMAGE_TOKEN * num_tokens + part)
125
+ text = "".join(expanded)
126
+
127
+ updated_medias, video_prompts = self.preprocess_medias(medias)
128
+ preprocessed = self.media_processor.preprocess(
129
+ updated_medias, return_tensors=return_tensors)
130
+ text = self.update_raw_text(text, video_prompts)
131
+ text_inputs = self.tokenizer([text]) # no return_tensors -> lists
132
+ data = {**text_inputs, **preprocessed.data}
133
+ # Qwen-convention key: downstream training forwards take
134
+ # `image_grid_thw` (same rename the SGLang wrapper applies).
135
+ if "grid_thws" in data:
136
+ data["image_grid_thw"] = data.pop("grid_thws")
137
+ return BatchFeature(data=data)
138
+
139
+ if messages is None and (medias is None or text is None):
140
+ raise ValueError(
141
+ "Provide either 'messages' or both 'medias' and 'text'")
142
+
143
+ if medias is not None and text is not None:
144
+ updated_medias, video_prompts = self.preprocess_medias(medias)
145
+ preprocessed = self.media_processor.preprocess(
146
+ updated_medias, return_tensors=return_tensors)
147
+ text = self.update_raw_text(text, video_prompts)
148
+ text_inputs = self.tokenizer(text, return_tensors=return_tensors)
149
+ return BatchFeature(data={**text_inputs, **preprocessed.data})
150
+
151
+ if medias is None:
152
+ medias = self._extract_medias_from_messages(messages)
153
+ updated_medias, video_prompts = self.preprocess_medias(medias)
154
+ preprocessed = self.media_processor.preprocess(
155
+ updated_medias, return_tensors=return_tensors)
156
+
157
+ # Generate text if not provided
158
+ if text is None:
159
+ text = self.tokenizer.apply_chat_template(messages, **kwargs)
160
+
161
+ text = self.update_raw_text(text, video_prompts)
162
+
163
+ text_inputs = self.tokenizer(text, return_tensors=return_tensors)
164
+ return BatchFeature(data={**text_inputs, **preprocessed.data})
165
+
166
+ @staticmethod
167
+ def _extract_medias_from_messages(messages: list[dict]) -> list[dict]:
168
+ """
169
+ Extract media items from messages in a single pass.
170
+
171
+ This is an optimized version that processes messages only once.
172
+ Kept as internal method since external callers should use __call__.
173
+ """
174
+ medias = []
175
+ for msg in messages:
176
+ if msg['role'] != 'user' or not msg.get('content'):
177
+ continue
178
+
179
+ for content_part in msg['content']:
180
+ if not isinstance(content_part, dict):
181
+ continue
182
+
183
+ content_type = content_part.get('type')
184
+ if content_type in ['video_url', 'video']:
185
+ medias.append({
186
+ 'type': 'video',
187
+ 'video': content_part['video_url']['url'],
188
+ 'first_frame_timestamp': 0.0
189
+ })
190
+ elif content_type in ['image_url', 'image']:
191
+ medias.append({
192
+ 'type': 'image',
193
+ 'image': content_part['image_url'],
194
+ })
195
+ return medias
196
+
197
+ def apply_chat_template(self, messages, **kwargs):
198
+ return self.tokenizer.apply_chat_template(messages, **kwargs)
199
+
200
+ def batch_decode(self, *args, **kwargs):
201
+ return self.tokenizer.batch_decode(*args, **kwargs)
202
+
203
+ def decode(self, *args, **kwargs):
204
+ return self.tokenizer.decode(*args, **kwargs)
205
+
206
+ @property
207
+ def model_input_names(self):
208
+ return ['input_ids', 'attention_mask', 'pixel_values', 'grid_thws']
kimi_k25_vision_processing.py ADDED
@@ -0,0 +1,251 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Image processor class for Kimi-K2.5.
2
+ """
3
+
4
+ import json
5
+ from typing import Any, Dict, Optional, Union
6
+
7
+ import numpy as np
8
+ import torch
9
+ from PIL import Image
10
+ from transformers.image_processing_utils import (BaseImageProcessor,
11
+ BatchFeature)
12
+ from transformers.utils import TensorType
13
+
14
+ from .media_utils import (MediaInput, VideoChunkInput, _to_tensor,
15
+ ensure_media_type, get_video_meta, image_to_np,
16
+ navit_patchify, navit_resize_image,
17
+ navit_resize_video, normalize,
18
+ real_sample_fps_and_max_num_frames, timestamp_as_str)
19
+
20
+ try:
21
+ from mecord import VideoReader
22
+ except ImportError:
23
+ VideoReader = None
24
+
25
+
26
+ def resampling(video_bytes: bytes,
27
+ sample_indices: list[int],
28
+ key_indices=None,
29
+ frame_time_info=None,
30
+ num_threads=4) -> str:
31
+ video = VideoReader(video_bytes,
32
+ num_threads=num_threads,
33
+ frame_time_info=frame_time_info,
34
+ key_indices=key_indices)
35
+ # extract target frames
36
+ frames = video[sample_indices]
37
+ frames = [Image.fromarray(frame) for frame in frames]
38
+ return frames
39
+
40
+
41
+ class KimiK25VisionProcessor(BaseImageProcessor):
42
+ model_type = "kimi_k25"
43
+
44
+ def __init__(
45
+ self,
46
+ media_proc_cfg: dict,
47
+ **kwargs,
48
+ ):
49
+ super().__init__(**kwargs)
50
+ self.media_proc_cfg = media_proc_cfg
51
+ self.num_frames_per_chunk = media_proc_cfg[
52
+ 'temporal_merge_kernel_size']
53
+
54
+ def media_tokens_calculator(self, media: MediaInput):
55
+ media = ensure_media_type(media)
56
+ ret = self.get_resize_config(media)
57
+ return ret['num_tokens']
58
+
59
+ @classmethod
60
+ def make_chunk_prompt(cls, timestamp_text: str) -> str:
61
+ return f"{timestamp_text}<|media_begin|>video<|media_content|><|media_pad|><|media_end|>"
62
+
63
+ def split_video_chunks(self,
64
+ video_url: str | bytes) -> list[list[Image.Image]]:
65
+ # video_url should be base64 str or bytes
66
+ video_spec = get_video_meta(video_url)
67
+ sample_fps = min(self.media_proc_cfg['sample_fps'], video_spec.fps)
68
+ sampled_nframes = max(
69
+ round(video_spec.num_frames * sample_fps / video_spec.fps), 1)
70
+ frame_inds = np.linspace(0, video_spec.num_frames - 1,
71
+ sampled_nframes).round().astype(int)
72
+ frame_inds = frame_inds.tolist()
73
+ sampled_frame_ids = []
74
+ temporal_merge_kernel_size = self.media_proc_cfg[
75
+ "temporal_merge_kernel_size"]
76
+ num_chunks = 0
77
+ chunk_timestamp = []
78
+ for i in range(0, len(frame_inds), temporal_merge_kernel_size):
79
+ sampled_frame_ids.extend(frame_inds[i:i +
80
+ temporal_merge_kernel_size])
81
+ start_time = frame_inds[i] / float(video_spec.fps)
82
+ timestamp_text = timestamp_as_str(
83
+ start_time, self.media_proc_cfg["timestamp_mode"])
84
+ chunk_timestamp.append(timestamp_text)
85
+ num_chunks += 1
86
+
87
+ sampled_frames = resampling(video_url, sampled_frame_ids)
88
+ chunks = []
89
+ for chunk_id in range(num_chunks):
90
+ chunk = sampled_frames[chunk_id *
91
+ temporal_merge_kernel_size:(chunk_id + 1) *
92
+ temporal_merge_kernel_size]
93
+ chunks.append(
94
+ VideoChunkInput(type="video_chunk",
95
+ video_chunk=chunk,
96
+ prompt=self.make_chunk_prompt(
97
+ chunk_timestamp[chunk_id])))
98
+ return chunks
99
+
100
+ def get_resize_config(self, media_input: MediaInput) -> dict:
101
+ if media_input['type'] == 'image':
102
+ w, h = media_input['image'].size
103
+ ret = navit_resize_image(
104
+ w, h, self.media_proc_cfg['patch_size'],
105
+ self.media_proc_cfg['merge_kernel_size'],
106
+ self.media_proc_cfg['in_patch_limit'],
107
+ self.media_proc_cfg['patch_limit_on_one_side'],
108
+ self.media_proc_cfg['fixed_output_tokens'])
109
+ return ret
110
+ elif media_input['type'] == 'video_chunk':
111
+ frame = media_input['video_chunk'][0]
112
+ width, height = frame.size
113
+ num_frames = len(media_input["video_chunk"])
114
+ fps = 1.0
115
+
116
+ sample_fps, max_num_frames_each_video = real_sample_fps_and_max_num_frames(
117
+ media_input["type"],
118
+ self.media_proc_cfg['sample_fps'],
119
+ self.media_proc_cfg['max_num_frames_each_video'],
120
+ )
121
+
122
+ in_patch_limit_each_frame = self.media_proc_cfg[
123
+ 'in_patch_limit_each_frame']
124
+ if in_patch_limit_each_frame is None:
125
+ in_patch_limit_each_frame = self.media_proc_cfg[
126
+ 'in_patch_limit']
127
+
128
+ ret = navit_resize_video(
129
+ width,
130
+ height,
131
+ num_frames,
132
+ fps,
133
+ sample_fps,
134
+ self.media_proc_cfg['patch_size'],
135
+ self.media_proc_cfg['merge_kernel_size'],
136
+ in_patch_limit_each_frame,
137
+ self.media_proc_cfg['patch_limit_on_one_side'],
138
+ self.media_proc_cfg['in_patch_limit_video'],
139
+ max_num_frames_each_video,
140
+ self.media_proc_cfg['fixed_output_tokens'],
141
+ )
142
+ return ret
143
+ else:
144
+ raise ValueError("Unsupported type: {}".format(
145
+ media_input['type']))
146
+
147
+ def resize_image(self, image: Image.Image, new_width: int, new_height: int,
148
+ pad_width: int, pad_height: int) -> np.ndarray:
149
+ image_np = image_to_np(image, (new_width, new_height), "resize")
150
+ image_np = np.pad(
151
+ image_np,
152
+ ((0, pad_height), (0, pad_width), (0, 0)),
153
+ mode="constant",
154
+ constant_values=0,
155
+ )
156
+ return image_np
157
+
158
+ def preprocess(
159
+ self,
160
+ medias: list[MediaInput],
161
+ return_tensors: Optional[Union[str, TensorType]] = None,
162
+ ) -> BatchFeature:
163
+ """
164
+ Preprocess a atom vision input (images/video_chunk) into model-ready tensors.
165
+
166
+ Args:
167
+ medias: List of MediaInput.
168
+ return_tensors: Desired output format ('pt', 'np', 'tf', or None).
169
+
170
+ Returns:
171
+ BatchFeature containing 'pixel_values' and 'grid_thws' tensors.
172
+ """
173
+ if not isinstance(medias, list):
174
+ medias = [medias]
175
+ if medias:
176
+ pixel_values = []
177
+ for item in medias:
178
+ item = ensure_media_type(item)
179
+ resize_config = self.get_resize_config(item)
180
+ new_width, new_height, pad_width, pad_height = resize_config[
181
+ 'new_width'], resize_config['new_height'], resize_config[
182
+ 'pad_width'], resize_config['pad_height']
183
+ if item['type'] == 'image':
184
+ image = item['image']
185
+ image_np = self.resize_image(image, new_width, new_height,
186
+ pad_width, pad_height)
187
+ pixel_values.append(np.expand_dims(image_np, axis=0))
188
+ elif item['type'] == 'video_chunk':
189
+ pixels = []
190
+ for frame in item['video_chunk']:
191
+ frame_np = self.resize_image(frame, new_width,
192
+ new_height, pad_width,
193
+ pad_height)
194
+ pixels.append(frame_np)
195
+ pixel_values.append(np.stack(pixels, axis=0))
196
+ else:
197
+ raise ValueError("Unsupported type: {}".format(
198
+ item['type']))
199
+ normalized_pixel_values = []
200
+ image_std_inv = 1.0 / np.array(self.media_proc_cfg['image_std'])
201
+ image_mean = np.array(self.media_proc_cfg['image_mean'])
202
+ for pixels in pixel_values:
203
+ pixels = normalize(pixels, image_mean, image_std_inv)
204
+ pixels_and_thw = navit_patchify(
205
+ pixels,
206
+ self.media_proc_cfg['patch_size'],
207
+ )
208
+ normalized_pixel_values.append(pixels_and_thw)
209
+
210
+ pixel_values = torch.cat([
211
+ _to_tensor(pixel_value['pixel_values'])
212
+ for pixel_value in normalized_pixel_values
213
+ ])
214
+ grid_thws = torch.cat([
215
+ _to_tensor(pixel_value['grid_thw'],
216
+ dtype=torch.int64).unsqueeze(0)
217
+ for pixel_value in normalized_pixel_values
218
+ ])
219
+
220
+ data = {
221
+ 'pixel_values': pixel_values,
222
+ 'grid_thws': grid_thws,
223
+ }
224
+
225
+ else:
226
+ data = {}
227
+
228
+ return BatchFeature(data=data, tensor_type=return_tensors)
229
+
230
+ def __repr__(self):
231
+ return f"KimiK25VisionProcessor(media_proc_cfg={self.media_proc_cfg})"
232
+
233
+ def to_dict(self) -> Dict[str, Any]:
234
+ output = super().to_dict()
235
+ output["media_proc_cfg"] = self.media_proc_cfg
236
+ if "media_processor" in output:
237
+ del output["media_processor"]
238
+ return output
239
+
240
+ @classmethod
241
+ def from_dict(cls, config_dict: Dict[str, Any], **kwargs):
242
+ config = config_dict.copy()
243
+ media_proc_cfg = config.pop("media_proc_cfg", {})
244
+ return cls(media_proc_cfg=media_proc_cfg, **config, **kwargs)
245
+
246
+ def to_json_string(self):
247
+ dictionary = self.to_dict()
248
+ for key, value in dictionary.items():
249
+ if hasattr(value, 'tolist'):
250
+ dictionary[key] = value.tolist()
251
+ return json.dumps(dictionary, indent=2, sort_keys=True) + "\n"
media_utils.py ADDED
@@ -0,0 +1,368 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import base64
2
+ import io
3
+ import math
4
+ import os
5
+ from datetime import datetime, timezone
6
+ from typing import List, Literal, Optional, TypedDict
7
+
8
+ import numpy as np
9
+ from PIL import Image
10
+ from pydantic import BaseModel, Field
11
+
12
+ try:
13
+ from mecord import VideoReader
14
+ except ImportError:
15
+ VideoReader = None
16
+
17
+
18
+ class VideoSpec(BaseModel):
19
+ media_type: str = Literal['video']
20
+ height: int = Field(..., gt=0, description="video frame height")
21
+ width: int = Field(..., gt=0, description="video frame width")
22
+ num_frames: int = Field(..., gt=0, description="num frames")
23
+ fps: float = Field(..., gt=0, description="average fps")
24
+
25
+ # optional, help to accelerate video reading
26
+ key_indices: list[int] = Field(None, description="key indices")
27
+ frame_time_info: dict = Field(None, description="frame time info")
28
+
29
+
30
+ class ImageInput(TypedDict):
31
+ type: Literal['image']
32
+ image: Image.Image
33
+
34
+
35
+ class VideoChunkInput(TypedDict):
36
+ type: Literal['video_chunk']
37
+ video_chunk: List[Image.Image]
38
+ prompt: Optional[str] = None
39
+
40
+
41
+ MediaInput = ImageInput | VideoChunkInput
42
+
43
+
44
+ def get_video_meta(video_src: bytes | str | os.PathLike,
45
+ accurate: bool = True) -> dict:
46
+ """Get the dimensions of a video."""
47
+ if isinstance(video_src, os.PathLike):
48
+ video_src = str(video_src)
49
+ # if b64 string, decode to bytes
50
+ if isinstance(video_src,
51
+ str) and video_src.startswith('data:video/mp4;base64,'):
52
+ video_src = base64.b64decode(video_src.split(',')[1])
53
+ video = VideoReader(video_src, auto_init=accurate, num_threads=1)
54
+ assert video.num_frames > 0, "Invalid video format."
55
+ assert video.original_width > 0 and video.original_height > 0, (
56
+ "Invalid video format.")
57
+ assert video.avg_fps > 0, "Invalid video format."
58
+ return VideoSpec(media_type='video',
59
+ height=video.original_height,
60
+ width=video.original_width,
61
+ num_frames=video.num_frames,
62
+ fps=video.avg_fps,
63
+ key_indices=video.key_indices,
64
+ frame_time_info=video.frame_time_info)
65
+
66
+
67
+ def timestamp_as_str(timestamp: float,
68
+ timestamp_mode: str = "hh:mm:ss.fff") -> str:
69
+ """Convert a timestamp to a string in the format of HH:MM:SS.mmm."""
70
+ if timestamp_mode == "hh:mm:ss.fff":
71
+ return (datetime.fromtimestamp(timestamp,
72
+ tz=timezone.utc).strftime("%H:%M:%S") +
73
+ f".{int((timestamp % 1) * 1000):03d}")
74
+ elif timestamp_mode == "mm:ss.fff":
75
+ return (datetime.fromtimestamp(timestamp,
76
+ tz=timezone.utc).strftime("%M:%S") +
77
+ f".{int((timestamp % 1) * 1000):03d}")
78
+ elif timestamp_mode == "mm:ss":
79
+ return datetime.fromtimestamp(timestamp,
80
+ tz=timezone.utc).strftime("%M:%S")
81
+ else:
82
+ raise ValueError(f"Invalid timestamp mode: {timestamp_mode}")
83
+
84
+
85
+ def navit_resize_image(
86
+ width: int,
87
+ height: int,
88
+ patch_size: int,
89
+ merge_kernel_size: int,
90
+ in_patch_limit: int,
91
+ patch_limit_on_one_side: int,
92
+ fixed_output_tokens: int | None,
93
+ ):
94
+ # Apply the patch limits.
95
+ s1 = math.sqrt(
96
+ in_patch_limit /
97
+ (max(1.0, width // patch_size) * max(1.0, height // patch_size)))
98
+ s2 = patch_limit_on_one_side * patch_size / width
99
+ s3 = patch_limit_on_one_side * patch_size / height
100
+ scale = min(1.0, s1, s2, s3)
101
+ new_w, new_h = max(1, int(width * scale)), max(1, int(height * scale))
102
+ new_w = min(new_w, patch_limit_on_one_side * patch_size)
103
+ new_h = min(new_h, patch_limit_on_one_side * patch_size)
104
+
105
+ # Calculate the padding to make the height and width divisible by the merge kernel size and patch size.
106
+ factor = merge_kernel_size * patch_size
107
+
108
+ pad_height = (factor - new_h % factor) % factor
109
+ pad_width = (factor - new_w % factor) % factor
110
+
111
+ if fixed_output_tokens is not None:
112
+ num_tokens = fixed_output_tokens
113
+ else:
114
+ # Calculate new dimensions after padding and patching
115
+ token_height = (new_h + pad_height) // factor
116
+ token_width = (new_w + pad_width) // factor
117
+
118
+ assert token_height * merge_kernel_size <= patch_limit_on_one_side, (
119
+ f"token_height {token_height} * merge_kernel_size {merge_kernel_size} > patch_limit_on_one_side {patch_limit_on_one_side}"
120
+ )
121
+ assert token_width * merge_kernel_size <= patch_limit_on_one_side, (
122
+ f"token_width {token_width} * merge_kernel_size {merge_kernel_size} > patch_limit_on_one_side {patch_limit_on_one_side}"
123
+ )
124
+
125
+ num_tokens = token_height * token_width
126
+ return {
127
+ "num_tokens": num_tokens,
128
+ "new_width": new_w,
129
+ "new_height": new_h,
130
+ "pad_width": pad_width,
131
+ "pad_height": pad_height,
132
+ "sampled_nframes": 1,
133
+ }
134
+
135
+
136
+ def navit_resize_video(
137
+ width: int,
138
+ height: int,
139
+ nframes: int,
140
+ avg_fps: float,
141
+ sample_fps: float,
142
+ patch_size: int,
143
+ merge_kernel_size: int,
144
+ in_patch_limit_each_frame: int,
145
+ patch_limit_on_one_side: int,
146
+ in_patch_limit_total: int | None,
147
+ max_num_frames_each_video: int | None,
148
+ fixed_output_tokens_each_frame: int | None,
149
+ ):
150
+ sample_fps = min(sample_fps, avg_fps)
151
+ # Calculate the number of frames to sample based on target FPS
152
+ sampled_nframes = max(round(nframes * sample_fps / avg_fps), 1)
153
+ if max_num_frames_each_video is not None:
154
+ sampled_nframes = min(sampled_nframes, max_num_frames_each_video)
155
+
156
+ if in_patch_limit_total is not None:
157
+ in_patch_limit_each_frame = min(
158
+ round(in_patch_limit_total / sampled_nframes),
159
+ in_patch_limit_each_frame)
160
+
161
+ ret = navit_resize_image(
162
+ width,
163
+ height,
164
+ patch_size,
165
+ merge_kernel_size,
166
+ in_patch_limit_each_frame,
167
+ patch_limit_on_one_side,
168
+ fixed_output_tokens_each_frame,
169
+ )
170
+ ret["sampled_nframes"] = sampled_nframes
171
+ return ret
172
+
173
+
174
+ def real_sample_fps_and_max_num_frames(
175
+ type_name: Literal["video", "video_chunk"],
176
+ sample_fps: float,
177
+ max_num_frames_each_video: int | None,
178
+ ) -> tuple[int, int | None]:
179
+ if type_name == "video":
180
+ return sample_fps, max_num_frames_each_video
181
+ elif type_name == "video_chunk":
182
+ max_num_frames_each_video = None
183
+ sample_fps = math.inf
184
+ return sample_fps, max_num_frames_each_video
185
+ else:
186
+ return math.inf, None
187
+
188
+
189
+ def _to_pil(data: str | bytes):
190
+ if isinstance(data, Image.Image):
191
+
192
+ return data.convert("RGB")
193
+ elif isinstance(data, str):
194
+ if data.startswith("data:"):
195
+ raw_base64 = data.split(",")[1]
196
+ return Image.open(io.BytesIO(
197
+ base64.b64decode(raw_base64))).convert("RGB")
198
+ else:
199
+ return Image.open(data).convert("RGB")
200
+ elif isinstance(data, bytes):
201
+ return Image.open(io.BytesIO(data)).convert("RGB")
202
+ else:
203
+ raise ValueError(f"Unsupported data type: {type(data)}")
204
+
205
+
206
+ def ensure_media_type(media: MediaInput) -> MediaInput:
207
+ if media['type'] == 'image':
208
+ media['image'] = _to_pil(media['image'])
209
+ return media
210
+ elif media['type'] == 'video_chunk':
211
+ media['video_chunk'] = [
212
+ _to_pil(frame) for frame in media['video_chunk']
213
+ ]
214
+ return media
215
+ else:
216
+ raise ValueError(f"Unsupported media type: {media['type']}")
217
+
218
+
219
+ def image_to_np(
220
+ image: Image.Image,
221
+ resize_to: tuple[int, int] | None = None,
222
+ mode: str = "resize",
223
+ raise_error_for_ill_resize: bool = True,
224
+ ) -> np.ndarray:
225
+ """Convert an image to a numpy array.
226
+
227
+ Args:
228
+ content: The image to convert.
229
+ resize_to: The size to resize the image to.
230
+ mode: The mode to resize the image to.
231
+ raise_error_for_ill_resize: Whether to raise an error for ill-sized resize.
232
+
233
+ Returns:
234
+ A numpy array.
235
+ """
236
+ assert isinstance(image, Image.Image), "image must be a PIL Image"
237
+ if resize_to is not None:
238
+ if mode == "resize":
239
+ image = image.resize(resize_to, resample=Image.Resampling.BICUBIC)
240
+
241
+ elif mode == "rescale_and_pad_to_center":
242
+ scale = min(resize_to[0] / image.width,
243
+ resize_to[1] / image.height, 1.0)
244
+ new_width = round(image.width * scale)
245
+ new_height = round(image.height * scale)
246
+ if new_width == 0 or new_height == 0:
247
+ if raise_error_for_ill_resize:
248
+ raise ValueError(
249
+ f"Invalid resize to: {resize_to}, from image size: {image.size}"
250
+ )
251
+ else:
252
+ return np.zeros((resize_to[1], resize_to[0], 3),
253
+ dtype=np.uint8)
254
+
255
+ image = image.resize((new_width, new_height),
256
+ resample=Image.Resampling.BICUBIC)
257
+ padding_left = (resize_to[0] - new_width) // 2
258
+ padding_right = resize_to[0] - new_width - padding_left
259
+ padding_top = (resize_to[1] - new_height) // 2
260
+ padding_bottom = resize_to[1] - new_height - padding_top
261
+ image = np.asarray(image)
262
+ image = np.pad(
263
+ image,
264
+ ((padding_top, padding_bottom), (padding_left, padding_right),
265
+ (0, 0)),
266
+ mode="constant",
267
+ constant_values=0,
268
+ )
269
+ assert image.shape == (resize_to[1], resize_to[0], 3)
270
+
271
+ elif mode == "rescale_and_pad_to_rightbottom":
272
+ scale = min(resize_to[0] / image.width,
273
+ resize_to[1] / image.height, 1.0)
274
+ new_width = round(image.width * scale)
275
+ new_height = round(image.height * scale)
276
+ if new_width == 0 or new_height == 0:
277
+ if raise_error_for_ill_resize:
278
+ raise ValueError(
279
+ f"Invalid resize to: {resize_to}, from image size: {image.size}"
280
+ )
281
+ else:
282
+ return np.zeros((resize_to[1], resize_to[0], 3),
283
+ dtype=np.uint8)
284
+
285
+ image = image.resize((new_width, new_height),
286
+ resample=Image.Resampling.BICUBIC)
287
+ padding_right = resize_to[0] - new_width
288
+ padding_bottom = resize_to[1] - new_height
289
+ image = np.asarray(image)
290
+ image = np.pad(
291
+ image,
292
+ ((0, padding_bottom), (0, padding_right), (0, 0)),
293
+ mode="constant",
294
+ constant_values=0,
295
+ )
296
+ assert image.shape == (resize_to[1], resize_to[0], 3)
297
+
298
+ else:
299
+ raise ValueError(f"Invalid mode: {mode}")
300
+
301
+ if isinstance(image, Image.Image):
302
+ return np.asarray(image)
303
+ else:
304
+ return image
305
+
306
+
307
+ def navit_patchify(pixel_values: np.ndarray,
308
+ patch_size: int) -> dict[str, np.ndarray]:
309
+ """Reshape the pixel values to a navit shape.
310
+
311
+ Args:
312
+ pixel_values: np.ndarray, shape (t, h, w, c)
313
+ patch_size: int
314
+
315
+ Returns:
316
+ dict[str, np.ndarray]
317
+ - patches: np.ndarray, shape (t * h//patch_size * w//patch_size, c, patch_size, patch_size)
318
+ - grid_thw: np.ndarray, (t, h//patch_size, w//patch_size)
319
+ """
320
+ T, H, W, C = pixel_values.shape
321
+ assert C == 3, "pixel_values must have 3 channels"
322
+
323
+ patches = pixel_values.reshape(T, H // patch_size, patch_size,
324
+ W // patch_size, patch_size, C)
325
+ # (T, H//patch_size, W//patch_size, C, patch_size, patch_size)
326
+ patches = patches.transpose(0, 1, 3, 5, 2, 4)
327
+ patches = patches.reshape(-1, C, patch_size, patch_size)
328
+ grid_thw = np.array([T, H // patch_size, W // patch_size])
329
+ return {"pixel_values": patches, "grid_thw": grid_thw}
330
+
331
+
332
+ def normalize(x: np.ndarray,
333
+ mean,
334
+ std_inv,
335
+ pixels_dtype: np.dtype = np.float32) -> np.ndarray:
336
+ """Normalize the image.
337
+
338
+ Args:
339
+ x: The image to normalize. The shape is (..., 3). The dtype is uint8. The range is [0, 255].
340
+ mean: The mean of the image.
341
+ std_inv: The inverse of the std of the image.
342
+ pixels_dtype: The dtype of the image.
343
+ Returns:
344
+ The normalized image. The shape is (..., 3). The dtype is determined by the pixels_dtype.
345
+ """
346
+ x = (x / 255.0).astype(pixels_dtype)
347
+ x -= mean
348
+ x *= std_inv
349
+ return x
350
+
351
+
352
+ def _to_tensor(data, **kwargs):
353
+ import torch
354
+
355
+ if isinstance(data, np.ndarray):
356
+ return torch.from_numpy(data).to(**kwargs)
357
+ elif isinstance(data, torch.Tensor):
358
+ return data.to(**kwargs)
359
+ elif isinstance(data, list):
360
+ return [_to_tensor(item, **kwargs) for item in data]
361
+ elif isinstance(data, tuple):
362
+ return tuple(_to_tensor(item, **kwargs) for item in data)
363
+ elif isinstance(data, dict):
364
+ return {k: _to_tensor(v, **kwargs) for k, v in data.items()}
365
+ elif data is None:
366
+ return None
367
+ else:
368
+ raise ValueError(f"Unsupported data type: {type(data)}")
mm_projector.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e7c6ce8c27424f292e708e7bbb48ade57ea9f1aaddd28bd6a1020a860d9db80c
3
+ size 99117136
model-00001-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd4b389324d8ed223c28a9e16a718228fa4d24ce09cdc0d0db92fd722df8e6d8
3
+ size 5363940952
model-00002-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:11b80a3a469fe8ab1d6bd2d88c9bb53cfeb3ce54357be876bbacc5f6fc26db30
3
+ size 5361736696
model-00003-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3960cfbfa5e2f7bbdd195c8becbe712a4eb9d2d68a2bb912bdba72dc7dafbe43
3
+ size 5363339120
model-00004-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bf88f32b6d8f43ec1c066d5953e120fc5bbc85dc99947f0971dc3b502199d4f0
3
+ size 5361736640
model-00005-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ff48f71f3a005abf34724f6db856071d3e9b0a3d1b80c94c26bbe06842108b2b
3
+ size 5366353560
model-00006-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e9ce4ee588a5c73ac16a7190b7df88115498f056f70fe164b47c2fe9963d658
3
+ size 5361736504
model-00007-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:99f92392fd0b9fc37a773029c313f27653749441623d7f5893c78497d64b9ee6
3
+ size 5366353704
model-00008-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:af41af526c10de5197c7aee9b92094c0adb6650d2ffcea333640cbbb66654e2f
3
+ size 5361736368
model-00009-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6b63348060bc303475fb41322c52ebb43022bfbb941535a140b98afa55ed030b
3
+ size 5366353832
model-00010-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:512b70194c08d5b36ce5e51968b75b424fd262abac21cd276aa41eeb0d34cddb
3
+ size 5361736232
model-00011-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a0acf28004cee9c111aa910a0e05e2f06f3131d1322452235e49362640e9c2a1
3
+ size 5363339600
model-00012-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5c9163d9d7ce3765055712880eae315ccb2bb88c8bc019b24f6b44e10aafdd08
3
+ size 5366353488
model-00013-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ae74ec25db1cedd85894edf5926cb4aae6d8bdf5daf463434497310ee86aa26a
3
+ size 5361736696
model-00014-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4dcc960bd41c7a2a183807af151524f7a547123244d2949e692f4117aa400d0d
3
+ size 5366353496
model-00015-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9b0749ad2ad5307543d1e9593518c614116f701f06618986f8d4574efd52c292
3
+ size 5361736680
model-00016-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d55105e496b79d3d5f10a676558002c66428098b88b438c67fba970d13fccb8f
3
+ size 5366353520
model-00017-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bcb77ae2f687e27b91fe04432515e18d3e19a5888be98b1dfd5df123e93de2d0
3
+ size 5361736544
model-00018-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:756113e9521d3955d38f684b6997fd68eaeab92c660270aa2ce1d57b119bd84d
3
+ size 5363339264
model-00019-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4dd4ec5b07f1d12b576049d435ac158fbf3442ff6335250bfea3f15577a83154
3
+ size 5361736408
model-00020-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5381c71742768c88e2692ab164ef07fcd37a85bd1cbc5333a763554d2a9e5a74
3
+ size 5364805840
model-00021-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:13aa866d1b48ee5341c020ba314954909ff16ee0b36f9c4bed21919e3ab2c958
3
+ size 5361736336
model-00022-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:99e5ca5f30d7129b6b10e60c941e38341704cddbf2d7c0acbc0003e0ddec0e31
3
+ size 5366353872
model-00023-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:23b63f18f5a670a309074c1fb826fae01ecb12da1869477fb4ec12559336eb9d
3
+ size 5357948328
model-00024-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fb99757b4ea90a1b18c6c080d56da37eebf4247ae88893adcb12db40ab058551
3
+ size 5357555688
model-00025-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e78dccea55fe0d54686ffefce8114278132e35b1b482b5fbde77eba405ec305e
3
+ size 5363339104
model-00026-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:12a4f637a50fe23663b52425fb5d75c50a3fcca4211485aecc4a4e7b1bba2817
3
+ size 5361736696
model-00027-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9beb92efca20325a0dea2d0bac9a3c131c6b533d89a416bbb5035bd9cf02c8c5
3
+ size 5366353504
model-00028-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:68d45c421cac6851303032f5ea4c93d74be84e70b8418ceb0d58fd0d95e2e46b
3
+ size 5361736648
model-00029-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fa8915c4907c0deca57944f8582d907898c82333c326f6d86690f82a5f8e825d
3
+ size 5366353560
model-00030-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:128b570a6fb59ae7b153f0da4cf713b1087aa89c6977a376fd359b72086256f2
3
+ size 5361736512
model-00031-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aea2c91014901cd0a30283943cdae4bf7686a8fd6d3c56d37f5b6389feacf6e3
3
+ size 5366353688
model-00032-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aa6396111a23f21191ab964d456e039562aee26415c193eee66818a65e371157
3
+ size 5361736376
model-00033-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eea8bf4e723abc1a409fbd3fcf3cd3cd26eb76d7684a5a5c288019bea102c9c1
3
+ size 5363339440
model-00034-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:140be56abe6eb707904e920186acde09f6f68a2516a05de8c0cddf74d31794b3
3
+ size 5361736240
model-00141-of-00141.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e1375c9156215b03d2c95ed4832810e6771ff9217f3ce5582b2368198a275272
3
+ size 4724454592
model.safetensors.index.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8c51579c9ca8c22b8da94c96fe6559cb7aa78a15418a2a2af5bf419bfdafca83
3
+ size 11385154
preprocessor_config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_map": {
3
+ "AutoProcessor": "kimi_k25_processor.KimiK25Processor",
4
+ "AutoImageProcessor": "kimi_k25_vision_processing.KimiK25VisionProcessor"
5
+ },
6
+ "media_proc_cfg": {
7
+ "in_patch_limit": 16384,
8
+ "patch_size": 14,
9
+ "image_mean": [
10
+ 0.5,
11
+ 0.5,
12
+ 0.5
13
+ ],
14
+ "image_std": [
15
+ 0.5,
16
+ 0.5,
17
+ 0.5
18
+ ],
19
+ "merge_kernel_size": 2,
20
+ "fixed_output_tokens": null,
21
+ "patch_limit_on_one_side": 512,
22
+ "in_patch_limit_each_frame": 16384,
23
+ "in_patch_limit_video": null,
24
+ "sample_fps": 2.0,
25
+ "max_num_frames_each_video": null,
26
+ "temporal_merge_kernel_size": 4,
27
+ "timestamp_mode": "hh:mm:ss.fff",
28
+ "config_type": "media_proc.processors.moonvit.MoonViTMediaProcessorConfig"
29
+ }
30
+ }
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:19e773648cb4e65de8660ea6365e10acca112d42a854923df93db4a6f333a82d
3
+ size 20217442
tokenizer_config.json ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "clean_up_tokenization_spaces": false,
4
+ "do_lower_case": false,
5
+ "eos_token": "<|endoftext|>",
6
+ "extra_special_tokens": [
7
+ "<|endoftext|>",
8
+ "[MASK]",
9
+ "[gMASK]",
10
+ "[sMASK]",
11
+ "<sop>",
12
+ "<eop>",
13
+ "<|system|>",
14
+ "<|user|>",
15
+ "<|assistant|>",
16
+ "<|observation|>",
17
+ "<|begin_of_image|>",
18
+ "<|end_of_image|>",
19
+ "<|begin_of_video|>",
20
+ "<|end_of_video|>",
21
+ "<|begin_of_audio|>",
22
+ "<|end_of_audio|>",
23
+ "<|begin_of_transcription|>",
24
+ "<|end_of_transcription|>"
25
+ ],
26
+ "is_local": true,
27
+ "model_max_length": 1048576,
28
+ "model_specific_special_tokens": {},
29
+ "pad_token": "<|endoftext|>",
30
+ "padding_side": "left",
31
+ "remove_space": false,
32
+ "tokenizer_class": "TokenizersBackend"
33
+ }