diff --git a/._configuration_glm5v.py b/._configuration_glm5v.py new file mode 100644 index 0000000000000000000000000000000000000000..c1429106c97fffc0ae7798165ebd641d1ec63601 Binary files /dev/null and b/._configuration_glm5v.py differ diff --git a/._kimi_k25_processor.py b/._kimi_k25_processor.py new file mode 100644 index 0000000000000000000000000000000000000000..c1429106c97fffc0ae7798165ebd641d1ec63601 Binary files /dev/null and b/._kimi_k25_processor.py differ diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..aa7aacd0134a92c3c1943fdecc75cd8b7420cce6 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +model.safetensors.index.json filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..dad101f5579602c103ef4f53dc872496dfbd64d9 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,121 @@ +[gMASK] +{%- set effective_reasoning_effort = 'high' if reasoning_effort is defined and reasoning_effort == 'high' else 'max' -%} +{%- if (enable_thinking is not defined or enable_thinking) and effective_reasoning_effort is not none -%}<|system|>Reasoning Effort: {{ effective_reasoning_effort | capitalize }}{%- endif -%} +{%- if tools -%} +{%- macro tool_to_json(tool) -%} + {%- set ns_tool = namespace(first=true) -%} + {{ '{' -}} + {%- for k, v in tool.items() -%} + {%- if k != 'defer_loading' and k != 'strict' -%} + {%- if not ns_tool.first -%}{{- ', ' -}}{%- endif -%} + {%- set ns_tool.first = false -%} + "{{ k }}": {{ v | tojson(ensure_ascii=False) }} + {%- endif -%} + {%- endfor -%} + {{- '}' -}} +{%- endmacro -%} +<|system|> +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{% for tool in tools %} +{%- if 'function' in tool -%} + {%- set tool = tool['function'] -%} +{%- endif -%} +{% if tool.defer_loading is not defined or not tool.defer_loading %} +{{ tool_to_json(tool) }} +{% endif %} +{% endfor %} + + +For each function call, output the function name and arguments within the following XML format: +{function-name}{arg-key-1}{arg-value-1}{arg-key-2}{arg-value-2}...{%- endif -%} +{%- macro visible_text(content) -%} + {%- if content is string -%} + {{- content }} + {%- elif content is iterable and content is not mapping -%} + {%- for item in content -%} + {%- if item is mapping and item.type == 'text' -%} + {{- item.text }} + {%- elif item is string -%} + {{- item }} + {%- elif item is mapping and item.type in ['image', 'image_url'] -%} + {{- '<|begin_of_image|><|image|><|end_of_image|>' }} + {%- elif item is mapping and item.type in ['video', 'video_url', 'audio', 'audio_url', 'input_audio'] -%} + {%- set media_type = item.type | replace('_url', '') | replace('input_', '') -%} + {{- "You are unable to process this " ~ media_type ~ " because you don't have multi-modal input ability. Try different methods." }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{- content }} + {%- endif -%} +{%- endmacro -%} +{%- set ns = namespace(last_user_index=-1) -%} +{%- for m in messages %} + {%- if m.role == 'user' %} + {%- set ns.last_user_index = loop.index0 -%} + {%- endif %} +{%- endfor %} +{%- for m in messages -%} +{%- if m.role == 'user' -%}<|user|>{{ visible_text(m.content) }} +{%- elif m.role == 'assistant' -%} +<|assistant|> +{%- set content = visible_text(m.content) %} +{%- if m.reasoning_content is string %} + {%- set reasoning_content = m.reasoning_content %} +{%- elif '' in content %} + {%- set reasoning_content = content.split('')[0].split('')[-1] %} + {%- set content = content.split('')[-1] %} +{%- endif %} +{%- if ((clear_thinking is defined and not clear_thinking) or loop.index0 > ns.last_user_index) and reasoning_content is defined -%} +{{ '' + reasoning_content + ''}} +{%- else -%} +{{ '' }} +{%- endif -%} +{%- if content.strip() -%} +{{ content.strip() }} +{%- endif -%} +{% if m.tool_calls %} +{% for tc in m.tool_calls %} +{%- if tc.function %} + {%- set tc = tc.function %} +{%- endif %} +{{- '' + tc.name -}} +{% set _args = tc.arguments %}{% for k, v in _args.items() %}{{ k }}{{ v | tojson(ensure_ascii=False) if v is not string else v }}{% endfor %}{% endfor %} +{% endif %} +{%- elif m.role == 'tool' -%} +{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|observation|>' -}} +{%- endif %} +{%- if m.content is string -%} + {{- '' + m.content + '' -}} +{%- elif m.content is iterable and m.content is not mapping and m.content and m.content.0.type == "tool_reference" -%} + {{- '\n' -}} + {% for tr in m.content %} + {%- for tool in tools -%} + {%- if 'function' in tool -%} + {%- set tool = tool['function'] -%} + {%- endif -%} + {%- if tool.name == tr.name -%} + {{- tool_to_json(tool) + '\n' -}} + {%- endif -%} + {%- endfor -%} + {%- endfor -%} + {{- '' -}} +{%- elif m.content is iterable and m.content is not mapping and m.content and m.content.0 is mapping and m.content.0.output is defined -%} + {%- for tr in m.content -%} + {{- '' + tr.output + '' -}} + {%- endfor -%} +{%- else -%} + {{- '' + visible_text(m.content) + '' -}} +{% endif -%} +{%- elif m.role == 'system' -%} +<|system|>{{ visible_text(m.content) }} +{%- endif -%} +{%- endfor -%} +{%- if add_generation_prompt -%} + <|assistant|>{{- '' if (enable_thinking is defined and not enable_thinking) else '' -}} +{%- endif -%} diff --git a/config.json b/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cb07c5587442a892774f780a9260002f71cd3402 --- /dev/null +++ b/config.json @@ -0,0 +1,1370 @@ +{ + "architectures": [ + "Glm5vForConditionalGeneration" + ], + "model_type": "glm5v", + "auto_map": { + "AutoConfig": "configuration_glm5v.Glm5vConfig" + }, + "text_config": { + "architectures": [ + "GlmMoeDsaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "dtype": "bfloat16", + "eos_token_id": [ + 154820, + 154827, + 154829 + ], + "ep_size": 1, + "first_k_dense_replace": 3, + "head_dim": 192, + "hidden_act": "silu", + "hidden_size": 6144, + "index_head_dim": 128, + "index_n_heads": 32, + "index_share_for_mtp_iteration": true, + "index_skip_topk_offset": 3, + "index_topk": 2048, + "index_topk_freq": 4, + "index_topk_pattern": null, + "indexer_rope_interleave": true, + "indexer_types": [ + "full", + "full", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared" + ], + "initializer_range": 0.02, + "intermediate_size": 12288, + "kv_lora_rank": 512, + "max_position_embeddings": 1048576, + "mlp_layer_types": [ + "dense", + "dense", + "dense", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse" + ], + "model_type": "glm_moe_dsa", + "moe_intermediate_size": 2048, + "moe_layer_freq": 1, + "n_group": 1, + "n_routed_experts": 256, + "n_shared_experts": 1, + "norm_topk_prob": true, + "num_attention_heads": 64, + "num_experts_per_tok": 8, + "num_hidden_layers": 78, + "num_key_value_heads": 64, + "num_nextn_predict_layers": 1, + "pad_token_id": 154820, + "pretraining_tp": 1, + "q_lora_rank": 2048, + "qk_head_dim": 256, + "qk_nope_head_dim": 192, + "qk_rope_head_dim": 64, + "rms_norm_eps": 1e-05, + "rope_interleave": true, + "rope_parameters": { + "rope_theta": 8000000, + "rope_type": "default" + }, + "routed_scaling_factor": 2.5, + "scoring_func": "sigmoid", + "tie_word_embeddings": false, + "topk_group": 1, + "topk_method": "noaux_tc", + "transformers_version": "5.12.0", + "use_cache": true, + "v_head_dim": 256, + "vocab_size": 154880, + "quantization_config": { + "activation_scheme": "dynamic", + "fmt": "e4m3", + "quant_method": "fp8", + "weight_block_size": [ + 128, + 128 + ], + "modules_to_not_convert": [ + "model.layers.59.post_attention_layernorm", + "model.layers.47.mlp.gate.e_score_correction_bias", + "model.layers.71.self_attn.kv_a_layernorm", + "model.layers.46.mlp.gate.e_score_correction_bias", + "model.layers.30.self_attn.kv_a_layernorm", + "model.layers.23.self_attn.kv_a_layernorm", + "model.layers.73.mlp.gate.e_score_correction_bias", + "model.layers.36.mlp.gate", + "model.layers.12.self_attn.q_a_layernorm", + "model.layers.51.mlp.gate", + "model.layers.47.self_attn.kv_a_layernorm", + "model.layers.45.self_attn.q_a_layernorm", + "model.layers.15.self_attn.q_a_layernorm", + "model.layers.39.input_layernorm", + "model.layers.50.self_attn.kv_a_layernorm", + "model.layers.19.post_attention_layernorm", + "model.layers.71.input_layernorm", + "model.layers.72.self_attn.kv_a_layernorm", + "model.layers.70.mlp.gate", + "model.layers.52.input_layernorm", + "model.layers.78.self_attn.kv_a_layernorm", + "model.layers.48.post_attention_layernorm", + "model.layers.4.self_attn.q_a_layernorm", + "model.layers.62.self_attn.q_a_layernorm", + "model.layers.4.post_attention_layernorm", + "model.layers.38.self_attn.indexers_proj", + "model.layers.61.mlp.gate.e_score_correction_bias", + "model.layers.14.mlp.gate.e_score_correction_bias", + "model.layers.36.self_attn.kv_a_layernorm", + "model.layers.65.mlp.gate", + "model.layers.13.self_attn.kv_a_layernorm", + "model.layers.57.mlp.gate", + "model.layers.65.self_attn.kv_a_layernorm", + "model.layers.68.input_layernorm", + "model.layers.70.self_attn.kv_a_layernorm", + "model.layers.24.mlp.gate.e_score_correction_bias", + "model.layers.4.input_layernorm", + "model.layers.20.input_layernorm", + "model.layers.65.self_attn.q_a_layernorm", + "model.layers.38.mlp.gate", + "model.layers.76.mlp.gate.e_score_correction_bias", + "model.layers.62.input_layernorm", + "model.layers.45.input_layernorm", + "model.layers.68.post_attention_layernorm", + "model.layers.72.input_layernorm", + "model.layers.32.mlp.gate.e_score_correction_bias", + "model.layers.27.input_layernorm", + "model.layers.16.post_attention_layernorm", + "model.layers.35.self_attn.q_a_layernorm", + "model.layers.58.self_attn.indexers_proj", + "model.layers.3.mlp.gate.e_score_correction_bias", + "model.layers.49.self_attn.q_a_layernorm", + "model.layers.66.mlp.gate.e_score_correction_bias", + "model.layers.10.self_attn.q_a_layernorm", + "model.layers.51.mlp.gate.e_score_correction_bias", + "model.layers.64.self_attn.q_a_layernorm", + "model.layers.70.input_layernorm", + "model.layers.71.post_attention_layernorm", + "model.layers.67.post_attention_layernorm", + "model.layers.6.self_attn.indexer.k_norm", + "model.layers.3.mlp.gate", + "model.layers.11.mlp.gate", + "model.layers.17.input_layernorm", + "model.layers.13.mlp.gate", + "model.layers.15.mlp.gate", + "model.layers.71.self_attn.q_a_layernorm", + "model.layers.33.mlp.gate.e_score_correction_bias", + "model.layers.72.mlp.gate.e_score_correction_bias", + "model.layers.65.post_attention_layernorm", + "model.layers.78.hnorm", + "model.layers.40.post_attention_layernorm", + "model.layers.12.mlp.gate.e_score_correction_bias", + "model.layers.8.input_layernorm", + "model.layers.73.mlp.gate", + "model.layers.11.post_attention_layernorm", + "model.layers.61.input_layernorm", + "model.layers.14.self_attn.kv_a_layernorm", + "model.layers.39.self_attn.kv_a_layernorm", + "model.layers.70.self_attn.indexer.k_norm", + "model.layers.45.mlp.gate", + "model.layers.51.self_attn.q_a_layernorm", + "model.layers.64.post_attention_layernorm", + "model.layers.20.post_attention_layernorm", + "model.layers.66.self_attn.kv_a_layernorm", + "model.layers.10.self_attn.indexer.k_norm.bias", + "model.layers.23.input_layernorm", + "model.layers.38.self_attn.indexer.k_norm", + "model.layers.19.self_attn.kv_a_layernorm", + "model.layers.40.self_attn.q_a_layernorm", + "model.layers.10.self_attn.indexer.k_norm", + "model.layers.44.post_attention_layernorm", + "model.layers.70.post_attention_layernorm", + "model.layers.41.mlp.gate.e_score_correction_bias", + "model.layers.36.post_attention_layernorm", + "model.layers.30.self_attn.indexer.k_norm.bias", + "model.layers.54.self_attn.q_a_layernorm", + "model.layers.59.mlp.gate", + "model.layers.69.post_attention_layernorm", + "lm_head", + "model.layers.77.self_attn.kv_a_layernorm", + "model.layers.75.self_attn.q_a_layernorm", + "model.layers.56.self_attn.q_a_layernorm", + "model.layers.50.self_attn.indexer.k_norm.bias", + "model.layers.54.mlp.gate", + "model.layers.6.self_attn.indexer.k_norm.bias", + "model.layers.15.input_layernorm", + "model.layers.29.self_attn.q_a_layernorm", + "model.layers.54.self_attn.indexers_proj", + "model.layers.73.self_attn.kv_a_layernorm", + "model.layers.0.self_attn.indexer.k_norm", + "model.layers.58.self_attn.indexer.k_norm", + "model.layers.36.input_layernorm", + "model.layers.47.self_attn.q_a_layernorm", + "model.layers.26.mlp.gate.e_score_correction_bias", + "model.layers.50.input_layernorm", + "model.layers.16.self_attn.q_a_layernorm", + "model.layers.19.mlp.gate.e_score_correction_bias", + "model.layers.78.input_layernorm", + "model.layers.27.mlp.gate", + "model.layers.19.self_attn.q_a_layernorm", + "model.layers.25.input_layernorm", + "model.layers.33.self_attn.kv_a_layernorm", + "model.layers.32.mlp.gate", + "model.layers.61.self_attn.kv_a_layernorm", + "model.layers.50.self_attn.indexers_proj", + "model.layers.34.self_attn.indexers_proj", + "model.layers.4.mlp.gate.e_score_correction_bias", + "model.layers.76.post_attention_layernorm", + "model.layers.57.post_attention_layernorm", + "model.layers.73.post_attention_layernorm", + "model.layers.68.mlp.gate.e_score_correction_bias", + "model.layers.34.self_attn.kv_a_layernorm", + "model.layers.78.enorm", + "model.layers.30.mlp.gate", + "model.layers.10.self_attn.indexers_proj", + "model.layers.34.input_layernorm", + "model.layers.34.post_attention_layernorm", + "model.layers.20.mlp.gate.e_score_correction_bias", + "model.layers.54.self_attn.indexer.k_norm.bias", + "model.layers.44.mlp.gate", + "model.layers.25.mlp.gate.e_score_correction_bias", + "model.layers.26.self_attn.indexers_proj", + "model.layers.66.self_attn.indexer.k_norm.bias", + "model.layers.41.post_attention_layernorm", + "model.layers.51.post_attention_layernorm", + "model.layers.22.self_attn.indexer.k_norm.bias", + "model.layers.21.mlp.gate.e_score_correction_bias", + "model.layers.15.self_attn.kv_a_layernorm", + "model.layers.25.self_attn.kv_a_layernorm", + "model.layers.55.mlp.gate", + "model.layers.11.self_attn.q_a_layernorm", + "model.layers.72.mlp.gate", + "model.layers.21.self_attn.kv_a_layernorm", + "model.layers.59.mlp.gate.e_score_correction_bias", + "model.layers.2.input_layernorm", + "model.layers.5.mlp.gate", + "model.layers.54.self_attn.kv_a_layernorm", + "model.layers.0.self_attn.indexers_proj", + "model.layers.71.mlp.gate.e_score_correction_bias", + "model.layers.6.self_attn.q_a_layernorm", + "model.layers.14.mlp.gate", + "model.layers.44.self_attn.kv_a_layernorm", + "model.layers.61.post_attention_layernorm", + "model.layers.64.self_attn.kv_a_layernorm", + "model.layers.1.post_attention_layernorm", + "model.layers.4.mlp.gate", + "model.layers.22.post_attention_layernorm", + "model.layers.21.self_attn.q_a_layernorm", + "model.layers.74.self_attn.q_a_layernorm", + "model.layers.18.self_attn.indexers_proj", + "model.layers.75.input_layernorm", + "model.layers.70.self_attn.q_a_layernorm", + "model.layers.22.self_attn.indexer.k_norm", + "model.layers.42.mlp.gate.e_score_correction_bias", + "model.layers.5.self_attn.kv_a_layernorm", + "model.layers.31.mlp.gate.e_score_correction_bias", + "model.layers.62.self_attn.indexer.k_norm.bias", + "model.layers.29.post_attention_layernorm", + "model.layers.6.self_attn.kv_a_layernorm", + "model.layers.48.input_layernorm", + "model.layers.74.self_attn.indexer.k_norm.bias", + "model.layers.7.input_layernorm", + "model.layers.39.self_attn.q_a_layernorm", + "model.layers.41.input_layernorm", + "model.layers.54.self_attn.indexer.k_norm", + "model.layers.32.self_attn.q_a_layernorm", + "model.layers.39.mlp.gate.e_score_correction_bias", + "model.layers.42.input_layernorm", + "model.layers.43.mlp.gate", + "model.layers.72.self_attn.q_a_layernorm", + "model.layers.37.mlp.gate.e_score_correction_bias", + "model.layers.5.post_attention_layernorm", + "model.layers.28.mlp.gate", + "model.layers.2.post_attention_layernorm", + "model.layers.49.post_attention_layernorm", + "model.layers.43.self_attn.kv_a_layernorm", + "model.layers.12.input_layernorm", + "model.layers.56.mlp.gate", + "model.layers.5.input_layernorm", + "model.layers.29.mlp.gate.e_score_correction_bias", + "model.layers.4.self_attn.kv_a_layernorm", + "model.layers.20.self_attn.q_a_layernorm", + "model.layers.59.self_attn.q_a_layernorm", + "model.layers.68.mlp.gate", + "model.layers.21.post_attention_layernorm", + "model.layers.26.self_attn.q_a_layernorm", + "model.layers.52.self_attn.q_a_layernorm", + "model.layers.29.self_attn.kv_a_layernorm", + "model.layers.24.input_layernorm", + "model.layers.34.self_attn.indexer.k_norm.bias", + "model.layers.43.input_layernorm", + "model.layers.27.self_attn.kv_a_layernorm", + "model.layers.31.input_layernorm", + "model.layers.69.mlp.gate", + "model.layers.26.post_attention_layernorm", + "model.layers.3.self_attn.q_a_layernorm", + "model.layers.37.post_attention_layernorm", + "model.layers.52.mlp.gate", + "model.layers.73.input_layernorm", + "model.layers.19.input_layernorm", + "model.layers.13.self_attn.q_a_layernorm", + "model.layers.76.input_layernorm", + "model.layers.8.mlp.gate", + "model.layers.63.post_attention_layernorm", + "model.layers.58.mlp.gate", + "model.layers.31.post_attention_layernorm", + "model.layers.23.mlp.gate.e_score_correction_bias", + "model.layers.34.self_attn.q_a_layernorm", + "model.layers.78.self_attn.indexer.k_norm.bias", + "model.layers.13.mlp.gate.e_score_correction_bias", + "model.layers.71.mlp.gate", + "model.layers.14.input_layernorm", + "model.layers.17.mlp.gate.e_score_correction_bias", + "model.layers.8.post_attention_layernorm", + "model.layers.45.self_attn.kv_a_layernorm", + "model.layers.46.self_attn.indexer.k_norm", + "model.layers.20.self_attn.kv_a_layernorm", + "model.layers.78.self_attn.indexer.k_norm", + "model.layers.37.mlp.gate", + "model.layers.7.mlp.gate", + "model.layers.30.mlp.gate.e_score_correction_bias", + "model.layers.67.mlp.gate", + "model.layers.73.self_attn.q_a_layernorm", + "model.layers.10.self_attn.kv_a_layernorm", + "model.layers.61.mlp.gate", + "model.layers.22.mlp.gate.e_score_correction_bias", + "model.layers.58.input_layernorm", + "model.layers.8.self_attn.kv_a_layernorm", + "model.layers.33.self_attn.q_a_layernorm", + "model.layers.55.self_attn.q_a_layernorm", + "model.layers.32.input_layernorm", + "model.layers.17.self_attn.kv_a_layernorm", + "model.layers.46.self_attn.indexer.k_norm.bias", + "model.layers.11.mlp.gate.e_score_correction_bias", + "model.layers.9.input_layernorm", + "model.layers.77.input_layernorm", + "model.layers.16.self_attn.kv_a_layernorm", + "model.layers.45.post_attention_layernorm", + "model.layers.74.mlp.gate", + "model.layers.68.self_attn.q_a_layernorm", + "model.layers.55.self_attn.kv_a_layernorm", + "model.layers.44.input_layernorm", + "model.layers.60.self_attn.kv_a_layernorm", + "model.layers.14.post_attention_layernorm", + "model.layers.62.self_attn.indexers_proj", + "model.layers.0.self_attn.q_a_layernorm", + "model.layers.72.post_attention_layernorm", + "model.layers.78.post_attention_layernorm", + "model.layers.60.mlp.gate.e_score_correction_bias", + "model.layers.26.self_attn.indexer.k_norm.bias", + "model.layers.38.post_attention_layernorm", + "model.layers.50.post_attention_layernorm", + "model.layers.28.input_layernorm", + "model.layers.30.self_attn.indexer.k_norm", + "model.layers.37.self_attn.q_a_layernorm", + "model.layers.56.input_layernorm", + "model.layers.27.post_attention_layernorm", + "model.layers.74.input_layernorm", + "model.layers.78.self_attn.q_a_layernorm", + "model.layers.42.self_attn.q_a_layernorm", + "model.layers.26.self_attn.indexer.k_norm", + "model.layers.42.self_attn.kv_a_layernorm", + "model.layers.1.self_attn.indexer.k_norm", + "model.layers.39.post_attention_layernorm", + "model.layers.48.mlp.gate.e_score_correction_bias", + "model.layers.74.post_attention_layernorm", + "model.layers.69.self_attn.q_a_layernorm", + "model.layers.13.input_layernorm", + "model.layers.74.self_attn.indexer.k_norm", + "model.layers.29.mlp.gate", + "model.layers.7.mlp.gate.e_score_correction_bias", + "model.layers.3.self_attn.kv_a_layernorm", + "model.layers.35.self_attn.kv_a_layernorm", + "model.layers.46.self_attn.indexers_proj", + "model.layers.0.self_attn.kv_a_layernorm", + "model.layers.43.self_attn.q_a_layernorm", + "model.norm", + "model.layers.1.self_attn.kv_a_layernorm", + "model.layers.38.self_attn.indexer.k_norm.bias", + "model.layers.30.self_attn.q_a_layernorm", + "model.layers.0.input_layernorm", + "model.layers.47.input_layernorm", + "model.layers.8.self_attn.q_a_layernorm", + "model.layers.61.self_attn.q_a_layernorm", + "model.layers.1.self_attn.indexer.k_norm.bias", + "model.layers.62.mlp.gate.e_score_correction_bias", + "model.layers.66.self_attn.q_a_layernorm", + "model.layers.46.self_attn.kv_a_layernorm", + "model.layers.66.mlp.gate", + "model.layers.69.input_layernorm", + "model.layers.69.self_attn.kv_a_layernorm", + "model.layers.22.input_layernorm", + "model.layers.30.post_attention_layernorm", + "model.layers.24.self_attn.q_a_layernorm", + "model.layers.48.mlp.gate", + "model.layers.7.self_attn.kv_a_layernorm", + "model.layers.67.input_layernorm", + "model.layers.27.self_attn.q_a_layernorm", + "model.layers.23.mlp.gate", + "model.layers.45.mlp.gate.e_score_correction_bias", + "model.layers.76.self_attn.kv_a_layernorm", + "model.layers.22.self_attn.q_a_layernorm", + "model.layers.78.mlp.gate", + "model.layers.9.mlp.gate", + "model.layers.57.self_attn.q_a_layernorm", + "model.layers.78.mlp.gate.e_score_correction_bias", + "model.layers.23.post_attention_layernorm", + "model.layers.38.self_attn.q_a_layernorm", + "model.layers.53.mlp.gate", + "model.layers.77.self_attn.q_a_layernorm", + "model.layers.62.self_attn.kv_a_layernorm", + "model.layers.12.mlp.gate", + "model.layers.34.self_attn.indexer.k_norm", + "model.layers.46.self_attn.q_a_layernorm", + "model.layers.36.self_attn.q_a_layernorm", + "model.layers.58.self_attn.indexer.k_norm.bias", + "model.layers.60.post_attention_layernorm", + "model.layers.28.post_attention_layernorm", + "model.layers.2.self_attn.kv_a_layernorm", + "model.layers.70.self_attn.indexers_proj", + "model.layers.16.mlp.gate.e_score_correction_bias", + "model.layers.2.self_attn.indexer.k_norm", + "model.layers.31.self_attn.kv_a_layernorm", + "model.layers.64.input_layernorm", + "model.layers.6.mlp.gate.e_score_correction_bias", + "model.layers.50.self_attn.indexer.k_norm", + "model.layers.58.self_attn.q_a_layernorm", + "model.layers.57.mlp.gate.e_score_correction_bias", + "model.layers.13.post_attention_layernorm", + "model.layers.28.self_attn.kv_a_layernorm", + "model.layers.31.self_attn.q_a_layernorm", + "model.layers.1.self_attn.q_a_layernorm", + "model.layers.17.mlp.gate", + "model.layers.15.post_attention_layernorm", + "model.layers.18.self_attn.indexer.k_norm.bias", + "model.layers.18.self_attn.indexer.k_norm", + "model.layers.43.mlp.gate.e_score_correction_bias", + "model.layers.11.input_layernorm", + "model.layers.9.post_attention_layernorm", + "model.layers.33.input_layernorm", + "model.layers.66.input_layernorm", + "model.layers.78.self_attn.indexers_proj", + "model.layers.54.post_attention_layernorm", + "model.layers.53.mlp.gate.e_score_correction_bias", + "model.layers.5.mlp.gate.e_score_correction_bias", + "model.layers.19.mlp.gate", + "model.layers.49.mlp.gate", + "model.layers.66.self_attn.indexers_proj", + "model.layers.14.self_attn.indexer.k_norm.bias", + "model.layers.41.mlp.gate", + "model.layers.42.self_attn.indexer.k_norm.bias", + "model.layers.65.input_layernorm", + "model.layers.75.mlp.gate", + "model.layers.12.self_attn.kv_a_layernorm", + "model.layers.53.input_layernorm", + "model.layers.58.self_attn.kv_a_layernorm", + "model.layers.60.self_attn.q_a_layernorm", + "model.layers.6.mlp.gate", + "model.layers.49.self_attn.kv_a_layernorm", + "model.layers.16.mlp.gate", + "model.layers.22.mlp.gate", + "model.layers.58.post_attention_layernorm", + "model.layers.15.mlp.gate.e_score_correction_bias", + "model.layers.26.input_layernorm", + "model.layers.3.post_attention_layernorm", + "model.layers.6.input_layernorm", + "model.layers.22.self_attn.kv_a_layernorm", + "model.layers.16.input_layernorm", + "model.layers.33.post_attention_layernorm", + "model.layers.17.post_attention_layernorm", + "model.layers.35.mlp.gate", + "model.layers.56.post_attention_layernorm", + "model.layers.7.self_attn.q_a_layernorm", + "model.layers.24.mlp.gate", + "model.layers.24.post_attention_layernorm", + "model.layers.36.mlp.gate.e_score_correction_bias", + "model.layers.56.mlp.gate.e_score_correction_bias", + "model.layers.32.post_attention_layernorm", + "model.layers.47.post_attention_layernorm", + "model.layers.40.mlp.gate", + "model.layers.41.self_attn.kv_a_layernorm", + "model.layers.52.self_attn.kv_a_layernorm", + "model.layers.9.self_attn.kv_a_layernorm", + "model.layers.77.mlp.gate.e_score_correction_bias", + "model.layers.78.shared_head.norm", + "model.layers.53.post_attention_layernorm", + "model.layers.63.self_attn.kv_a_layernorm", + "model.layers.48.self_attn.kv_a_layernorm", + "model.layers.28.mlp.gate.e_score_correction_bias", + "model.layers.8.mlp.gate.e_score_correction_bias", + "model.layers.52.mlp.gate.e_score_correction_bias", + "model.layers.52.post_attention_layernorm", + "model.layers.18.self_attn.q_a_layernorm", + "model.layers.50.mlp.gate.e_score_correction_bias", + "model.layers.64.mlp.gate.e_score_correction_bias", + "model.layers.38.mlp.gate.e_score_correction_bias", + "model.layers.59.input_layernorm", + "model.layers.21.input_layernorm", + "model.layers.39.mlp.gate", + "model.layers.77.post_attention_layernorm", + "model.layers.76.self_attn.q_a_layernorm", + "model.layers.12.post_attention_layernorm", + "model.layers.43.post_attention_layernorm", + "model.layers.35.input_layernorm", + "model.layers.62.mlp.gate", + "model.layers.38.self_attn.kv_a_layernorm", + "model.layers.63.mlp.gate", + "model.layers.27.mlp.gate.e_score_correction_bias", + "model.layers.18.mlp.gate", + "model.layers.25.post_attention_layernorm", + "model.layers.55.mlp.gate.e_score_correction_bias", + "model.layers.66.self_attn.indexer.k_norm", + "model.layers.48.self_attn.q_a_layernorm", + "model.layers.78.eh_proj", + "model.layers.77.mlp.gate", + "model.layers.6.self_attn.indexers_proj", + "model.layers.14.self_attn.q_a_layernorm", + "model.layers.2.self_attn.q_a_layernorm", + "model.layers.42.mlp.gate", + "model.layers.54.mlp.gate.e_score_correction_bias", + "model.layers.60.input_layernorm", + "model.layers.40.self_attn.kv_a_layernorm", + "model.layers.51.self_attn.kv_a_layernorm", + "model.layers.18.input_layernorm", + "model.layers.30.self_attn.indexers_proj", + "model.layers.53.self_attn.kv_a_layernorm", + "model.layers.6.post_attention_layernorm", + "model.layers.70.self_attn.indexer.k_norm.bias", + "model.layers.54.input_layernorm", + "model.layers.41.self_attn.q_a_layernorm", + "model.layers.18.mlp.gate.e_score_correction_bias", + "model.layers.34.mlp.gate.e_score_correction_bias", + "model.layers.47.mlp.gate", + "model.layers.46.mlp.gate", + "model.layers.25.self_attn.q_a_layernorm", + "model.layers.63.mlp.gate.e_score_correction_bias", + "model.layers.66.post_attention_layernorm", + "model.layers.76.mlp.gate", + "model.embed_tokens", + "model.layers.67.self_attn.kv_a_layernorm", + "model.layers.67.mlp.gate.e_score_correction_bias", + "model.layers.26.self_attn.kv_a_layernorm", + "model.layers.31.mlp.gate", + "model.layers.18.self_attn.kv_a_layernorm", + "model.layers.34.mlp.gate", + "model.layers.55.post_attention_layernorm", + "model.layers.30.input_layernorm", + "model.layers.50.mlp.gate", + "model.layers.63.input_layernorm", + "model.layers.11.self_attn.kv_a_layernorm", + "model.layers.24.self_attn.kv_a_layernorm", + "model.layers.75.mlp.gate.e_score_correction_bias", + "model.layers.10.post_attention_layernorm", + "model.layers.1.self_attn.indexers_proj", + "model.layers.42.self_attn.indexers_proj", + "model.layers.46.input_layernorm", + "model.layers.23.self_attn.q_a_layernorm", + "model.layers.63.self_attn.q_a_layernorm", + "model.layers.14.self_attn.indexer.k_norm", + "model.layers.14.self_attn.indexers_proj", + "model.layers.60.mlp.gate", + "model.layers.70.mlp.gate.e_score_correction_bias", + "model.layers.53.self_attn.q_a_layernorm", + "model.layers.0.post_attention_layernorm", + "model.layers.35.mlp.gate.e_score_correction_bias", + "model.layers.75.self_attn.kv_a_layernorm", + "model.layers.1.input_layernorm", + "model.layers.42.post_attention_layernorm", + "model.layers.37.input_layernorm", + "model.layers.9.mlp.gate.e_score_correction_bias", + "model.layers.46.post_attention_layernorm", + "model.layers.26.mlp.gate", + "model.layers.65.mlp.gate.e_score_correction_bias", + "model.layers.67.self_attn.q_a_layernorm", + "model.layers.44.mlp.gate.e_score_correction_bias", + "model.layers.33.mlp.gate", + "model.layers.29.input_layernorm", + "model.layers.18.post_attention_layernorm", + "model.layers.75.post_attention_layernorm", + "model.layers.44.self_attn.q_a_layernorm", + "model.layers.10.mlp.gate", + "model.layers.21.mlp.gate", + "model.layers.58.mlp.gate.e_score_correction_bias", + "model.layers.7.post_attention_layernorm", + "model.layers.50.self_attn.q_a_layernorm", + "model.layers.9.self_attn.q_a_layernorm", + "model.layers.2.self_attn.indexers_proj", + "model.layers.0.self_attn.indexer.k_norm.bias", + "model.layers.74.self_attn.kv_a_layernorm", + "model.layers.74.mlp.gate.e_score_correction_bias", + "model.layers.49.mlp.gate.e_score_correction_bias", + "model.layers.62.post_attention_layernorm", + "model.layers.57.self_attn.kv_a_layernorm", + "model.layers.40.input_layernorm", + "model.layers.69.mlp.gate.e_score_correction_bias", + "model.layers.35.post_attention_layernorm", + "model.layers.74.self_attn.indexers_proj", + "model.layers.55.input_layernorm", + "model.layers.49.input_layernorm", + "model.layers.40.mlp.gate.e_score_correction_bias", + "model.layers.25.mlp.gate", + "model.layers.2.self_attn.indexer.k_norm.bias", + "model.layers.68.self_attn.kv_a_layernorm", + "model.layers.51.input_layernorm", + "model.layers.22.self_attn.indexers_proj", + "model.layers.10.input_layernorm", + "model.layers.42.self_attn.indexer.k_norm", + "model.layers.56.self_attn.kv_a_layernorm", + "model.layers.17.self_attn.q_a_layernorm", + "model.layers.59.self_attn.kv_a_layernorm", + "model.layers.3.input_layernorm", + "model.layers.57.input_layernorm", + "model.layers.38.input_layernorm", + "model.layers.62.self_attn.indexer.k_norm", + "model.layers.20.mlp.gate", + "model.layers.64.mlp.gate", + "model.layers.28.self_attn.q_a_layernorm", + "model.layers.32.self_attn.kv_a_layernorm", + "model.layers.10.mlp.gate.e_score_correction_bias", + "model.layers.5.self_attn.q_a_layernorm", + "model.layers.37.self_attn.kv_a_layernorm" + ] + } + }, + "vision_config": { + "patch_size": 14, + "init_pos_emb_height": 64, + "init_pos_emb_width": 64, + "init_pos_emb_time": 4, + "pos_emb_type": "divided_fixed", + "num_attention_heads": 16, + "num_hidden_layers": 27, + "hidden_size": 1152, + "intermediate_size": 4304, + "vt_num_attention_heads": 16, + "vt_num_hidden_layers": 27, + "vt_hidden_size": 1152, + "vt_intermediate_size": 4304, + "merge_kernel_size": [ + 2, + 2 + ], + "video_attn_type": "spatial_temporal", + "merge_type": "sd2_tpool", + "mm_projector_type": "patchmerger", + "mm_hidden_size": 1152, + "projector_hidden_act": "gelu", + "projector_ln_eps": 1e-05, + "text_hidden_size": 6144 + }, + "ignore_index": -100, + "media_placeholder_token_id": 154854, + "pad_token_id": 154820, + "use_unified_vision_chunk": true, + "video_placeholder": "<|glm5v_video_placeholder|>", + "encoder_only": false, + "language_only": false, + "tie_word_embeddings": false, + "quantization_config": { + "activation_scheme": "dynamic", + "fmt": "e4m3", + "quant_method": "fp8", + "weight_block_size": [ + 128, + 128 + ], + "modules_to_not_convert": [ + "model.layers.59.post_attention_layernorm", + "model.layers.47.mlp.gate.e_score_correction_bias", + "model.layers.71.self_attn.kv_a_layernorm", + "model.layers.46.mlp.gate.e_score_correction_bias", + "model.layers.30.self_attn.kv_a_layernorm", + "model.layers.23.self_attn.kv_a_layernorm", + "model.layers.73.mlp.gate.e_score_correction_bias", + "model.layers.36.mlp.gate", + "model.layers.12.self_attn.q_a_layernorm", + "model.layers.51.mlp.gate", + "model.layers.47.self_attn.kv_a_layernorm", + "model.layers.45.self_attn.q_a_layernorm", + "model.layers.15.self_attn.q_a_layernorm", + "model.layers.39.input_layernorm", + "model.layers.50.self_attn.kv_a_layernorm", + "model.layers.19.post_attention_layernorm", + "model.layers.71.input_layernorm", + "model.layers.72.self_attn.kv_a_layernorm", + "model.layers.70.mlp.gate", + "model.layers.52.input_layernorm", + "model.layers.78.self_attn.kv_a_layernorm", + "model.layers.48.post_attention_layernorm", + "model.layers.4.self_attn.q_a_layernorm", + "model.layers.62.self_attn.q_a_layernorm", + "model.layers.4.post_attention_layernorm", + "model.layers.38.self_attn.indexers_proj", + "model.layers.61.mlp.gate.e_score_correction_bias", + "model.layers.14.mlp.gate.e_score_correction_bias", + "model.layers.36.self_attn.kv_a_layernorm", + "model.layers.65.mlp.gate", + "model.layers.13.self_attn.kv_a_layernorm", + "model.layers.57.mlp.gate", + "model.layers.65.self_attn.kv_a_layernorm", + "model.layers.68.input_layernorm", + "model.layers.70.self_attn.kv_a_layernorm", + "model.layers.24.mlp.gate.e_score_correction_bias", + "model.layers.4.input_layernorm", + "model.layers.20.input_layernorm", + "model.layers.65.self_attn.q_a_layernorm", + "model.layers.38.mlp.gate", + "model.layers.76.mlp.gate.e_score_correction_bias", + "model.layers.62.input_layernorm", + "model.layers.45.input_layernorm", + "model.layers.68.post_attention_layernorm", + "model.layers.72.input_layernorm", + "model.layers.32.mlp.gate.e_score_correction_bias", + "model.layers.27.input_layernorm", + "model.layers.16.post_attention_layernorm", + "model.layers.35.self_attn.q_a_layernorm", + "model.layers.58.self_attn.indexers_proj", + "model.layers.3.mlp.gate.e_score_correction_bias", + "model.layers.49.self_attn.q_a_layernorm", + "model.layers.66.mlp.gate.e_score_correction_bias", + "model.layers.10.self_attn.q_a_layernorm", + "model.layers.51.mlp.gate.e_score_correction_bias", + "model.layers.64.self_attn.q_a_layernorm", + "model.layers.70.input_layernorm", + "model.layers.71.post_attention_layernorm", + "model.layers.67.post_attention_layernorm", + "model.layers.6.self_attn.indexer.k_norm", + "model.layers.3.mlp.gate", + "model.layers.11.mlp.gate", + "model.layers.17.input_layernorm", + "model.layers.13.mlp.gate", + "model.layers.15.mlp.gate", + "model.layers.71.self_attn.q_a_layernorm", + "model.layers.33.mlp.gate.e_score_correction_bias", + "model.layers.72.mlp.gate.e_score_correction_bias", + "model.layers.65.post_attention_layernorm", + "model.layers.78.hnorm", + "model.layers.40.post_attention_layernorm", + "model.layers.12.mlp.gate.e_score_correction_bias", + "model.layers.8.input_layernorm", + "model.layers.73.mlp.gate", + "model.layers.11.post_attention_layernorm", + "model.layers.61.input_layernorm", + "model.layers.14.self_attn.kv_a_layernorm", + "model.layers.39.self_attn.kv_a_layernorm", + "model.layers.70.self_attn.indexer.k_norm", + "model.layers.45.mlp.gate", + "model.layers.51.self_attn.q_a_layernorm", + "model.layers.64.post_attention_layernorm", + "model.layers.20.post_attention_layernorm", + "model.layers.66.self_attn.kv_a_layernorm", + "model.layers.10.self_attn.indexer.k_norm.bias", + "model.layers.23.input_layernorm", + "model.layers.38.self_attn.indexer.k_norm", + "model.layers.19.self_attn.kv_a_layernorm", + "model.layers.40.self_attn.q_a_layernorm", + "model.layers.10.self_attn.indexer.k_norm", + "model.layers.44.post_attention_layernorm", + "model.layers.70.post_attention_layernorm", + "model.layers.41.mlp.gate.e_score_correction_bias", + "model.layers.36.post_attention_layernorm", + "model.layers.30.self_attn.indexer.k_norm.bias", + "model.layers.54.self_attn.q_a_layernorm", + "model.layers.59.mlp.gate", + "model.layers.69.post_attention_layernorm", + "lm_head", + "model.layers.77.self_attn.kv_a_layernorm", + "model.layers.75.self_attn.q_a_layernorm", + "model.layers.56.self_attn.q_a_layernorm", + "model.layers.50.self_attn.indexer.k_norm.bias", + "model.layers.54.mlp.gate", + "model.layers.6.self_attn.indexer.k_norm.bias", + "model.layers.15.input_layernorm", + "model.layers.29.self_attn.q_a_layernorm", + "model.layers.54.self_attn.indexers_proj", + "model.layers.73.self_attn.kv_a_layernorm", + "model.layers.0.self_attn.indexer.k_norm", + "model.layers.58.self_attn.indexer.k_norm", + "model.layers.36.input_layernorm", + "model.layers.47.self_attn.q_a_layernorm", + "model.layers.26.mlp.gate.e_score_correction_bias", + "model.layers.50.input_layernorm", + "model.layers.16.self_attn.q_a_layernorm", + "model.layers.19.mlp.gate.e_score_correction_bias", + "model.layers.78.input_layernorm", + "model.layers.27.mlp.gate", + "model.layers.19.self_attn.q_a_layernorm", + "model.layers.25.input_layernorm", + "model.layers.33.self_attn.kv_a_layernorm", + "model.layers.32.mlp.gate", + "model.layers.61.self_attn.kv_a_layernorm", + "model.layers.50.self_attn.indexers_proj", + "model.layers.34.self_attn.indexers_proj", + "model.layers.4.mlp.gate.e_score_correction_bias", + "model.layers.76.post_attention_layernorm", + "model.layers.57.post_attention_layernorm", + "model.layers.73.post_attention_layernorm", + "model.layers.68.mlp.gate.e_score_correction_bias", + "model.layers.34.self_attn.kv_a_layernorm", + "model.layers.78.enorm", + "model.layers.30.mlp.gate", + "model.layers.10.self_attn.indexers_proj", + "model.layers.34.input_layernorm", + "model.layers.34.post_attention_layernorm", + "model.layers.20.mlp.gate.e_score_correction_bias", + "model.layers.54.self_attn.indexer.k_norm.bias", + "model.layers.44.mlp.gate", + "model.layers.25.mlp.gate.e_score_correction_bias", + "model.layers.26.self_attn.indexers_proj", + "model.layers.66.self_attn.indexer.k_norm.bias", + "model.layers.41.post_attention_layernorm", + "model.layers.51.post_attention_layernorm", + "model.layers.22.self_attn.indexer.k_norm.bias", + "model.layers.21.mlp.gate.e_score_correction_bias", + "model.layers.15.self_attn.kv_a_layernorm", + "model.layers.25.self_attn.kv_a_layernorm", + "model.layers.55.mlp.gate", + "model.layers.11.self_attn.q_a_layernorm", + "model.layers.72.mlp.gate", + "model.layers.21.self_attn.kv_a_layernorm", + "model.layers.59.mlp.gate.e_score_correction_bias", + "model.layers.2.input_layernorm", + "model.layers.5.mlp.gate", + "model.layers.54.self_attn.kv_a_layernorm", + "model.layers.0.self_attn.indexers_proj", + "model.layers.71.mlp.gate.e_score_correction_bias", + "model.layers.6.self_attn.q_a_layernorm", + "model.layers.14.mlp.gate", + "model.layers.44.self_attn.kv_a_layernorm", + "model.layers.61.post_attention_layernorm", + "model.layers.64.self_attn.kv_a_layernorm", + "model.layers.1.post_attention_layernorm", + "model.layers.4.mlp.gate", + "model.layers.22.post_attention_layernorm", + "model.layers.21.self_attn.q_a_layernorm", + "model.layers.74.self_attn.q_a_layernorm", + "model.layers.18.self_attn.indexers_proj", + "model.layers.75.input_layernorm", + "model.layers.70.self_attn.q_a_layernorm", + "model.layers.22.self_attn.indexer.k_norm", + "model.layers.42.mlp.gate.e_score_correction_bias", + "model.layers.5.self_attn.kv_a_layernorm", + "model.layers.31.mlp.gate.e_score_correction_bias", + "model.layers.62.self_attn.indexer.k_norm.bias", + "model.layers.29.post_attention_layernorm", + "model.layers.6.self_attn.kv_a_layernorm", + "model.layers.48.input_layernorm", + "model.layers.74.self_attn.indexer.k_norm.bias", + "model.layers.7.input_layernorm", + "model.layers.39.self_attn.q_a_layernorm", + "model.layers.41.input_layernorm", + "model.layers.54.self_attn.indexer.k_norm", + "model.layers.32.self_attn.q_a_layernorm", + "model.layers.39.mlp.gate.e_score_correction_bias", + "model.layers.42.input_layernorm", + "model.layers.43.mlp.gate", + "model.layers.72.self_attn.q_a_layernorm", + "model.layers.37.mlp.gate.e_score_correction_bias", + "model.layers.5.post_attention_layernorm", + "model.layers.28.mlp.gate", + "model.layers.2.post_attention_layernorm", + "model.layers.49.post_attention_layernorm", + "model.layers.43.self_attn.kv_a_layernorm", + "model.layers.12.input_layernorm", + "model.layers.56.mlp.gate", + "model.layers.5.input_layernorm", + "model.layers.29.mlp.gate.e_score_correction_bias", + "model.layers.4.self_attn.kv_a_layernorm", + "model.layers.20.self_attn.q_a_layernorm", + "model.layers.59.self_attn.q_a_layernorm", + "model.layers.68.mlp.gate", + "model.layers.21.post_attention_layernorm", + "model.layers.26.self_attn.q_a_layernorm", + "model.layers.52.self_attn.q_a_layernorm", + "model.layers.29.self_attn.kv_a_layernorm", + "model.layers.24.input_layernorm", + "model.layers.34.self_attn.indexer.k_norm.bias", + "model.layers.43.input_layernorm", + "model.layers.27.self_attn.kv_a_layernorm", + "model.layers.31.input_layernorm", + "model.layers.69.mlp.gate", + "model.layers.26.post_attention_layernorm", + "model.layers.3.self_attn.q_a_layernorm", + "model.layers.37.post_attention_layernorm", + "model.layers.52.mlp.gate", + "model.layers.73.input_layernorm", + "model.layers.19.input_layernorm", + "model.layers.13.self_attn.q_a_layernorm", + "model.layers.76.input_layernorm", + "model.layers.8.mlp.gate", + "model.layers.63.post_attention_layernorm", + "model.layers.58.mlp.gate", + "model.layers.31.post_attention_layernorm", + "model.layers.23.mlp.gate.e_score_correction_bias", + "model.layers.34.self_attn.q_a_layernorm", + "model.layers.78.self_attn.indexer.k_norm.bias", + "model.layers.13.mlp.gate.e_score_correction_bias", + "model.layers.71.mlp.gate", + "model.layers.14.input_layernorm", + "model.layers.17.mlp.gate.e_score_correction_bias", + "model.layers.8.post_attention_layernorm", + "model.layers.45.self_attn.kv_a_layernorm", + "model.layers.46.self_attn.indexer.k_norm", + "model.layers.20.self_attn.kv_a_layernorm", + "model.layers.78.self_attn.indexer.k_norm", + "model.layers.37.mlp.gate", + "model.layers.7.mlp.gate", + "model.layers.30.mlp.gate.e_score_correction_bias", + "model.layers.67.mlp.gate", + "model.layers.73.self_attn.q_a_layernorm", + "model.layers.10.self_attn.kv_a_layernorm", + "model.layers.61.mlp.gate", + "model.layers.22.mlp.gate.e_score_correction_bias", + "model.layers.58.input_layernorm", + "model.layers.8.self_attn.kv_a_layernorm", + "model.layers.33.self_attn.q_a_layernorm", + "model.layers.55.self_attn.q_a_layernorm", + "model.layers.32.input_layernorm", + "model.layers.17.self_attn.kv_a_layernorm", + "model.layers.46.self_attn.indexer.k_norm.bias", + "model.layers.11.mlp.gate.e_score_correction_bias", + "model.layers.9.input_layernorm", + "model.layers.77.input_layernorm", + "model.layers.16.self_attn.kv_a_layernorm", + "model.layers.45.post_attention_layernorm", + "model.layers.74.mlp.gate", + "model.layers.68.self_attn.q_a_layernorm", + "model.layers.55.self_attn.kv_a_layernorm", + "model.layers.44.input_layernorm", + "model.layers.60.self_attn.kv_a_layernorm", + "model.layers.14.post_attention_layernorm", + "model.layers.62.self_attn.indexers_proj", + "model.layers.0.self_attn.q_a_layernorm", + "model.layers.72.post_attention_layernorm", + "model.layers.78.post_attention_layernorm", + "model.layers.60.mlp.gate.e_score_correction_bias", + "model.layers.26.self_attn.indexer.k_norm.bias", + "model.layers.38.post_attention_layernorm", + "model.layers.50.post_attention_layernorm", + "model.layers.28.input_layernorm", + "model.layers.30.self_attn.indexer.k_norm", + "model.layers.37.self_attn.q_a_layernorm", + "model.layers.56.input_layernorm", + "model.layers.27.post_attention_layernorm", + "model.layers.74.input_layernorm", + "model.layers.78.self_attn.q_a_layernorm", + "model.layers.42.self_attn.q_a_layernorm", + "model.layers.26.self_attn.indexer.k_norm", + "model.layers.42.self_attn.kv_a_layernorm", + "model.layers.1.self_attn.indexer.k_norm", + "model.layers.39.post_attention_layernorm", + "model.layers.48.mlp.gate.e_score_correction_bias", + "model.layers.74.post_attention_layernorm", + "model.layers.69.self_attn.q_a_layernorm", + "model.layers.13.input_layernorm", + "model.layers.74.self_attn.indexer.k_norm", + "model.layers.29.mlp.gate", + "model.layers.7.mlp.gate.e_score_correction_bias", + "model.layers.3.self_attn.kv_a_layernorm", + "model.layers.35.self_attn.kv_a_layernorm", + "model.layers.46.self_attn.indexers_proj", + "model.layers.0.self_attn.kv_a_layernorm", + "model.layers.43.self_attn.q_a_layernorm", + "model.norm", + "model.layers.1.self_attn.kv_a_layernorm", + "model.layers.38.self_attn.indexer.k_norm.bias", + "model.layers.30.self_attn.q_a_layernorm", + "model.layers.0.input_layernorm", + "model.layers.47.input_layernorm", + "model.layers.8.self_attn.q_a_layernorm", + "model.layers.61.self_attn.q_a_layernorm", + "model.layers.1.self_attn.indexer.k_norm.bias", + "model.layers.62.mlp.gate.e_score_correction_bias", + "model.layers.66.self_attn.q_a_layernorm", + "model.layers.46.self_attn.kv_a_layernorm", + "model.layers.66.mlp.gate", + "model.layers.69.input_layernorm", + "model.layers.69.self_attn.kv_a_layernorm", + "model.layers.22.input_layernorm", + "model.layers.30.post_attention_layernorm", + "model.layers.24.self_attn.q_a_layernorm", + "model.layers.48.mlp.gate", + "model.layers.7.self_attn.kv_a_layernorm", + "model.layers.67.input_layernorm", + "model.layers.27.self_attn.q_a_layernorm", + "model.layers.23.mlp.gate", + "model.layers.45.mlp.gate.e_score_correction_bias", + "model.layers.76.self_attn.kv_a_layernorm", + "model.layers.22.self_attn.q_a_layernorm", + "model.layers.78.mlp.gate", + "model.layers.9.mlp.gate", + "model.layers.57.self_attn.q_a_layernorm", + "model.layers.78.mlp.gate.e_score_correction_bias", + "model.layers.23.post_attention_layernorm", + "model.layers.38.self_attn.q_a_layernorm", + "model.layers.53.mlp.gate", + "model.layers.77.self_attn.q_a_layernorm", + "model.layers.62.self_attn.kv_a_layernorm", + "model.layers.12.mlp.gate", + "model.layers.34.self_attn.indexer.k_norm", + "model.layers.46.self_attn.q_a_layernorm", + "model.layers.36.self_attn.q_a_layernorm", + "model.layers.58.self_attn.indexer.k_norm.bias", + "model.layers.60.post_attention_layernorm", + "model.layers.28.post_attention_layernorm", + "model.layers.2.self_attn.kv_a_layernorm", + "model.layers.70.self_attn.indexers_proj", + "model.layers.16.mlp.gate.e_score_correction_bias", + "model.layers.2.self_attn.indexer.k_norm", + "model.layers.31.self_attn.kv_a_layernorm", + "model.layers.64.input_layernorm", + "model.layers.6.mlp.gate.e_score_correction_bias", + "model.layers.50.self_attn.indexer.k_norm", + "model.layers.58.self_attn.q_a_layernorm", + "model.layers.57.mlp.gate.e_score_correction_bias", + "model.layers.13.post_attention_layernorm", + "model.layers.28.self_attn.kv_a_layernorm", + "model.layers.31.self_attn.q_a_layernorm", + "model.layers.1.self_attn.q_a_layernorm", + "model.layers.17.mlp.gate", + "model.layers.15.post_attention_layernorm", + "model.layers.18.self_attn.indexer.k_norm.bias", + "model.layers.18.self_attn.indexer.k_norm", + "model.layers.43.mlp.gate.e_score_correction_bias", + "model.layers.11.input_layernorm", + "model.layers.9.post_attention_layernorm", + "model.layers.33.input_layernorm", + "model.layers.66.input_layernorm", + "model.layers.78.self_attn.indexers_proj", + "model.layers.54.post_attention_layernorm", + "model.layers.53.mlp.gate.e_score_correction_bias", + "model.layers.5.mlp.gate.e_score_correction_bias", + "model.layers.19.mlp.gate", + "model.layers.49.mlp.gate", + "model.layers.66.self_attn.indexers_proj", + "model.layers.14.self_attn.indexer.k_norm.bias", + "model.layers.41.mlp.gate", + "model.layers.42.self_attn.indexer.k_norm.bias", + "model.layers.65.input_layernorm", + "model.layers.75.mlp.gate", + "model.layers.12.self_attn.kv_a_layernorm", + "model.layers.53.input_layernorm", + "model.layers.58.self_attn.kv_a_layernorm", + "model.layers.60.self_attn.q_a_layernorm", + "model.layers.6.mlp.gate", + "model.layers.49.self_attn.kv_a_layernorm", + "model.layers.16.mlp.gate", + "model.layers.22.mlp.gate", + "model.layers.58.post_attention_layernorm", + "model.layers.15.mlp.gate.e_score_correction_bias", + "model.layers.26.input_layernorm", + "model.layers.3.post_attention_layernorm", + "model.layers.6.input_layernorm", + "model.layers.22.self_attn.kv_a_layernorm", + "model.layers.16.input_layernorm", + "model.layers.33.post_attention_layernorm", + "model.layers.17.post_attention_layernorm", + "model.layers.35.mlp.gate", + "model.layers.56.post_attention_layernorm", + "model.layers.7.self_attn.q_a_layernorm", + "model.layers.24.mlp.gate", + "model.layers.24.post_attention_layernorm", + "model.layers.36.mlp.gate.e_score_correction_bias", + "model.layers.56.mlp.gate.e_score_correction_bias", + "model.layers.32.post_attention_layernorm", + "model.layers.47.post_attention_layernorm", + "model.layers.40.mlp.gate", + "model.layers.41.self_attn.kv_a_layernorm", + "model.layers.52.self_attn.kv_a_layernorm", + "model.layers.9.self_attn.kv_a_layernorm", + "model.layers.77.mlp.gate.e_score_correction_bias", + "model.layers.78.shared_head.norm", + "model.layers.53.post_attention_layernorm", + "model.layers.63.self_attn.kv_a_layernorm", + "model.layers.48.self_attn.kv_a_layernorm", + "model.layers.28.mlp.gate.e_score_correction_bias", + "model.layers.8.mlp.gate.e_score_correction_bias", + "model.layers.52.mlp.gate.e_score_correction_bias", + "model.layers.52.post_attention_layernorm", + "model.layers.18.self_attn.q_a_layernorm", + "model.layers.50.mlp.gate.e_score_correction_bias", + "model.layers.64.mlp.gate.e_score_correction_bias", + "model.layers.38.mlp.gate.e_score_correction_bias", + "model.layers.59.input_layernorm", + "model.layers.21.input_layernorm", + "model.layers.39.mlp.gate", + "model.layers.77.post_attention_layernorm", + "model.layers.76.self_attn.q_a_layernorm", + "model.layers.12.post_attention_layernorm", + "model.layers.43.post_attention_layernorm", + "model.layers.35.input_layernorm", + "model.layers.62.mlp.gate", + "model.layers.38.self_attn.kv_a_layernorm", + "model.layers.63.mlp.gate", + "model.layers.27.mlp.gate.e_score_correction_bias", + "model.layers.18.mlp.gate", + "model.layers.25.post_attention_layernorm", + "model.layers.55.mlp.gate.e_score_correction_bias", + "model.layers.66.self_attn.indexer.k_norm", + "model.layers.48.self_attn.q_a_layernorm", + "model.layers.78.eh_proj", + "model.layers.77.mlp.gate", + "model.layers.6.self_attn.indexers_proj", + "model.layers.14.self_attn.q_a_layernorm", + "model.layers.2.self_attn.q_a_layernorm", + "model.layers.42.mlp.gate", + "model.layers.54.mlp.gate.e_score_correction_bias", + "model.layers.60.input_layernorm", + "model.layers.40.self_attn.kv_a_layernorm", + "model.layers.51.self_attn.kv_a_layernorm", + "model.layers.18.input_layernorm", + "model.layers.30.self_attn.indexers_proj", + "model.layers.53.self_attn.kv_a_layernorm", + "model.layers.6.post_attention_layernorm", + "model.layers.70.self_attn.indexer.k_norm.bias", + "model.layers.54.input_layernorm", + "model.layers.41.self_attn.q_a_layernorm", + "model.layers.18.mlp.gate.e_score_correction_bias", + "model.layers.34.mlp.gate.e_score_correction_bias", + "model.layers.47.mlp.gate", + "model.layers.46.mlp.gate", + "model.layers.25.self_attn.q_a_layernorm", + "model.layers.63.mlp.gate.e_score_correction_bias", + "model.layers.66.post_attention_layernorm", + "model.layers.76.mlp.gate", + "model.embed_tokens", + "model.layers.67.self_attn.kv_a_layernorm", + "model.layers.67.mlp.gate.e_score_correction_bias", + "model.layers.26.self_attn.kv_a_layernorm", + "model.layers.31.mlp.gate", + "model.layers.18.self_attn.kv_a_layernorm", + "model.layers.34.mlp.gate", + "model.layers.55.post_attention_layernorm", + "model.layers.30.input_layernorm", + "model.layers.50.mlp.gate", + "model.layers.63.input_layernorm", + "model.layers.11.self_attn.kv_a_layernorm", + "model.layers.24.self_attn.kv_a_layernorm", + "model.layers.75.mlp.gate.e_score_correction_bias", + "model.layers.10.post_attention_layernorm", + "model.layers.1.self_attn.indexers_proj", + "model.layers.42.self_attn.indexers_proj", + "model.layers.46.input_layernorm", + "model.layers.23.self_attn.q_a_layernorm", + "model.layers.63.self_attn.q_a_layernorm", + "model.layers.14.self_attn.indexer.k_norm", + "model.layers.14.self_attn.indexers_proj", + "model.layers.60.mlp.gate", + "model.layers.70.mlp.gate.e_score_correction_bias", + "model.layers.53.self_attn.q_a_layernorm", + "model.layers.0.post_attention_layernorm", + "model.layers.35.mlp.gate.e_score_correction_bias", + "model.layers.75.self_attn.kv_a_layernorm", + "model.layers.1.input_layernorm", + "model.layers.42.post_attention_layernorm", + "model.layers.37.input_layernorm", + "model.layers.9.mlp.gate.e_score_correction_bias", + "model.layers.46.post_attention_layernorm", + "model.layers.26.mlp.gate", + "model.layers.65.mlp.gate.e_score_correction_bias", + "model.layers.67.self_attn.q_a_layernorm", + "model.layers.44.mlp.gate.e_score_correction_bias", + "model.layers.33.mlp.gate", + "model.layers.29.input_layernorm", + "model.layers.18.post_attention_layernorm", + "model.layers.75.post_attention_layernorm", + "model.layers.44.self_attn.q_a_layernorm", + "model.layers.10.mlp.gate", + "model.layers.21.mlp.gate", + "model.layers.58.mlp.gate.e_score_correction_bias", + "model.layers.7.post_attention_layernorm", + "model.layers.50.self_attn.q_a_layernorm", + "model.layers.9.self_attn.q_a_layernorm", + "model.layers.2.self_attn.indexers_proj", + "model.layers.0.self_attn.indexer.k_norm.bias", + "model.layers.74.self_attn.kv_a_layernorm", + "model.layers.74.mlp.gate.e_score_correction_bias", + "model.layers.49.mlp.gate.e_score_correction_bias", + "model.layers.62.post_attention_layernorm", + "model.layers.57.self_attn.kv_a_layernorm", + "model.layers.40.input_layernorm", + "model.layers.69.mlp.gate.e_score_correction_bias", + "model.layers.35.post_attention_layernorm", + "model.layers.74.self_attn.indexers_proj", + "model.layers.55.input_layernorm", + "model.layers.49.input_layernorm", + "model.layers.40.mlp.gate.e_score_correction_bias", + "model.layers.25.mlp.gate", + "model.layers.2.self_attn.indexer.k_norm.bias", + "model.layers.68.self_attn.kv_a_layernorm", + "model.layers.51.input_layernorm", + "model.layers.22.self_attn.indexers_proj", + "model.layers.10.input_layernorm", + "model.layers.42.self_attn.indexer.k_norm", + "model.layers.56.self_attn.kv_a_layernorm", + "model.layers.17.self_attn.q_a_layernorm", + "model.layers.59.self_attn.kv_a_layernorm", + "model.layers.3.input_layernorm", + "model.layers.57.input_layernorm", + "model.layers.38.input_layernorm", + "model.layers.62.self_attn.indexer.k_norm", + "model.layers.20.mlp.gate", + "model.layers.64.mlp.gate", + "model.layers.28.self_attn.q_a_layernorm", + "model.layers.32.self_attn.kv_a_layernorm", + "model.layers.10.mlp.gate.e_score_correction_bias", + "model.layers.5.self_attn.q_a_layernorm", + "model.layers.37.self_attn.kv_a_layernorm" + ] + } +} \ No newline at end of file diff --git a/configuration_glm5v.py b/configuration_glm5v.py new file mode 100644 index 0000000000000000000000000000000000000000..85bfa11489cb90502bdbb8036df4c927db119f16 --- /dev/null +++ b/configuration_glm5v.py @@ -0,0 +1,152 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Glm5vConfig — remote-code config carried inside the assembled GLM5V SGLang +checkpoint (referenced by config.json ``auto_map``; loaded with +``--trust-remote-code``, which the checkpoint already requires for the Kimi +image-processor remote code). + +Self-contained: depends only on ``transformers``. Mirrors SGLang's in-tree +``KimiK25Config`` structure (``vision_config`` + ``text_config`` + media +placeholder fields) with GLM-5.2 as the text model: + +* ``text_config`` -> ``GlmMoeDsaConfig`` (transformers-native ``glm_moe_dsa``). +* ``vision_config``-> MoonViT fields; ``text_hidden_size`` (projector output + dim) retargeted to GLM hidden 6144. +* ``media_placeholder_token_id`` -> GLM ``<|image|>`` = 154854. +""" + +from transformers import AutoConfig +from transformers.configuration_utils import PretrainedConfig + + +class Glm5vVisionConfig(PretrainedConfig): + """MoonViT vision tower + PatchMerger projector config. + + Field names/defaults mirror SGLang's ``KimiK25VisionConfig`` (declared + names like ``hidden_size``) while the official Kimi checkpoint's ``vt_*`` + names arrive via **kwargs and are stored as attributes — SGLang's model + code reads both families (tower: ``hidden_size``; projector: + ``vt_hidden_size``/``text_hidden_size``). + """ + + model_type = "glm5v_vision" + + def __init__( + self, + # Vision tower + patch_size: int = 14, + init_pos_emb_height: int = 64, + init_pos_emb_width: int = 64, + init_pos_emb_time: int = 4, + pos_emb_type: str = "divided_fixed", + num_attention_heads: int = 16, + num_hidden_layers: int = 27, + hidden_size: int = 1152, + intermediate_size: int = 4304, + merge_kernel_size=(2, 2), + video_attn_type: str = "spatial_temporal", + merge_type: str = "sd2_tpool", + # MM projector + mm_projector_type: str = "patchmerger", + mm_hidden_size: int | None = None, + projector_hidden_act: str = "gelu", + projector_ln_eps: float = 1e-5, + text_hidden_size: int = 6144, # GLM-5.2 hidden (Kimi default is 7168) + **kwargs, + ): + super().__init__(**kwargs) + self.patch_size = patch_size + self.init_pos_emb_height = init_pos_emb_height + self.init_pos_emb_width = init_pos_emb_width + self.init_pos_emb_time = init_pos_emb_time + self.pos_emb_type = pos_emb_type + self.num_attention_heads = num_attention_heads + self.num_hidden_layers = num_hidden_layers + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.merge_kernel_size = merge_kernel_size + self.video_attn_type = video_attn_type + self.merge_type = merge_type + self.mm_projector_type = mm_projector_type + self.mm_hidden_size = mm_hidden_size if mm_hidden_size is not None else hidden_size + self.projector_hidden_act = projector_hidden_act + self.projector_ln_eps = projector_ln_eps + self.text_hidden_size = text_hidden_size + + +class Glm5vConfig(PretrainedConfig): + """glm5v top-level config: MoonViT ``vision_config`` + GLM-5.2 ``text_config``.""" + + model_type = "glm5v" + + def __init__( + self, + text_config=None, + vision_config=None, + ignore_index: int = -100, + media_placeholder_token_id: int = 154854, # GLM <|image|> + pad_token_id: int = 154820, + use_unified_vision_chunk: bool = True, + video_placeholder: str = "<|glm5v_video_placeholder|>", + encoder_only: bool = False, + language_only: bool = False, + **kwargs, + ): + # Vision config (MoonViT). + if vision_config is None: + self.vision_config = Glm5vVisionConfig() + elif isinstance(vision_config, dict): + self.vision_config = Glm5vVisionConfig(**vision_config) + else: + self.vision_config = vision_config + + # Text config (GLM-5.2 / glm_moe_dsa), built via AutoConfig so the + # transformers-native GlmMoeDsaConfig class is used. + raw_text = dict(text_config) if isinstance(text_config, dict) else None + if text_config is None: + self.text_config = AutoConfig.for_model("glm_moe_dsa") + elif isinstance(text_config, dict): + tc = dict(text_config) + tc.setdefault("model_type", "glm_moe_dsa") + self.text_config = AutoConfig.for_model(**tc) + else: + self.text_config = text_config + + # transformers 5.8.x GlmMoeDsaConfig drops/clobbers raw DSA fields the + # sparse-attention path needs. SGLang applies this same restore for + # bare GlmMoeDsaForCausalLM checkpoints (see its HfModelConfigParser; + # fixed upstream by transformers PR #46338, gone once >= 5.10); our + # top-level arch is Glm5v so we replicate it here. + if raw_text is not None: + for key in ("qk_rope_head_dim", "index_topk_freq"): + if key in raw_text: + setattr(self.text_config, key, raw_text[key]) + if hasattr(self.text_config, "qk_nope_head_dim") and hasattr( + self.text_config, "qk_rope_head_dim" + ): + self.text_config.qk_head_dim = ( + self.text_config.qk_nope_head_dim + + self.text_config.qk_rope_head_dim + ) + + self.ignore_index = ignore_index + self.media_placeholder_token_id = media_placeholder_token_id + self.use_unified_vision_chunk = use_unified_vision_chunk + self.video_placeholder = video_placeholder + self.encoder_only = encoder_only + self.language_only = language_only + + # Propagate quantization config from the text model (Kimi pattern): + # only the GLM text Linears are FP8; vision/projector stay bf16 by + # construction in the model code. + if getattr(self.text_config, "quantization_config", None) is not None: + self.quantization_config = self.text_config.quantization_config + + super().__init__(pad_token_id=pad_token_id, **kwargs) + + @property + def hidden_size(self) -> int: + return self.text_config.hidden_size + + @property + def vocab_size(self) -> int: + return self.text_config.vocab_size diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..39454c37be77190c555221dfbbd48d92233f908d --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "_from_model_config": true, + "eos_token_id": [ + 154820, + 154827, + 154829 + ], + "pad_token_id": 154820, + "temperature": 1.0, + "top_p": 0.95, + "transformers_version": "5.12.0" +} \ No newline at end of file diff --git a/kimi_k25_processor.py b/kimi_k25_processor.py new file mode 100644 index 0000000000000000000000000000000000000000..b47390de3f2c5add9266b63b7bd308d14e4b60e1 --- /dev/null +++ b/kimi_k25_processor.py @@ -0,0 +1,208 @@ +from transformers.feature_extraction_utils import BatchFeature +from transformers.processing_utils import ProcessorMixin +from transformers.utils import logging + +logger = logging.get_logger(__name__) + + +class KimiK25Processor(ProcessorMixin): + r""" + Constructs a KimiK25 processor which wraps a KimiK25 image processor and a tokenizer into a single processor. + + [`KimiK25Processor`] offers all the functionalities of [`KimiK25ImageProcessor`] and [`TikTokenTokenizer`]. See the + [`~KimiK25Processor.__call__`] and [`~KimiK25Processor.decode`] for more information. + + Args: + image_processor ([`KimiK25ImageProcessor`], *optional*): + The image processor is a required input. + tokenizer ([`TikTokenTokenizer`], *optional*): + The tokenizer is a required input. + chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages + in a chat into a tokenizable string. + """ + + attributes = ["image_processor", "tokenizer"] + valid_kwargs = ["chat_template"] + image_processor_class = "AutoImageProcessor" + tokenizer_class = "AutoTokenizer" + + def __init__( + self, + image_processor=None, + tokenizer=None, + chat_template=None, + **kwargs, + ): + super().__init__(image_processor, + tokenizer, + chat_template=chat_template) + self.media_processor = image_processor + # A special temporal placeholder to be replaced by actual video placeholders + self.video_placeholder = "<|kimi_k25_video_placeholder|>" + + def update_raw_text(self, text: str, video_prompts: list[str]) -> str: + # replace video prompt in text with video chunk prompts + video_count = text.count(self.video_placeholder) + if video_count == 0: + return text + assert video_count == len(video_prompts) + text_parts = text.split(self.video_placeholder) + assert len(text_parts) == len(video_prompts) + 1 + text = "".join([ + text_parts[i] + video_prompts[i] for i in range(len(video_prompts)) + ]) + text += text_parts[-1] + return text + + def preprocess_medias(self, medias: list[dict]) -> list[dict]: + updated_medias = [] + video_prompts = [] + for media in medias: + if media['type'] == 'image': + updated_medias.append(media) + elif media['type'] == 'video': + video_chunks = self.media_processor.split_video_chunks( + media['video']) + updated_medias.extend(video_chunks) + video_prompts.append("".join( + [vc['prompt'] for vc in video_chunks])) + else: + raise ValueError(f"unsupported media type: {media['type']}") + return updated_medias, video_prompts + + # glm5v: the image placeholder expanded per patch (GLM <|image|> = 154854). + # The chat template wraps it as <|begin_of_image|><|image|><|end_of_image|>. + GLM5V_IMAGE_TOKEN = "<|image|>" + + def __call__(self, + messages: list[dict] = None, + medias: list[dict] = None, + text: str = None, + images: list = None, + return_tensors: str = "pt", + **kwargs) -> BatchFeature: + """ + Process multimodal inputs for Kimi-K2.5 model. + + This processor accepts ordered messages and extracts both media and text in a single pass. + text will be automatically updated if video input detected in messages + + Args: + messages: List of message dicts with 'role' and 'content' fields. + If provided, medias and text will be extracted automatically. + medias: Pre-extracted list of media dicts. If None, extracted from messages. + text: Pre-formatted text string. If None, generated via apply_chat_template. + images: Standard HF VLM API (``processor(text=..., images=[...])``), as + called by generic drivers (e.g. slime's rollout prompt prep). + Converted to ``medias`` and each ``<|image|>`` placeholder in + ``text`` is expanded to that image's per-patch token count, so + the returned ``input_ids`` align with ``pixel_values`` (same + semantics as Qwen-family processors and the serving-layer + wrappers in vLLM/SGLang). + return_tensors: Format of returned tensors ('pt', 'np', 'tf'). Default: 'pt'. + **kwargs: Additional arguments passed to tokenizer.apply_chat_template. + + Returns: + BatchFeature with fields: input_ids, attention_mask, pixel_values, grid_thws. + """ + if images is not None and medias is None and text is not None: + # Standard HF call: expand placeholders, run the media preprocess, and + # return with STANDARD-HF dtypes: input_ids/attention_mask as python + # lists (callers like slime's rollout do `sample.tokens += tokens`), + # media tensors as `return_tensors` (default pt) for the train side. + if not isinstance(images, (list, tuple)): + images = [images] + medias = [{"type": "image", "image": img} for img in images] + parts = text.split(self.GLM5V_IMAGE_TOKEN) + if len(parts) - 1 != len(images): + raise ValueError( + f"got {len(images)} images but {len(parts) - 1} " + f"{self.GLM5V_IMAGE_TOKEN!r} placeholders in text") + expanded = [parts[0]] + for media, part in zip(medias, parts[1:]): + num_tokens = self.media_processor.media_tokens_calculator(media) + expanded.append(self.GLM5V_IMAGE_TOKEN * num_tokens + part) + text = "".join(expanded) + + updated_medias, video_prompts = self.preprocess_medias(medias) + preprocessed = self.media_processor.preprocess( + updated_medias, return_tensors=return_tensors) + text = self.update_raw_text(text, video_prompts) + text_inputs = self.tokenizer([text]) # no return_tensors -> lists + data = {**text_inputs, **preprocessed.data} + # Qwen-convention key: downstream training forwards take + # `image_grid_thw` (same rename the SGLang wrapper applies). + if "grid_thws" in data: + data["image_grid_thw"] = data.pop("grid_thws") + return BatchFeature(data=data) + + if messages is None and (medias is None or text is None): + raise ValueError( + "Provide either 'messages' or both 'medias' and 'text'") + + if medias is not None and text is not None: + updated_medias, video_prompts = self.preprocess_medias(medias) + preprocessed = self.media_processor.preprocess( + updated_medias, return_tensors=return_tensors) + text = self.update_raw_text(text, video_prompts) + text_inputs = self.tokenizer(text, return_tensors=return_tensors) + return BatchFeature(data={**text_inputs, **preprocessed.data}) + + if medias is None: + medias = self._extract_medias_from_messages(messages) + updated_medias, video_prompts = self.preprocess_medias(medias) + preprocessed = self.media_processor.preprocess( + updated_medias, return_tensors=return_tensors) + + # Generate text if not provided + if text is None: + text = self.tokenizer.apply_chat_template(messages, **kwargs) + + text = self.update_raw_text(text, video_prompts) + + text_inputs = self.tokenizer(text, return_tensors=return_tensors) + return BatchFeature(data={**text_inputs, **preprocessed.data}) + + @staticmethod + def _extract_medias_from_messages(messages: list[dict]) -> list[dict]: + """ + Extract media items from messages in a single pass. + + This is an optimized version that processes messages only once. + Kept as internal method since external callers should use __call__. + """ + medias = [] + for msg in messages: + if msg['role'] != 'user' or not msg.get('content'): + continue + + for content_part in msg['content']: + if not isinstance(content_part, dict): + continue + + content_type = content_part.get('type') + if content_type in ['video_url', 'video']: + medias.append({ + 'type': 'video', + 'video': content_part['video_url']['url'], + 'first_frame_timestamp': 0.0 + }) + elif content_type in ['image_url', 'image']: + medias.append({ + 'type': 'image', + 'image': content_part['image_url'], + }) + return medias + + def apply_chat_template(self, messages, **kwargs): + return self.tokenizer.apply_chat_template(messages, **kwargs) + + def batch_decode(self, *args, **kwargs): + return self.tokenizer.batch_decode(*args, **kwargs) + + def decode(self, *args, **kwargs): + return self.tokenizer.decode(*args, **kwargs) + + @property + def model_input_names(self): + return ['input_ids', 'attention_mask', 'pixel_values', 'grid_thws'] diff --git a/kimi_k25_vision_processing.py b/kimi_k25_vision_processing.py new file mode 100644 index 0000000000000000000000000000000000000000..fdf3ab2f100f7c28a1f1e7295297e54b515d0b53 --- /dev/null +++ b/kimi_k25_vision_processing.py @@ -0,0 +1,251 @@ +"""Image processor class for Kimi-K2.5. +""" + +import json +from typing import Any, Dict, Optional, Union + +import numpy as np +import torch +from PIL import Image +from transformers.image_processing_utils import (BaseImageProcessor, + BatchFeature) +from transformers.utils import TensorType + +from .media_utils import (MediaInput, VideoChunkInput, _to_tensor, + ensure_media_type, get_video_meta, image_to_np, + navit_patchify, navit_resize_image, + navit_resize_video, normalize, + real_sample_fps_and_max_num_frames, timestamp_as_str) + +try: + from mecord import VideoReader +except ImportError: + VideoReader = None + + +def resampling(video_bytes: bytes, + sample_indices: list[int], + key_indices=None, + frame_time_info=None, + num_threads=4) -> str: + video = VideoReader(video_bytes, + num_threads=num_threads, + frame_time_info=frame_time_info, + key_indices=key_indices) + # extract target frames + frames = video[sample_indices] + frames = [Image.fromarray(frame) for frame in frames] + return frames + + +class KimiK25VisionProcessor(BaseImageProcessor): + model_type = "kimi_k25" + + def __init__( + self, + media_proc_cfg: dict, + **kwargs, + ): + super().__init__(**kwargs) + self.media_proc_cfg = media_proc_cfg + self.num_frames_per_chunk = media_proc_cfg[ + 'temporal_merge_kernel_size'] + + def media_tokens_calculator(self, media: MediaInput): + media = ensure_media_type(media) + ret = self.get_resize_config(media) + return ret['num_tokens'] + + @classmethod + def make_chunk_prompt(cls, timestamp_text: str) -> str: + return f"{timestamp_text}<|media_begin|>video<|media_content|><|media_pad|><|media_end|>" + + def split_video_chunks(self, + video_url: str | bytes) -> list[list[Image.Image]]: + # video_url should be base64 str or bytes + video_spec = get_video_meta(video_url) + sample_fps = min(self.media_proc_cfg['sample_fps'], video_spec.fps) + sampled_nframes = max( + round(video_spec.num_frames * sample_fps / video_spec.fps), 1) + frame_inds = np.linspace(0, video_spec.num_frames - 1, + sampled_nframes).round().astype(int) + frame_inds = frame_inds.tolist() + sampled_frame_ids = [] + temporal_merge_kernel_size = self.media_proc_cfg[ + "temporal_merge_kernel_size"] + num_chunks = 0 + chunk_timestamp = [] + for i in range(0, len(frame_inds), temporal_merge_kernel_size): + sampled_frame_ids.extend(frame_inds[i:i + + temporal_merge_kernel_size]) + start_time = frame_inds[i] / float(video_spec.fps) + timestamp_text = timestamp_as_str( + start_time, self.media_proc_cfg["timestamp_mode"]) + chunk_timestamp.append(timestamp_text) + num_chunks += 1 + + sampled_frames = resampling(video_url, sampled_frame_ids) + chunks = [] + for chunk_id in range(num_chunks): + chunk = sampled_frames[chunk_id * + temporal_merge_kernel_size:(chunk_id + 1) * + temporal_merge_kernel_size] + chunks.append( + VideoChunkInput(type="video_chunk", + video_chunk=chunk, + prompt=self.make_chunk_prompt( + chunk_timestamp[chunk_id]))) + return chunks + + def get_resize_config(self, media_input: MediaInput) -> dict: + if media_input['type'] == 'image': + w, h = media_input['image'].size + ret = navit_resize_image( + w, h, self.media_proc_cfg['patch_size'], + self.media_proc_cfg['merge_kernel_size'], + self.media_proc_cfg['in_patch_limit'], + self.media_proc_cfg['patch_limit_on_one_side'], + self.media_proc_cfg['fixed_output_tokens']) + return ret + elif media_input['type'] == 'video_chunk': + frame = media_input['video_chunk'][0] + width, height = frame.size + num_frames = len(media_input["video_chunk"]) + fps = 1.0 + + sample_fps, max_num_frames_each_video = real_sample_fps_and_max_num_frames( + media_input["type"], + self.media_proc_cfg['sample_fps'], + self.media_proc_cfg['max_num_frames_each_video'], + ) + + in_patch_limit_each_frame = self.media_proc_cfg[ + 'in_patch_limit_each_frame'] + if in_patch_limit_each_frame is None: + in_patch_limit_each_frame = self.media_proc_cfg[ + 'in_patch_limit'] + + ret = navit_resize_video( + width, + height, + num_frames, + fps, + sample_fps, + self.media_proc_cfg['patch_size'], + self.media_proc_cfg['merge_kernel_size'], + in_patch_limit_each_frame, + self.media_proc_cfg['patch_limit_on_one_side'], + self.media_proc_cfg['in_patch_limit_video'], + max_num_frames_each_video, + self.media_proc_cfg['fixed_output_tokens'], + ) + return ret + else: + raise ValueError("Unsupported type: {}".format( + media_input['type'])) + + def resize_image(self, image: Image.Image, new_width: int, new_height: int, + pad_width: int, pad_height: int) -> np.ndarray: + image_np = image_to_np(image, (new_width, new_height), "resize") + image_np = np.pad( + image_np, + ((0, pad_height), (0, pad_width), (0, 0)), + mode="constant", + constant_values=0, + ) + return image_np + + def preprocess( + self, + medias: list[MediaInput], + return_tensors: Optional[Union[str, TensorType]] = None, + ) -> BatchFeature: + """ + Preprocess a atom vision input (images/video_chunk) into model-ready tensors. + + Args: + medias: List of MediaInput. + return_tensors: Desired output format ('pt', 'np', 'tf', or None). + + Returns: + BatchFeature containing 'pixel_values' and 'grid_thws' tensors. + """ + if not isinstance(medias, list): + medias = [medias] + if medias: + pixel_values = [] + for item in medias: + item = ensure_media_type(item) + resize_config = self.get_resize_config(item) + new_width, new_height, pad_width, pad_height = resize_config[ + 'new_width'], resize_config['new_height'], resize_config[ + 'pad_width'], resize_config['pad_height'] + if item['type'] == 'image': + image = item['image'] + image_np = self.resize_image(image, new_width, new_height, + pad_width, pad_height) + pixel_values.append(np.expand_dims(image_np, axis=0)) + elif item['type'] == 'video_chunk': + pixels = [] + for frame in item['video_chunk']: + frame_np = self.resize_image(frame, new_width, + new_height, pad_width, + pad_height) + pixels.append(frame_np) + pixel_values.append(np.stack(pixels, axis=0)) + else: + raise ValueError("Unsupported type: {}".format( + item['type'])) + normalized_pixel_values = [] + image_std_inv = 1.0 / np.array(self.media_proc_cfg['image_std']) + image_mean = np.array(self.media_proc_cfg['image_mean']) + for pixels in pixel_values: + pixels = normalize(pixels, image_mean, image_std_inv) + pixels_and_thw = navit_patchify( + pixels, + self.media_proc_cfg['patch_size'], + ) + normalized_pixel_values.append(pixels_and_thw) + + pixel_values = torch.cat([ + _to_tensor(pixel_value['pixel_values']) + for pixel_value in normalized_pixel_values + ]) + grid_thws = torch.cat([ + _to_tensor(pixel_value['grid_thw'], + dtype=torch.int64).unsqueeze(0) + for pixel_value in normalized_pixel_values + ]) + + data = { + 'pixel_values': pixel_values, + 'grid_thws': grid_thws, + } + + else: + data = {} + + return BatchFeature(data=data, tensor_type=return_tensors) + + def __repr__(self): + return f"KimiK25VisionProcessor(media_proc_cfg={self.media_proc_cfg})" + + def to_dict(self) -> Dict[str, Any]: + output = super().to_dict() + output["media_proc_cfg"] = self.media_proc_cfg + if "media_processor" in output: + del output["media_processor"] + return output + + @classmethod + def from_dict(cls, config_dict: Dict[str, Any], **kwargs): + config = config_dict.copy() + media_proc_cfg = config.pop("media_proc_cfg", {}) + return cls(media_proc_cfg=media_proc_cfg, **config, **kwargs) + + def to_json_string(self): + dictionary = self.to_dict() + for key, value in dictionary.items(): + if hasattr(value, 'tolist'): + dictionary[key] = value.tolist() + return json.dumps(dictionary, indent=2, sort_keys=True) + "\n" diff --git a/media_utils.py b/media_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8795e06f381700d6420798f82174e3f9647e9f89 --- /dev/null +++ b/media_utils.py @@ -0,0 +1,368 @@ +import base64 +import io +import math +import os +from datetime import datetime, timezone +from typing import List, Literal, Optional, TypedDict + +import numpy as np +from PIL import Image +from pydantic import BaseModel, Field + +try: + from mecord import VideoReader +except ImportError: + VideoReader = None + + +class VideoSpec(BaseModel): + media_type: str = Literal['video'] + height: int = Field(..., gt=0, description="video frame height") + width: int = Field(..., gt=0, description="video frame width") + num_frames: int = Field(..., gt=0, description="num frames") + fps: float = Field(..., gt=0, description="average fps") + + # optional, help to accelerate video reading + key_indices: list[int] = Field(None, description="key indices") + frame_time_info: dict = Field(None, description="frame time info") + + +class ImageInput(TypedDict): + type: Literal['image'] + image: Image.Image + + +class VideoChunkInput(TypedDict): + type: Literal['video_chunk'] + video_chunk: List[Image.Image] + prompt: Optional[str] = None + + +MediaInput = ImageInput | VideoChunkInput + + +def get_video_meta(video_src: bytes | str | os.PathLike, + accurate: bool = True) -> dict: + """Get the dimensions of a video.""" + if isinstance(video_src, os.PathLike): + video_src = str(video_src) + # if b64 string, decode to bytes + if isinstance(video_src, + str) and video_src.startswith('data:video/mp4;base64,'): + video_src = base64.b64decode(video_src.split(',')[1]) + video = VideoReader(video_src, auto_init=accurate, num_threads=1) + assert video.num_frames > 0, "Invalid video format." + assert video.original_width > 0 and video.original_height > 0, ( + "Invalid video format.") + assert video.avg_fps > 0, "Invalid video format." + return VideoSpec(media_type='video', + height=video.original_height, + width=video.original_width, + num_frames=video.num_frames, + fps=video.avg_fps, + key_indices=video.key_indices, + frame_time_info=video.frame_time_info) + + +def timestamp_as_str(timestamp: float, + timestamp_mode: str = "hh:mm:ss.fff") -> str: + """Convert a timestamp to a string in the format of HH:MM:SS.mmm.""" + if timestamp_mode == "hh:mm:ss.fff": + return (datetime.fromtimestamp(timestamp, + tz=timezone.utc).strftime("%H:%M:%S") + + f".{int((timestamp % 1) * 1000):03d}") + elif timestamp_mode == "mm:ss.fff": + return (datetime.fromtimestamp(timestamp, + tz=timezone.utc).strftime("%M:%S") + + f".{int((timestamp % 1) * 1000):03d}") + elif timestamp_mode == "mm:ss": + return datetime.fromtimestamp(timestamp, + tz=timezone.utc).strftime("%M:%S") + else: + raise ValueError(f"Invalid timestamp mode: {timestamp_mode}") + + +def navit_resize_image( + width: int, + height: int, + patch_size: int, + merge_kernel_size: int, + in_patch_limit: int, + patch_limit_on_one_side: int, + fixed_output_tokens: int | None, +): + # Apply the patch limits. + s1 = math.sqrt( + in_patch_limit / + (max(1.0, width // patch_size) * max(1.0, height // patch_size))) + s2 = patch_limit_on_one_side * patch_size / width + s3 = patch_limit_on_one_side * patch_size / height + scale = min(1.0, s1, s2, s3) + new_w, new_h = max(1, int(width * scale)), max(1, int(height * scale)) + new_w = min(new_w, patch_limit_on_one_side * patch_size) + new_h = min(new_h, patch_limit_on_one_side * patch_size) + + # Calculate the padding to make the height and width divisible by the merge kernel size and patch size. + factor = merge_kernel_size * patch_size + + pad_height = (factor - new_h % factor) % factor + pad_width = (factor - new_w % factor) % factor + + if fixed_output_tokens is not None: + num_tokens = fixed_output_tokens + else: + # Calculate new dimensions after padding and patching + token_height = (new_h + pad_height) // factor + token_width = (new_w + pad_width) // factor + + assert token_height * merge_kernel_size <= patch_limit_on_one_side, ( + f"token_height {token_height} * merge_kernel_size {merge_kernel_size} > patch_limit_on_one_side {patch_limit_on_one_side}" + ) + assert token_width * merge_kernel_size <= patch_limit_on_one_side, ( + f"token_width {token_width} * merge_kernel_size {merge_kernel_size} > patch_limit_on_one_side {patch_limit_on_one_side}" + ) + + num_tokens = token_height * token_width + return { + "num_tokens": num_tokens, + "new_width": new_w, + "new_height": new_h, + "pad_width": pad_width, + "pad_height": pad_height, + "sampled_nframes": 1, + } + + +def navit_resize_video( + width: int, + height: int, + nframes: int, + avg_fps: float, + sample_fps: float, + patch_size: int, + merge_kernel_size: int, + in_patch_limit_each_frame: int, + patch_limit_on_one_side: int, + in_patch_limit_total: int | None, + max_num_frames_each_video: int | None, + fixed_output_tokens_each_frame: int | None, +): + sample_fps = min(sample_fps, avg_fps) + # Calculate the number of frames to sample based on target FPS + sampled_nframes = max(round(nframes * sample_fps / avg_fps), 1) + if max_num_frames_each_video is not None: + sampled_nframes = min(sampled_nframes, max_num_frames_each_video) + + if in_patch_limit_total is not None: + in_patch_limit_each_frame = min( + round(in_patch_limit_total / sampled_nframes), + in_patch_limit_each_frame) + + ret = navit_resize_image( + width, + height, + patch_size, + merge_kernel_size, + in_patch_limit_each_frame, + patch_limit_on_one_side, + fixed_output_tokens_each_frame, + ) + ret["sampled_nframes"] = sampled_nframes + return ret + + +def real_sample_fps_and_max_num_frames( + type_name: Literal["video", "video_chunk"], + sample_fps: float, + max_num_frames_each_video: int | None, +) -> tuple[int, int | None]: + if type_name == "video": + return sample_fps, max_num_frames_each_video + elif type_name == "video_chunk": + max_num_frames_each_video = None + sample_fps = math.inf + return sample_fps, max_num_frames_each_video + else: + return math.inf, None + + +def _to_pil(data: str | bytes): + if isinstance(data, Image.Image): + + return data.convert("RGB") + elif isinstance(data, str): + if data.startswith("data:"): + raw_base64 = data.split(",")[1] + return Image.open(io.BytesIO( + base64.b64decode(raw_base64))).convert("RGB") + else: + return Image.open(data).convert("RGB") + elif isinstance(data, bytes): + return Image.open(io.BytesIO(data)).convert("RGB") + else: + raise ValueError(f"Unsupported data type: {type(data)}") + + +def ensure_media_type(media: MediaInput) -> MediaInput: + if media['type'] == 'image': + media['image'] = _to_pil(media['image']) + return media + elif media['type'] == 'video_chunk': + media['video_chunk'] = [ + _to_pil(frame) for frame in media['video_chunk'] + ] + return media + else: + raise ValueError(f"Unsupported media type: {media['type']}") + + +def image_to_np( + image: Image.Image, + resize_to: tuple[int, int] | None = None, + mode: str = "resize", + raise_error_for_ill_resize: bool = True, +) -> np.ndarray: + """Convert an image to a numpy array. + + Args: + content: The image to convert. + resize_to: The size to resize the image to. + mode: The mode to resize the image to. + raise_error_for_ill_resize: Whether to raise an error for ill-sized resize. + + Returns: + A numpy array. + """ + assert isinstance(image, Image.Image), "image must be a PIL Image" + if resize_to is not None: + if mode == "resize": + image = image.resize(resize_to, resample=Image.Resampling.BICUBIC) + + elif mode == "rescale_and_pad_to_center": + scale = min(resize_to[0] / image.width, + resize_to[1] / image.height, 1.0) + new_width = round(image.width * scale) + new_height = round(image.height * scale) + if new_width == 0 or new_height == 0: + if raise_error_for_ill_resize: + raise ValueError( + f"Invalid resize to: {resize_to}, from image size: {image.size}" + ) + else: + return np.zeros((resize_to[1], resize_to[0], 3), + dtype=np.uint8) + + image = image.resize((new_width, new_height), + resample=Image.Resampling.BICUBIC) + padding_left = (resize_to[0] - new_width) // 2 + padding_right = resize_to[0] - new_width - padding_left + padding_top = (resize_to[1] - new_height) // 2 + padding_bottom = resize_to[1] - new_height - padding_top + image = np.asarray(image) + image = np.pad( + image, + ((padding_top, padding_bottom), (padding_left, padding_right), + (0, 0)), + mode="constant", + constant_values=0, + ) + assert image.shape == (resize_to[1], resize_to[0], 3) + + elif mode == "rescale_and_pad_to_rightbottom": + scale = min(resize_to[0] / image.width, + resize_to[1] / image.height, 1.0) + new_width = round(image.width * scale) + new_height = round(image.height * scale) + if new_width == 0 or new_height == 0: + if raise_error_for_ill_resize: + raise ValueError( + f"Invalid resize to: {resize_to}, from image size: {image.size}" + ) + else: + return np.zeros((resize_to[1], resize_to[0], 3), + dtype=np.uint8) + + image = image.resize((new_width, new_height), + resample=Image.Resampling.BICUBIC) + padding_right = resize_to[0] - new_width + padding_bottom = resize_to[1] - new_height + image = np.asarray(image) + image = np.pad( + image, + ((0, padding_bottom), (0, padding_right), (0, 0)), + mode="constant", + constant_values=0, + ) + assert image.shape == (resize_to[1], resize_to[0], 3) + + else: + raise ValueError(f"Invalid mode: {mode}") + + if isinstance(image, Image.Image): + return np.asarray(image) + else: + return image + + +def navit_patchify(pixel_values: np.ndarray, + patch_size: int) -> dict[str, np.ndarray]: + """Reshape the pixel values to a navit shape. + + Args: + pixel_values: np.ndarray, shape (t, h, w, c) + patch_size: int + + Returns: + dict[str, np.ndarray] + - patches: np.ndarray, shape (t * h//patch_size * w//patch_size, c, patch_size, patch_size) + - grid_thw: np.ndarray, (t, h//patch_size, w//patch_size) + """ + T, H, W, C = pixel_values.shape + assert C == 3, "pixel_values must have 3 channels" + + patches = pixel_values.reshape(T, H // patch_size, patch_size, + W // patch_size, patch_size, C) + # (T, H//patch_size, W//patch_size, C, patch_size, patch_size) + patches = patches.transpose(0, 1, 3, 5, 2, 4) + patches = patches.reshape(-1, C, patch_size, patch_size) + grid_thw = np.array([T, H // patch_size, W // patch_size]) + return {"pixel_values": patches, "grid_thw": grid_thw} + + +def normalize(x: np.ndarray, + mean, + std_inv, + pixels_dtype: np.dtype = np.float32) -> np.ndarray: + """Normalize the image. + + Args: + x: The image to normalize. The shape is (..., 3). The dtype is uint8. The range is [0, 255]. + mean: The mean of the image. + std_inv: The inverse of the std of the image. + pixels_dtype: The dtype of the image. + Returns: + The normalized image. The shape is (..., 3). The dtype is determined by the pixels_dtype. + """ + x = (x / 255.0).astype(pixels_dtype) + x -= mean + x *= std_inv + return x + + +def _to_tensor(data, **kwargs): + import torch + + if isinstance(data, np.ndarray): + return torch.from_numpy(data).to(**kwargs) + elif isinstance(data, torch.Tensor): + return data.to(**kwargs) + elif isinstance(data, list): + return [_to_tensor(item, **kwargs) for item in data] + elif isinstance(data, tuple): + return tuple(_to_tensor(item, **kwargs) for item in data) + elif isinstance(data, dict): + return {k: _to_tensor(v, **kwargs) for k, v in data.items()} + elif data is None: + return None + else: + raise ValueError(f"Unsupported data type: {type(data)}") diff --git a/mm_projector.safetensors b/mm_projector.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d9b7cb79f25966d061c878402c061da568c75c0a --- /dev/null +++ b/mm_projector.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7c6ce8c27424f292e708e7bbb48ade57ea9f1aaddd28bd6a1020a860d9db80c +size 99117136 diff --git a/model-00001-of-00141.safetensors b/model-00001-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6418f8f157b41a2c2ca9bcf06ac4deebc6c70bdb --- /dev/null +++ b/model-00001-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd4b389324d8ed223c28a9e16a718228fa4d24ce09cdc0d0db92fd722df8e6d8 +size 5363940952 diff --git a/model-00002-of-00141.safetensors b/model-00002-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..67972547dbc2e63b595090fb27cce74e0d114e49 --- /dev/null +++ b/model-00002-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11b80a3a469fe8ab1d6bd2d88c9bb53cfeb3ce54357be876bbacc5f6fc26db30 +size 5361736696 diff --git a/model-00003-of-00141.safetensors b/model-00003-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d7abcb210cc6413506434fdf2c7fa2b77f2fccee --- /dev/null +++ b/model-00003-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3960cfbfa5e2f7bbdd195c8becbe712a4eb9d2d68a2bb912bdba72dc7dafbe43 +size 5363339120 diff --git a/model-00004-of-00141.safetensors b/model-00004-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d70c6b29ae5c18d369e891d93af7a89065d27160 --- /dev/null +++ b/model-00004-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf88f32b6d8f43ec1c066d5953e120fc5bbc85dc99947f0971dc3b502199d4f0 +size 5361736640 diff --git a/model-00005-of-00141.safetensors b/model-00005-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5694214cacf9c25d73f1e52b9a53d936b7efb203 --- /dev/null +++ b/model-00005-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff48f71f3a005abf34724f6db856071d3e9b0a3d1b80c94c26bbe06842108b2b +size 5366353560 diff --git a/model-00006-of-00141.safetensors b/model-00006-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8c9d6a08b4f9b5652ab6d2e0e574e21dd413e2f6 --- /dev/null +++ b/model-00006-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e9ce4ee588a5c73ac16a7190b7df88115498f056f70fe164b47c2fe9963d658 +size 5361736504 diff --git a/model-00007-of-00141.safetensors b/model-00007-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..da611f29452a9c8d25d6fae4a4a8b52258295864 --- /dev/null +++ b/model-00007-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99f92392fd0b9fc37a773029c313f27653749441623d7f5893c78497d64b9ee6 +size 5366353704 diff --git a/model-00008-of-00141.safetensors b/model-00008-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fda61a2d0eaa78e440554ba4762d86bfc6a2af9b --- /dev/null +++ b/model-00008-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af41af526c10de5197c7aee9b92094c0adb6650d2ffcea333640cbbb66654e2f +size 5361736368 diff --git a/model-00009-of-00141.safetensors b/model-00009-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9440a603e508b5e0bfa9d808c91e5c3ebb4a3ca1 --- /dev/null +++ b/model-00009-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b63348060bc303475fb41322c52ebb43022bfbb941535a140b98afa55ed030b +size 5366353832 diff --git a/model-00010-of-00141.safetensors b/model-00010-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0caf2c1eaf40250c595655546fe36625b1715453 --- /dev/null +++ b/model-00010-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:512b70194c08d5b36ce5e51968b75b424fd262abac21cd276aa41eeb0d34cddb +size 5361736232 diff --git a/model-00011-of-00141.safetensors b/model-00011-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e209fb84312cbb4e5e47eabed097ce3c5cdf9222 --- /dev/null +++ b/model-00011-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0acf28004cee9c111aa910a0e05e2f06f3131d1322452235e49362640e9c2a1 +size 5363339600 diff --git a/model-00012-of-00141.safetensors b/model-00012-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9222b8fc03b1c1ebafb2f0c13faea3d8d8555a29 --- /dev/null +++ b/model-00012-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c9163d9d7ce3765055712880eae315ccb2bb88c8bc019b24f6b44e10aafdd08 +size 5366353488 diff --git a/model-00013-of-00141.safetensors b/model-00013-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e3d92e9565fe29503a928b93078220f28a1b3756 --- /dev/null +++ b/model-00013-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae74ec25db1cedd85894edf5926cb4aae6d8bdf5daf463434497310ee86aa26a +size 5361736696 diff --git a/model-00014-of-00141.safetensors b/model-00014-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fbcfee05f4bc0ff81b27b732141842776c3fc9c5 --- /dev/null +++ b/model-00014-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dcc960bd41c7a2a183807af151524f7a547123244d2949e692f4117aa400d0d +size 5366353496 diff --git a/model-00015-of-00141.safetensors b/model-00015-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..96e7301673a090f5a50cf667419110119423e897 --- /dev/null +++ b/model-00015-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b0749ad2ad5307543d1e9593518c614116f701f06618986f8d4574efd52c292 +size 5361736680 diff --git a/model-00016-of-00141.safetensors b/model-00016-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..217641db4c5501767e88458cc463c5fbfffec6fc --- /dev/null +++ b/model-00016-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d55105e496b79d3d5f10a676558002c66428098b88b438c67fba970d13fccb8f +size 5366353520 diff --git a/model-00017-of-00141.safetensors b/model-00017-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..14fab035f6a3a6e5186e0074eb737b719e83a6ed --- /dev/null +++ b/model-00017-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcb77ae2f687e27b91fe04432515e18d3e19a5888be98b1dfd5df123e93de2d0 +size 5361736544 diff --git a/model-00018-of-00141.safetensors b/model-00018-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1b29bc29cafa7a9e94cdea883c4b49683725cf76 --- /dev/null +++ b/model-00018-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:756113e9521d3955d38f684b6997fd68eaeab92c660270aa2ce1d57b119bd84d +size 5363339264 diff --git a/model-00019-of-00141.safetensors b/model-00019-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d7e67afbbf661e96874062d869f5315333ee75fa --- /dev/null +++ b/model-00019-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dd4ec5b07f1d12b576049d435ac158fbf3442ff6335250bfea3f15577a83154 +size 5361736408 diff --git a/model-00020-of-00141.safetensors b/model-00020-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d737be4488957d8f45fa7295fbae8c988e958472 --- /dev/null +++ b/model-00020-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5381c71742768c88e2692ab164ef07fcd37a85bd1cbc5333a763554d2a9e5a74 +size 5364805840 diff --git a/model-00021-of-00141.safetensors b/model-00021-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7e39f457d8f9134f4509d41730b96d4a9d6fa3c5 --- /dev/null +++ b/model-00021-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13aa866d1b48ee5341c020ba314954909ff16ee0b36f9c4bed21919e3ab2c958 +size 5361736336 diff --git a/model-00022-of-00141.safetensors b/model-00022-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d79a570f37319ca52860e91c44e1406e15992ef3 --- /dev/null +++ b/model-00022-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99e5ca5f30d7129b6b10e60c941e38341704cddbf2d7c0acbc0003e0ddec0e31 +size 5366353872 diff --git a/model-00023-of-00141.safetensors b/model-00023-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a8798705681aa58ff90b850c3c19ba8e7c4fce4b --- /dev/null +++ b/model-00023-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23b63f18f5a670a309074c1fb826fae01ecb12da1869477fb4ec12559336eb9d +size 5357948328 diff --git a/model-00024-of-00141.safetensors b/model-00024-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4702a9203f6b378fbcfdf426436c3b352ce25ee2 --- /dev/null +++ b/model-00024-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb99757b4ea90a1b18c6c080d56da37eebf4247ae88893adcb12db40ab058551 +size 5357555688 diff --git a/model-00025-of-00141.safetensors b/model-00025-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b125efc709761e3a0f4102477a600ae3b954a722 --- /dev/null +++ b/model-00025-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e78dccea55fe0d54686ffefce8114278132e35b1b482b5fbde77eba405ec305e +size 5363339104 diff --git a/model-00026-of-00141.safetensors b/model-00026-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..21836651665a38a5ad42669f0e3060b6eb5c4ded --- /dev/null +++ b/model-00026-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12a4f637a50fe23663b52425fb5d75c50a3fcca4211485aecc4a4e7b1bba2817 +size 5361736696 diff --git a/model-00027-of-00141.safetensors b/model-00027-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..30cceae87fddf94602e84fe7b9bff6070294fd45 --- /dev/null +++ b/model-00027-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9beb92efca20325a0dea2d0bac9a3c131c6b533d89a416bbb5035bd9cf02c8c5 +size 5366353504 diff --git a/model-00028-of-00141.safetensors b/model-00028-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e86383f276a18338465b4774b67cd90236246cfe --- /dev/null +++ b/model-00028-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68d45c421cac6851303032f5ea4c93d74be84e70b8418ceb0d58fd0d95e2e46b +size 5361736648 diff --git a/model-00029-of-00141.safetensors b/model-00029-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..edd476f0b19607c50640fe2fa1d7deb0651bfd56 --- /dev/null +++ b/model-00029-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa8915c4907c0deca57944f8582d907898c82333c326f6d86690f82a5f8e825d +size 5366353560 diff --git a/model-00030-of-00141.safetensors b/model-00030-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..300e894348c38f2608d7598ab184025b48ae2f2f --- /dev/null +++ b/model-00030-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:128b570a6fb59ae7b153f0da4cf713b1087aa89c6977a376fd359b72086256f2 +size 5361736512 diff --git a/model-00031-of-00141.safetensors b/model-00031-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bac06d2291d929866c4f2d6906040337228b6df4 --- /dev/null +++ b/model-00031-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aea2c91014901cd0a30283943cdae4bf7686a8fd6d3c56d37f5b6389feacf6e3 +size 5366353688 diff --git a/model-00032-of-00141.safetensors b/model-00032-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..54c20064b17be32183ed928a59c5f7e4a8c6aa62 --- /dev/null +++ b/model-00032-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa6396111a23f21191ab964d456e039562aee26415c193eee66818a65e371157 +size 5361736376 diff --git a/model-00033-of-00141.safetensors b/model-00033-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..abea797a44656caae16bcf7c2aab82f006ea0687 --- /dev/null +++ b/model-00033-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eea8bf4e723abc1a409fbd3fcf3cd3cd26eb76d7684a5a5c288019bea102c9c1 +size 5363339440 diff --git a/model-00034-of-00141.safetensors b/model-00034-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3392b6c1750befa6a386de1382ce18cc97b517f1 --- /dev/null +++ b/model-00034-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:140be56abe6eb707904e920186acde09f6f68a2516a05de8c0cddf74d31794b3 +size 5361736240 diff --git a/model-00141-of-00141.safetensors b/model-00141-of-00141.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e8e1a68b344810d9be30ef1447403026a8c290cc --- /dev/null +++ b/model-00141-of-00141.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1375c9156215b03d2c95ed4832810e6771ff9217f3ce5582b2368198a275272 +size 4724454592 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..bd63043e72a947d5cb3c4828e916ab6e5796ab39 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c51579c9ca8c22b8da94c96fe6559cb7aa78a15418a2a2af5bf419bfdafca83 +size 11385154 diff --git a/preprocessor_config.json b/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ddcc61425f0473502e7c46134a35c6efa2e84a7a --- /dev/null +++ b/preprocessor_config.json @@ -0,0 +1,30 @@ +{ + "auto_map": { + "AutoProcessor": "kimi_k25_processor.KimiK25Processor", + "AutoImageProcessor": "kimi_k25_vision_processing.KimiK25VisionProcessor" + }, + "media_proc_cfg": { + "in_patch_limit": 16384, + "patch_size": 14, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "merge_kernel_size": 2, + "fixed_output_tokens": null, + "patch_limit_on_one_side": 512, + "in_patch_limit_each_frame": 16384, + "in_patch_limit_video": null, + "sample_fps": 2.0, + "max_num_frames_each_video": null, + "temporal_merge_kernel_size": 4, + "timestamp_mode": "hh:mm:ss.fff", + "config_type": "media_proc.processors.moonvit.MoonViTMediaProcessorConfig" + } +} \ No newline at end of file diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..aba40197a4cdb5607f4ab7a05fb0a4ee8054fd6d --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19e773648cb4e65de8660ea6365e10acca112d42a854923df93db4a6f333a82d +size 20217442 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e375fa0a4e16660ce0e2026e39374d35dad4c079 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,33 @@ +{ + "backend": "tokenizers", + "clean_up_tokenization_spaces": false, + "do_lower_case": false, + "eos_token": "<|endoftext|>", + "extra_special_tokens": [ + "<|endoftext|>", + "[MASK]", + "[gMASK]", + "[sMASK]", + "", + "", + "<|system|>", + "<|user|>", + "<|assistant|>", + "<|observation|>", + "<|begin_of_image|>", + "<|end_of_image|>", + "<|begin_of_video|>", + "<|end_of_video|>", + "<|begin_of_audio|>", + "<|end_of_audio|>", + "<|begin_of_transcription|>", + "<|end_of_transcription|>" + ], + "is_local": true, + "model_max_length": 1048576, + "model_specific_special_tokens": {}, + "pad_token": "<|endoftext|>", + "padding_side": "left", + "remove_space": false, + "tokenizer_class": "TokenizersBackend" +} diff --git a/vision_tower.safetensors b/vision_tower.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..989f897873bd023446e7d9765b519017bd7c1883 --- /dev/null +++ b/vision_tower.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af1c927919bd073c2ab893bf2c6d9826100a06828c3b40f0128fabb4660a610d +size 833769904