{ "architectures": [ "KimiK3ForConditionalGeneration" ], "auto_map": { "AutoConfig": "configuration_kimi_k3.KimiK3Config", "AutoModel": "modeling_kimi_k3.KimiK3ForConditionalGeneration", "AutoModelForCausalLM": "modeling_kimi_k3.KimiK3ForConditionalGeneration" }, "dtype": "bfloat16", "ignore_index": -100, "media_placeholder_token_id": 163605, "model_type": "kimi_k3", "pad_token_id": 0, "quantization_config": { "config_groups": { "group_0": { "format": "mxfp4-pack-quantized", "input_activations": { "actorder": null, "block_structure": null, "dynamic": true, "group_size": 32, "num_bits": 4, "observer": null, "observer_kwargs": {}, "scale_dtype": "torch.uint8", "strategy": "group", "symmetric": true, "type": "float", "zp_dtype": null }, "output_activations": null, "targets": [ "re:.*block_sparse_moe.*" ], "weights": { "actorder": null, "block_structure": null, "dynamic": false, "group_size": 32, "num_bits": 4, "observer": "memoryless_minmax", "observer_kwargs": {}, "scale_dtype": "torch.uint8", "strategy": "group", "symmetric": true, "type": "float", "zp_dtype": null } } }, "format": "mxfp4-pack-quantized", "global_compression_ratio": null, "ignore": [ "vision_tower.encoder.blocks.0.mlp.fc0", "vision_tower.encoder.blocks.0.mlp.fc1", "vision_tower.encoder.blocks.0.wqkv", "vision_tower.encoder.blocks.0.wo", "vision_tower.encoder.blocks.1.mlp.fc0", "vision_tower.encoder.blocks.1.mlp.fc1", "vision_tower.encoder.blocks.1.wqkv", "vision_tower.encoder.blocks.1.wo", "mm_projector.proj.0", "mm_projector.proj.2", "language_model.model.layers.0.self_attn.q_proj", "language_model.model.layers.0.self_attn.k_proj", "language_model.model.layers.0.self_attn.v_proj", "language_model.model.layers.0.self_attn.f_a_proj", "language_model.model.layers.0.self_attn.f_b_proj", "language_model.model.layers.0.self_attn.b_proj", "language_model.model.layers.0.self_attn.g_proj", "language_model.model.layers.0.self_attn.o_norm", "language_model.model.layers.0.self_attn.o_proj", "language_model.model.layers.0.mlp.gate_proj", "language_model.model.layers.0.mlp.up_proj", "language_model.model.layers.0.mlp.down_proj", "language_model.model.layers.0.self_attention_res_proj", "language_model.model.layers.0.mlp_res_proj", "language_model.model.layers.1.self_attn.q_proj", "language_model.model.layers.1.self_attn.k_proj", "language_model.model.layers.1.self_attn.v_proj", "language_model.model.layers.1.self_attn.f_a_proj", "language_model.model.layers.1.self_attn.f_b_proj", "language_model.model.layers.1.self_attn.b_proj", "language_model.model.layers.1.self_attn.g_proj", "language_model.model.layers.1.self_attn.o_norm", "language_model.model.layers.1.self_attn.o_proj", "language_model.model.layers.1.block_sparse_moe.gate", "language_model.model.layers.1.self_attention_res_proj", "language_model.model.layers.1.mlp_res_proj", "language_model.model.layers.2.self_attn.q_proj", "language_model.model.layers.2.self_attn.k_proj", "language_model.model.layers.2.self_attn.v_proj", "language_model.model.layers.2.self_attn.f_a_proj", "language_model.model.layers.2.self_attn.f_b_proj", "language_model.model.layers.2.self_attn.b_proj", "language_model.model.layers.2.self_attn.g_proj", "language_model.model.layers.2.self_attn.o_norm", "language_model.model.layers.2.self_attn.o_proj", "language_model.model.layers.2.block_sparse_moe.gate", "language_model.model.layers.2.self_attention_res_proj", "language_model.model.layers.2.mlp_res_proj", "language_model.model.layers.3.self_attn.q_a_proj", "language_model.model.layers.3.self_attn.q_b_proj", "language_model.model.layers.3.self_attn.kv_a_proj_with_mqa", "language_model.model.layers.3.self_attn.kv_b_proj", "language_model.model.layers.3.self_attn.o_proj", "language_model.model.layers.3.self_attn.g_proj", "language_model.model.layers.3.block_sparse_moe.gate", "language_model.model.layers.3.self_attention_res_proj", "language_model.model.layers.3.mlp_res_proj", "language_model.model.layers.4.self_attn.q_proj", "language_model.model.layers.4.self_attn.k_proj", "language_model.model.layers.4.self_attn.v_proj", "language_model.model.layers.4.self_attn.f_a_proj", "language_model.model.layers.4.self_attn.f_b_proj", "language_model.model.layers.4.self_attn.b_proj", "language_model.model.layers.4.self_attn.g_proj", "language_model.model.layers.4.self_attn.o_norm", "language_model.model.layers.4.self_attn.o_proj", "language_model.model.layers.4.block_sparse_moe.gate", "language_model.model.layers.4.self_attention_res_proj", "language_model.model.layers.4.mlp_res_proj", "language_model.model.layers.5.self_attn.q_proj", "language_model.model.layers.5.self_attn.k_proj", "language_model.model.layers.5.self_attn.v_proj", "language_model.model.layers.5.self_attn.f_a_proj", "language_model.model.layers.5.self_attn.f_b_proj", "language_model.model.layers.5.self_attn.b_proj", "language_model.model.layers.5.self_attn.g_proj", "language_model.model.layers.5.self_attn.o_norm", "language_model.model.layers.5.self_attn.o_proj", "language_model.model.layers.5.block_sparse_moe.gate", "language_model.model.layers.5.self_attention_res_proj", "language_model.model.layers.5.mlp_res_proj", "language_model.model.layers.6.self_attn.q_proj", "language_model.model.layers.6.self_attn.k_proj", "language_model.model.layers.6.self_attn.v_proj", "language_model.model.layers.6.self_attn.f_a_proj", "language_model.model.layers.6.self_attn.f_b_proj", "language_model.model.layers.6.self_attn.b_proj", "language_model.model.layers.6.self_attn.g_proj", "language_model.model.layers.6.self_attn.o_norm", "language_model.model.layers.6.self_attn.o_proj", "language_model.model.layers.6.block_sparse_moe.gate", "language_model.model.layers.6.self_attention_res_proj", "language_model.model.layers.6.mlp_res_proj", "language_model.model.layers.7.self_attn.q_a_proj", "language_model.model.layers.7.self_attn.q_b_proj", "language_model.model.layers.7.self_attn.kv_a_proj_with_mqa", "language_model.model.layers.7.self_attn.kv_b_proj", "language_model.model.layers.7.self_attn.o_proj", "language_model.model.layers.7.self_attn.g_proj", "language_model.model.layers.7.block_sparse_moe.gate", "language_model.model.layers.7.self_attention_res_proj", "language_model.model.layers.7.mlp_res_proj", "language_model.model.output_attn_res_proj", "language_model.lm_head" ], "kv_cache_scheme": null, "quant_method": "compressed-tensors", "quantization_status": "compressed", "sparsity_config": {}, "transform_config": {}, "version": "0.17.2.dev32+g1f3b632.d20260723" }, "text_config": { "_name_or_path": "", "activation_situ_beta": 4.0, "activation_situ_linear_beta": 25.0, "architectures": null, "attn_res_block_size": 4, "auto_map": { "AutoConfig": "configuration_kimi_k3.KimiLinearConfig", "AutoModel": "modeling_kimi_linear.KimiLinearModel", "AutoModelForCausalLM": "modeling_kimi_linear.KimiLinearForCausalLM" }, "bos_token_id": 1, "chunk_size_feed_forward": 0, "dtype": null, "eos_token_id": 2, "first_k_dense_replace": 1, "head_dim": 74, "hidden_act": "situ", "hidden_size": 1024, "id2label": { "0": "LABEL_0", "1": "LABEL_1" }, "initializer_range": 0.02, "intermediate_size": 2048, "is_encoder_decoder": false, "kv_lora_rank": 128, "label2id": { "LABEL_0": 0, "LABEL_1": 1 }, "latent_moe_use_norm": true, "linear_attn_config": { "full_attn_layers": [ 4, 8 ], "head_dim": 32, "kda_layers": [ 1, 2, 3, 5, 6, 7 ], "num_heads": 8, "short_conv_kernel_size": 4, "use_full_rank_gate": true }, "max_position_embeddings": 4096, "mla_use_nope": true, "mla_use_output_gate": true, "model_type": "kimi_linear", "moe_intermediate_size": 256, "moe_layer_freq": 1, "moe_renormalize": true, "moe_router_activation_func": "sigmoid", "num_attention_heads": 8, "num_expert_group": 1, "num_experts": 8, "num_experts_per_token": 2, "num_hidden_layers": 8, "num_key_value_heads": 8, "num_nextn_predict_layers": 0, "num_shared_experts": 1, "output_attentions": false, "output_hidden_states": false, "pad_token_id": 0, "problem_type": null, "q_lora_rank": 256, "qk_nope_head_dim": 64, "qk_rope_head_dim": 32, "return_dict": true, "rms_norm_eps": 1e-05, "rope_parameters": { "rope_theta": 10000.0, "rope_type": "default" }, "rope_theta": 10000.0, "routed_expert_hidden_size": 512, "routed_scaling_factor": 1.0, "tie_word_embeddings": false, "topk_group": 1, "topk_method": "noaux_tc", "use_cache": true, "use_grouped_topk": true, "v_head_dim": 64, "vocab_size": 163840 }, "transformers_version": "5.15.0.dev0", "vision_config": { "_name_or_path": "", "activation_func": "gelu_pytorch_tanh", "architectures": null, "attn_bias": false, "chunk_size_feed_forward": 0, "dtype": null, "id2label": { "0": "LABEL_0", "1": "LABEL_1" }, "init_pos_emb_height": 64, "init_pos_emb_time": 4, "init_pos_emb_width": 64, "is_encoder_decoder": false, "label2id": { "LABEL_0": 0, "LABEL_1": 1 }, "linear_bias": false, "merge_kernel_size": [ 2, 2 ], "merge_type": "sd2_tpool", "mlp_type": "mlp2", "mm_hidden_size": 256, "mm_projector_type": "patchmergerv2", "model_type": "", "norm_type": "rmsnorm", "output_attentions": false, "output_hidden_states": false, "patch_embed_proj_bias": false, "patch_size": 14, "pos_emb_interpolation_mode": "bilinear", "pos_emb_type": "divided_fixed", "problem_type": null, "projector_hidden_act": "gelu", "projector_ln_eps": 1e-05, "qkv_hidden_size": 1536, "return_dict": true, "text_hidden_size": 1024, "vt_hidden_size": 256, "vt_intermediate_size": 512, "vt_num_attention_heads": 4, "vt_num_hidden_layers": 2 } }