{
  "name_or_path": "tclf90/Kimi-K3-Cubic-2.5Bit",
  "architectures": [
    "KimiK3ForConditionalGeneration"
  ],
  "auto_map": {
    "AutoConfig": "configuration_kimi_k3.KimiK3Config",
    "AutoModel": "modeling_kimi_k3.KimiK3ForConditionalGeneration",
    "AutoModelForCausalLM": "modeling_kimi_k3.KimiK3ForConditionalGeneration"
  },
  "bos_token_id": 163584,
  "dtype": "bfloat16",
  "eos_token_id": 163586,
  "ignore_index": -100,
  "image_placeholder": "<|kimi_image_placeholder|>",
  "media_placeholder_token_id": 163605,
  "model_type": "kimi_k3",
  "pad_token_id": 163839,
  "text_config": {
    "_name_or_path": "",
    "activation_situ_beta": 4.0,
    "activation_situ_linear_beta": 25.0,
    "add_cross_attention": false,
    "architectures": [
      "KimiLinearForCausalLM"
    ],
    "attn_res_block_size": 12,
    "auto_map": {
      "AutoConfig": "configuration_kimi_k3.KimiLinearConfig",
      "AutoModel": "modeling_kimi_linear.KimiLinearModel",
      "AutoModelForCausalLM": "modeling_kimi_linear.KimiLinearForCausalLM"
    },
    "bad_words_ids": null,
    "begin_suppress_tokens": null,
    "bos_token_id": 163584,
    "chunk_size_feed_forward": 0,
    "cross_attention_hidden_size": null,
    "decoder_start_token_id": null,
    "diversity_penalty": 0.0,
    "do_sample": false,
    "dtype": "bfloat16",
    "early_stopping": false,
    "encoder_no_repeat_ngram_size": 0,
    "eos_token_id": 163586,
    "exponential_decay_length_penalty": null,
    "finetuning_task": null,
    "first_k_dense_replace": 1,
    "forced_bos_token_id": null,
    "forced_eos_token_id": null,
    "hidden_act": "situ",
    "hidden_size": 7168,
    "id2label": {
      "0": "LABEL_0",
      "1": "LABEL_1"
    },
    "initializer_range": 0.02,
    "intermediate_size": 33792,
    "is_decoder": false,
    "is_encoder_decoder": false,
    "kv_lora_rank": 512,
    "label2id": {
      "LABEL_0": 0,
      "LABEL_1": 1
    },
    "latent_moe_use_norm": true,
    "length_penalty": 1.0,
    "linear_attn_config": {
      "full_attn_layers": [
        4,
        8,
        12,
        16,
        20,
        24,
        28,
        32,
        36,
        40,
        44,
        48,
        52,
        56,
        60,
        64,
        68,
        72,
        76,
        80,
        84,
        88,
        92,
        93
      ],
      "gate_lower_bound": -5.0,
      "head_dim": 128,
      "kda_layers": [
        1,
        2,
        3,
        5,
        6,
        7,
        9,
        10,
        11,
        13,
        14,
        15,
        17,
        18,
        19,
        21,
        22,
        23,
        25,
        26,
        27,
        29,
        30,
        31,
        33,
        34,
        35,
        37,
        38,
        39,
        41,
        42,
        43,
        45,
        46,
        47,
        49,
        50,
        51,
        53,
        54,
        55,
        57,
        58,
        59,
        61,
        62,
        63,
        65,
        66,
        67,
        69,
        70,
        71,
        73,
        74,
        75,
        77,
        78,
        79,
        81,
        82,
        83,
        85,
        86,
        87,
        89,
        90,
        91
      ],
      "num_heads": 96,
      "short_conv_kernel_size": 4,
      "use_full_rank_gate": true
    },
    "max_length": 20,
    "max_position_embeddings": 1048576,
    "min_length": 0,
    "mla_use_nope": true,
    "mla_use_output_gate": true,
    "model_type": "kimi_linear",
    "moe_intermediate_size": 3072,
    "moe_layer_freq": 1,
    "moe_renormalize": true,
    "moe_router_activation_func": "sigmoid",
    "no_repeat_ngram_size": 0,
    "num_attention_heads": 96,
    "num_beam_groups": 1,
    "num_beams": 1,
    "num_expert_group": 1,
    "num_experts": 896,
    "num_experts_per_token": 16,
    "num_hidden_layers": 93,
    "num_key_value_heads": 96,
    "num_nextn_predict_layers": 0,
    "num_return_sequences": 1,
    "num_shared_experts": 2,
    "output_attentions": false,
    "output_hidden_states": false,
    "output_scores": false,
    "pad_token_id": 163839,
    "prefix": null,
    "problem_type": null,
    "pruned_heads": {},
    "q_lora_rank": 1536,
    "qk_nope_head_dim": 128,
    "qk_rope_head_dim": 64,
    "quantization_config": {
      "quant_method": "cubic",
      "format": "cubic-pack-quantized",
      "quantization_status": "compressed",
      "config_groups": {
        "moe_layers_1_3": {
          "targets": [
            "re:.*\\.layers\\.(?:1|2|3)\\.block_sparse_moe\\.experts"
          ],
          "input_activations": null,
          "output_activations": null,
          "weights": {
            "num_bits": 3,
            "group_size": 256,
            "strategy": "group",
            "symmetric": true,
            "dynamic": false,
            "scale_dtype": "torch.float32",
            "param_dtype": "torch.float16",
            "reserved_code": "zero",
            "packing": "little-endian-bitstream"
          }
        },
        "moe_layers_4_32": {
          "targets": [
            "re:.*\\.layers\\.(?:4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32)\\.block_sparse_moe\\.experts"
          ],
          "input_activations": null,
          "output_activations": null,
          "weights": {
            "num_bits": 3,
            "group_size": 512,
            "strategy": "group",
            "symmetric": true,
            "dynamic": false,
            "scale_dtype": "torch.float32",
            "param_dtype": "torch.float16",
            "reserved_code": "zero",
            "packing": "little-endian-bitstream"
          }
        },
        "moe_layers_33_91": {
          "targets": [
            "re:.*\\.layers\\.(?:33|34|35|36|37|38|39|40|41|42|43|44|45|46|47|48|49|50|51|52|53|54|55|56|57|58|59|60|61|62|63|64|65|66|67|68|69|70|71|72|73|74|75|76|77|78|79|80|81|82|83|84|85|86|87|88|89|90|91)\\.block_sparse_moe\\.experts"
          ],
          "input_activations": null,
          "output_activations": null,
          "weights": {
            "num_bits": 2,
            "group_size": 512,
            "strategy": "group",
            "symmetric": true,
            "dynamic": false,
            "scale_dtype": "torch.float32",
            "param_dtype": "torch.float16",
            "reserved_code": "zero",
            "packing": "little-endian-bitstream"
          }
        },
        "moe_layers_92_92": {
          "targets": [
            "re:.*\\.layers\\.(?:92)\\.block_sparse_moe\\.experts"
          ],
          "input_activations": null,
          "output_activations": null,
          "weights": {
            "num_bits": 4,
            "group_size": 512,
            "strategy": "group",
            "symmetric": true,
            "dynamic": false,
            "scale_dtype": "torch.float32",
            "param_dtype": "torch.float16",
            "reserved_code": "zero",
            "packing": "little-endian-bitstream"
          }
        }
      },
      "ignore": [
        "re:.*self_attn.*",
        "re:.*shared_experts.*",
        "re:.*mlp\\.(gate|up|gate_up|down)_proj.*",
        "re:.*lm_head.*",
        "re:.*vision_tower.*",
        "re:.*mm_projector.*"
      ],
      "runtime_weight_storage": "native_packed_bitstream",
      "layer_bit_schedule": [
        {
          "start_layer": 1,
          "end_layer": 3,
          "num_bits": 3,
          "group_size": 256
        },
        {
          "start_layer": 4,
          "end_layer": 32,
          "num_bits": 3,
          "group_size": 512
        },
        {
          "start_layer": 33,
          "end_layer": 91,
          "num_bits": 2,
          "group_size": 512
        },
        {
          "start_layer": 92,
          "end_layer": 92,
          "num_bits": 4,
          "group_size": 512
        }
      ],
      "layer_group_size_overrides": {
        "1": 256,
        "2": 256,
        "3": 256,
        "4": 512,
        "5": 512,
        "6": 512,
        "7": 512,
        "8": 512,
        "9": 512,
        "10": 512,
        "11": 512,
        "12": 512,
        "13": 512,
        "14": 512,
        "15": 512,
        "16": 512,
        "17": 512,
        "18": 512,
        "19": 512,
        "20": 512,
        "21": 512,
        "22": 512,
        "23": 512,
        "24": 512,
        "25": 512,
        "26": 512,
        "27": 512,
        "28": 512,
        "29": 512,
        "30": 512,
        "31": 512,
        "32": 512,
        "33": 512,
        "34": 512,
        "35": 512,
        "36": 512,
        "37": 512,
        "38": 512,
        "39": 512,
        "40": 512,
        "41": 512,
        "42": 512,
        "43": 512,
        "44": 512,
        "45": 512,
        "46": 512,
        "47": 512,
        "48": 512,
        "49": 512,
        "50": 512,
        "51": 512,
        "52": 512,
        "53": 512,
        "54": 512,
        "55": 512,
        "56": 512,
        "57": 512,
        "58": 512,
        "59": 512,
        "60": 512,
        "61": 512,
        "62": 512,
        "63": 512,
        "64": 512,
        "65": 512,
        "66": 512,
        "67": 512,
        "68": 512,
        "69": 512,
        "70": 512,
        "71": 512,
        "72": 512,
        "73": 512,
        "74": 512,
        "75": 512,
        "76": 512,
        "77": 512,
        "78": 512,
        "79": 512,
        "80": 512,
        "81": 512,
        "82": 512,
        "83": 512,
        "84": 512,
        "85": 512,
        "86": 512,
        "87": 512,
        "88": 512,
        "89": 512,
        "90": 512,
        "91": 512,
        "92": 512
      },
      "tensor_bit_overrides": [],
      "expert_payload_bits": 2.369565217391304,
      "expert_effective_bits": 2.498641304347826,
      "converted_tensor_effective_bits": 2.498641304347826
    },
    "remove_invalid_values": false,
    "repetition_penalty": 1.0,
    "return_dict": true,
    "return_dict_in_generate": false,
    "rms_norm_eps": 1e-05,
    "routed_expert_hidden_size": 3584,
    "routed_scaling_factor": 1.0,
    "sep_token_id": null,
    "suppress_tokens": null,
    "task_specific_params": null,
    "temperature": 1.0,
    "tf_legacy_loss": false,
    "tie_encoder_decoder": false,
    "tie_word_embeddings": false,
    "tokenizer_class": null,
    "top_k": 50,
    "top_p": 1.0,
    "topk_group": 1,
    "topk_method": "noaux_tc",
    "torchscript": false,
    "transformers_version": "4.56.2",
    "typical_p": 1.0,
    "use_bfloat16": false,
    "use_cache": true,
    "use_grouped_topk": true,
    "v_head_dim": 128,
    "vocab_size": 163840
  },
  "tie_word_embeddings": false,
  "vision_config": {
    "_attn_implementation": "flash_attention_2",
    "activation_func": "gelu_pytorch_tanh",
    "attn_bias": false,
    "init_pos_emb_height": 64,
    "init_pos_emb_time": 4,
    "init_pos_emb_width": 64,
    "linear_bias": false,
    "merge_kernel_size": [
      2,
      2
    ],
    "merge_type": "sd2_tpool",
    "mlp_type": "mlp2",
    "mm_hidden_size": 1024,
    "mm_projector_type": "patchmergerv2",
    "norm_type": "rmsnorm",
    "patch_embed_proj_bias": false,
    "patch_size": 14,
    "pos_emb_interpolation_mode": "bilinear",
    "pos_emb_type": "divided_fixed",
    "projector_hidden_act": "gelu",
    "projector_ln_eps": 1e-05,
    "qkv_hidden_size": 1536,
    "text_hidden_size": 7168,
    "vt_hidden_size": 1024,
    "vt_intermediate_size": 4096,
    "vt_num_attention_heads": 12,
    "vt_num_hidden_layers": 27
  }
}