Paris99's picture
Inkling-2-Mini MLX 3bit (source 6e7c824, inkling2-mlx v0.1.0): weights, DSpark draft, card, Apache-2.0 LICENSE and NOTICE
9bba054 verified
Raw History Blame Contribute Delete
92.4 kB
{ "architectures": [ "Inkling2LoopForConditionalGeneration" ], "do_sample": true, "eos_token_id": 200006, "generation_config": { "do_sample": true, "temperature": 1.0, "eos_token_id": 200006, "pad_token_id": 199999 }, "model_type": "inkling2_loop", "mtp_config": { "hidden_size": 2304, "num_hidden_layers": 3, "num_attention_heads": 24, "num_key_value_heads": 8, "head_dim": 128, "intermediate_size": 9216, "max_position_embeddings": 1048576, "torch_dtype": "bfloat16", "q_bias": false, "o_bias": false, "rms_norm_eps": 1e-06, "rope_theta": 1000000.0, "local_layer_ids": [ 0, 2 ], "sliding_window_size": 1024, "block_size": 8, "markov_rank": 256, "markov_head_type": "rnn_hidden", "target_layer_ids": [ 5, 23, 41, 48 ] }, "temperature": 1.0, "text_config": { "model_type": "inkling2_loop_text", "model_max_length": 1048576, "torch_dtype": "bfloat16", "hidden_size": 2304, "num_hidden_layers": 24, "vocab_size": 201024, "num_attention_heads": 24, "num_key_value_heads": 8, "head_dim": 128, "d_rel": 16, "rel_extent": 1024, "q_bias": false, "o_bias": false, "log_scaling_n_floor": 8192, "log_scaling_alpha": 0.1, "rms_norm_eps": 1e-06, "use_value_norm_no_gamma": true, "qk_l2_clip_threshold": 22.627416997969522, "rel_l2_clip_threshold": 8.0, "local_layer_ids": [ 0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22 ], "sconv_kernel_size": 4, "unpadded_vocab_size": 200058, "final_logit_softcapping": 30.0, "swa_head_dim": 128, "swa_num_attention_heads": 24, "swa_num_key_value_heads": 8, "sliding_window_size": 1024, "n_layers_per_attn_res_block": 6, "attn_res_normalize_values": true, "n_routed_experts": 128, "num_experts_per_tok": 8, "n_shared_experts": 0, "intermediate_size": 1152, "mlp_gate_softcap": 4.0, "mlp_up_softcap": 25.0, "route_scale": 8.0, "use_gate_bias": true, "loop_config": { "core_start": 6, "core_end": 18, "num_passes": 3, "gate_rank": 64, "is_varloop": false } }, "vision_config": { "model_type": "inkling2_vision", "text_hidden_size": 2304, "temporal_patch_size": 2, "native_patch_size": 20, "adapter_merge_size": 2, "projection_hidden_dim": 4096, "projector_bias": false, "hidden_size": 1024, "num_hidden_layers": 23, "num_attention_heads": 16, "mlp_ratio": 4.0, "posemb_grid_size": 32, "layer_norm_eps": 1e-05 }, "quantization": { "group_size": 64, "bits": 3, "mode": "affine", "language_model.model.embed_tokens": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.0.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.0.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.0.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.0.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.0.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.0.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.0.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.0.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.0.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.1.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.1.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.1.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.1.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.1.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.1.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.1.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.1.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.1.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.2.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.2.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.2.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.2.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.2.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.2.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.2.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.2.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.2.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.3.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.3.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.3.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.3.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.3.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.3.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.3.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.3.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.3.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.4.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.4.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.4.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.4.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.4.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.4.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.4.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.4.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.4.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.5.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.5.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.5.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.5.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.5.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.5.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.5.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.5.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.5.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.6.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.6.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.6.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.6.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.6.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.6.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.6.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.6.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.6.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.7.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.7.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.7.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.7.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.7.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.7.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.7.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.7.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.7.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.8.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.8.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.8.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.8.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.8.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.8.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.8.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.8.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.8.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.9.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.9.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.9.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.9.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.9.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.9.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.9.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.9.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.9.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.10.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.10.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.10.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.10.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.10.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.10.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.10.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.10.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.10.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.11.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.11.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.11.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.11.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.11.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.11.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.11.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.11.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.11.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.12.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.12.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.12.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.12.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.12.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.12.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.12.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.12.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.12.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.13.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.13.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.13.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.13.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.13.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.13.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.13.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.13.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.13.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.14.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.14.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.14.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.14.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.14.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.14.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.14.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.14.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.14.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.15.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.15.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.15.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.15.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.15.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.15.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.15.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.15.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.15.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.16.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.16.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.16.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.16.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.16.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.16.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.16.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.16.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.16.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.17.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.17.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.17.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.17.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.17.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.17.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.17.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.17.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.17.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.18.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.18.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.18.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.18.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.18.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.18.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.18.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.18.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.18.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.19.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.19.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.19.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.19.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.19.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.19.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.19.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.19.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.19.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.20.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.20.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.20.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.20.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.20.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.20.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.20.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.20.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.20.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.21.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.21.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.21.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.21.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.21.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.21.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.21.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.21.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.21.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.22.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.22.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.22.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.22.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.22.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.22.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.22.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.22.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.22.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.23.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.23.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.23.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.23.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.23.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.23.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.23.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.23.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.23.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.lm_head": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.0.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.0.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.0.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.0.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.0.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.0.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.1.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.1.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.1.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.1.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.1.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.1.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.2.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.2.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.2.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.2.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.2.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.2.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.3.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.3.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.3.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.3.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.3.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.3.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.4.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.4.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.4.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.4.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.4.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.4.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.5.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.5.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.5.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.5.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.5.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.5.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.6.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.6.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.6.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.6.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.6.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.6.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.7.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.7.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.7.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.7.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.7.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.7.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.8.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.8.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.8.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.8.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.8.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.8.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.9.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.9.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.9.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.9.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.9.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.9.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.10.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.10.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.10.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.10.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.10.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.10.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.11.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.11.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.11.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.11.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.11.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.11.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.12.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.12.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.12.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.12.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.12.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.12.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.13.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.13.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.13.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.13.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.13.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.13.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.14.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.14.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.14.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.14.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.14.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.14.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.15.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.15.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.15.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.15.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.15.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.15.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.16.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.16.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.16.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.16.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.16.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.16.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.17.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.17.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.17.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.17.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.17.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.17.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.18.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.18.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.18.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.18.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.18.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.18.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.19.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.19.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.19.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.19.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.19.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.19.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.20.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.20.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.20.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.20.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.20.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.20.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.21.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.21.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.21.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.21.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.21.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.21.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.22.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.22.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.22.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.22.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.22.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.22.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.proj_in": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.proj_out": { "group_size": 64, "bits": 4, "mode": "affine" } }, "quantization_config": { "group_size": 64, "bits": 3, "mode": "affine", "language_model.model.embed_tokens": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.0.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.0.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.0.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.0.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.0.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.0.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.0.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.0.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.0.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.1.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.1.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.1.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.1.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.1.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.1.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.1.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.1.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.1.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.2.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.2.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.2.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.2.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.2.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.2.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.2.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.2.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.2.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.3.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.3.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.3.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.3.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.3.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.3.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.3.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.3.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.3.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.4.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.4.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.4.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.4.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.4.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.4.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.4.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.4.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.4.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.5.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.5.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.5.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.5.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.5.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.5.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.5.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.5.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.5.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.6.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.6.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.6.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.6.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.6.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.6.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.6.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.6.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.6.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.7.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.7.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.7.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.7.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.7.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.7.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.7.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.7.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.7.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.8.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.8.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.8.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.8.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.8.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.8.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.8.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.8.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.8.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.9.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.9.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.9.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.9.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.9.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.9.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.9.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.9.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.9.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.10.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.10.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.10.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.10.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.10.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.10.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.10.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.10.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.10.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.11.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.11.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.11.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.11.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.11.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.11.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.11.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.11.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.11.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.12.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.12.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.12.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.12.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.12.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.12.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.12.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.12.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.12.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.13.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.13.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.13.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.13.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.13.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.13.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.13.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.13.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.13.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.14.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.14.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.14.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.14.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.14.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.14.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.14.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.14.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.14.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.15.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.15.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.15.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.15.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.15.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.15.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.15.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.15.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.15.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.16.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.16.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.16.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.16.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.16.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.16.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.16.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.16.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.16.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.17.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.17.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.17.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.17.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.17.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.17.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.17.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.17.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.17.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.18.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.18.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.18.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.18.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.18.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.18.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.18.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.18.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.18.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.19.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.19.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.19.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.19.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.19.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.19.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.19.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.19.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.19.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 3, "mode": "affine" }, "language_model.model.layers.20.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.20.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.20.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.20.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.20.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.20.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.20.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.20.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.20.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.21.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.21.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.21.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.21.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.21.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.21.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.21.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.21.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.21.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.22.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.22.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.22.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.22.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.22.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.22.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.22.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.22.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.22.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.23.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.23.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.23.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.23.self_attn.r_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.23.self_attn.o_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.23.self_attn.gate_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "language_model.model.layers.23.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.23.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.model.layers.23.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "language_model.lm_head": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.0.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.0.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.0.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.0.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.0.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.0.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.1.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.1.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.1.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.1.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.1.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.1.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.2.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.2.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.2.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.2.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.2.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.2.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.3.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.3.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.3.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.3.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.3.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.3.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.4.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.4.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.4.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.4.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.4.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.4.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.5.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.5.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.5.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.5.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.5.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.5.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.6.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.6.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.6.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.6.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.6.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.6.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.7.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.7.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.7.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.7.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.7.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.7.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.8.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.8.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.8.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.8.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.8.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.8.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.9.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.9.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.9.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.9.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.9.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.9.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.10.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.10.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.10.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.10.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.10.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.10.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.11.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.11.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.11.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.11.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.11.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.11.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.12.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.12.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.12.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.12.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.12.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.12.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.13.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.13.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.13.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.13.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.13.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.13.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.14.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.14.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.14.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.14.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.14.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.14.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.15.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.15.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.15.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.15.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.15.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.15.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.16.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.16.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.16.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.16.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.16.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.16.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.17.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.17.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.17.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.17.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.17.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.17.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.18.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.18.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.18.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.18.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.18.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.18.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.19.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.19.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.19.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.19.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.19.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.19.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.20.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.20.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.20.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.20.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.20.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.20.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.21.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.21.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.21.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.21.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.21.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.21.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.22.self_attn.q_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.22.self_attn.k_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.22.self_attn.v_proj": { "group_size": 64, "bits": 6, "mode": "affine" }, "vision_tower.layers.22.self_attn.out_proj": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.22.mlp.fc1": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.layers.22.mlp.fc2": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.proj_in": { "group_size": 64, "bits": 4, "mode": "affine" }, "vision_tower.proj_out": { "group_size": 64, "bits": 4, "mode": "affine" } } }
{
"architectures": [
"Inkling2LoopForConditionalGeneration"
],
"do_sample": true,
"eos_token_id": 200006,
"generation_config": {
"do_sample": true,
"temperature": 1.0,
"eos_token_id": 200006,
"pad_token_id": 199999
},
"model_type": "inkling2_loop",
"mtp_config": {
"hidden_size": 2304,
"num_hidden_layers": 3,
"num_attention_heads": 24,
"num_key_value_heads": 8,
"head_dim": 128,
"intermediate_size": 9216,
"max_position_embeddings": 1048576,
"torch_dtype": "bfloat16",
"q_bias": false,
"o_bias": false,
"rms_norm_eps": 1e-06,
"rope_theta": 1000000.0,
"local_layer_ids": [
0,
2
],
"sliding_window_size": 1024,
"block_size": 8,
"markov_rank": 256,
"markov_head_type": "rnn_hidden",
"target_layer_ids": [
5,
23,
41,
48
]
},
"temperature": 1.0,
"text_config": {
"model_type": "inkling2_loop_text",
"model_max_length": 1048576,
"torch_dtype": "bfloat16",
"hidden_size": 2304,
"num_hidden_layers": 24,
"vocab_size": 201024,
"num_attention_heads": 24,
"num_key_value_heads": 8,
"head_dim": 128,
"d_rel": 16,
"rel_extent": 1024,
"q_bias": false,
"o_bias": false,
"log_scaling_n_floor": 8192,
"log_scaling_alpha": 0.1,
"rms_norm_eps": 1e-06,
"use_value_norm_no_gamma": true,
"qk_l2_clip_threshold": 22.627416997969522,
"rel_l2_clip_threshold": 8.0,
"local_layer_ids": [
0,
1,
2,
3,
4,
6,
7,
8,
9,
10,
12,
13,
14,
15,
16,
18,
19,
20,
21,
22
],
"sconv_kernel_size": 4,
"unpadded_vocab_size": 200058,
"final_logit_softcapping": 30.0,
"swa_head_dim": 128,
"swa_num_attention_heads": 24,
"swa_num_key_value_heads": 8,
"sliding_window_size": 1024,
"n_layers_per_attn_res_block": 6,
"attn_res_normalize_values": true,
"n_routed_experts": 128,
"num_experts_per_tok": 8,
"n_shared_experts": 0,
"intermediate_size": 1152,
"mlp_gate_softcap": 4.0,
"mlp_up_softcap": 25.0,
"route_scale": 8.0,