{ "architectures": [ "Inkling2ForConditionalGeneration" ], "model_type": "inkling2", "eos_token_id": 200006, "text_config": { "model_max_length": 1048576, "torch_dtype": "bfloat16", "hidden_size": 4608, "num_hidden_layers": 48, "vocab_size": 201024, "num_attention_heads": 48, "num_key_value_heads": 8, "head_dim": 128, "d_rel": 16, "rel_extent": 1024, "q_bias": false, "o_bias": false, "log_scaling_n_floor": 128000, "log_scaling_alpha": 0.1, "rms_norm_eps": 1e-06, "pre_norm_pre_affine": false, "final_norm_pre_affine": false, "use_value_norm_no_gamma": true, "qk_l2_clip_threshold": 22.627416997969522, "rel_l2_clip_threshold": 8.0, "local_layer_ids": [ 0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 30, 31, 32, 33, 34, 36, 37, 38, 39, 40, 42, 43, 44, 45, 46 ], "sconv_kernel_size": 4, "unpadded_vocab_size": 200058, "final_logit_softcapping": 30.0, "swa_head_dim": 128, "swa_num_attention_heads": 48, "swa_num_key_value_heads": 8, "sliding_window_size": 1024, "n_layers_per_attn_res_block": 6, "attn_res_normalize_values": true, "n_routed_experts": 384, "num_experts_per_tok": 8, "n_shared_experts": 0, "intermediate_size": 2304, "mlp_gate_softcap": 4.0, "mlp_up_softcap": 25.0, "route_scale": 8.0, "use_gate_bias": true }, "vision_config": { "temporal_patch_size": 2, "native_patch_size": 20, "adapter_merge_size": 2, "projection_hidden_dim": 4096, "projector_bias": false, "hidden_size": 1024, "num_hidden_layers": 23, "num_attention_heads": 16, "mlp_ratio": 4.0, "posemb_grid_size": 32, "layer_norm_eps": 1e-05 }, "mtp_config": { "hidden_size": 4608, "num_hidden_layers": 5, "num_attention_heads": 48, "num_key_value_heads": 8, "head_dim": 128, "intermediate_size": 18432, "max_position_embeddings": 1048576, "torch_dtype": "bfloat16", "q_bias": false, "o_bias": false, "rms_norm_eps": 1e-06, "rope_theta": 1000000.0, "local_layer_ids": [ 0, 1, 3, 4 ], "sliding_window_size": 1024, "block_size": 8, "markov_rank": 256, "markov_head_type": "rnn_hidden", "target_layer_ids": [ 5, 11, 17, 23, 29, 35, 41, 48 ] }, "quantization": { "group_size": 64, "bits": 2, "mode": "affine", "model.embed": { "bits": 8, "group_size": 64, "mode": "affine" }, "lm_head": { "bits": 8, "group_size": 64, "mode": "affine" }, "model.layers.0.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.0.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.1.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.1.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.10.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.10.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.11.attn.q_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.11.attn.k_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.11.attn.v_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.11.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.12.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.12.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.13.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.13.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.14.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.14.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.15.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.15.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.16.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.16.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.17.attn.q_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.17.attn.k_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.17.attn.v_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.17.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.18.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.18.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.19.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.19.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.2.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.2.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.20.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.20.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.21.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.21.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.22.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.22.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.23.attn.q_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.23.attn.k_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.23.attn.v_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.23.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.24.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.24.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.25.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.25.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.26.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.26.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.27.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.27.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.28.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.28.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.29.attn.q_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.29.attn.k_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.29.attn.v_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.29.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.3.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.3.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.30.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.30.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.31.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.31.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.32.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.32.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.33.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.33.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.34.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.34.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.35.attn.q_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.35.attn.k_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.35.attn.v_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.35.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.36.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.36.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.37.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.37.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.38.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.38.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.39.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.39.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.4.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.4.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.40.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.40.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.41.attn.q_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.41.attn.k_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.41.attn.v_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.41.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.42.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.42.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.43.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.43.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.44.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.44.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.45.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.45.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.46.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.46.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.47.attn.q_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.47.attn.k_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.47.attn.v_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.47.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.5.attn.q_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.5.attn.k_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.5.attn.v_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.5.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.6.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.6.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.7.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.7.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.8.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.8.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.9.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.9.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" } }, "quantization_config": { "group_size": 64, "bits": 2, "mode": "affine", "model.embed": { "bits": 8, "group_size": 64, "mode": "affine" }, "lm_head": { "bits": 8, "group_size": 64, "mode": "affine" }, "model.layers.0.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.0.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.1.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.1.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.10.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.10.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.11.attn.q_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.11.attn.k_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.11.attn.v_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.11.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.12.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.12.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.13.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.13.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.14.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.14.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.15.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.15.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.16.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.16.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.17.attn.q_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.17.attn.k_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.17.attn.v_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.17.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.18.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.18.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.19.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.19.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.2.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.2.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.20.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.20.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.21.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.21.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.22.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.22.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.23.attn.q_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.23.attn.k_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.23.attn.v_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.23.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.24.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.24.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.25.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.25.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.26.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.26.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.27.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.27.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.28.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.28.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.29.attn.q_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.29.attn.k_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.29.attn.v_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.29.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.3.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.3.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.30.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.30.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.31.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.31.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.32.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.32.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.33.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.33.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.34.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.34.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.35.attn.q_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.35.attn.k_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.35.attn.v_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.35.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.36.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.36.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.37.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.37.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.38.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.38.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.39.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.39.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.4.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.4.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.40.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.40.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.41.attn.q_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.41.attn.k_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.41.attn.v_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.41.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.42.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.42.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.43.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.43.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.44.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.44.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.45.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.45.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.46.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.46.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.47.attn.q_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.47.attn.k_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.47.attn.v_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.47.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.5.attn.q_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.5.attn.k_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.5.attn.v_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.5.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.6.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.6.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.7.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.7.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.8.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.8.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.9.attn.in_proj": { "bits": 8, "group_size": 128, "mode": "affine" }, "model.layers.9.attn.o_proj": { "bits": 8, "group_size": 128, "mode": "affine" } } }