| { |
| "architectures": ["OpenPanguV2ForCausalLM"], |
| "attention_dropout": 0.0, |
| "auto_map": { |
| "AutoConfig": "configuration_openpangu_v2.OpenPanguV2Config" |
| }, |
| "_attn_implementation":"eager", |
| "block_post_layernorm_idx": [0, 4, 9, 14, 19, 24, 29, 34, 39], |
| "bos_token_id": 148899, |
| "dsa_layers": [0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 33, 36, 39, 42, 45], |
| "dtype": "bfloat16", |
| "eos_token_id": 148902, |
| "first_k_dense_replace": 2, |
| "hidden_act": "silu", |
| "hidden_size": 2560, |
| "index_head_dim": 128, |
| "index_n_heads": 24, |
| "index_topk": 2048, |
| "intermediate_size": 9216, |
| "kv_lora_rank": 512, |
| "max_position_embeddings": 524288, |
| "mhc_num_stream": 4, |
| "mhc_recur_norm": 20, |
| "mhc_use_gamma": true, |
| "model_type": "openpangu_v2", |
| "moe_intermediate_size": 1024, |
| "n_routed_experts": 256, |
| "n_shared_experts": 1, |
| "norm_topk_prob": true, |
| "num_attention_heads": 48, |
| "num_experts_per_tok": 8, |
| "num_hidden_layers": 46, |
| "num_nextn_predict_layers": 3, |
| "pad_token_id": 148902, |
| "param_sink_number": 128, |
| "q_lora_rank": 1024, |
| "qk_nope_head_dim": 128, |
| "qk_rope_head_dim": 64, |
| "rms_norm_eps": 1e-05, |
| "rope_theta": 6400000, |
| "rope_interleave": false, |
| "router_enable_expert_bias": true, |
| "routed_scaling_factor": 2.5, |
| "router_sliding_window": 3, |
| "sandwich_norm": true, |
| "sliding_window": 512, |
| "sliding_window_list": [512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 2048, 2048, 2048], |
| "swa_layers": [1, 2, 4, 5, 7, 8, 10, 11, 13, 14, 16, 17, 19, 20, 22, 23, 25, 26, 28, 29, 31, 32, 34, 35, 37, 38, 40, 41, 43, 44, 46, 47, 48], |
| "tie_word_embeddings": false, |
| "transformers_version": "5.0.0", |
| "use_cache": true, |
| "use_mhc": true, |
| "use_mome": true, |
| "v_head_dim": 128, |
| "vocab_size": 151552 |
| } |
|
|
