File size: 1,190 Bytes
5be5643 97a2e7c 5be5643 97a2e7c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 | {
"architectures": [
"RavenGuardForCausalLM"
],
"model_type": "ravenguard",
"auto_map": {
"AutoConfig": "configuration_ravenguard.RavenGuardConfig",
"AutoModelForCausalLM": "modeling_ravenguard.RavenGuardForCausalLM"
},
"torch_dtype": "float32",
"tie_word_embeddings": false,
"transformers_version": "4.57.3",
"backbone_repo_id": "netis-ai/RavenGuard-gen",
"sequence_len": 2048,
"vocab_size": 65536,
"n_layer": 20,
"n_head": 10,
"n_kv_head": 10,
"n_embd": 1280,
"window_pattern": "SSSL",
"msa_block_size": 64,
"msa_top_k_blocks": 16,
"msa_local_window": 128,
"msa_sink_blocks": 1,
"msa_index_mode": "pooled_k",
"msa_kernel": "sdpa",
"sparse_token_budget": 512,
"sparse_block_size": 64,
"sparse_block_budget": 16,
"sparse_index_mode": "proxy_qk",
"sparse_index_heads": 1,
"sparse_index_dim": 64,
"ve_n_unique": 3,
"rope_base": 100000,
"doc_mask": false,
"bos_token_id": -1,
"grad_checkpoint": false,
"n_experts": 0,
"n_experts_active": 0,
"n_shared_experts": 0,
"moe_ffn_mult": 2.0,
"moe_aux_coef": 0.01,
"moe_zloss_coef": 0.001,
"moe_norm_topk": true,
"emo_doc_pool": false,
"emo_bos_id": -1
}
|