| { | |
| "architectures": [ | |
| "RavenGuardForCausalLM" | |
| ], | |
| "model_type": "ravenguard", | |
| "auto_map": { | |
| "AutoConfig": "configuration_ravenguard.RavenGuardConfig", | |
| "AutoModelForCausalLM": "modeling_ravenguard.RavenGuardForCausalLM" | |
| }, | |
| "torch_dtype": "float32", | |
| "tie_word_embeddings": false, | |
| "transformers_version": "4.57.3", | |
| "backbone_repo_id": "netis-ai/RavenGuard-gen", | |
| "sequence_len": 2048, | |
| "vocab_size": 65536, | |
| "n_layer": 20, | |
| "n_head": 10, | |
| "n_kv_head": 10, | |
| "n_embd": 1280, | |
| "window_pattern": "SSSL", | |
| "msa_block_size": 64, | |
| "msa_top_k_blocks": 16, | |
| "msa_local_window": 128, | |
| "msa_sink_blocks": 1, | |
| "msa_index_mode": "pooled_k", | |
| "msa_kernel": "sdpa", | |
| "sparse_token_budget": 512, | |
| "sparse_block_size": 64, | |
| "sparse_block_budget": 16, | |
| "sparse_index_mode": "proxy_qk", | |
| "sparse_index_heads": 1, | |
| "sparse_index_dim": 64, | |
| "ve_n_unique": 3, | |
| "rope_base": 100000, | |
| "doc_mask": false, | |
| "bos_token_id": -1, | |
| "grad_checkpoint": false, | |
| "n_experts": 0, | |
| "n_experts_active": 0, | |
| "n_shared_experts": 0, | |
| "moe_ffn_mult": 2.0, | |
| "moe_aux_coef": 0.01, | |
| "moe_zloss_coef": 0.001, | |
| "moe_norm_topk": true, | |
| "emo_doc_pool": false, | |
| "emo_bos_id": -1 | |
| } | |