{
  "architectures": [
    "RavenGuardForCausalLM"
  ],
  "model_type": "ravenguard",
  "auto_map": {
    "AutoConfig": "configuration_ravenguard.RavenGuardConfig",
    "AutoModelForCausalLM": "modeling_ravenguard.RavenGuardForCausalLM"
  },
  "torch_dtype": "float32",
  "tie_word_embeddings": false,
  "transformers_version": "4.57.3",
  "backbone_repo_id": "netis-ai/RavenGuard-gen",
  "sequence_len": 2048,
  "vocab_size": 65536,
  "n_layer": 20,
  "n_head": 10,
  "n_kv_head": 10,
  "n_embd": 1280,
  "window_pattern": "SSSL",
  "msa_block_size": 64,
  "msa_top_k_blocks": 16,
  "msa_local_window": 128,
  "msa_sink_blocks": 1,
  "msa_index_mode": "pooled_k",
  "msa_kernel": "sdpa",
  "sparse_token_budget": 512,
  "sparse_block_size": 64,
  "sparse_block_budget": 16,
  "sparse_index_mode": "proxy_qk",
  "sparse_index_heads": 1,
  "sparse_index_dim": 64,
  "ve_n_unique": 3,
  "rope_base": 100000,
  "doc_mask": false,
  "bos_token_id": -1,
  "grad_checkpoint": false,
  "n_experts": 0,
  "n_experts_active": 0,
  "n_shared_experts": 0,
  "moe_ffn_mult": 2.0,
  "moe_aux_coef": 0.01,
  "moe_zloss_coef": 0.001,
  "moe_norm_topk": true,
  "emo_doc_pool": false,
  "emo_bos_id": -1
}