config.json

{
  "architectures": [
    "MetisMoRLMHeadModel"
  ],
  "model_type": "metis_multihead_latent_moe",
  "name": "Metis-1.5",
  "architecture": "metis_multihead_latent_moe_decoder",
  "vocab_size": 32768,
  "block_size": 1024,
  "d_model": 1536,
  "n_layer": 19,
  "n_heads": 24,
  "n_kv_heads": 8,
  "head_dim": 64,
  "intermediate_size": 4096,
  "hidden_act": "swiglu",
  "attn_cfg": {},
  "bos_token_id": 1,
  "eos_token_id": 2,
  "pad_token_id": 0,
  "unk_token_id": 3,
  "rms_norm": true,
  "residual_in_fp32": false,
  "fused_add_norm": false,
  "pad_vocab_size_multiple": 16,
  "tie_embeddings": true,
  "torch_dtype": "bfloat16",
  "estimated_params": 950973312,
  "attention_bias": false,
  "mlp_bias": false,
  "attention_dropout": 0.0,
  "rope_theta": 10000.0,
  "attention_backend": "sdpa",
  "fp8_pad_multiple": 16,
  "mor_max_depth": 3,
  "mor_router_hidden_dim": 384,
  "mor_router_aux_loss_coef": 0.0,
  "mor_target_avg_depth": 1.0
}