{
  "architectures": [
    "MicroGlotModel"
  ],
  "attention_dropout_prob": 0.0,
  "attn_output_dropout_prob": 0.0,
  "auto_map": {
    "AutoConfig": "modeling_microglot.MicroGlotConfig",
    "AutoModel": "modeling_microglot.MicroGlotModel",
    "AutoModelForCausalLM": "modeling_microglot.MicroGlotForCausalLM"
  },
  "bos_token_id": 1,
  "classifier_dropout_prob": 0.0,
  "eos_token_id": 2,
  "ffn_dropout_prob": 0.0,
  "gate_softmax_over_all_experts": true,
  "gradient_checkpointing": false,
  "hidden_size": 1024,
  "init_scale": 0.1,
  "intermediate_size": 2816,
  "max_trained_length": 8192,
  "model_type": "microglot",
  "moe_capacity_factor": 1.25,
  "moe_dispatch": "sorted",
  "moe_drop_tokens": true,
  "moe_eval_capacity_factor": 2.0,
  "moe_expert_dropout_prob": 0.0,
  "moe_layer_experts": [
    4,
    64,
    4,
    32,
    4,
    16,
    4,
    8,
    4,
    8,
    4,
    8,
    4,
    8,
    4,
    8,
    4,
    16,
    4,
    32,
    4,
    64,
    4
  ],
  "moe_min_capacity": 4,
  "moe_noisy_gate_policy": null,
  "moe_output_dropout_prob": 0.0,
  "moe_use_residual": true,
  "moe_use_rts": true,
  "num_experts_per_tok": 1,
  "num_hidden_layers": 23,
  "num_kv_heads": 8,
  "num_query_heads": 16,
  "num_species": 99700,
  "pad_token_id": 0,
  "prepend_species_token": true,
  "rope_theta": 500000.0,
  "router_aux_loss_coef": 0.001,
  "species_emb_dim": 32,
  "species_encoder_embedding_dim": 32,
  "species_encoder_pooling_strategy": "eos",
  "species_encoder_repo": null,
  "species_encoder_revision": null,
  "species_encoder_subfolder": "species_encoder",
  "species_vocab_sha256": "7f46fd7aebc6980c7cb4d5b3505009bee87b3f93e1ee7fdbc4146bf6b56e61e1",
  "tie_word_embeddings": true,
  "tokenizer_class": "MicroGlotTokenizerFast",
  "torch_dtype": "bfloat16",
  "transformers_version": "4.51.3",
  "use_flash_attention": true,
  "use_swiglu": true,
  "vocab_size": 8192
}