| { |
| "architectures": [ |
| "MicroGlotModel" |
| ], |
| "attention_dropout_prob": 0.0, |
| "attn_output_dropout_prob": 0.0, |
| "auto_map": { |
| "AutoConfig": "modeling_microglot.MicroGlotConfig", |
| "AutoModel": "modeling_microglot.MicroGlotModel", |
| "AutoModelForCausalLM": "modeling_microglot.MicroGlotForCausalLM" |
| }, |
| "bos_token_id": 1, |
| "classifier_dropout_prob": 0.0, |
| "eos_token_id": 2, |
| "ffn_dropout_prob": 0.0, |
| "gate_softmax_over_all_experts": true, |
| "gradient_checkpointing": false, |
| "hidden_size": 1024, |
| "init_scale": 0.1, |
| "intermediate_size": 2816, |
| "max_trained_length": 8192, |
| "model_type": "microglot", |
| "moe_capacity_factor": 1.25, |
| "moe_dispatch": "sorted", |
| "moe_drop_tokens": true, |
| "moe_eval_capacity_factor": 2.0, |
| "moe_expert_dropout_prob": 0.0, |
| "moe_layer_experts": [ |
| 4, |
| 64, |
| 4, |
| 32, |
| 4, |
| 16, |
| 4, |
| 8, |
| 4, |
| 8, |
| 4, |
| 8, |
| 4, |
| 8, |
| 4, |
| 8, |
| 4, |
| 16, |
| 4, |
| 32, |
| 4, |
| 64, |
| 4 |
| ], |
| "moe_min_capacity": 4, |
| "moe_noisy_gate_policy": null, |
| "moe_output_dropout_prob": 0.0, |
| "moe_use_residual": true, |
| "moe_use_rts": true, |
| "num_experts_per_tok": 1, |
| "num_hidden_layers": 23, |
| "num_kv_heads": 8, |
| "num_query_heads": 16, |
| "num_species": 99700, |
| "pad_token_id": 0, |
| "prepend_species_token": true, |
| "rope_theta": 500000.0, |
| "router_aux_loss_coef": 0.001, |
| "species_emb_dim": 32, |
| "species_encoder_embedding_dim": 32, |
| "species_encoder_pooling_strategy": "eos", |
| "species_encoder_repo": null, |
| "species_encoder_revision": null, |
| "species_encoder_subfolder": "species_encoder", |
| "species_vocab_sha256": "7f46fd7aebc6980c7cb4d5b3505009bee87b3f93e1ee7fdbc4146bf6b56e61e1", |
| "tie_word_embeddings": true, |
| "tokenizer_class": "MicroGlotTokenizerFast", |
| "torch_dtype": "bfloat16", |
| "transformers_version": "4.51.3", |
| "use_flash_attention": true, |
| "use_swiglu": true, |
| "vocab_size": 8192 |
| } |
|
|
