| { |
| "_name_or_path": "ScrapeGoat-v1", |
| "model_type": "scrapegoat", |
| "vocab_size": 248320, |
| "hidden_size": 4096, |
| "num_hidden_layers": 81, |
| "hidden_act": "silu", |
| "max_position_embeddings": 262144, |
| "initializer_range": 0.02, |
| "rms_norm_eps": 1e-06, |
| "use_cache": true, |
| "pad_token_id": 0, |
| "bos_token_id": 1, |
| "eos_token_id": 2, |
| "tie_word_embeddings": false, |
| "rope_theta": 10000.0, |
| "rope_scaling": null, |
| "attention_bias": false, |
| "attention_dropout": 0.0, |
| "track_a_num_attention_heads": 32, |
| "track_a_num_key_value_heads": 2, |
| "track_a_head_dim": 256, |
| "track_a_num_experts": 512, |
| "track_a_moe_intermediate_size": 1024, |
| "track_b_num_attention_heads": 64, |
| "track_b_num_key_value_heads": 8, |
| "track_b_head_dim": 128, |
| "track_b_num_experts": 192, |
| "num_experts": 704, |
| "track_b_moe_intermediate_size": 1536, |
| "track_b_intermediate_size": 13312, |
| "num_experts_per_tok": 8, |
| "output_router_logits": false, |
| "router_aux_loss_coef": 0.001, |
| "kda_head_dim": 256, |
| "kda_conv_kernel": 3, |
| "kda_gqa_layers": [ |
| 0, |
| 4, |
| 8, |
| 12, |
| 16, |
| 20, |
| 24, |
| 28, |
| 32, |
| 36, |
| 40, |
| 44, |
| 48, |
| 52, |
| 56, |
| 60, |
| 64, |
| 68, |
| 72, |
| 76, |
| 80 |
| ], |
| "quantile_balancing": true, |
| "qb_iterations": 5, |
| "attn_residual": false, |
| "attn_res_blocks": 8, |
| "stable_moe_stage": 1, |
| "mom_enabled": true, |
| "mom_num_memories": 4, |
| "mom_active_memories": 2, |
| "mom_shared_memory": true, |
| "mom_load_balancing": true, |
| "stable_moe_r3": false, |
| "stable_moe_r3_cache": true, |
| "dspark_block_size": 6, |
| "dspark_noise_token_id": 0, |
| "dspark_target_layer_ids": [], |
| "dspark_markov_rank": 256, |
| "num_attention_heads": 32, |
| "num_key_value_heads": 8, |
| "architectures": [ |
| "ScrapeGoatForCausalLM" |
| ], |
| "return_dict": true |
| } |
