{ "architectures": [ "PrunedOlmoeForCausalLM" ], "attention_bias": false, "attention_dropout": 0.0, "auto_map": { "AutoConfig": "configuration_pruned_olmoe.PrunedOlmoeConfig", "AutoModelForCausalLM": "modeling_pruned_olmoe.PrunedOlmoeForCausalLM" }, "clip_qkv": null, "dtype": "bfloat16", "eos_token_id": 50279, "expert_widths": [ [ 1024, 384, 256, 256, 768, 1024, 128, 768, 384, 128, 1024, 384, 384, 128, 640, 896, 896, 256, 768, 128, 768, 768, 768, 768, 640, 896, 256, 128, 384, 896, 256, 768, 512, 640, 768, 896, 1024, 384, 384, 512, 384, 256, 768, 640, 896, 896, 512, 768, 512, 384, 1024, 896, 896, 768, 128 ], [ 768, 896, 896, 256, 896, 384, 768, 768, 1024, 384, 512, 1024, 768, 768, 1024, 640, 128, 768, 384, 768, 128, 256, 512, 384, 896, 256, 1024, 1024, 896, 1024, 896, 768, 256, 768, 128, 1024, 768, 768, 640, 384, 1024, 384, 640, 896, 1024, 512, 512, 512, 640 ], [ 768, 1024, 512, 384, 384, 768, 256, 384, 1024, 896, 640, 256, 1024, 1024, 1024, 640, 1024, 896, 512, 768, 128, 768, 1024, 768, 256, 1024, 896, 1024, 896, 512, 640, 384, 896, 384, 512, 640, 256, 768, 256, 768, 768, 640, 1024, 640, 896, 768, 768, 256 ], [ 640, 640, 896, 512, 640, 896, 896, 384, 896, 896, 384, 896, 640, 256, 640, 1024, 1024, 768, 1024, 896, 1024, 768, 896, 256, 512, 768, 1024, 256, 768, 512, 256, 640, 1024, 1024, 512, 1024, 768, 256, 1024, 384, 896, 1024, 896, 1024, 384 ], [ 256, 640, 640, 896, 768, 896, 768, 768, 896, 896, 1024, 256, 512, 1024, 640, 896, 512, 512, 384, 384, 256, 384, 384, 896, 896, 768, 640, 896, 768, 1024, 512, 640, 512, 640, 896, 512, 512, 384, 640, 896, 896, 896, 1024, 640, 1024, 640, 1024 ], [ 1024, 512, 1024, 1024, 640, 896, 640, 1024, 896, 384, 1024, 128, 896, 768, 1024, 768, 640, 896, 768, 640, 512, 896, 512, 640, 256, 768, 640, 768, 384, 896, 512, 512, 256, 512, 896, 256, 384, 640, 512, 640, 896, 512, 1024, 256, 768, 1024, 768, 256, 256 ], [ 640, 896, 1024, 896, 1024, 1024, 1024, 512, 256, 256, 1024, 768, 512, 768, 1024, 1024, 1024, 384, 512, 1024, 512, 1024, 128, 640, 640, 896, 768, 128, 256, 256, 256, 256, 896, 1024, 1024, 384, 896, 256, 896, 640, 1024, 384, 640, 256, 1024, 1024, 1024 ], [ 1024, 384, 1024, 1024, 256, 128, 256, 384, 256, 384, 896, 768, 896, 896, 512, 896, 640, 384, 384, 896, 768, 384, 896, 768, 768, 512, 896, 768, 768, 896, 128, 896, 512, 256, 768, 128, 384, 256, 896, 896, 384, 768, 512, 640, 256, 768, 640, 896, 384, 512, 1024, 768, 384 ], [ 512, 768, 512, 256, 128, 640, 384, 640, 768, 896, 640, 768, 256, 384, 1024, 896, 256, 896, 512, 256, 896, 768, 256, 896, 896, 384, 896, 640, 768, 512, 768, 768, 1024, 768, 640, 768, 384, 256, 512, 896, 128, 384, 256, 768, 384, 256, 1024, 1024, 896, 256, 1024, 256, 128, 896 ], [ 640, 640, 896, 256, 1024, 512, 1024, 768, 384, 512, 256, 768, 896, 768, 512, 768, 768, 640, 384, 768, 512, 768, 768, 512, 768, 128, 896, 512, 768, 1024, 128, 384, 768, 768, 768, 384, 512, 640, 768, 512, 768, 1024, 640, 896, 256, 1024, 384, 768, 768, 768 ], [ 896, 512, 896, 768, 384, 384, 768, 512, 768, 512, 1024, 640, 896, 896, 256, 640, 1024, 256, 896, 128, 128, 128, 768, 896, 384, 896, 512, 896, 384, 256, 640, 640, 896, 768, 640, 256, 896, 896, 512, 128, 896, 256, 256, 640, 896, 896, 128, 1024, 256, 384, 1024, 640, 896 ], [ 768, 384, 640, 896, 256, 128, 384, 896, 128, 128, 896, 256, 384, 896, 512, 768, 768, 512, 512, 768, 896, 640, 768, 896, 896, 896, 640, 896, 640, 512, 896, 256, 512, 128, 512, 384, 768, 768, 1024, 256, 768, 256, 768, 512, 640, 1024, 128, 896, 896, 896, 896, 1024 ], [ 512, 384, 768, 640, 640, 768, 1024, 896, 512, 256, 640, 768, 640, 896, 128, 256, 896, 1024, 256, 640, 512, 256, 128, 512, 256, 640, 768, 768, 128, 128, 768, 640, 1024, 1024, 768, 512, 896, 768, 896, 1024, 896, 896, 512, 640, 1024, 512, 1024, 512, 512, 512, 768 ], [ 896, 896, 1024, 1024, 896, 128, 768, 256, 1024, 256, 1024, 640, 384, 256, 256, 512, 768, 896, 512, 768, 384, 1024, 896, 896, 1024, 896, 768, 896, 640, 1024, 512, 896, 512, 1024, 512, 512, 256, 256, 256, 512, 768, 128, 384, 512, 896, 896, 1024, 256, 384, 640 ], [ 896, 640, 384, 512, 256, 640, 1024, 384, 1024, 1024, 768, 256, 1024, 768, 512, 896, 256, 1024, 768, 768, 768, 384, 384, 256, 1024, 384, 384, 384, 896, 768, 640, 768, 512, 896, 896, 896, 896, 256, 384, 128, 1024, 896, 256, 256, 768, 640, 896, 384, 768, 512, 640 ], [ 896, 1024, 768, 1024, 896, 256, 768, 128, 128, 768, 512, 896, 384, 768, 1024, 256, 768, 768, 256, 512, 512, 640, 512, 256, 768, 896, 384, 1024, 640, 1024, 512, 512, 384, 512, 512, 1024, 384, 896, 768, 384, 384, 128, 384, 1024, 896, 640, 768, 768, 256, 640, 512, 640, 384 ] ], "glean_metadata": { "base_model": "allenai/OLMoE-1B-7B-0125-Instruct", "block_size": 128, "criterion": "reap", "dead_experts": 217, "keep_fraction": 0.5, "min_width": 128, "params": 3697491968, "scores": "outputs/scores_0125inst_dolmino-math/scores.pt" }, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 1024, "max_position_embeddings": 4096, "model_type": "pruned_olmoe", "norm_topk_prob": false, "num_attention_heads": 16, "num_experts": 64, "num_experts_per_tok": 8, "num_hidden_layers": 16, "num_key_value_heads": 16, "output_router_logits": false, "pad_token_id": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000.0, "router_aux_loss_coef": 0.01, "tie_word_embeddings": false, "transformers_version": "4.57.6", "use_cache": false, "vocab_size": 50304 }