| { |
| "alpha_init": 1.5, |
| "architectures": [ |
| "MoAMetricLM" |
| ], |
| "attn_drop": 0.1, |
| "attn_heads": 16, |
| "bos_token_id": 0, |
| "conv_kernel": 5, |
| "conv_mult": 2, |
| "dim": 512, |
| "discrepancy_modulation": true, |
| "drop_path": 0.0, |
| "dtype": "float32", |
| "enable_feature_gates": true, |
| "enable_router_gates": true, |
| "energy_amplification": 3.1415, |
| "eos_token_id": 0, |
| "ff_mult": 3, |
| "ffn_hidden": 1536, |
| "hidden_size": 512, |
| "intermediate_size": 1536, |
| "layer_scale_init_value": 0.0001, |
| "learn_alpha": true, |
| "learn_radius": true, |
| "lm_attn_heads": 16, |
| "lm_ffn_hidden": 1536, |
| "lm_intermediate_size": 1536, |
| "lm_mixer_hidden": 768, |
| "lm_mqa_q_heads": 16, |
| "lm_num_attention_heads": 16, |
| "lm_num_key_value_heads": 16, |
| "lm_proj_drop": 0.1, |
| "lm_router_dropout": 0.1, |
| "lm_router_hidden": 128, |
| "lm_router_temperature": 1.5, |
| "lr_rank": 32, |
| "maha_init": 1.0, |
| "max_position_embeddings": 2048, |
| "max_seq_len_cached": 2048, |
| "metric": "maha_diag", |
| "mixer_hidden": 768, |
| "model_type": "moa_metric", |
| "mqa_q_heads": 16, |
| "n_branches": 3, |
| "num_attention_heads": 16, |
| "num_hidden_layers": 4, |
| "num_key_value_heads": 16, |
| "num_layers": 4, |
| "origin_init_scale": 0.0, |
| "pad_token_id": 0, |
| "proj_drop": 0.1, |
| "r_basis": 16, |
| "radius_init": 3.5, |
| "router_dropout": 0.1, |
| "router_hidden": 128, |
| "router_temperature": 2.0, |
| "router_topk": 2, |
| "theta_base": 10000.0, |
| "ti_reg_samples": 16, |
| "ti_reg_weight": 0.01, |
| "tie_word_embeddings": true, |
| "transformers_version": "5.0.0", |
| "use_balls": true, |
| "vocab_size": 50277 |
| } |
|
|
