| { | |
| "vocab_size": 10000, | |
| "dim": 1536, | |
| "num_layers": 4, | |
| "num_heads": 4, | |
| "max_recursion": 4, | |
| "num_experts": 4, | |
| "ffn_expansion": 4, | |
| "max_position_embeddings": 2048, | |
| "model_type": "MoR", | |
| "architecture": "MixtureOfRecursions", | |
| "hidden_act": "gelu", | |
| "_merge_method": "SLERP(t=0.5)", | |
| "_merged_models": [ | |
| "liminerity/MoR-v4", | |
| "liminerity/MoR-v3" | |
| ] | |
| } |