| { | |
| "name": "german-commons-350m", | |
| "vocab_size": 32768, | |
| "d_model": 1024, | |
| "n_layers": 24, | |
| "n_heads": 16, | |
| "n_kv_heads": 16, | |
| "d_ff": 2816, | |
| "context_len": 2048, | |
| "rope_theta": 10000.0, | |
| "norm": "rmsnorm", | |
| "mlp": "swiglu", | |
| "tie_embeddings": true, | |
| "dropout": 0.0 | |
| } |