| { | |
| "vocab_size": 8192, | |
| "context_length": 128, | |
| "d_model": 384, | |
| "n_heads": 6, | |
| "n_layers": 7, | |
| "d_ff": 960, | |
| "dropout": 0.0, | |
| "norm": "rmsnorm", | |
| "norm_eps": 1e-05, | |
| "activation": "swiglu", | |
| "bias": false, | |
| "tie_embeddings": true, | |
| "position_embedding": "learned" | |
| } | |