File size: 280 Bytes
c83c617 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 | {
"vocab_size": 8192,
"context_length": 128,
"d_model": 384,
"n_heads": 6,
"n_layers": 7,
"d_ff": 960,
"dropout": 0.0,
"norm": "rmsnorm",
"norm_eps": 1e-05,
"activation": "swiglu",
"bias": false,
"tie_embeddings": true,
"position_embedding": "learned"
}
|