| # Component 4 model config. | |
| # You can switch the preset name or directly edit dimensions below. | |
| preset: medium_420m | |
| model: | |
| vocab_size: 50000 | |
| max_seq_len: 2048 | |
| d_model: 1152 | |
| n_layers: 23 | |
| n_heads: 16 | |
| d_ff: 4608 | |
| dropout: 0.1 | |
| tie_embeddings: true | |
| gradient_checkpointing: false | |
| init_std: 0.02 | |
| rms_norm_eps: 0.00001 | |