| checkpoint_ssm_layers: true | |
| d_ff: 3072 | |
| d_inner: 2048 | |
| d_model: 1024 | |
| dtype: float16 | |
| gradient_checkpointing: true | |
| load_balancing_coef: 0.01 | |
| max_seq_len: 2048 | |
| n_layer: 24 | |
| num_experts: 4 | |
| router_z_loss_coef: 0.001 | |
| ssm_d_state: 16 | |
| ssm_expand: 2 | |
| top_k: 1 | |
| use_cpu_offload: false | |
| use_flash_attention: false | |
| vocab_size: 261019 | |