| { |
| "LanguageModel": { |
| "input": { |
| "vocab_size": 51200, |
| "pad_vocab_size_multiple": 8 |
| } |
| }, |
| "MixerModel": { |
| "input": { |
| "d_model": 2048, |
| "n_layer": 24, |
| "lm_head_prenorm": "layer" |
| } |
| }, |
| "Block1": { |
| "n_layers": 24, |
| "BlockType": "modules.phi_block", |
| "block_input": { |
| "resid_dropout": 0.0 |
| }, |
| "CoreType": "modules.mixers.discrete_mamba2", |
| "core_input": { |
| "d_state": 64, |
| "n_v_heads": 32, |
| "n_qk_heads": 32, |
| "d_conv": 4, |
| "conv_bias": true, |
| "expand": 1, |
| "chunk_size": 128, |
| "activation": "identity", |
| "bias": false |
| } |
| } |
| } |
|
|
