| { | |
| "vocab_size": 32000, | |
| "d_model": 1280, | |
| "n_layers": 26, | |
| "n_heads": 20, | |
| "n_kv_heads": 5, | |
| "ffn_hidden": 3584, | |
| "max_seq_len": 2048, | |
| "rope_theta": 10000.0, | |
| "dropout": 0.0, | |
| "tie_embeddings": true | |
| } | |
| { | |
| "vocab_size": 32000, | |
| "d_model": 1280, | |
| "n_layers": 26, | |
| "n_heads": 20, | |
| "n_kv_heads": 5, | |
| "ffn_hidden": 3584, | |
| "max_seq_len": 2048, | |
| "rope_theta": 10000.0, | |
| "dropout": 0.0, | |
| "tie_embeddings": true | |
| } | |