| { | |
| "attention_pattern": "full", | |
| "attention_sink_size": 0, | |
| "context_length": 1024, | |
| "dropout": 0.0, | |
| "ffn_hidden_multiplier": 2.6875, | |
| "final_logit_softcap": 30.0, | |
| "global_attention_every": 6, | |
| "n_embd": 1024, | |
| "n_head": 16, | |
| "n_kv_head": 8, | |
| "n_layer": 24, | |
| "native_gqa": false, | |
| "qk_norm": true, | |
| "sliding_window_size": 0, | |
| "swiglu_clamp": 10.0, | |
| "tie_embeddings": true, | |
| "vocab_size": 32000 | |
| } | |