| { | |
| "model_config": { | |
| "sequence_len": 2048, | |
| "vocab_size": 32768, | |
| "n_layer": 24, | |
| "n_head": 12, | |
| "n_kv_head": 12, | |
| "n_embd": 1536, | |
| "window_pattern": "SSSL" | |
| }, | |
| "user_config": { | |
| "source": "sft", | |
| "model_tag": "d24_decoderstack_w1", | |
| "model_step": null, | |
| "unembedding_lr": 0.004, | |
| "embedding_lr": 0.2, | |
| "matrix_lr": 0.02, | |
| "scalar_lr": 0.5, | |
| "init_lr_frac": 0.02 | |
| } | |
| } |