| { | |
| "vocab_size": 12288, | |
| "n_layers": 8, | |
| "d_model": 384, | |
| "n_heads": 6, | |
| "context": 1024, | |
| "dropout": 0.1, | |
| "attn_dropout": 0.0, | |
| "ple": true, | |
| "d_ple": 64, | |
| "qk_norm": true, | |
| "architecture": "decoder-only transformer (RoPE + RMSNorm + SwiGLU)", | |
| "tie_word_embeddings": true, | |
| "trained_epoch": 8, | |
| "train_loss": 1.1231083750724793, | |
| "val_loss": 6.794381237030029, | |
| "params": 25763264 | |
| } | |