| { | |
| "vocab_size": 4096, | |
| "n_layers": 6, | |
| "d_model": 256, | |
| "n_heads": 4, | |
| "context": 512, | |
| "dropout": 0.1, | |
| "attn_dropout": 0.0, | |
| "architecture": "decoder-only transformer (RoPE + RMSNorm + SwiGLU)", | |
| "tie_word_embeddings": true, | |
| "trained_epoch": 12, | |
| "train_loss": 3.5534612417221068, | |
| "val_loss": 4.055904150009155, | |
| "params": 5868800 | |
| } | |