| { | |
| "vocab_size": 12288, | |
| "n_layers": 8, | |
| "d_model": 384, | |
| "n_heads": 6, | |
| "context": 1024, | |
| "dropout": 0.1, | |
| "attn_dropout": 0.0, | |
| "architecture": "decoder-only transformer (RoPE + RMSNorm + SwiGLU)", | |
| "tie_word_embeddings": true, | |
| "trained_epoch": 8, | |
| "train_loss": 2.4910502552986147, | |
| "val_loss": 5.017208409309387, | |
| "params": 18880896 | |
| } | |