| { | |
| "vocab_size": 4096, | |
| "n_layers": 8, | |
| "d_model": 384, | |
| "n_heads": 6, | |
| "context": 1024, | |
| "dropout": 0.1, | |
| "attn_dropout": 0.0, | |
| "architecture": "decoder-only transformer (RoPE + RMSNorm + SwiGLU)", | |
| "tie_word_embeddings": true, | |
| "trained_epoch": 5, | |
| "train_loss": 2.598654329776764, | |
| "val_loss": 4.0741489887237545, | |
| "params": 15735168 | |
| } | |