| { | |
| "architecture": "NativeRefusalGPT", | |
| "d_model": 384, | |
| "n_layers": 16, | |
| "n_heads": 8, | |
| "head_dim": 48, | |
| "ffn_hidden": 1536, | |
| "vocab_size": 50257, | |
| "max_seq": 256, | |
| "refusal_fraction": 0.0, | |
| "total_tokens_trained": 300000000, | |
| "optimizer": "AdamW", | |
| "lr": 0.0003, | |
| "weight_decay": 0.1, | |
| "tied_embeddings": true | |
| } |