| { | |
| "vocab_size": 50257, | |
| "emb_dim": 768, | |
| "hidden_dim": 2048, | |
| "num_layers": 12, | |
| "num_heads": 12, | |
| "num_kv_heads": 4, | |
| "max_seq_len": 1024, | |
| "window_size": 1024, | |
| "sliding_window_ratio": 0.75, | |
| "rope_theta": 10000.0, | |
| "dtype": "torch.float16", | |
| "bias": false, | |
| "dropout": 0.0 | |
| } |