File size: 336 Bytes
8620460
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
{
 "vocab_size": 16384,
 "d_model": 1024,
 "n_layers": 24,
 "max_seq_len": 4096,
 "attn_every": 4,
 "n_heads": 8,
 "n_kv_heads": 2,
 "head_dim": 128,
 "rope_theta": 50000.0,
 "dn_k_heads": 4,
 "dn_v_heads": 8,
 "dn_head_dim": 128,
 "dn_conv_size": 4,
 "ffn_hidden": 2816,
 "norm_eps": 1e-06,
 "tie_embeddings": true,
 "init_std": 0.02
}