File size: 503 Bytes
4991727
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
{
  "model": {
    "attention_dropout": 0.0,
    "drop_path_rate": 0.0,
    "embedding_dropout": 0.0,
    "embedding_size": 768,
    "ffn_dropout": 0.0,
    "mlp_hidden_size": 1536,
    "num_attention_heads": 16,
    "num_cls": 12,
    "num_layers": 8,
    "num_outputs": 5000,
    "positional_damping_factor": 0.1,
    "residual_scale_init": 1.0,
    "rms_norm_eps": 1e-06,
    "rope_base": 10000.0,
    "rope_fraction": 0.5,
    "use_absolute_positional_embeddings": false,
    "use_rope": true
  }
}