File size: 503 Bytes
4991727 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 | {
"model": {
"attention_dropout": 0.0,
"drop_path_rate": 0.0,
"embedding_dropout": 0.0,
"embedding_size": 768,
"ffn_dropout": 0.0,
"mlp_hidden_size": 1536,
"num_attention_heads": 16,
"num_cls": 12,
"num_layers": 8,
"num_outputs": 5000,
"positional_damping_factor": 0.1,
"residual_scale_init": 1.0,
"rms_norm_eps": 1e-06,
"rope_base": 10000.0,
"rope_fraction": 0.5,
"use_absolute_positional_embeddings": false,
"use_rope": true
}
}
|