MatrixFormer / config.json
Tabimpute's picture
Release single MatrixFormer checkpoint after 250k additional synthetic matrices
4991727 verified
Raw History Blame Contribute Delete
503 Bytes
{
"model": {
"attention_dropout": 0.0,
"drop_path_rate": 0.0,
"embedding_dropout": 0.0,
"embedding_size": 768,
"ffn_dropout": 0.0,
"mlp_hidden_size": 1536,
"num_attention_heads": 16,
"num_cls": 12,
"num_layers": 8,
"num_outputs": 5000,
"positional_damping_factor": 0.1,
"residual_scale_init": 1.0,
"rms_norm_eps": 1e-06,
"rope_base": 10000.0,
"rope_fraction": 0.5,
"use_absolute_positional_embeddings": false,
"use_rope": true
}
}