flcmind / params.json
FLCgeigeigei's picture
Add params.json for model metadata and download tracking
d31da26 verified
Raw History Blame Contribute Delete
613 Bytes
{
"model": "FLCMind",
"framework": "pytorch",
"hidden_size": 768,
"num_hidden_layers": 8,
"num_attention_heads": 8,
"num_key_value_heads": 4,
"intermediate_size": 2432,
"vocab_size": 6400,
"max_position_embeddings": 32768,
"rope_theta": 1000000,
"rms_norm_eps": 1e-06,
"num_experts": 4,
"num_experts_per_tok": 1,
"variants": {
"dense": {
"file": "full_sft_768.pth",
"use_moe": false,
"parameters": "63.9M"
},
"moe": {
"file": "moe_sft_768_moe.pth",
"use_moe": true,
"parameters": "198.4M",
"active_parameters": "63.9M"
}
}
}