{ "model": "FLCMind", "framework": "pytorch", "hidden_size": 768, "num_hidden_layers": 8, "num_attention_heads": 8, "num_key_value_heads": 4, "intermediate_size": 2432, "vocab_size": 6400, "max_position_embeddings": 32768, "rope_theta": 1000000, "rms_norm_eps": 1e-06, "num_experts": 4, "num_experts_per_tok": 1, "variants": { "dense": { "file": "full_sft_768.pth", "use_moe": false, "parameters": "63.9M" }, "moe": { "file": "moe_sft_768_moe.pth", "use_moe": true, "parameters": "198.4M", "active_parameters": "63.9M" } } }