Upload config.json with huggingface_hub

config.json

@@ -0,0 +1,39 @@
+{
+  "vocab_size": 50304,
+  "d_model": 768,
+  "n_layers": 16,
+  "n_dense_layers": 1,
+  "n_q_heads": 12,
+  "n_kv_heads": 3,
+  "head_dim": 128,
+  "qk_norm": true,
+  "rope_theta": 10000.0,
+  "attn_softcap": 0.0,
+  "dense_ffn": 2304,
+  "expert_ffn": 320,
+  "n_experts": 36,
+  "top_k": 6,
+  "n_shared": 1,
+  "gating": "sigmoid",
+  "norm_topk_prob": false,
+  "balancing": "aux_free",
+  "aux_loss_coef": 0.001,
+  "z_loss_coef": 0.001,
+  "bias_update_rate": 0.001,
+  "router_init_std": 0.02,
+  "tie_embeddings": true,
+  "scale_embeddings": false,
+  "final_z_loss_coef": 0.0001,
+  "logit_softcap": 0.0,
+  "n_mtp": 0,
+  "mtp_weight": 0.1,
+  "init_std": 0.02,
+  "expert_backend": "grouped",
+  "fused_ce": true,
+  "ce_chunk": 4096,
+  "fp8_head": false,
+  "fp8_x_scale": 1.0,
+  "fp8_w_scale": 1.0,
+  "fp8_grad_scale": 1.0,
+  "preset": "500M"
+}