config.json

{
  "model_type": "tinyv4",
  "architectures": ["TinyV4"],
  "auto_map": {
    "AutoModel": "modeling_tinyv4.TinyV4",
    "AutoModelForCausalLM": "modeling_tinyv4.TinyV4"
  },
  "vocab_size": 32000,
  "dim": 128,
  "depth": 6,
  "n_hc": 2,
  "n_routed": 4,
  "n_active": 2,
  "n_shared": 1,
  "expert_intermediate": 192,
  "csa_m": 4,
  "csa_topk": 16,
  "hca_m": 8,
  "n_win": 16,
  "n_q_head": 4,
  "head_dim": 48,
  "d_c": 64,
  "n_idx_head": 4,
  "idx_head_dim": 48,
  "n_out_group": 2,
  "d_g": 64,
  "rope_dim": 24,
  "mtp_depth": 1,
  "hash_layers": 2,
  "max_len": 512,
  "sinkhorn_iters": 20,
  "aux_bias_update": 0.001,
  "bal_loss_weight": 0.0001,
  "step": 5000,
  "best_loss": 3.9723572731018066
}