config.json

{
  "version": "1.0",
  "vocab_size": 844,
  "dim": 256,
  "n_layers": 6,
  "n_heads": 8,
  "n_experts": 8,
  "n_experts_per_token": 2,
  "ffn_hidden": 512,
  "max_seq_len": 128,
  "rope_theta": 10000.0,
  "aux_loss_coef": 0.01,
  "tie_embeddings": true,
  "model_type": "ecoaco",
  "name": "ECOACO",
  "architectures": [
    "EcoacoForCausalLM"
  ]
}