config.json

{
  "activation": "gelu",
  "bias_free_ln": false,
  "cell_m": 4096,
  "col_feature_group_size": 3,
  "col_nhead": 8,
  "col_num_blocks": 3,
  "col_num_inds": 128,
  "dropout": 0.0,
  "embed_dim": 128,
  "ff_factor": 2,
  "icl_nhead": 8,
  "icl_num_blocks": 12,
  "max_classes": 10,
  "row_nhead": 8,
  "row_num_blocks": 3,
  "row_num_cls": 4,
  "row_rope_base": 100000,
  "zero_init": false
}