config.json

{
  "model_type": "tolm",
  "architectures": [
    "TOLMForMaskedLM"
  ],
  "auto_map": {
    "AutoConfig": "tolm.TOLMConfig",
    "AutoModel": "tolm.TOLMModel",
    "AutoModelForMaskedLM": "tolm.TOLMForMaskedLM",
    "AutoModelForCausalLM": "tolm.TOLMForCausalLM"
  },
  "vocab_size": 16000,
  "max_seq_len": 512,
  "hidden_size": 512,
  "num_hidden_layers": 12,
  "num_attention_heads": 8,
  "intermediate_size": 1536,
  "position_buckets": 32,
  "dropout": 0.1,
  "attention_dropout": 0.1,
  "initializer_range": 0.03227486121839514,
  "value_gating": true,
  "residual_mixing": true,
  "pad_token_id": 1,
  "bos_token_id": 2,
  "eos_token_id": 3,
  "mask_token_id": 4,
  "absolute_positions": false,
  "use_rope": false,
  "use_alibi": false,
  "recurrent_steps": 1,
  "num_experts": 1,
  "experts_per_token": 1,
  "expert_intermediate_size": null,
  "future_offsets": [],
  "state_mixer_kernel": 0,
  "rtd_auxiliary": false,
  "geometry_lexical_dim": 0,
  "geometry_curvature": 1.0,
  "cognitive_readout_layer": 0,
  "cognitive_readout_weight": 0.0,
  "direct_sum_dims": [
    96,
    128,
    288
  ],
  "direct_sum_heads": [
    3,
    4,
    6
  ],
  "direct_sum_intermediate_sizes": [
    512,
    640,
    1344
  ]
}