{
  "architectures": [
    "RexForCausalLM"
  ],
  "auto_map": {
    "AutoConfig": "configuration_rex.RexConfig",
    "AutoModelForCausalLM": "modeling_rex.RexForCausalLM"
  },
  "d_model": 1536,
  "dropout": 0.0,
  "ffn_dim": 3968,
  "initializer_range": 0.02,
  "max_seq_len": 1024,
  "model_type": "rex",
  "n_heads": 16,
  "n_layers": 8,
  "norm_eps": 1e-05,
  "recurrence_steps": 4,
  "tie_embeddings": true,
  "tokenizer_name": "gpt2",
  "torch_dtype": "float32",
  "transformers_version": "4.55.0",
  "use_step_embeddings": true,
  "vocab_size": 50257
}