File size: 561 Bytes
a61b335 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 | {
"architectures": [
"RexForCausalLM"
],
"auto_map": {
"AutoConfig": "configuration_rex.RexConfig",
"AutoModelForCausalLM": "modeling_rex.RexForCausalLM"
},
"d_model": 1536,
"dropout": 0.0,
"ffn_dim": 3968,
"initializer_range": 0.02,
"max_seq_len": 1024,
"model_type": "rex",
"n_heads": 16,
"n_layers": 8,
"norm_eps": 1e-05,
"recurrence_steps": 4,
"tie_embeddings": true,
"tokenizer_name": "gpt2",
"torch_dtype": "float32",
"transformers_version": "4.55.0",
"use_step_embeddings": true,
"vocab_size": 50257
}
|