| { |
| "_class_name": "TransformerConfig", |
| "architectures": [ |
| "TransformerForCausalLM" |
| ], |
| "auto_map": { |
| "AutoConfig": "config.TransformerConfig", |
| "AutoModel": "model.TransformerModel", |
| "AutoModelForCausalLM": "model.TransformerForCausalLM" |
| }, |
|
|
| "vocab_size": 50257, |
| "hidden_size": 1024, |
| "embedding_size": 1024, |
| "num_hidden_layers": 24, |
| "num_attention_heads": 16, |
| "intermediate_size": 4096, |
| "max_position_embeddings": 1024, |
|
|
| "use_causal_attention": true, |
| "use_cache": false, |
|
|
| "initializer_range": 0.02, |
|
|
| "bos_token_id": 50256, |
| "eos_token_id": 50256, |
| "pad_token_id": 50256, |
|
|
| "transformers_version": "4.57.1" |
| } |
