config.json

{
    "architectures": [
        "TinyGPTForCausalLM"
    ],
    "auto_map": {
        "AutoConfig": "configuration_tinygpt.TinyGPTConfig",
        "AutoModelForCausalLM": "modeling_tinygpt.TinyGPTForCausalLM"
    },
    "vocab_size": 32000,
    "hidden_size": 384,
    "num_hidden_layers": 12,
    "num_attention_heads": 8,
    "intermediate_size": 1024,
    "max_position_embeddings": 1024,
    "rms_norm_eps": 1e-06,
    "model_type": "tinygpt",
    "torch_dtype": "float32"
}