{
  "architectures": [
    "TinyLLMForCausalLM"
  ],
  "auto_map": {
    "AutoConfig": "configuration_tinyllm.TinyLLMConfig",
    "AutoModelForCausalLM": "modeling_tinyllm.TinyLLMForCausalLM"
  },
  "hidden_size": 512,
  "max_position_embeddings": 512,
  "model_cfg": {
    "checkpoint_name": null,
    "context_window": 512,
    "core_model": {
      "attn": {
        "attn_type": "generic",
        "bias": false,
        "group_size": 4,
        "is_causal": true,
        "normalization": "rms_norm",
        "num_heads": 16
      },
      "ffn": {
        "bias": false,
        "ffn_dim": 1320,
        "ffn_type": "swiglu",
        "normalization": "rms_norm"
      },
      "num_layers": 8
    },
    "embedding_weight_tying": true,
    "hidden_dim": 512,
    "lm_head": {
      "bias": false,
      "normalization": "rms_norm"
    },
    "positional_encoding_type": "rope",
    "vocab_size": 50257
  },
  "model_type": "tinyllm",
  "num_attention_heads": 16,
  "num_hidden_layers": 8,
  "torch_dtype": "float32",
  "transformers_version": "4.51.3",
  "vocab_size": 50257
}