{
  "architectures": [
    "ForgeLM"
  ],
  "model_type": "forgelm",
  "base_model": "Qwen/Qwen2.5-Coder-1.5B-Instruct",
  "vocab_size": 151936,
  "hidden_size": 1536,
  "num_hidden_layers": 28,
  "num_attention_heads": 12,
  "num_key_value_heads": 2,
  "intermediate_size": 8960,
  "hidden_act": "silu",
  "max_position_embeddings": 32768,
  "rms_norm_eps": 1e-06,
  "rope_theta": 1000000.0,
  "tie_word_embeddings": false,
  "torch_dtype": "bfloat16",
  "use_cache": true,
  "attention_bias": true,
  "bias": true,
  "attention_type": "mla",
  "kv_compression_dim": 512,
  "ffn_type": "moe",
  "moe_num_experts": 4,
  "moe_top_k": 4,
  "moe_shared_expert": true,
  "moe_expert_size": 1792,
  "keystack_features": [
    "mla",
    "moe",
    "mrl",
    "quarot",
    "value_residual",
    "rotorquant",
    "mtp",
    "airllm"
  ],
  "keystack_cosine_similarity": 1.0,
  "training_required": false,
  "vibe_coded": true,
  "vibe_coded_by": "Devin Desktop (Cognition)",
  "auto_map": {
    "AutoModelForCausalLM": "modeling_forgelm.ForgeLMForCausalLM"
  },
  "transformers_version": "4.51.3"
}