pixel_model_config.json

{
  "name": "pixel_100m",
  "vocab_size": 1262,
  "context_length": 1024,
  "num_layers": 12,
  "hidden_size": 768,
  "num_attention_heads": 12,
  "num_key_value_heads": 4,
  "intermediate_size": 2048,
  "rope_base": 500000,
  "tie_word_embeddings": true,
  "dropout": 0.0,
  "attention_dropout": 0.0,
  "rms_norm_eps": 1e-05,
  "use_moe": false,
  "moe": {
    "enabled": false,
    "num_experts": 4,
    "top_k": 2,
    "expert_interval": 4
  },
  "lora": {
    "enabled": false,
    "rank": 8,
    "alpha": 16,
    "dropout": 0.05,
    "target_modules": [
      "q_proj",
      "k_proj",
      "v_proj",
      "o_proj",
      "gate_proj",
      "up_proj",
      "down_proj"
    ]
  }
}