config.json

{
  "activation_type": "silu",
  "additional_vocab_size": 128,
  "architectures": [
    "MolmoForCausalLM"
  ],
  "attention_dropout": 0.0,
  "attention_type": "sdpa",
  "auto_map": {
    "AutoConfig": "config_molmo.MolmoConfig",
    "AutoModelForCausalLM": "modeling_molmo.MolmoForCausalLM"
  },
  "bias_for_layer_norm": false,
  "clip_qkv": null,
  "embedding_dropout": 0.0,
  "embedding_size": 152064,
  "float32_attention": true,
  "hidden_size": 3584,
  "image_feature_dropout": 0.0,
  "image_padding_embed": "pad_and_partial_pad",
  "initializer_range": 0.02,
  "intermediate_size": 37888,
  "layer_norm_eps": 1e-06,
  "layer_norm_type": "rms",
  "max_position_embeddings": 4096,
  "model_type": "molmo",
  "moe_num_experts": 0,
  "moe_top_k": 2,
  "norm_after": false,
  "normalize_input_embeds": false,
  "num_attention_heads": 28,
  "num_hidden_layers": 28,
  "num_key_value_heads": 4,
  "qk_layer_norm": false,
  "qkv_bias": true,
  "residual_dropout": 0.0,
  "rope_theta": 1000000.0,
  "scale_logits": false,
  "tie_word_embeddings": false,
  "torch_dtype": "float32",
  "transformers_version": "4.50.3",
  "use_cache": true,
  "use_position_ids": true,
  "vision_config": {
    "attention_dropout": 0.0,
    "float32_attention": true,
    "image_default_input_size": [
      336,
      336
    ],
    "image_emb_dim": 1024,
    "image_head_dim": 64,
    "image_mlp_activations": "quick_gelu",
    "image_mlp_dim": 4096,
    "image_norm_eps": 1e-05,
    "image_num_heads": 16,
    "image_num_key_value_heads": 16,
    "image_num_layers": 23,
    "image_num_pos": 577,
    "image_patch_size": 14,
    "image_pos_patch_size": 14,
    "initializer_range": 0.02,
    "model_type": "",
    "residual_dropout": 0.0
  },
  "vit_layers": [
    -2,
    -9
  ],
  "vocab_size": 152064,
  "weight_tying": false
}