| { |
| "activation_type": "silu", |
| "additional_vocab_size": 128, |
| "architectures": [ |
| "MolmoForCausalLM" |
| ], |
| "attention_dropout": 0.0, |
| "attention_type": "sdpa", |
| "auto_map": { |
| "AutoConfig": "config_molmo.MolmoConfig", |
| "AutoModelForCausalLM": "modeling_molmo.MolmoForCausalLM" |
| }, |
| "bias_for_layer_norm": false, |
| "clip_qkv": null, |
| "embedding_dropout": 0.0, |
| "embedding_size": 152064, |
| "float32_attention": true, |
| "hidden_size": 3584, |
| "image_feature_dropout": 0.0, |
| "image_padding_embed": "pad_and_partial_pad", |
| "initializer_range": 0.02, |
| "intermediate_size": 37888, |
| "layer_norm_eps": 1e-06, |
| "layer_norm_type": "rms", |
| "max_position_embeddings": 4096, |
| "model_type": "molmo", |
| "moe_num_experts": 0, |
| "moe_top_k": 2, |
| "norm_after": false, |
| "normalize_input_embeds": false, |
| "num_attention_heads": 28, |
| "num_hidden_layers": 28, |
| "num_key_value_heads": 4, |
| "qk_layer_norm": false, |
| "qkv_bias": true, |
| "residual_dropout": 0.0, |
| "rope_theta": 1000000.0, |
| "scale_logits": false, |
| "tie_word_embeddings": false, |
| "torch_dtype": "float32", |
| "transformers_version": "4.50.3", |
| "use_cache": true, |
| "use_position_ids": true, |
| "vision_config": { |
| "attention_dropout": 0.0, |
| "float32_attention": true, |
| "image_default_input_size": [ |
| 336, |
| 336 |
| ], |
| "image_emb_dim": 1024, |
| "image_head_dim": 64, |
| "image_mlp_activations": "quick_gelu", |
| "image_mlp_dim": 4096, |
| "image_norm_eps": 1e-05, |
| "image_num_heads": 16, |
| "image_num_key_value_heads": 16, |
| "image_num_layers": 23, |
| "image_num_pos": 577, |
| "image_patch_size": 14, |
| "image_pos_patch_size": 14, |
| "initializer_range": 0.02, |
| "model_type": "", |
| "residual_dropout": 0.0 |
| }, |
| "vit_layers": [ |
| -2, |
| -9 |
| ], |
| "vocab_size": 152064, |
| "weight_tying": false |
| } |
