config.json

{
  "architectures": [
    "MOSSv1d6ImageAsPrefixVLModel"
  ],
  "compressor_config": {
    "input_channels": 1024,
    "intermediate_size": 1536,
    "output_channels": 1024,
    "ratio": 2
  },
  "decoder_config": {
    "abs_pe": false,
    "abs_pe_max_length": 8192,
    "addition_eos_token_ls": null,
    "attention_dropout": 0.0,
    "attn_bias": true,
    "attn_o_bias": false,
    "bos_token_id": 1,
    "cross_attn_layers": [],
    "cross_attn_num_key_value_heads": 2,
    "eos_token_id": 2,
    "gradient_checkpointing": false,
    "head_dim": 128,
    "hidden_act": "silu",
    "hidden_size": 1024,
    "img_token_id": 67839,
    "initializer_range": 0.02,
    "intermediate_size": 1536,
    "kwargs": {},
    "max_length": 4096,
    "max_position_embeddings": 8192,
    "model_path": null,
    "model_type": "",
    "no_bos_sampling": false,
    "num_attention_heads": 8,
    "num_hidden_layers": 12,
    "num_key_value_heads": 2,
    "output_cross_attentions": false,
    "partial_rope": false,
    "return_dict": true,
    "rms_norm_eps": 1e-06,
    "rope_theta": 1000000.0,
    "scale_embedding": false,
    "self_attn_layers": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7,
      8,
      9,
      10,
      11
    ],
    "sliding_window": null,
    "use_qk_norm": true,
    "vocab_size": 67840
  },
  "dtype": "float32",
  "encoder_config": {
    "eps": 1e-05,
    "ffn_type": "swiglu",
    "gradient_checkpointing": false,
    "grid_size": [
      96,
      96
    ],
    "head_dim": 128,
    "height": 1536,
    "hidden_act": "silu",
    "hidden_size": 1024,
    "image_size": [
      1536,
      1536
    ],
    "intermediate_size": 1536,
    "merge_size": 2,
    "model_path": null,
    "model_type": "",
    "num_attention_heads": 8,
    "num_channels": 3,
    "num_hidden_layers": 12,
    "num_key_value_heads": 4,
    "partial_rope": false,
    "patch_size": 16,
    "qk_norm": false,
    "rope_theta": 10000.0,
    "stride": 16,
    "use_post_norm": false,
    "use_pre_norm": true,
    "use_qk_norm": true,
    "width": 1536
  },
  "linear_adapter": false,
  "model_type": "moss_v1d6",
  "transformers_version": "4.57.6",
  "auto_map": {
    "AutoConfig": "configuration_moss_v1d6.MossV1d6Config",
    "AutoModelForCausalLM": "modeling_moss_v1d6.MossV1d6ForCausalLM"
  }
}