config.json

{
  "architectures": [
    "MiniMaxM3SparseForConditionalGeneration"
  ],
  "auto_map": {
    "AutoConfig": "configuration_minimax_m3_vl.MiniMaxM3VLConfig"
  },
  "model_type": "minimax_m3_vl",
  "text_config": {
    "hidden_size": 6144,
    "intermediate_size": 3072,
    "num_hidden_layers": 60,
    "num_attention_heads": 64,
    "num_key_value_heads": 4,
    "head_dim": 128,
    "vocab_size": 200064,
    "max_position_embeddings": 1048576,
    "rms_norm_eps": 1e-06,
    "use_gemma_norm": true,
    "attention_output_gate": false,
    "rope_theta": 5000000,
    "rotary_dim": 64,
    "partial_rotary_factor": 0.5,
    "hidden_act": "swigluoai",
    "use_qk_norm": true,
    "tie_word_embeddings": false,
    "dense_intermediate_size": 12288,
    "shared_intermediate_size": 3072,
    "num_local_experts": 128,
    "num_experts_per_tok": 4,
    "n_shared_experts": 1,
    "scoring_func": "sigmoid",
    "use_routing_bias": true,
    "moe_layer_freq": [
      0,
      0,
      0,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1,
      1
    ],
    "qk_norm_type": "per_head",
    "num_mtp_modules": 7,
    "num_nextn_predict_layers": 1,
    "swiglu_alpha": 1.702,
    "swiglu_limit": 7.0,
    "routed_scaling_factor": 2.0,
    "sparse_attention_config": {
      "use_sparse_attention": true,
      "sparse_index_dim": 128,
      "sparse_num_index_heads": 4,
      "sparse_topk_blocks": 16,
      "sparse_block_size": 128,
      "sparse_disable_index_value": [
        0,
        0,
        0,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1
      ],
      "sparse_score_type": "max",
      "sparse_init_block": 0,
      "sparse_local_block": 1,
      "sparse_attention_freq": [
        0,
        0,
        0,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1
      ]
    },
    "architectures": [
      "MiniMaxM3SparseForCausalLM"
    ]
  },
  "vision_config": {
    "hidden_size": 1280,
    "num_attention_heads": 16,
    "num_hidden_layers": 32,
    "intermediate_size": 5120,
    "patch_size": 14,
    "image_size": 2016,
    "projection_dim": 6144,
    "position_embedding_type": "rope",
    "rope_mode": "3d",
    "rope_theta": 10000.0,
    "attention_dropout": 0.0,
    "hidden_act": "gelu",
    "initializer_factor": 1.0,
    "initializer_range": 0.02,
    "layer_norm_eps": 1e-05,
    "model_type": "clip_vision_model",
    "num_channels": 3,
    "vocab_size": 32000,
    "img_token_compression_config": {
      "image_token_compression_method": "patch_merge",
      "spatial_merge_size": 2,
      "temporal_patch_size": 2
    },
    "vision_segment_max_frames": 4
  },
  "img_token_compression_config": {
    "image_token_compression_method": "patch_merge",
    "spatial_merge_size": 2,
    "temporal_patch_size": 2
  },
  "image_grid_pinpoints": "[(336, 336), (336, 672), (336, 1008), (336, 1344), (336, 1680), (336, 2016), (672, 336), (672, 672), (672, 1008), (672, 1344), (672, 1680), (672, 2016), (1008, 336), (1008, 672), (1008, 1008), (1008, 1344), (1008, 1680), (1008, 2016), (1344, 336), (1344, 672), (1344, 1008), (1344, 1344), (1344, 1680), (1344, 2016), (1680, 336), (1680, 672), (1680, 1008), (1680, 1344), (1680, 1680), (1680, 2016), (2016, 336), (2016, 672), (2016, 1008), (2016, 1344), (2016, 1680), (2016, 2016)]",
  "image_seq_length": 576,
  "image_token_index": 200025,
  "video_token_index": 200026,
  "multimodal_projector_bias": true,
  "num_reward_heads": 0,
  "process_image_mode": "dynamic_res",
  "projector_hidden_act": "gelu",
  "vision_feature_layer": -1,
  "vision_feature_select_strategy": "full",
  "torch_dtype": "bfloat16",
  "transformers_version": "4.52.4",
  "projector_hidden_size": 6144
}