{ "_external_rope_config_kwargs": {}, "add_cross_attention": false, "architectures": [ "Gemma4ForConditionalGeneration" ], "attn_mechanism": "vanilla", "audio_config": { "_name_or_path": "", "architectures": null, "attention_chunk_size": 12, "attention_context_left": 13, "attention_context_right": 0, "attention_invalid_logits_value": -1000000000.0, "attention_logit_cap": 50.0, "chunk_size_feed_forward": 0, "conv_kernel_size": 5, "dtype": "bfloat16", "gradient_clipping": 10000000000.0, "hidden_act": "silu", "hidden_size": 1024, "id2label": { "0": "LABEL_0", "1": "LABEL_1" }, "initializer_range": 0.02, "is_encoder_decoder": false, "label2id": { "LABEL_0": 0, "LABEL_1": 1 }, "model_type": "gemma4_audio", "num_attention_heads": 8, "num_hidden_layers": 12, "output_attentions": false, "output_hidden_states": false, "output_proj_dims": 1536, "problem_type": null, "residual_weight": 0.5, "return_dict": true, "rms_norm_eps": 1e-06, "subsampling_conv_channels": [ 128, 32 ], "use_clipped_linears": true }, "audio_token_id": 258881, "backend": null, "bits": null, "blocksize_b": 1, "blocksize_k": 512, "blocksize_q": 512, "boa_token_id": 256000, "boi_token_id": 255999, "bos_token_id": null, "cross_attention_hidden_size": null, "decode_attn_mechanism": null, "decoder_start_token_id": null, "dtype": "bfloat16", "easy_method": "train", "eoa_token_id": 258883, "eoa_token_index": 258883, "eoi_token_id": 258882, "eos_token_id": null, "fcm_max_ratio": 0.0, "fcm_min_ratio": 0.0, "flash_attention_backward_pass_impl": "triton", "fsdp_is_ep_bound": true, "gradient_checkpointing": "", "gradient_checkpointing_targets": null, "hardware_abstraction": false, "image_token_id": 258880, "initializer_range": 0.02, "is_decoder": false, "kv_cache_quantization_config": null, "kv_cache_sharding_sequence_axis_name": "sp", "kvdtype": "bfloat16", "lmhead_chunksize": null, "max_position_embeddings": null, "mla_attn_dtype": "bfloat16", "mla_attn_mechanism": "auto", "mla_attn_softmax_dtype": "float32", "model_type": "gemma4", "moe_force_xla_gmm": false, "moe_method": "fused_moe", "moe_tiling_size_batch": 4, "moe_tiling_size_dim": 128, "moe_tiling_size_seqlen": 128, "operation_configs": null, "pad_token_id": null, "pallas_k_block_size": 128, "pallas_m_block_size": 128, "pallas_n_block_size": 128, "partition_axis": { "attention_dim_axis": null, "attention_kv_dim_axis": null, "batch_axis": [ "fsdp", "dp" ], "bias_head_sequence_axis": null, "bias_key_sequence_axis": null, "data_parallel_axis": "dp", "decode_attention_dim_axis": null, "decode_attention_kv_dim_axis": null, "decode_batch_axis": [ "fsdp", "dp" ], "decode_head_axis": "tp", "decode_key_sequence_axis": "sp", "decode_kv_head_axis": "tp", "decode_query_sequence_axis": null, "expert_axis": "ep", "expert_gate_axis": null, "expert_parallel_axis": "ep", "fully_sharded_data_parallel_axis": "fsdp", "head_axis": "tp", "hidden_state_axis": "tp", "key_sequence_axis": "sp", "kv_head_axis": "tp", "mlp_intermediate_axis": "tp", "query_sequence_axis": "sp", "sequence_axis": "sp", "sequence_parallel_axis": "sp", "tensor_parallel_axis": "tp", "vocab_axis": "tp" }, "platform": null, "precompute_masks": true, "pretraining_tp": 1, "qmm_platform_override": null, "qmm_tpu_path_override": null, "quantization_config": null, "scan_attention_layers": false, "scan_mlp_chunk_size": 1024, "scan_ring_attention": true, "sep_token_id": null, "sequence_axis_name": "sp", "sharding_axis_dims": [ 1, -1, 1, 1, 1 ], "sharding_axis_names": [ "dp", "fsdp", "ep", "tp", "sp" ], "sharding_dcn_axis_dims": null, "sp_is_ep_bound": true, "text_config": { "_external_rope_config_kwargs": {}, "add_cross_attention": false, "attention_bias": false, "attention_dropout": 0.0, "attention_k_eq_v": false, "attn_dtype": "bfloat16", "attn_mechanism": "vanilla", "attn_softmax_dtype": "float32", "backend": null, "bits": null, "blocksize_b": 1, "blocksize_k": 512, "blocksize_q": 512, "bos_token_id": 2, "cross_attention_hidden_size": null, "decode_attn_mechanism": null, "decoder_start_token_id": null, "dtype": "bfloat16", "easy_method": "train", "enable_moe_block": false, "eos_token_id": 1, "expert_intermediate_size": null, "fcm_max_ratio": 0.0, "fcm_min_ratio": 0.0, "final_logit_softcapping": 30.0, "flash_attention_backward_pass_impl": "triton", "fsdp_is_ep_bound": true, "global_head_dim": 512, "gradient_checkpointing": "", "gradient_checkpointing_targets": null, "hardware_abstraction": false, "head_dim": 256, "hidden_activation": "gelu_pytorch_tanh", "hidden_size": 1536, "hidden_size_per_layer_input": 256, "initializer_range": 0.02, "intermediate_size": 6144, "is_decoder": false, "kv_cache_quantization_config": null, "kv_cache_sharding_sequence_axis_name": "sp", "kvdtype": "bfloat16", "layer_types": [ "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention" ], "lmhead_chunksize": null, "max_position_embeddings": 131072, "mla_attn_dtype": "bfloat16", "mla_attn_mechanism": "auto", "mla_attn_softmax_dtype": "float32", "model_type": "gemma4_text", "moe_force_xla_gmm": false, "moe_intermediate_size": null, "moe_method": "fused_moe", "moe_tiling_size_batch": 4, "moe_tiling_size_dim": 128, "moe_tiling_size_seqlen": 128, "num_attention_heads": 8, "num_experts": null, "num_global_key_value_heads": null, "num_hidden_layers": 35, "num_key_value_heads": 1, "num_kv_shared_layers": 20, "num_local_experts": null, "operation_configs": null, "pad_token_id": 0, "pallas_k_block_size": 128, "pallas_m_block_size": 128, "pallas_n_block_size": 128, "partition_axis": { "attention_dim_axis": null, "attention_kv_dim_axis": null, "batch_axis": [ "fsdp", "dp" ], "bias_head_sequence_axis": null, "bias_key_sequence_axis": null, "data_parallel_axis": "dp", "decode_attention_dim_axis": null, "decode_attention_kv_dim_axis": null, "decode_batch_axis": [ "fsdp", "dp" ], "decode_head_axis": "tp", "decode_key_sequence_axis": "sp", "decode_kv_head_axis": "tp", "decode_query_sequence_axis": null, "expert_axis": "ep", "expert_gate_axis": null, "expert_parallel_axis": "ep", "fully_sharded_data_parallel_axis": "fsdp", "head_axis": "tp", "hidden_state_axis": "tp", "key_sequence_axis": "sp", "kv_head_axis": "tp", "mlp_intermediate_axis": "tp", "query_sequence_axis": "sp", "sequence_axis": "sp", "sequence_parallel_axis": "sp", "tensor_parallel_axis": "tp", "vocab_axis": "tp" }, "platform": null, "precompute_masks": true, "pretraining_tp": 1, "qmm_platform_override": null, "qmm_tpu_path_override": null, "quantization_config": null, "rms_norm_eps": 1e-06, "rope_parameters": { "full_attention": { "partial_rotary_factor": 0.25, "rope_theta": 1000000.0, "rope_type": "proportional", "type": "proportional" }, "sliding_attention": { "rope_theta": 10000.0, "rope_type": "default", "type": "default" } }, "scan_attention_layers": false, "scan_layers": false, "scan_mlp_chunk_size": 1024, "scan_ring_attention": true, "sep_token_id": null, "sequence_axis_name": "sp", "sharding_axis_dims": [ 1, -1, 1, 1, 1 ], "sharding_axis_names": [ "dp", "fsdp", "ep", "tp", "sp" ], "sharding_dcn_axis_dims": null, "sliding_window": 512, "sp_is_ep_bound": true, "tie_encoder_decoder": false, "tie_word_embeddings": true, "top_k_experts": null, "use_bidirectional_attention": null, "use_cache": true, "use_double_wide_mlp": true, "use_expert_tensor_mode": false, "use_qmm_best_config": false, "use_ring_of_experts": false, "use_scan_mlp": false, "use_sharded_kv_caching": false, "use_sharding_constraint": false, "vocab_size": 262144, "vocab_size_per_layer_input": 262144 }, "tie_encoder_decoder": false, "tie_word_embeddings": true, "transformers_version": "5.5.0", "use_expert_tensor_mode": false, "use_qmm_best_config": false, "use_ring_of_experts": false, "use_scan_mlp": false, "use_sharded_kv_caching": false, "use_sharding_constraint": false, "video_token_id": 258884, "vision_config": { "_external_rope_config_kwargs": {}, "add_cross_attention": false, "attention_bias": false, "attention_dropout": 0.0, "attn_dtype": "bfloat16", "attn_mechanism": "vanilla", "attn_softmax_dtype": "float32", "backend": null, "bits": null, "blocksize_b": 1, "blocksize_k": 512, "blocksize_q": 512, "bos_token_id": null, "cross_attention_hidden_size": null, "decode_attn_mechanism": null, "decoder_start_token_id": null, "default_output_length": 280, "dtype": "bfloat16", "easy_method": "train", "eos_token_id": null, "fcm_max_ratio": 0.0, "fcm_min_ratio": 0.0, "flash_attention_backward_pass_impl": "triton", "fsdp_is_ep_bound": true, "global_head_dim": 64, "gradient_checkpointing": "", "gradient_checkpointing_targets": null, "hardware_abstraction": false, "head_dim": 64, "hidden_activation": "gelu_pytorch_tanh", "hidden_size": 768, "initializer_range": 0.02, "intermediate_size": 3072, "is_decoder": false, "kv_cache_quantization_config": null, "kv_cache_sharding_sequence_axis_name": "sp", "kvdtype": "bfloat16", "lmhead_chunksize": null, "max_position_embeddings": 131072, "mla_attn_dtype": "bfloat16", "mla_attn_mechanism": "auto", "mla_attn_softmax_dtype": "float32", "model_type": "gemma4_vision", "moe_force_xla_gmm": false, "moe_method": "fused_moe", "moe_tiling_size_batch": 4, "moe_tiling_size_dim": 128, "moe_tiling_size_seqlen": 128, "num_attention_heads": 12, "num_hidden_layers": 16, "num_key_value_heads": 12, "operation_configs": null, "pad_token_id": null, "pallas_k_block_size": 128, "pallas_m_block_size": 128, "pallas_n_block_size": 128, "partition_axis": { "attention_dim_axis": null, "attention_kv_dim_axis": null, "batch_axis": [ "fsdp", "dp" ], "bias_head_sequence_axis": null, "bias_key_sequence_axis": null, "data_parallel_axis": "dp", "decode_attention_dim_axis": null, "decode_attention_kv_dim_axis": null, "decode_batch_axis": [ "fsdp", "dp" ], "decode_head_axis": "tp", "decode_key_sequence_axis": "sp", "decode_kv_head_axis": "tp", "decode_query_sequence_axis": null, "expert_axis": "ep", "expert_gate_axis": null, "expert_parallel_axis": "ep", "fully_sharded_data_parallel_axis": "fsdp", "head_axis": "tp", "hidden_state_axis": "tp", "key_sequence_axis": "sp", "kv_head_axis": "tp", "mlp_intermediate_axis": "tp", "query_sequence_axis": "sp", "sequence_axis": "sp", "sequence_parallel_axis": "sp", "tensor_parallel_axis": "tp", "vocab_axis": "tp" }, "patch_size": 16, "platform": null, "pooling_kernel_size": 3, "position_embedding_size": 10240, "precompute_masks": true, "pretraining_tp": 1, "qmm_platform_override": null, "qmm_tpu_path_override": null, "quantization_config": null, "rms_norm_eps": 1e-06, "rope_parameters": { "rope_theta": 100.0, "rope_type": "default", "type": "default" }, "scan_attention_layers": false, "scan_mlp_chunk_size": 1024, "scan_ring_attention": true, "sep_token_id": null, "sequence_axis_name": "sp", "sharding_axis_dims": [ 1, -1, 1, 1, 1 ], "sharding_axis_names": [ "dp", "fsdp", "ep", "tp", "sp" ], "sharding_dcn_axis_dims": null, "sp_is_ep_bound": true, "standardize": false, "tie_encoder_decoder": false, "tie_word_embeddings": true, "use_clipped_linears": true, "use_expert_tensor_mode": false, "use_qmm_best_config": false, "use_ring_of_experts": false, "use_scan_mlp": false, "use_sharded_kv_caching": false, "use_sharding_constraint": false }, "vision_soft_tokens_per_image": 280 }