{ "architecture": { "attention_variant": "GQA", "family": "image_text_to_text", "is_moe": false, "mixer": "attention", "positional": "rope", "sliding_window": 512, "tie_word_embeddings": true, "towers": [ { "attention_variant": "GQA", "config_class": "Gemma4TextConfig", "hidden_size": 2304, "name": "text", "num_attention_heads": 8, "num_hidden_layers": 30, "positional": "rope" } ], "view": "multimodal" }, "capabilities": { "attention_backends": [ "eager", "sdpa", "flash_attention", "flex_attention" ], "attention_patterns": [ "sliding", "causal" ], "attention_schedule": [ "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention" ], "task_heads": [ "causal_lm", "image_text_to_text", "multimodal_lm", "pretraining" ], "tensor_parallel": true }, "components": [ { "children": [ "language_model" ], "class_name": "Gemma4Model", "id": "model", "kind": "model", "path_pattern": "model" }, { "children": [ "language_model.embed_tokens", "decoder_layers", "language_model.norm", "language_model.rotary_emb", "language_model.embed_tokens_per_layer", "language_model.per_layer_projection_norm" ], "class_name": "Gemma4TextModel", "id": "language_model", "kind": "model", "path_pattern": "model.language_model" }, { "attributes": { "embedding_dim": "config.hidden_size", "num_embeddings": "config.vocab_size", "tied_lm_head": true }, "class_name": "Gemma4TextScaledWordEmbedding", "id": "language_model.embed_tokens", "kind": "embedding", "path_pattern": "model.language_model.embed_tokens" }, { "attributes": { "norm_type": "rms" }, "class_name": "Gemma4RMSNorm", "id": "language_model.norm", "kind": "normalization", "path_pattern": "model.language_model.norm" }, { "attributes": { "head_dim": "config.head_dim", "scheme": "rope" }, "class_name": "Gemma4TextRotaryEmbedding", "id": "language_model.rotary_emb", "kind": "position", "path_pattern": "model.language_model.rotary_emb" }, { "attributes": { "embedding_dim": "config.hidden_size", "num_embeddings": "config.vocab_size" }, "class_name": "Gemma4TextScaledWordEmbedding", "id": "language_model.embed_tokens_per_layer", "kind": "embedding", "path_pattern": "model.language_model.embed_tokens_per_layer" }, { "attributes": { "norm_type": "rms" }, "class_name": "Gemma4RMSNorm", "id": "language_model.per_layer_projection_norm", "kind": "normalization", "path_pattern": "model.language_model.per_layer_projection_norm" } ], "config": { "class_name": "Gemma4Config", "model_type": "gemma4", "module": "transformers.models.gemma4.configuration_gemma4", "referenced_fields": {}, "salient_fields": { "is_encoder_decoder": false, "tie_word_embeddings": true } }, "edges": [ { "kind": "data", "source": "language_model.embed_tokens", "target": "decoder_layers" }, { "kind": "data", "source": "decoder_layers", "target": "language_model.norm" }, { "kind": "data", "source": "language_model.norm", "target": "language_model.embed_tokens_per_layer" }, { "kind": "data", "source": "decoder_layer.self_attn", "target": "decoder_layer.mlp" }, { "kind": "data", "source": "decoder_layer.mlp", "target": "decoder_layer.input_layernorm" }, { "kind": "data", "source": "decoder_layer.input_layernorm", "target": "decoder_layer.post_attention_layernorm" }, { "kind": "data", "source": "decoder_layer.post_attention_layernorm", "target": "decoder_layer.pre_feedforward_layernorm" }, { "kind": "data", "source": "decoder_layer.pre_feedforward_layernorm", "target": "decoder_layer.post_feedforward_layernorm" }, { "kind": "data", "source": "decoder_layer.self_attn.k_norm", "target": "decoder_layer.self_attn.v_norm" }, { "kind": "mask", "source": "input:attention_mask", "target": "decoder_layer.self_attn" }, { "kind": "residual", "source": "decoder_layer", "target": "decoder_layer.self_attn" }, { "kind": "residual", "source": "decoder_layer", "target": "decoder_layer.mlp" }, { "kind": "position", "source": "language_model.rotary_emb", "target": "decoder_layer.self_attn" }, { "kind": "data", "provenance": "intra_module", "source": "decoder_layer.self_attn", "target": "decoder_layer.self_attn.q_proj" }, { "kind": "data", "provenance": "intra_module", "source": "decoder_layer.self_attn", "target": "decoder_layer.self_attn.k_proj" }, { "kind": "data", "provenance": "intra_module", "source": "decoder_layer.self_attn", "target": "decoder_layer.self_attn.v_proj" }, { "kind": "data", "provenance": "intra_module", "source": "decoder_layer.self_attn.q_proj", "target": "decoder_layer.self_attn.q_norm" }, { "kind": "data", "provenance": "intra_module", "source": "decoder_layer.self_attn.k_proj", "target": "decoder_layer.self_attn.k_norm" }, { "kind": "data", "provenance": "intra_module", "source": "decoder_layer.self_attn.q_norm", "target": "decoder_layer.self_attn.o_proj" }, { "kind": "data", "provenance": "intra_module", "source": "decoder_layer.self_attn.k_norm", "target": "decoder_layer.self_attn.o_proj" }, { "kind": "data", "provenance": "intra_module", "source": "decoder_layer.self_attn.v_proj", "target": "decoder_layer.self_attn.o_proj" }, { "kind": "data", "provenance": "intra_module", "source": "decoder_layer.mlp", "target": "decoder_layer.mlp.gate_proj" }, { "kind": "data", "provenance": "intra_module", "source": "decoder_layer.mlp", "target": "decoder_layer.mlp.up_proj" }, { "kind": "data", "provenance": "intra_module", "source": "decoder_layer.mlp.gate_proj", "target": "decoder_layer.mlp.down_proj" }, { "kind": "data", "provenance": "intra_module", "source": "decoder_layer.mlp.up_proj", "target": "decoder_layer.mlp.down_proj" } ], "extends": "gemma3", "model_type": "gemma4", "patches": [ { "added": { "attrs": [ "shared_kv_states" ] }, "component_kind": null, "parent_class": "Gemma3nModelOutputWithPast", "parent_model": "gemma3n", "relation": "inherits", "target_class": "Gemma4ModelOutputWithPast" }, { "added": { "attrs": [ "shared_kv_states" ] }, "component_kind": null, "parent_class": "Gemma3nCausalLMOutputWithPast", "parent_model": "gemma3n", "relation": "inherits", "target_class": "Gemma4CausalLMOutputWithPast" }, { "added": { "attrs": [ "shared_kv_states" ] }, "component_kind": null, "parent_class": "BaseModelOutputWithPast", "relation": "new", "target_class": "Gemma4TextModelOutputWithPast" }, { "added": { "attrs": [ "attention_mask" ] }, "component_kind": null, "parent_class": "BaseModelOutputWithPooling", "relation": "new", "target_class": "Gemma4AudioModelOutput" }, { "added": { "methods": [ "__init__", "forward" ] }, "component_kind": null, "parent_class": "Module", "relation": "new", "target_class": "Gemma4ClippableLinear" }, { "added": { "attrs": [ "inv_timescales" ], "methods": [ "__init__", "forward" ] }, "component_kind": null, "parent_class": "Module", "relation": "new", "target_class": "Gemma4AudioRelPositionalEncoding" }, { "added": { "methods": [ "__init__", "_convert_to_block", "_extract_block_context", "_rel_shift", "forward" ] }, "component_kind": "attention", "parent_class": "Module", "relation": "new", "target_class": "Gemma4AudioAttention" }, { "added": { "methods": [ "__init__", "forward" ] }, "component_kind": "transformer_block", "parent_class": "Module", "relation": "new", "target_class": "Gemma4AudioSubSampleConvProjectionLayer" }, { "added": { "methods": [ "__init__", "forward" ] }, "component_kind": null, "parent_class": "Module", "relation": "new", "target_class": "Gemma4AudioSubSampleConvProjection" }, { "added": { "methods": [ "__init__", "forward" ] }, "component_kind": "feed_forward", "parent_class": "Module", "relation": "new", "target_class": "Gemma4AudioFeedForward" }, { "added": { "methods": [ "forward", "left_pad" ] }, "component_kind": null, "parent_class": "Conv1d", "relation": "new", "target_class": "Gemma4AudioCausalConv1d" }, { "added": { "methods": [ "__init__", "forward" ] }, "component_kind": null, "parent_class": "Module", "relation": "new", "target_class": "Gemma4AudioLightConv1d" }, { "added": { "methods": [ "__init__", "forward" ] }, "component_kind": "transformer_block", "parent_class": "Module", "relation": "new", "target_class": "Gemma4AudioLayer" }, { "added": { "methods": [ "__init__", "_position_embeddings", "forward" ] }, "component_kind": null, "parent_class": "Module", "relation": "new", "target_class": "Gemma4VisionPatchEmbedder" }, { "added": { "methods": [ "__init__", "_avg_pool_by_positions", "forward" ] }, "component_kind": "pooler", "parent_class": "Module", "relation": "new", "target_class": "Gemma4VisionPooler" }, { "component_kind": "feed_forward", "overridden": { "methods": [ "__init__" ] }, "parent_class": "Gemma3MLP", "relation": "inherits", "target_class": "Gemma4VisionMLP" }, { "component_kind": "position", "overridden": { "methods": [ "compute_default_rope_parameters", "forward" ] }, "parent_class": "LlamaRotaryEmbedding", "parent_model": "llama", "relation": "inherits", "target_class": "Gemma4VisionRotaryEmbedding" }, { "component_kind": "attention", "overridden": { "methods": [ "__init__", "forward" ] }, "parent_class": "Gemma3Attention", "relation": "inherits", "target_class": "Gemma4VisionAttention" }, { "component_kind": "transformer_block", "overridden": { "methods": [ "__init__", "forward" ] }, "parent_class": "Gemma3DecoderLayer", "relation": "inherits", "target_class": "Gemma4VisionEncoderLayer" }, { "added": { "methods": [ "__init__", "forward" ] }, "component_kind": "encoder", "parent_class": "Module", "relation": "new", "target_class": "Gemma4VisionEncoder" }, { "component_kind": "feed_forward", "overridden": { "methods": [ "__init__" ] }, "parent_class": "Gemma3MLP", "relation": "inherits", "target_class": "Gemma4TextMLP" }, { "component_kind": "position", "overridden": { "methods": [ "__init__" ] }, "parent_class": "Gemma3RotaryEmbedding", "relation": "inherits", "target_class": "Gemma4TextRotaryEmbedding" }, { "added": { "methods": [ "__init__", "forward" ] }, "component_kind": "attention", "parent_class": "Module", "relation": "new", "target_class": "Gemma4TextAttention" }, { "component_kind": "moe", "overridden": { "methods": [ "__init__" ] }, "parent_class": "MixtralExperts", "parent_model": "mixtral", "relation": "inherits", "target_class": "Gemma4TextExperts" }, { "added": { "methods": [ "__init__", "forward" ] }, "component_kind": "moe", "parent_class": "Module", "relation": "new", "target_class": "Gemma4TextRouter" }, { "component_kind": "transformer_block", "overridden": { "methods": [ "__init__", "forward" ] }, "parent_class": "Gemma3DecoderLayer", "relation": "inherits", "target_class": "Gemma4TextDecoderLayer" }, { "component_kind": "model", "overridden": { "attrs": [ "_can_record_outputs", "_no_split_modules", "input_modalities" ], "methods": [ "_init_weights" ] }, "parent_class": "Gemma3nPreTrainedModel", "parent_model": "gemma3n", "relation": "inherits", "target_class": "Gemma4PreTrainedModel" }, { "added": { "attrs": [ "_can_record_outputs" ], "methods": [ "get_per_layer_inputs", "project_per_layer_inputs" ] }, "component_kind": "model", "overridden": { "attrs": [ "config" ], "methods": [ "__init__", "forward" ] }, "parent_class": "Gemma3TextModel", "relation": "inherits", "target_class": "Gemma4TextModel" }, { "added": { "attrs": [ "base_model_prefix" ] }, "component_kind": "lm_head", "overridden": { "methods": [ "forward" ] }, "parent_class": "Gemma3ForCausalLM", "relation": "inherits", "target_class": "Gemma4ForCausalLM" }, { "added": { "attrs": [ "_can_record_outputs", "base_model_prefix", "config", "main_input_name" ], "methods": [ "__init__", "_convert_4d_mask_to_blocked_5d", "forward" ] }, "component_kind": "model", "parent_class": "Gemma4PreTrainedModel", "relation": "new", "target_class": "Gemma4AudioModel" }, { "added": { "attrs": [ "_can_record_outputs", "config" ], "methods": [ "__init__", "forward" ] }, "component_kind": "model", "parent_class": "Gemma4PreTrainedModel", "relation": "new", "target_class": "Gemma4VisionModel" }, { "component_kind": null, "overridden": { "methods": [ "__init__", "forward" ] }, "parent_class": "Gemma3nMultimodalEmbedder", "parent_model": "gemma3n", "relation": "inherits", "target_class": "Gemma4MultimodalEmbedder" }, { "added": { "methods": [ "get_video_features" ] }, "component_kind": "model", "overridden": { "methods": [ "__init__", "forward", "get_audio_features", "get_image_features", "get_per_layer_input_embeddings", "get_placeholder_mask", "set_per_layer_input_embeddings" ] }, "parent_class": "Gemma3nModel", "parent_model": "gemma3n", "relation": "inherits", "target_class": "Gemma4Model" }, { "added": { "attrs": [ "base_model_prefix" ], "methods": [ "create_masks_for_generate" ] }, "component_kind": "lm_head", "overridden": { "methods": [ "forward", "get_image_features", "get_per_layer_input_embeddings", "prepare_inputs_for_generation", "set_per_layer_input_embeddings" ] }, "parent_class": "Gemma3nForConditionalGeneration", "parent_model": "gemma3n", "relation": "inherits", "target_class": "Gemma4ForConditionalGeneration" } ], "provenance": { "config_class": "Gemma4Config", "config_module": "transformers.models.gemma4.configuration_gemma4", "model_class": "Gemma4Model", "model_module": "transformers.models.gemma4.modeling_gemma4" }, "repeats": [ { "body": "decoder_layer", "container_path_pattern": "model.language_model.layers", "count": 30, "count_expr": "30", "count_source": "module_tree", "id": "decoder_layers", "index_symbol": "i", "item_path_pattern": "model.language_model.layers.{i}", "kind": "symbolic_repeat", "provenance": { "class_name": "Gemma4TextDecoderLayer", "container_path_pattern": "model.language_model.layers", "item_path_pattern": "model.language_model.layers.{i}", "source": "module_tree_repeat_collapse" }, "repeated_class_name": "Gemma4TextDecoderLayer" } ], "schema_version": "architecture-template-v0", "templates": [ { "children": [ "decoder_layer.self_attn", "decoder_layer.mlp", "decoder_layer.input_layernorm", "decoder_layer.post_attention_layernorm", "decoder_layer.pre_feedforward_layernorm", "decoder_layer.post_feedforward_layernorm", "decoder_layer.post_per_layer_input_norm" ], "class_name": "Gemma4TextDecoderLayer", "id": "decoder_layer", "kind": "transformer_block", "path_pattern": "model.language_model.layers.{i}" }, { "attributes": { "head_dim": 256, "n_heads": 8, "n_kv_heads": 4, "qk_norm": true, "rope": true, "sliding_window": 512, "variant": "GQA" }, "children": [ "decoder_layer.self_attn.q_proj", "decoder_layer.self_attn.q_norm", "decoder_layer.self_attn.k_norm", "decoder_layer.self_attn.v_norm", "decoder_layer.self_attn.k_proj", "decoder_layer.self_attn.v_proj", "decoder_layer.self_attn.o_proj" ], "class_name": "Gemma4TextAttention", "id": "decoder_layer.self_attn", "kind": "attention", "path_pattern": "model.language_model.layers.{i}.self_attn" }, { "attributes": { "in_features": "config.hidden_size", "out_features": 2048 }, "class_name": "Linear", "id": "decoder_layer.self_attn.q_proj", "kind": "projection", "path_pattern": "model.language_model.layers.{i}.self_attn.q_proj" }, { "attributes": { "norm_type": "rms" }, "class_name": "Gemma4RMSNorm", "id": "decoder_layer.self_attn.q_norm", "kind": "normalization", "path_pattern": "model.language_model.layers.{i}.self_attn.q_norm" }, { "attributes": { "norm_type": "rms" }, "class_name": "Gemma4RMSNorm", "id": "decoder_layer.self_attn.k_norm", "kind": "normalization", "path_pattern": "model.language_model.layers.{i}.self_attn.k_norm" }, { "attributes": { "norm_type": "rms" }, "class_name": "Gemma4RMSNorm", "id": "decoder_layer.self_attn.v_norm", "kind": "normalization", "path_pattern": "model.language_model.layers.{i}.self_attn.v_norm" }, { "attributes": { "in_features": "config.hidden_size", "out_features": 1024 }, "class_name": "Linear", "id": "decoder_layer.self_attn.k_proj", "kind": "projection", "path_pattern": "model.language_model.layers.{i}.self_attn.k_proj" }, { "attributes": { "in_features": "config.hidden_size", "out_features": 1024 }, "class_name": "Linear", "id": "decoder_layer.self_attn.v_proj", "kind": "projection", "path_pattern": "model.language_model.layers.{i}.self_attn.v_proj" }, { "attributes": { "in_features": 2048, "out_features": "config.hidden_size" }, "class_name": "Linear", "id": "decoder_layer.self_attn.o_proj", "kind": "projection", "path_pattern": "model.language_model.layers.{i}.self_attn.o_proj" }, { "attributes": { "hidden_size": 2304, "intermediate_size": 9216 }, "children": [ "decoder_layer.mlp.gate_proj", "decoder_layer.mlp.up_proj", "decoder_layer.mlp.down_proj" ], "class_name": "Gemma4TextMLP", "id": "decoder_layer.mlp", "kind": "feed_forward", "path_pattern": "model.language_model.layers.{i}.mlp" }, { "attributes": { "in_features": "config.hidden_size", "out_features": "config.intermediate_size" }, "class_name": "Linear", "id": "decoder_layer.mlp.gate_proj", "kind": "projection", "path_pattern": "model.language_model.layers.{i}.mlp.gate_proj" }, { "attributes": { "in_features": "config.hidden_size", "out_features": "config.intermediate_size" }, "class_name": "Linear", "id": "decoder_layer.mlp.up_proj", "kind": "projection", "path_pattern": "model.language_model.layers.{i}.mlp.up_proj" }, { "attributes": { "in_features": "config.intermediate_size", "out_features": "config.hidden_size" }, "class_name": "Linear", "id": "decoder_layer.mlp.down_proj", "kind": "projection", "path_pattern": "model.language_model.layers.{i}.mlp.down_proj" }, { "attributes": { "norm_type": "rms" }, "class_name": "Gemma4RMSNorm", "id": "decoder_layer.input_layernorm", "kind": "normalization", "path_pattern": "model.language_model.layers.{i}.input_layernorm" }, { "attributes": { "norm_type": "rms" }, "class_name": "Gemma4RMSNorm", "id": "decoder_layer.post_attention_layernorm", "kind": "normalization", "path_pattern": "model.language_model.layers.{i}.post_attention_layernorm" }, { "attributes": { "norm_type": "rms" }, "class_name": "Gemma4RMSNorm", "id": "decoder_layer.pre_feedforward_layernorm", "kind": "normalization", "path_pattern": "model.language_model.layers.{i}.pre_feedforward_layernorm" }, { "attributes": { "norm_type": "rms" }, "class_name": "Gemma4RMSNorm", "id": "decoder_layer.post_feedforward_layernorm", "kind": "normalization", "path_pattern": "model.language_model.layers.{i}.post_feedforward_layernorm" }, { "attributes": { "norm_type": "rms" }, "class_name": "Gemma4RMSNorm", "id": "decoder_layer.post_per_layer_input_norm", "kind": "normalization", "path_pattern": "model.language_model.layers.{i}.post_per_layer_input_norm" } ] }