| { |
| "architecture": { |
| "attention_variant": "MHA", |
| "family": "image_text_to_text", |
| "is_moe": false, |
| "mixer": "attention", |
| "positional": "learned", |
| "tie_word_embeddings": true, |
| "towers": [ |
| { |
| "attention_variant": "MHA", |
| "config_class": "BartConfig", |
| "hidden_size": 1024, |
| "name": "text", |
| "num_attention_heads": 16, |
| "num_hidden_layers": 12, |
| "positional": "learned" |
| }, |
| { |
| "config_class": "Florence2VisionConfig", |
| "hidden_size": [ |
| 128, |
| 256, |
| 512, |
| 1024 |
| ], |
| "name": "vision", |
| "num_attention_heads": [ |
| 4, |
| 8, |
| 16, |
| 32 |
| ], |
| "positional": "learned" |
| } |
| ], |
| "view": "multimodal" |
| }, |
| "capabilities": { |
| "attention_backends": [ |
| "eager", |
| "sdpa", |
| "flash_attention", |
| "flex_attention" |
| ], |
| "attention_patterns": [ |
| "causal" |
| ], |
| "attention_schedule": null, |
| "task_heads": [ |
| "image_text_to_text", |
| "multimodal_lm", |
| "pretraining" |
| ], |
| "tensor_parallel": false |
| }, |
| "components": [ |
| { |
| "children": [ |
| "vision_tower", |
| "multi_modal_projector", |
| "language_model" |
| ], |
| "class_name": "Florence2Model", |
| "id": "model", |
| "kind": "model", |
| "path_pattern": "model" |
| }, |
| { |
| "children": [ |
| "florence2_vision_conv_embeds", |
| "blocks" |
| ], |
| "class_name": "Florence2VisionBackbone", |
| "id": "vision_tower", |
| "kind": "model", |
| "path_pattern": "model.vision_tower" |
| }, |
| { |
| "children": [ |
| "multi_modal_projector.image_proj_norm", |
| "multi_modal_projector.image_position_embed", |
| "multi_modal_projector.visual_temporal_embed" |
| ], |
| "class_name": "Florence2MultiModalProjector", |
| "id": "multi_modal_projector", |
| "kind": "module", |
| "path_pattern": "model.multi_modal_projector" |
| }, |
| { |
| "attributes": { |
| "norm_type": "layer" |
| }, |
| "class_name": "LayerNorm", |
| "id": "multi_modal_projector.image_proj_norm", |
| "kind": "normalization", |
| "path_pattern": "model.multi_modal_projector.image_proj_norm" |
| }, |
| { |
| "attributes": {}, |
| "children": [ |
| "multi_modal_projector.image_position_embed.row_embeddings", |
| "multi_modal_projector.image_position_embed.column_embeddings" |
| ], |
| "class_name": "Florence2VisionLearnedAbsolutePositionEmbedding2D", |
| "id": "multi_modal_projector.image_position_embed", |
| "kind": "position", |
| "path_pattern": "model.multi_modal_projector.image_position_embed" |
| }, |
| { |
| "attributes": {}, |
| "class_name": "Embedding", |
| "id": "multi_modal_projector.image_position_embed.row_embeddings", |
| "kind": "position", |
| "path_pattern": "model.multi_modal_projector.image_position_embed.row_embeddings" |
| }, |
| { |
| "attributes": {}, |
| "class_name": "Embedding", |
| "id": "multi_modal_projector.image_position_embed.column_embeddings", |
| "kind": "position", |
| "path_pattern": "model.multi_modal_projector.image_position_embed.column_embeddings" |
| }, |
| { |
| "attributes": { |
| "max_position_embeddings": "config.max_position_embeddings", |
| "scheme": "learned" |
| }, |
| "class_name": "Florence2VisionPositionalEmbeddingCosine1D", |
| "id": "multi_modal_projector.visual_temporal_embed", |
| "kind": "position", |
| "path_pattern": "model.multi_modal_projector.visual_temporal_embed" |
| }, |
| { |
| "children": [ |
| "language_model.shared", |
| "language_model.encoder", |
| "language_model.decoder" |
| ], |
| "class_name": "BartModel", |
| "id": "language_model", |
| "kind": "model", |
| "path_pattern": "model.language_model" |
| }, |
| { |
| "attributes": { |
| "embedding_dim": "config.hidden_size", |
| "num_embeddings": "config.vocab_size", |
| "tied_lm_head": true |
| }, |
| "class_name": "BartScaledWordEmbedding", |
| "id": "language_model.shared", |
| "kind": "embedding", |
| "path_pattern": "model.language_model.shared" |
| }, |
| { |
| "children": [ |
| "language_model.encoder.embed_tokens", |
| "language_model.encoder.embed_positions", |
| "encoder_layers", |
| "language_model.encoder.layernorm_embedding" |
| ], |
| "class_name": "BartEncoder", |
| "id": "language_model.encoder", |
| "kind": "encoder", |
| "path_pattern": "model.language_model.encoder" |
| }, |
| { |
| "attributes": { |
| "embedding_dim": "config.hidden_size", |
| "num_embeddings": "config.vocab_size", |
| "tied_lm_head": true |
| }, |
| "class_name": "BartScaledWordEmbedding", |
| "id": "language_model.encoder.embed_tokens", |
| "kind": "embedding", |
| "path_pattern": "model.language_model.encoder.embed_tokens" |
| }, |
| { |
| "attributes": { |
| "max_position_embeddings": "config.hidden_size", |
| "scheme": "learned" |
| }, |
| "class_name": "BartLearnedPositionalEmbedding", |
| "id": "language_model.encoder.embed_positions", |
| "kind": "position", |
| "path_pattern": "model.language_model.encoder.embed_positions" |
| }, |
| { |
| "attributes": { |
| "norm_type": "layer" |
| }, |
| "class_name": "LayerNorm", |
| "id": "language_model.encoder.layernorm_embedding", |
| "kind": "normalization", |
| "path_pattern": "model.language_model.encoder.layernorm_embedding" |
| }, |
| { |
| "children": [ |
| "language_model.decoder.embed_tokens", |
| "language_model.decoder.embed_positions", |
| "decoder_layers", |
| "language_model.decoder.layernorm_embedding" |
| ], |
| "class_name": "BartDecoder", |
| "id": "language_model.decoder", |
| "kind": "decoder", |
| "path_pattern": "model.language_model.decoder" |
| }, |
| { |
| "attributes": { |
| "embedding_dim": "config.hidden_size", |
| "num_embeddings": "config.vocab_size", |
| "tied_lm_head": true |
| }, |
| "class_name": "BartScaledWordEmbedding", |
| "id": "language_model.decoder.embed_tokens", |
| "kind": "embedding", |
| "path_pattern": "model.language_model.decoder.embed_tokens" |
| }, |
| { |
| "attributes": { |
| "max_position_embeddings": "config.hidden_size", |
| "scheme": "learned" |
| }, |
| "class_name": "BartLearnedPositionalEmbedding", |
| "id": "language_model.decoder.embed_positions", |
| "kind": "position", |
| "path_pattern": "model.language_model.decoder.embed_positions" |
| }, |
| { |
| "attributes": { |
| "norm_type": "layer" |
| }, |
| "class_name": "LayerNorm", |
| "id": "language_model.decoder.layernorm_embedding", |
| "kind": "normalization", |
| "path_pattern": "model.language_model.decoder.layernorm_embedding" |
| } |
| ], |
| "config": { |
| "class_name": "Florence2Config", |
| "model_type": "florence2", |
| "module": "transformers.models.florence2.configuration_florence2", |
| "referenced_fields": {}, |
| "salient_fields": { |
| "is_encoder_decoder": true, |
| "tie_word_embeddings": true |
| } |
| }, |
| "edges": [ |
| { |
| "kind": "data", |
| "source": "florence2_vision_conv_embeds", |
| "target": "blocks" |
| }, |
| { |
| "kind": "data", |
| "source": "block.0.spatial_block", |
| "target": "block.0.channel_block" |
| }, |
| { |
| "kind": "data", |
| "source": "block.0.spatial_block.norm1", |
| "target": "block.0.spatial_block.window_attn" |
| }, |
| { |
| "kind": "data", |
| "source": "block.0.spatial_block.norm2", |
| "target": "block.0.spatial_block.ffn" |
| }, |
| { |
| "kind": "data", |
| "source": "block.0.channel_block.norm1", |
| "target": "block.0.channel_block.channel_attn" |
| }, |
| { |
| "kind": "data", |
| "source": "block.0.channel_block.norm2", |
| "target": "block.0.channel_block.ffn" |
| }, |
| { |
| "kind": "data", |
| "source": "block_2.spatial_block", |
| "target": "block_2.channel_block" |
| }, |
| { |
| "kind": "data", |
| "source": "block_2.spatial_block.norm1", |
| "target": "block_2.spatial_block.window_attn" |
| }, |
| { |
| "kind": "data", |
| "source": "block_2.spatial_block.norm2", |
| "target": "block_2.spatial_block.ffn" |
| }, |
| { |
| "kind": "data", |
| "source": "block_2.channel_block.norm1", |
| "target": "block_2.channel_block.channel_attn" |
| }, |
| { |
| "kind": "data", |
| "source": "block_2.channel_block.norm2", |
| "target": "block_2.channel_block.ffn" |
| }, |
| { |
| "kind": "data", |
| "source": "language_model.shared", |
| "target": "language_model.encoder" |
| }, |
| { |
| "kind": "data", |
| "source": "language_model.encoder", |
| "target": "language_model.decoder" |
| }, |
| { |
| "kind": "data", |
| "source": "language_model.encoder.embed_tokens", |
| "target": "encoder_layers" |
| }, |
| { |
| "kind": "data", |
| "source": "encoder_layers", |
| "target": "language_model.encoder.layernorm_embedding" |
| }, |
| { |
| "kind": "data", |
| "source": "encoder_layer.self_attn", |
| "target": "encoder_layer.self_attn_layer_norm" |
| }, |
| { |
| "kind": "data", |
| "source": "language_model.decoder.embed_tokens", |
| "target": "decoder_layers" |
| }, |
| { |
| "kind": "data", |
| "source": "decoder_layers", |
| "target": "language_model.decoder.layernorm_embedding" |
| }, |
| { |
| "kind": "data", |
| "source": "decoder_layer.self_attn_layer_norm", |
| "target": "decoder_layer.encoder_attn" |
| }, |
| { |
| "kind": "data", |
| "source": "decoder_layer.encoder_attn", |
| "target": "decoder_layer.encoder_attn_layer_norm" |
| }, |
| { |
| "kind": "mask", |
| "source": "input:attention_mask", |
| "target": "block.0.spatial_block.window_attn" |
| }, |
| { |
| "kind": "mask", |
| "source": "input:attention_mask", |
| "target": "block.0.channel_block.channel_attn" |
| }, |
| { |
| "kind": "mask", |
| "source": "input:attention_mask", |
| "target": "block_2.spatial_block.window_attn" |
| }, |
| { |
| "kind": "mask", |
| "source": "input:attention_mask", |
| "target": "block_2.channel_block.channel_attn" |
| }, |
| { |
| "kind": "mask", |
| "source": "input:attention_mask", |
| "target": "encoder_layer.self_attn" |
| }, |
| { |
| "kind": "mask", |
| "source": "input:attention_mask", |
| "target": "decoder_layer.self_attn" |
| }, |
| { |
| "kind": "mask", |
| "source": "input:attention_mask", |
| "target": "decoder_layer.encoder_attn" |
| }, |
| { |
| "kind": "residual", |
| "source": "block.0", |
| "target": "block.0.spatial_block.window_attn" |
| }, |
| { |
| "kind": "residual", |
| "source": "block.0", |
| "target": "block.0.spatial_block.ffn" |
| }, |
| { |
| "kind": "residual", |
| "source": "block.0.spatial_block", |
| "target": "block.0.spatial_block.window_attn" |
| }, |
| { |
| "kind": "residual", |
| "source": "block.0.spatial_block", |
| "target": "block.0.spatial_block.ffn" |
| }, |
| { |
| "kind": "residual", |
| "source": "block.0.channel_block", |
| "target": "block.0.channel_block.channel_attn" |
| }, |
| { |
| "kind": "residual", |
| "source": "block.0.channel_block", |
| "target": "block.0.channel_block.ffn" |
| }, |
| { |
| "kind": "residual", |
| "source": "block_2", |
| "target": "block_2.spatial_block.window_attn" |
| }, |
| { |
| "kind": "residual", |
| "source": "block_2", |
| "target": "block_2.spatial_block.ffn" |
| }, |
| { |
| "kind": "residual", |
| "source": "block_2.spatial_block", |
| "target": "block_2.spatial_block.window_attn" |
| }, |
| { |
| "kind": "residual", |
| "source": "block_2.spatial_block", |
| "target": "block_2.spatial_block.ffn" |
| }, |
| { |
| "kind": "residual", |
| "source": "block_2.channel_block", |
| "target": "block_2.channel_block.channel_attn" |
| }, |
| { |
| "kind": "residual", |
| "source": "block_2.channel_block", |
| "target": "block_2.channel_block.ffn" |
| }, |
| { |
| "kind": "residual", |
| "source": "encoder_layer", |
| "target": "encoder_layer.self_attn" |
| }, |
| { |
| "kind": "residual", |
| "source": "decoder_layer", |
| "target": "decoder_layer.self_attn" |
| }, |
| { |
| "kind": "residual", |
| "source": "decoder_layer", |
| "target": "decoder_layer.encoder_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.image_position_embed", |
| "target": "block.0.spatial_block.window_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.image_position_embed", |
| "target": "block.0.channel_block.channel_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.image_position_embed", |
| "target": "block_2.spatial_block.window_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.image_position_embed", |
| "target": "block_2.channel_block.channel_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.image_position_embed", |
| "target": "encoder_layer.self_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.image_position_embed", |
| "target": "decoder_layer.self_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.image_position_embed", |
| "target": "decoder_layer.encoder_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.image_position_embed.row_embeddings", |
| "target": "block.0.spatial_block.window_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.image_position_embed.row_embeddings", |
| "target": "block.0.channel_block.channel_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.image_position_embed.row_embeddings", |
| "target": "block_2.spatial_block.window_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.image_position_embed.row_embeddings", |
| "target": "block_2.channel_block.channel_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.image_position_embed.row_embeddings", |
| "target": "encoder_layer.self_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.image_position_embed.row_embeddings", |
| "target": "decoder_layer.self_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.image_position_embed.row_embeddings", |
| "target": "decoder_layer.encoder_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.image_position_embed.column_embeddings", |
| "target": "block.0.spatial_block.window_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.image_position_embed.column_embeddings", |
| "target": "block.0.channel_block.channel_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.image_position_embed.column_embeddings", |
| "target": "block_2.spatial_block.window_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.image_position_embed.column_embeddings", |
| "target": "block_2.channel_block.channel_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.image_position_embed.column_embeddings", |
| "target": "encoder_layer.self_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.image_position_embed.column_embeddings", |
| "target": "decoder_layer.self_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.image_position_embed.column_embeddings", |
| "target": "decoder_layer.encoder_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.visual_temporal_embed", |
| "target": "block.0.spatial_block.window_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.visual_temporal_embed", |
| "target": "block.0.channel_block.channel_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.visual_temporal_embed", |
| "target": "block_2.spatial_block.window_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.visual_temporal_embed", |
| "target": "block_2.channel_block.channel_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.visual_temporal_embed", |
| "target": "encoder_layer.self_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.visual_temporal_embed", |
| "target": "decoder_layer.self_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "multi_modal_projector.visual_temporal_embed", |
| "target": "decoder_layer.encoder_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "language_model.encoder.embed_positions", |
| "target": "encoder_layer.self_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "language_model.decoder.embed_positions", |
| "target": "decoder_layer.self_attn" |
| }, |
| { |
| "kind": "position", |
| "source": "language_model.decoder.embed_positions", |
| "target": "decoder_layer.encoder_attn" |
| }, |
| { |
| "kind": "cross_attention", |
| "source": "language_model.encoder", |
| "target": "decoder_layer.encoder_attn" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "block.0.spatial_block.window_attn", |
| "target": "block.0.spatial_block.window_attn.qkv" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "block.0.spatial_block.window_attn", |
| "target": "block.0.spatial_block.window_attn.proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "block.0.spatial_block.ffn", |
| "target": "block.0.spatial_block.ffn.fc1" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "block.0.spatial_block.ffn.fc1", |
| "target": "block.0.spatial_block.ffn.fc2" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "block.0.channel_block.channel_attn", |
| "target": "block.0.channel_block.channel_attn.qkv" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "block.0.channel_block.channel_attn", |
| "target": "block.0.channel_block.channel_attn.proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "block.0.channel_block.ffn", |
| "target": "block.0.channel_block.ffn.fc1" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "block.0.channel_block.ffn.fc1", |
| "target": "block.0.channel_block.ffn.fc2" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "block_2.spatial_block.window_attn", |
| "target": "block_2.spatial_block.window_attn.qkv" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "block_2.spatial_block.window_attn", |
| "target": "block_2.spatial_block.window_attn.proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "block_2.spatial_block.ffn", |
| "target": "block_2.spatial_block.ffn.fc1" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "block_2.spatial_block.ffn.fc1", |
| "target": "block_2.spatial_block.ffn.fc2" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "block_2.channel_block.channel_attn", |
| "target": "block_2.channel_block.channel_attn.qkv" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "block_2.channel_block.channel_attn", |
| "target": "block_2.channel_block.channel_attn.proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "block_2.channel_block.ffn", |
| "target": "block_2.channel_block.ffn.fc1" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "block_2.channel_block.ffn.fc1", |
| "target": "block_2.channel_block.ffn.fc2" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "encoder_layer.self_attn", |
| "target": "encoder_layer.self_attn.k_proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "encoder_layer.self_attn", |
| "target": "encoder_layer.self_attn.v_proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "encoder_layer.self_attn", |
| "target": "encoder_layer.self_attn.q_proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "encoder_layer.self_attn.k_proj", |
| "target": "encoder_layer.self_attn.out_proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "encoder_layer.self_attn.v_proj", |
| "target": "encoder_layer.self_attn.out_proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "encoder_layer.self_attn.q_proj", |
| "target": "encoder_layer.self_attn.out_proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "decoder_layer.self_attn", |
| "target": "decoder_layer.self_attn.k_proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "decoder_layer.self_attn", |
| "target": "decoder_layer.self_attn.v_proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "decoder_layer.self_attn", |
| "target": "decoder_layer.self_attn.q_proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "decoder_layer.self_attn.k_proj", |
| "target": "decoder_layer.self_attn.out_proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "decoder_layer.self_attn.v_proj", |
| "target": "decoder_layer.self_attn.out_proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "decoder_layer.self_attn.q_proj", |
| "target": "decoder_layer.self_attn.out_proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "decoder_layer.encoder_attn", |
| "target": "decoder_layer.encoder_attn.k_proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "decoder_layer.encoder_attn", |
| "target": "decoder_layer.encoder_attn.v_proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "decoder_layer.encoder_attn", |
| "target": "decoder_layer.encoder_attn.q_proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "decoder_layer.encoder_attn.k_proj", |
| "target": "decoder_layer.encoder_attn.out_proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "decoder_layer.encoder_attn.v_proj", |
| "target": "decoder_layer.encoder_attn.out_proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "decoder_layer.encoder_attn.q_proj", |
| "target": "decoder_layer.encoder_attn.out_proj" |
| } |
| ], |
| "extends": "llava", |
| "model_type": "florence2", |
| "patches": [ |
| { |
| "added": { |
| "attrs": [ |
| "activation_function", |
| "depths", |
| "drop_path_rate", |
| "embed_dim", |
| "in_channels", |
| "initializer_range", |
| "max_position_embeddings", |
| "max_temporal_embeddings", |
| "mlp_ratio", |
| "model_type", |
| "num_groups", |
| "num_heads", |
| "patch_padding", |
| "patch_prenorm", |
| "patch_size", |
| "patch_stride", |
| "projection_dim", |
| "qkv_bias", |
| "window_size" |
| ] |
| }, |
| "component_kind": "config", |
| "parent_class": "PreTrainedConfig", |
| "relation": "new", |
| "target_class": "Florence2VisionConfig" |
| }, |
| { |
| "added": { |
| "attrs": [ |
| "image_token_id", |
| "is_encoder_decoder", |
| "model_type", |
| "sub_configs", |
| "text_config", |
| "tie_word_embeddings", |
| "vision_config" |
| ], |
| "methods": [ |
| "__post_init__" |
| ] |
| }, |
| "component_kind": "config", |
| "parent_class": "PreTrainedConfig", |
| "relation": "new", |
| "target_class": "Florence2Config" |
| }, |
| { |
| "component_kind": null, |
| "parent_class": "LlavaProcessorKwargs", |
| "relation": "new", |
| "target_class": "Florence2ProcessorKwargs" |
| }, |
| { |
| "added": { |
| "methods": [ |
| "__call__", |
| "__init__", |
| "_construct_prompts", |
| "_get_num_multimodal_tokens", |
| "batch_decode", |
| "decode", |
| "model_input_names", |
| "post_process_generation", |
| "post_process_image_text_to_text" |
| ] |
| }, |
| "component_kind": null, |
| "parent_class": "ProcessorMixin", |
| "relation": "new", |
| "target_class": "Florence2Processor" |
| }, |
| { |
| "added": { |
| "methods": [ |
| "__call__", |
| "__init__", |
| "_find_matched_token_indices", |
| "decode_with_spans", |
| "dequantize", |
| "parse_description_with_bboxes_from_text_and_spans", |
| "parse_description_with_polygons_from_text_and_spans", |
| "parse_ocr_from_text_and_spans", |
| "parse_phrase_grounding_from_text_and_spans", |
| "quantize" |
| ] |
| }, |
| "component_kind": null, |
| "parent_class": null, |
| "relation": "new", |
| "target_class": "Florence2PostProcessor" |
| }, |
| { |
| "added": { |
| "methods": [ |
| "__init__", |
| "forward" |
| ] |
| }, |
| "component_kind": null, |
| "parent_class": "Module", |
| "relation": "new", |
| "target_class": "Florence2VisionLearnedAbsolutePositionEmbedding2D" |
| }, |
| { |
| "added": { |
| "methods": [ |
| "__init__", |
| "forward", |
| "get_sinusoid_embeddings" |
| ] |
| }, |
| "component_kind": null, |
| "parent_class": "Module", |
| "relation": "new", |
| "target_class": "Florence2VisionPositionalEmbeddingCosine1D" |
| }, |
| { |
| "component_kind": "feed_forward", |
| "overridden": { |
| "methods": [ |
| "__init__" |
| ] |
| }, |
| "parent_class": "Llama4VisionMLP", |
| "parent_model": "llama4", |
| "relation": "inherits", |
| "target_class": "Florence2VisionMLP" |
| }, |
| { |
| "added": { |
| "methods": [ |
| "__init__", |
| "forward" |
| ] |
| }, |
| "component_kind": null, |
| "parent_class": "Module", |
| "relation": "new", |
| "target_class": "Florence2VisionConvEmbed" |
| }, |
| { |
| "added": { |
| "methods": [ |
| "__init__", |
| "forward" |
| ] |
| }, |
| "component_kind": "attention", |
| "parent_class": "Module", |
| "relation": "new", |
| "target_class": "Florence2VisionChannelAttention" |
| }, |
| { |
| "added": { |
| "methods": [ |
| "__init__", |
| "forward" |
| ] |
| }, |
| "component_kind": "transformer_block", |
| "parent_class": "Module", |
| "relation": "new", |
| "target_class": "Florence2VisionChannelBlock" |
| }, |
| { |
| "added": { |
| "methods": [ |
| "__init__", |
| "forward" |
| ] |
| }, |
| "component_kind": "attention", |
| "parent_class": "Module", |
| "relation": "new", |
| "target_class": "Florence2VisionWindowAttention" |
| }, |
| { |
| "added": { |
| "methods": [ |
| "__init__", |
| "forward" |
| ] |
| }, |
| "component_kind": "transformer_block", |
| "parent_class": "Module", |
| "relation": "new", |
| "target_class": "Florence2VisionSpatialBlock" |
| }, |
| { |
| "added": { |
| "methods": [ |
| "__init__", |
| "forward" |
| ] |
| }, |
| "component_kind": "transformer_block", |
| "parent_class": "Module", |
| "relation": "new", |
| "target_class": "Florence2VisionBlock" |
| }, |
| { |
| "added": { |
| "attrs": [ |
| "_can_compile_fullgraph", |
| "_can_record_outputs", |
| "_supports_flash_attn", |
| "_supports_flex_attn", |
| "_supports_sdpa", |
| "config_class", |
| "input_modalities", |
| "main_input_name" |
| ] |
| }, |
| "component_kind": "model", |
| "parent_class": "PreTrainedModel", |
| "relation": "new", |
| "target_class": "Florence2VisionPreTrainedModel" |
| }, |
| { |
| "added": { |
| "methods": [ |
| "__init__", |
| "forward" |
| ] |
| }, |
| "component_kind": null, |
| "parent_class": "Florence2VisionPreTrainedModel", |
| "relation": "new", |
| "target_class": "Florence2VisionBackbone" |
| }, |
| { |
| "added": { |
| "methods": [ |
| "__init__", |
| "forward" |
| ] |
| }, |
| "component_kind": null, |
| "parent_class": "Module", |
| "relation": "new", |
| "target_class": "Florence2MultiModalProjector" |
| }, |
| { |
| "added": { |
| "attrs": [ |
| "image_hidden_states" |
| ] |
| }, |
| "component_kind": null, |
| "parent_class": "Seq2SeqModelOutput", |
| "relation": "new", |
| "target_class": "Florence2Seq2SeqModelOutput" |
| }, |
| { |
| "added": { |
| "attrs": [ |
| "image_hidden_states" |
| ] |
| }, |
| "component_kind": null, |
| "parent_class": "Seq2SeqLMOutput", |
| "relation": "new", |
| "target_class": "Florence2Seq2SeqLMOutput" |
| }, |
| { |
| "added": { |
| "attrs": [ |
| "config_class" |
| ], |
| "methods": [ |
| "_init_weights" |
| ] |
| }, |
| "component_kind": "model", |
| "overridden": { |
| "attrs": [ |
| "_supports_attention_backend", |
| "base_model_prefix" |
| ] |
| }, |
| "parent_class": "LlavaPreTrainedModel", |
| "relation": "inherits", |
| "target_class": "Florence2PreTrainedModel" |
| }, |
| { |
| "added": { |
| "methods": [ |
| "get_encoder" |
| ] |
| }, |
| "component_kind": "model", |
| "overridden": { |
| "methods": [ |
| "__init__", |
| "forward", |
| "get_image_features" |
| ] |
| }, |
| "parent_class": "LlavaModel", |
| "relation": "inherits", |
| "target_class": "Florence2Model" |
| }, |
| { |
| "added": { |
| "methods": [ |
| "_prepare_encoder_decoder_kwargs_for_generation", |
| "get_placeholder_mask" |
| ] |
| }, |
| "component_kind": "lm_head", |
| "overridden": { |
| "attrs": [ |
| "_tied_weights_keys" |
| ], |
| "methods": [ |
| "forward", |
| "get_image_features" |
| ] |
| }, |
| "parent_class": "LlavaForConditionalGeneration", |
| "relation": "inherits", |
| "target_class": "Florence2ForConditionalGeneration" |
| } |
| ], |
| "provenance": { |
| "config_class": "Florence2Config", |
| "config_module": "transformers.models.florence2.configuration_florence2", |
| "model_class": "Florence2Model", |
| "model_module": "transformers.models.florence2.modeling_florence2" |
| }, |
| "repeats": [ |
| { |
| "body": "florence2_vision_conv_embed", |
| "container_path_pattern": "model.vision_tower.convs", |
| "count": 4, |
| "count_expr": "4", |
| "count_source": "module_tree", |
| "id": "florence2_vision_conv_embeds", |
| "index_symbol": "i", |
| "item_path_pattern": "model.vision_tower.convs.{i}", |
| "kind": "symbolic_repeat", |
| "provenance": { |
| "class_name": "Florence2VisionConvEmbed", |
| "container_path_pattern": "model.vision_tower.convs", |
| "item_path_pattern": "model.vision_tower.convs.{i}", |
| "source": "module_tree_repeat_collapse" |
| }, |
| "repeated_class_name": "Florence2VisionConvEmbed" |
| }, |
| { |
| "body": "block", |
| "container_path_pattern": "model.vision_tower.blocks", |
| "count": 4, |
| "count_expr": "4", |
| "count_source": "module_tree", |
| "id": "blocks", |
| "index_symbol": "i", |
| "item_path_pattern": "model.vision_tower.blocks.{i}", |
| "kind": "symbolic_repeat", |
| "provenance": { |
| "class_name": "ModuleList", |
| "container_path_pattern": "model.vision_tower.blocks", |
| "item_path_pattern": "model.vision_tower.blocks.{i}", |
| "source": "module_tree_repeat_collapse" |
| }, |
| "repeated_class_name": "ModuleList" |
| }, |
| { |
| "body": "block_2", |
| "container_path_pattern": "model.vision_tower.blocks.{i}", |
| "count": 9, |
| "count_expr": "9", |
| "count_source": "module_tree", |
| "id": "block_2s", |
| "index_symbol": "j", |
| "item_path_pattern": "model.vision_tower.blocks.{i}.{j}", |
| "kind": "symbolic_repeat", |
| "provenance": { |
| "class_name": "Florence2VisionBlock", |
| "container_path_pattern": "model.vision_tower.blocks.{i}", |
| "item_path_pattern": "model.vision_tower.blocks.{i}.{j}", |
| "source": "module_tree_repeat_collapse" |
| }, |
| "repeated_class_name": "Florence2VisionBlock" |
| }, |
| { |
| "body": "encoder_layer", |
| "container_path_pattern": "model.language_model.encoder.layers", |
| "count": 12, |
| "count_expr": "12", |
| "count_source": "module_tree", |
| "id": "encoder_layers", |
| "index_symbol": "i", |
| "item_path_pattern": "model.language_model.encoder.layers.{i}", |
| "kind": "symbolic_repeat", |
| "provenance": { |
| "class_name": "BartEncoderLayer", |
| "container_path_pattern": "model.language_model.encoder.layers", |
| "item_path_pattern": "model.language_model.encoder.layers.{i}", |
| "source": "module_tree_repeat_collapse" |
| }, |
| "repeated_class_name": "BartEncoderLayer" |
| }, |
| { |
| "body": "decoder_layer", |
| "container_path_pattern": "model.language_model.decoder.layers", |
| "count": 12, |
| "count_expr": "12", |
| "count_source": "module_tree", |
| "id": "decoder_layers", |
| "index_symbol": "i", |
| "item_path_pattern": "model.language_model.decoder.layers.{i}", |
| "kind": "symbolic_repeat", |
| "provenance": { |
| "class_name": "BartDecoderLayer", |
| "container_path_pattern": "model.language_model.decoder.layers", |
| "item_path_pattern": "model.language_model.decoder.layers.{i}", |
| "source": "module_tree_repeat_collapse" |
| }, |
| "repeated_class_name": "BartDecoderLayer" |
| } |
| ], |
| "schema_version": "architecture-template-v0", |
| "templates": [ |
| { |
| "children": [ |
| "florence2_vision_conv_embed.conv", |
| "florence2_vision_conv_embed.norm" |
| ], |
| "class_name": "Florence2VisionConvEmbed", |
| "id": "florence2_vision_conv_embed", |
| "kind": "module", |
| "path_pattern": "model.vision_tower.convs.{i}" |
| }, |
| { |
| "class_name": "Conv2d", |
| "id": "florence2_vision_conv_embed.conv", |
| "kind": "convolution", |
| "path_pattern": "model.vision_tower.convs.{i}.conv" |
| }, |
| { |
| "attributes": { |
| "norm_type": "layer" |
| }, |
| "class_name": "LayerNorm", |
| "id": "florence2_vision_conv_embed.norm", |
| "kind": "normalization", |
| "path_pattern": "model.vision_tower.convs.{i}.norm" |
| }, |
| { |
| "children": [ |
| "block.0" |
| ], |
| "class_name": "ModuleList", |
| "id": "block", |
| "kind": "repeated_container", |
| "path_pattern": "model.vision_tower.blocks.{i}" |
| }, |
| { |
| "children": [ |
| "block.0.spatial_block", |
| "block.0.channel_block" |
| ], |
| "class_name": "Florence2VisionBlock", |
| "id": "block.0", |
| "kind": "transformer_block", |
| "path_pattern": "model.vision_tower.blocks.{i}.0" |
| }, |
| { |
| "children": [ |
| "block.0.spatial_block.conv1", |
| "block.0.spatial_block.norm1", |
| "block.0.spatial_block.window_attn", |
| "block.0.spatial_block.conv2", |
| "block.0.spatial_block.norm2", |
| "block.0.spatial_block.ffn" |
| ], |
| "class_name": "Florence2VisionSpatialBlock", |
| "id": "block.0.spatial_block", |
| "kind": "transformer_block", |
| "path_pattern": "model.vision_tower.blocks.{i}.0.spatial_block" |
| }, |
| { |
| "class_name": "Conv2d", |
| "id": "block.0.spatial_block.conv1", |
| "kind": "convolution", |
| "path_pattern": "model.vision_tower.blocks.{i}.0.spatial_block.conv1" |
| }, |
| { |
| "attributes": { |
| "norm_type": "layer" |
| }, |
| "class_name": "LayerNorm", |
| "id": "block.0.spatial_block.norm1", |
| "kind": "normalization", |
| "path_pattern": "model.vision_tower.blocks.{i}.0.spatial_block.norm1" |
| }, |
| { |
| "attributes": { |
| "rope": false |
| }, |
| "children": [ |
| "block.0.spatial_block.window_attn.qkv", |
| "block.0.spatial_block.window_attn.proj" |
| ], |
| "class_name": "Florence2VisionWindowAttention", |
| "id": "block.0.spatial_block.window_attn", |
| "kind": "attention", |
| "path_pattern": "model.vision_tower.blocks.{i}.0.spatial_block.window_attn" |
| }, |
| { |
| "attributes": { |
| "in_features": 128, |
| "out_features": 384 |
| }, |
| "class_name": "Linear", |
| "id": "block.0.spatial_block.window_attn.qkv", |
| "kind": "projection", |
| "path_pattern": "model.vision_tower.blocks.{i}.0.spatial_block.window_attn.qkv" |
| }, |
| { |
| "attributes": { |
| "in_features": 128, |
| "out_features": 128 |
| }, |
| "class_name": "Linear", |
| "id": "block.0.spatial_block.window_attn.proj", |
| "kind": "projection", |
| "path_pattern": "model.vision_tower.blocks.{i}.0.spatial_block.window_attn.proj" |
| }, |
| { |
| "class_name": "Conv2d", |
| "id": "block.0.spatial_block.conv2", |
| "kind": "convolution", |
| "path_pattern": "model.vision_tower.blocks.{i}.0.spatial_block.conv2" |
| }, |
| { |
| "attributes": { |
| "norm_type": "layer" |
| }, |
| "class_name": "LayerNorm", |
| "id": "block.0.spatial_block.norm2", |
| "kind": "normalization", |
| "path_pattern": "model.vision_tower.blocks.{i}.0.spatial_block.norm2" |
| }, |
| { |
| "attributes": { |
| "activation": "gelu" |
| }, |
| "children": [ |
| "block.0.spatial_block.ffn.fc1", |
| "block.0.spatial_block.ffn.fc2" |
| ], |
| "class_name": "Florence2VisionMLP", |
| "id": "block.0.spatial_block.ffn", |
| "kind": "feed_forward", |
| "path_pattern": "model.vision_tower.blocks.{i}.0.spatial_block.ffn" |
| }, |
| { |
| "attributes": { |
| "in_features": 128, |
| "out_features": 512 |
| }, |
| "class_name": "Linear", |
| "id": "block.0.spatial_block.ffn.fc1", |
| "kind": "projection", |
| "path_pattern": "model.vision_tower.blocks.{i}.0.spatial_block.ffn.fc1" |
| }, |
| { |
| "attributes": { |
| "in_features": 512, |
| "out_features": 128 |
| }, |
| "class_name": "Linear", |
| "id": "block.0.spatial_block.ffn.fc2", |
| "kind": "projection", |
| "path_pattern": "model.vision_tower.blocks.{i}.0.spatial_block.ffn.fc2" |
| }, |
| { |
| "children": [ |
| "block.0.channel_block.conv1", |
| "block.0.channel_block.norm1", |
| "block.0.channel_block.channel_attn", |
| "block.0.channel_block.conv2", |
| "block.0.channel_block.norm2", |
| "block.0.channel_block.ffn" |
| ], |
| "class_name": "Florence2VisionChannelBlock", |
| "id": "block.0.channel_block", |
| "kind": "transformer_block", |
| "path_pattern": "model.vision_tower.blocks.{i}.0.channel_block" |
| }, |
| { |
| "class_name": "Conv2d", |
| "id": "block.0.channel_block.conv1", |
| "kind": "convolution", |
| "path_pattern": "model.vision_tower.blocks.{i}.0.channel_block.conv1" |
| }, |
| { |
| "attributes": { |
| "norm_type": "layer" |
| }, |
| "class_name": "LayerNorm", |
| "id": "block.0.channel_block.norm1", |
| "kind": "normalization", |
| "path_pattern": "model.vision_tower.blocks.{i}.0.channel_block.norm1" |
| }, |
| { |
| "attributes": { |
| "rope": false |
| }, |
| "children": [ |
| "block.0.channel_block.channel_attn.qkv", |
| "block.0.channel_block.channel_attn.proj" |
| ], |
| "class_name": "Florence2VisionChannelAttention", |
| "id": "block.0.channel_block.channel_attn", |
| "kind": "attention", |
| "path_pattern": "model.vision_tower.blocks.{i}.0.channel_block.channel_attn" |
| }, |
| { |
| "attributes": { |
| "in_features": 128, |
| "out_features": 384 |
| }, |
| "class_name": "Linear", |
| "id": "block.0.channel_block.channel_attn.qkv", |
| "kind": "projection", |
| "path_pattern": "model.vision_tower.blocks.{i}.0.channel_block.channel_attn.qkv" |
| }, |
| { |
| "attributes": { |
| "in_features": 128, |
| "out_features": 128 |
| }, |
| "class_name": "Linear", |
| "id": "block.0.channel_block.channel_attn.proj", |
| "kind": "projection", |
| "path_pattern": "model.vision_tower.blocks.{i}.0.channel_block.channel_attn.proj" |
| }, |
| { |
| "class_name": "Conv2d", |
| "id": "block.0.channel_block.conv2", |
| "kind": "convolution", |
| "path_pattern": "model.vision_tower.blocks.{i}.0.channel_block.conv2" |
| }, |
| { |
| "attributes": { |
| "norm_type": "layer" |
| }, |
| "class_name": "LayerNorm", |
| "id": "block.0.channel_block.norm2", |
| "kind": "normalization", |
| "path_pattern": "model.vision_tower.blocks.{i}.0.channel_block.norm2" |
| }, |
| { |
| "attributes": { |
| "activation": "gelu" |
| }, |
| "children": [ |
| "block.0.channel_block.ffn.fc1", |
| "block.0.channel_block.ffn.fc2" |
| ], |
| "class_name": "Florence2VisionMLP", |
| "id": "block.0.channel_block.ffn", |
| "kind": "feed_forward", |
| "path_pattern": "model.vision_tower.blocks.{i}.0.channel_block.ffn" |
| }, |
| { |
| "attributes": { |
| "in_features": 128, |
| "out_features": 512 |
| }, |
| "class_name": "Linear", |
| "id": "block.0.channel_block.ffn.fc1", |
| "kind": "projection", |
| "path_pattern": "model.vision_tower.blocks.{i}.0.channel_block.ffn.fc1" |
| }, |
| { |
| "attributes": { |
| "in_features": 512, |
| "out_features": 128 |
| }, |
| "class_name": "Linear", |
| "id": "block.0.channel_block.ffn.fc2", |
| "kind": "projection", |
| "path_pattern": "model.vision_tower.blocks.{i}.0.channel_block.ffn.fc2" |
| }, |
| { |
| "children": [ |
| "block_2.spatial_block", |
| "block_2.channel_block" |
| ], |
| "class_name": "Florence2VisionBlock", |
| "id": "block_2", |
| "kind": "transformer_block", |
| "path_pattern": "model.vision_tower.blocks.{i}.{j}" |
| }, |
| { |
| "children": [ |
| "block_2.spatial_block.conv1", |
| "block_2.spatial_block.norm1", |
| "block_2.spatial_block.window_attn", |
| "block_2.spatial_block.conv2", |
| "block_2.spatial_block.norm2", |
| "block_2.spatial_block.ffn" |
| ], |
| "class_name": "Florence2VisionSpatialBlock", |
| "id": "block_2.spatial_block", |
| "kind": "transformer_block", |
| "path_pattern": "model.vision_tower.blocks.{i}.{j}.spatial_block" |
| }, |
| { |
| "class_name": "Conv2d", |
| "id": "block_2.spatial_block.conv1", |
| "kind": "convolution", |
| "path_pattern": "model.vision_tower.blocks.{i}.{j}.spatial_block.conv1" |
| }, |
| { |
| "attributes": { |
| "norm_type": "layer" |
| }, |
| "class_name": "LayerNorm", |
| "id": "block_2.spatial_block.norm1", |
| "kind": "normalization", |
| "path_pattern": "model.vision_tower.blocks.{i}.{j}.spatial_block.norm1" |
| }, |
| { |
| "attributes": { |
| "rope": false |
| }, |
| "children": [ |
| "block_2.spatial_block.window_attn.qkv", |
| "block_2.spatial_block.window_attn.proj" |
| ], |
| "class_name": "Florence2VisionWindowAttention", |
| "id": "block_2.spatial_block.window_attn", |
| "kind": "attention", |
| "path_pattern": "model.vision_tower.blocks.{i}.{j}.spatial_block.window_attn" |
| }, |
| { |
| "attributes": { |
| "in_features": 512, |
| "out_features": 1536 |
| }, |
| "class_name": "Linear", |
| "id": "block_2.spatial_block.window_attn.qkv", |
| "kind": "projection", |
| "path_pattern": "model.vision_tower.blocks.{i}.{j}.spatial_block.window_attn.qkv" |
| }, |
| { |
| "attributes": { |
| "in_features": 512, |
| "out_features": 512 |
| }, |
| "class_name": "Linear", |
| "id": "block_2.spatial_block.window_attn.proj", |
| "kind": "projection", |
| "path_pattern": "model.vision_tower.blocks.{i}.{j}.spatial_block.window_attn.proj" |
| }, |
| { |
| "class_name": "Conv2d", |
| "id": "block_2.spatial_block.conv2", |
| "kind": "convolution", |
| "path_pattern": "model.vision_tower.blocks.{i}.{j}.spatial_block.conv2" |
| }, |
| { |
| "attributes": { |
| "norm_type": "layer" |
| }, |
| "class_name": "LayerNorm", |
| "id": "block_2.spatial_block.norm2", |
| "kind": "normalization", |
| "path_pattern": "model.vision_tower.blocks.{i}.{j}.spatial_block.norm2" |
| }, |
| { |
| "attributes": { |
| "activation": "gelu" |
| }, |
| "children": [ |
| "block_2.spatial_block.ffn.fc1", |
| "block_2.spatial_block.ffn.fc2" |
| ], |
| "class_name": "Florence2VisionMLP", |
| "id": "block_2.spatial_block.ffn", |
| "kind": "feed_forward", |
| "path_pattern": "model.vision_tower.blocks.{i}.{j}.spatial_block.ffn" |
| }, |
| { |
| "attributes": { |
| "in_features": 512, |
| "out_features": 2048 |
| }, |
| "class_name": "Linear", |
| "id": "block_2.spatial_block.ffn.fc1", |
| "kind": "projection", |
| "path_pattern": "model.vision_tower.blocks.{i}.{j}.spatial_block.ffn.fc1" |
| }, |
| { |
| "attributes": { |
| "in_features": 2048, |
| "out_features": 512 |
| }, |
| "class_name": "Linear", |
| "id": "block_2.spatial_block.ffn.fc2", |
| "kind": "projection", |
| "path_pattern": "model.vision_tower.blocks.{i}.{j}.spatial_block.ffn.fc2" |
| }, |
| { |
| "children": [ |
| "block_2.channel_block.conv1", |
| "block_2.channel_block.norm1", |
| "block_2.channel_block.channel_attn", |
| "block_2.channel_block.conv2", |
| "block_2.channel_block.norm2", |
| "block_2.channel_block.ffn" |
| ], |
| "class_name": "Florence2VisionChannelBlock", |
| "id": "block_2.channel_block", |
| "kind": "transformer_block", |
| "path_pattern": "model.vision_tower.blocks.{i}.{j}.channel_block" |
| }, |
| { |
| "class_name": "Conv2d", |
| "id": "block_2.channel_block.conv1", |
| "kind": "convolution", |
| "path_pattern": "model.vision_tower.blocks.{i}.{j}.channel_block.conv1" |
| }, |
| { |
| "attributes": { |
| "norm_type": "layer" |
| }, |
| "class_name": "LayerNorm", |
| "id": "block_2.channel_block.norm1", |
| "kind": "normalization", |
| "path_pattern": "model.vision_tower.blocks.{i}.{j}.channel_block.norm1" |
| }, |
| { |
| "attributes": { |
| "rope": false |
| }, |
| "children": [ |
| "block_2.channel_block.channel_attn.qkv", |
| "block_2.channel_block.channel_attn.proj" |
| ], |
| "class_name": "Florence2VisionChannelAttention", |
| "id": "block_2.channel_block.channel_attn", |
| "kind": "attention", |
| "path_pattern": "model.vision_tower.blocks.{i}.{j}.channel_block.channel_attn" |
| }, |
| { |
| "attributes": { |
| "in_features": 512, |
| "out_features": 1536 |
| }, |
| "class_name": "Linear", |
| "id": "block_2.channel_block.channel_attn.qkv", |
| "kind": "projection", |
| "path_pattern": "model.vision_tower.blocks.{i}.{j}.channel_block.channel_attn.qkv" |
| }, |
| { |
| "attributes": { |
| "in_features": 512, |
| "out_features": 512 |
| }, |
| "class_name": "Linear", |
| "id": "block_2.channel_block.channel_attn.proj", |
| "kind": "projection", |
| "path_pattern": "model.vision_tower.blocks.{i}.{j}.channel_block.channel_attn.proj" |
| }, |
| { |
| "class_name": "Conv2d", |
| "id": "block_2.channel_block.conv2", |
| "kind": "convolution", |
| "path_pattern": "model.vision_tower.blocks.{i}.{j}.channel_block.conv2" |
| }, |
| { |
| "attributes": { |
| "norm_type": "layer" |
| }, |
| "class_name": "LayerNorm", |
| "id": "block_2.channel_block.norm2", |
| "kind": "normalization", |
| "path_pattern": "model.vision_tower.blocks.{i}.{j}.channel_block.norm2" |
| }, |
| { |
| "attributes": { |
| "activation": "gelu" |
| }, |
| "children": [ |
| "block_2.channel_block.ffn.fc1", |
| "block_2.channel_block.ffn.fc2" |
| ], |
| "class_name": "Florence2VisionMLP", |
| "id": "block_2.channel_block.ffn", |
| "kind": "feed_forward", |
| "path_pattern": "model.vision_tower.blocks.{i}.{j}.channel_block.ffn" |
| }, |
| { |
| "attributes": { |
| "in_features": 512, |
| "out_features": 2048 |
| }, |
| "class_name": "Linear", |
| "id": "block_2.channel_block.ffn.fc1", |
| "kind": "projection", |
| "path_pattern": "model.vision_tower.blocks.{i}.{j}.channel_block.ffn.fc1" |
| }, |
| { |
| "attributes": { |
| "in_features": 2048, |
| "out_features": 512 |
| }, |
| "class_name": "Linear", |
| "id": "block_2.channel_block.ffn.fc2", |
| "kind": "projection", |
| "path_pattern": "model.vision_tower.blocks.{i}.{j}.channel_block.ffn.fc2" |
| }, |
| { |
| "children": [ |
| "encoder_layer.self_attn", |
| "encoder_layer.self_attn_layer_norm", |
| "encoder_layer.final_layer_norm" |
| ], |
| "class_name": "BartEncoderLayer", |
| "id": "encoder_layer", |
| "kind": "transformer_block", |
| "path_pattern": "model.language_model.encoder.layers.{i}" |
| }, |
| { |
| "attributes": { |
| "head_dim": 64, |
| "n_heads": 16, |
| "n_kv_heads": 16, |
| "rope": false, |
| "variant": "MHA" |
| }, |
| "children": [ |
| "encoder_layer.self_attn.k_proj", |
| "encoder_layer.self_attn.v_proj", |
| "encoder_layer.self_attn.q_proj", |
| "encoder_layer.self_attn.out_proj" |
| ], |
| "class_name": "BartAttention", |
| "id": "encoder_layer.self_attn", |
| "kind": "attention", |
| "path_pattern": "model.language_model.encoder.layers.{i}.self_attn" |
| }, |
| { |
| "attributes": { |
| "in_features": "config.hidden_size", |
| "out_features": "config.hidden_size" |
| }, |
| "class_name": "Linear", |
| "id": "encoder_layer.self_attn.k_proj", |
| "kind": "projection", |
| "path_pattern": "model.language_model.encoder.layers.{i}.self_attn.k_proj" |
| }, |
| { |
| "attributes": { |
| "in_features": "config.hidden_size", |
| "out_features": "config.hidden_size" |
| }, |
| "class_name": "Linear", |
| "id": "encoder_layer.self_attn.v_proj", |
| "kind": "projection", |
| "path_pattern": "model.language_model.encoder.layers.{i}.self_attn.v_proj" |
| }, |
| { |
| "attributes": { |
| "in_features": "config.hidden_size", |
| "out_features": "config.hidden_size" |
| }, |
| "class_name": "Linear", |
| "id": "encoder_layer.self_attn.q_proj", |
| "kind": "projection", |
| "path_pattern": "model.language_model.encoder.layers.{i}.self_attn.q_proj" |
| }, |
| { |
| "attributes": { |
| "in_features": "config.hidden_size", |
| "out_features": "config.hidden_size" |
| }, |
| "class_name": "Linear", |
| "id": "encoder_layer.self_attn.out_proj", |
| "kind": "projection", |
| "path_pattern": "model.language_model.encoder.layers.{i}.self_attn.out_proj" |
| }, |
| { |
| "attributes": { |
| "norm_type": "layer" |
| }, |
| "class_name": "LayerNorm", |
| "id": "encoder_layer.self_attn_layer_norm", |
| "kind": "normalization", |
| "path_pattern": "model.language_model.encoder.layers.{i}.self_attn_layer_norm" |
| }, |
| { |
| "attributes": { |
| "norm_type": "layer" |
| }, |
| "class_name": "LayerNorm", |
| "id": "encoder_layer.final_layer_norm", |
| "kind": "normalization", |
| "path_pattern": "model.language_model.encoder.layers.{i}.final_layer_norm" |
| }, |
| { |
| "children": [ |
| "decoder_layer.self_attn", |
| "decoder_layer.self_attn_layer_norm", |
| "decoder_layer.encoder_attn", |
| "decoder_layer.encoder_attn_layer_norm", |
| "decoder_layer.final_layer_norm" |
| ], |
| "class_name": "BartDecoderLayer", |
| "id": "decoder_layer", |
| "kind": "transformer_block", |
| "path_pattern": "model.language_model.decoder.layers.{i}" |
| }, |
| { |
| "attributes": { |
| "head_dim": 64, |
| "n_heads": 16, |
| "n_kv_heads": 16, |
| "rope": false, |
| "variant": "MHA" |
| }, |
| "children": [ |
| "decoder_layer.self_attn.k_proj", |
| "decoder_layer.self_attn.v_proj", |
| "decoder_layer.self_attn.q_proj", |
| "decoder_layer.self_attn.out_proj" |
| ], |
| "class_name": "BartAttention", |
| "id": "decoder_layer.self_attn", |
| "kind": "attention", |
| "path_pattern": "model.language_model.decoder.layers.{i}.self_attn" |
| }, |
| { |
| "attributes": { |
| "in_features": "config.hidden_size", |
| "out_features": "config.hidden_size" |
| }, |
| "class_name": "Linear", |
| "id": "decoder_layer.self_attn.k_proj", |
| "kind": "projection", |
| "path_pattern": "model.language_model.decoder.layers.{i}.self_attn.k_proj" |
| }, |
| { |
| "attributes": { |
| "in_features": "config.hidden_size", |
| "out_features": "config.hidden_size" |
| }, |
| "class_name": "Linear", |
| "id": "decoder_layer.self_attn.v_proj", |
| "kind": "projection", |
| "path_pattern": "model.language_model.decoder.layers.{i}.self_attn.v_proj" |
| }, |
| { |
| "attributes": { |
| "in_features": "config.hidden_size", |
| "out_features": "config.hidden_size" |
| }, |
| "class_name": "Linear", |
| "id": "decoder_layer.self_attn.q_proj", |
| "kind": "projection", |
| "path_pattern": "model.language_model.decoder.layers.{i}.self_attn.q_proj" |
| }, |
| { |
| "attributes": { |
| "in_features": "config.hidden_size", |
| "out_features": "config.hidden_size" |
| }, |
| "class_name": "Linear", |
| "id": "decoder_layer.self_attn.out_proj", |
| "kind": "projection", |
| "path_pattern": "model.language_model.decoder.layers.{i}.self_attn.out_proj" |
| }, |
| { |
| "attributes": { |
| "norm_type": "layer" |
| }, |
| "class_name": "LayerNorm", |
| "id": "decoder_layer.self_attn_layer_norm", |
| "kind": "normalization", |
| "path_pattern": "model.language_model.decoder.layers.{i}.self_attn_layer_norm" |
| }, |
| { |
| "attributes": { |
| "head_dim": 64, |
| "n_heads": 16, |
| "n_kv_heads": 16, |
| "rope": false, |
| "variant": "MHA" |
| }, |
| "children": [ |
| "decoder_layer.encoder_attn.k_proj", |
| "decoder_layer.encoder_attn.v_proj", |
| "decoder_layer.encoder_attn.q_proj", |
| "decoder_layer.encoder_attn.out_proj" |
| ], |
| "class_name": "BartAttention", |
| "id": "decoder_layer.encoder_attn", |
| "kind": "cross_attention", |
| "path_pattern": "model.language_model.decoder.layers.{i}.encoder_attn" |
| }, |
| { |
| "attributes": { |
| "in_features": "config.hidden_size", |
| "out_features": "config.hidden_size" |
| }, |
| "class_name": "Linear", |
| "id": "decoder_layer.encoder_attn.k_proj", |
| "kind": "projection", |
| "path_pattern": "model.language_model.decoder.layers.{i}.encoder_attn.k_proj" |
| }, |
| { |
| "attributes": { |
| "in_features": "config.hidden_size", |
| "out_features": "config.hidden_size" |
| }, |
| "class_name": "Linear", |
| "id": "decoder_layer.encoder_attn.v_proj", |
| "kind": "projection", |
| "path_pattern": "model.language_model.decoder.layers.{i}.encoder_attn.v_proj" |
| }, |
| { |
| "attributes": { |
| "in_features": "config.hidden_size", |
| "out_features": "config.hidden_size" |
| }, |
| "class_name": "Linear", |
| "id": "decoder_layer.encoder_attn.q_proj", |
| "kind": "projection", |
| "path_pattern": "model.language_model.decoder.layers.{i}.encoder_attn.q_proj" |
| }, |
| { |
| "attributes": { |
| "in_features": "config.hidden_size", |
| "out_features": "config.hidden_size" |
| }, |
| "class_name": "Linear", |
| "id": "decoder_layer.encoder_attn.out_proj", |
| "kind": "projection", |
| "path_pattern": "model.language_model.decoder.layers.{i}.encoder_attn.out_proj" |
| }, |
| { |
| "attributes": { |
| "norm_type": "layer" |
| }, |
| "class_name": "LayerNorm", |
| "id": "decoder_layer.encoder_attn_layer_norm", |
| "kind": "normalization", |
| "path_pattern": "model.language_model.decoder.layers.{i}.encoder_attn_layer_norm" |
| }, |
| { |
| "attributes": { |
| "norm_type": "layer" |
| }, |
| "class_name": "LayerNorm", |
| "id": "decoder_layer.final_layer_norm", |
| "kind": "normalization", |
| "path_pattern": "model.language_model.decoder.layers.{i}.final_layer_norm" |
| } |
| ] |
| } |
|
|
