| { |
| "architecture": { |
| "attention_variant": "MHA", |
| "family": "audio_classification", |
| "is_moe": false, |
| "mixer": "attention", |
| "view": "encoder" |
| }, |
| "capabilities": { |
| "attention_backends": [ |
| "eager", |
| "sdpa", |
| "flash_attention", |
| "flex_attention" |
| ], |
| "attention_patterns": [ |
| "bidirectional" |
| ], |
| "attention_schedule": null, |
| "task_heads": [ |
| "audio_classification" |
| ], |
| "tensor_parallel": false |
| }, |
| "components": [ |
| { |
| "children": [ |
| "embeddings", |
| "layers", |
| "layernorm" |
| ], |
| "class_name": "ASTModel", |
| "id": "model", |
| "kind": "model", |
| "path_pattern": "model" |
| }, |
| { |
| "attributes": { |
| "embedding_dim": "config.hidden_size" |
| }, |
| "children": [ |
| "embeddings.patch_embeddings" |
| ], |
| "class_name": "ASTEmbeddings", |
| "id": "embeddings", |
| "kind": "embedding", |
| "path_pattern": "model.embeddings" |
| }, |
| { |
| "attributes": { |
| "embedding_dim": "config.hidden_size" |
| }, |
| "children": [ |
| "embeddings.patch_embeddings.projection" |
| ], |
| "class_name": "ASTPatchEmbeddings", |
| "id": "embeddings.patch_embeddings", |
| "kind": "embedding", |
| "path_pattern": "model.embeddings.patch_embeddings" |
| }, |
| { |
| "class_name": "Conv2d", |
| "id": "embeddings.patch_embeddings.projection", |
| "kind": "convolution", |
| "path_pattern": "model.embeddings.patch_embeddings.projection" |
| }, |
| { |
| "attributes": { |
| "norm_type": "layer" |
| }, |
| "class_name": "LayerNorm", |
| "id": "layernorm", |
| "kind": "normalization", |
| "path_pattern": "model.layernorm" |
| } |
| ], |
| "config": { |
| "class_name": "ASTConfig", |
| "model_type": "audio-spectrogram-transformer", |
| "module": "transformers.models.audio_spectrogram_transformer.configuration_audio_spectrogram_transformer", |
| "referenced_fields": { |
| "num_hidden_layers": 12 |
| }, |
| "salient_fields": { |
| "hidden_act": "gelu", |
| "hidden_size": 768, |
| "intermediate_size": 3072, |
| "is_encoder_decoder": false, |
| "num_attention_heads": 12, |
| "num_hidden_layers": 12 |
| } |
| }, |
| "edges": [ |
| { |
| "kind": "data", |
| "source": "embeddings", |
| "target": "layers" |
| }, |
| { |
| "kind": "data", |
| "source": "layers", |
| "target": "layernorm" |
| }, |
| { |
| "kind": "data", |
| "source": "layer.attention", |
| "target": "layer.layernorm_before" |
| }, |
| { |
| "kind": "data", |
| "source": "layer.layernorm_before", |
| "target": "layer.layernorm_after" |
| }, |
| { |
| "kind": "data", |
| "source": "layer.layernorm_after", |
| "target": "layer.mlp" |
| }, |
| { |
| "kind": "mask", |
| "source": "input:attention_mask", |
| "target": "layer.attention" |
| }, |
| { |
| "kind": "residual", |
| "source": "layer", |
| "target": "layer.attention" |
| }, |
| { |
| "kind": "residual", |
| "source": "layer", |
| "target": "layer.mlp" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "layer.attention", |
| "target": "layer.attention.q_proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "layer.attention", |
| "target": "layer.attention.k_proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "layer.attention", |
| "target": "layer.attention.v_proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "layer.attention.q_proj", |
| "target": "layer.attention.o_proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "layer.attention.k_proj", |
| "target": "layer.attention.o_proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "layer.attention.v_proj", |
| "target": "layer.attention.o_proj" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "layer.mlp", |
| "target": "layer.mlp.fc1" |
| }, |
| { |
| "kind": "data", |
| "provenance": "intra_module", |
| "source": "layer.mlp.fc1", |
| "target": "layer.mlp.fc2" |
| } |
| ], |
| "extends": "vit", |
| "model_type": "audio-spectrogram-transformer", |
| "patches": [ |
| { |
| "added": { |
| "methods": [ |
| "__init__", |
| "forward" |
| ] |
| }, |
| "component_kind": "embedding", |
| "parent_class": "Module", |
| "relation": "new", |
| "target_class": "ASTPatchEmbeddings" |
| }, |
| { |
| "added": { |
| "methods": [ |
| "__init__", |
| "forward", |
| "get_shape" |
| ] |
| }, |
| "component_kind": "embedding", |
| "parent_class": "Module", |
| "relation": "new", |
| "target_class": "ASTEmbeddings" |
| }, |
| { |
| "component_kind": "model", |
| "overridden": { |
| "attrs": [ |
| "_can_record_outputs", |
| "_no_split_modules", |
| "base_model_prefix", |
| "config", |
| "input_modalities", |
| "main_input_name" |
| ], |
| "methods": [ |
| "_init_weights" |
| ] |
| }, |
| "parent_class": "ViTPreTrainedModel", |
| "relation": "inherits", |
| "target_class": "ASTPreTrainedModel" |
| }, |
| { |
| "added": { |
| "methods": [ |
| "__init__", |
| "forward", |
| "get_input_embeddings" |
| ] |
| }, |
| "component_kind": "model", |
| "parent_class": "ASTPreTrainedModel", |
| "relation": "new", |
| "target_class": "ASTModel" |
| }, |
| { |
| "added": { |
| "methods": [ |
| "__init__", |
| "forward" |
| ] |
| }, |
| "component_kind": "feed_forward", |
| "parent_class": "Module", |
| "relation": "new", |
| "target_class": "ASTMLPHead" |
| }, |
| { |
| "added": { |
| "methods": [ |
| "__init__", |
| "forward" |
| ] |
| }, |
| "component_kind": null, |
| "parent_class": "ASTPreTrainedModel", |
| "relation": "new", |
| "target_class": "ASTForAudioClassification" |
| } |
| ], |
| "provenance": { |
| "config_class": "ASTConfig", |
| "config_module": "transformers.models.audio_spectrogram_transformer.configuration_audio_spectrogram_transformer", |
| "model_class": "ASTModel", |
| "model_module": "transformers.models.audio_spectrogram_transformer.modeling_audio_spectrogram_transformer" |
| }, |
| "repeats": [ |
| { |
| "body": "layer", |
| "container_path_pattern": "model.layers", |
| "count": 12, |
| "count_expr": "config.num_hidden_layers", |
| "count_source": "config", |
| "id": "layers", |
| "index_symbol": "i", |
| "item_path_pattern": "model.layers.{i}", |
| "kind": "symbolic_repeat", |
| "provenance": { |
| "class_name": "ASTLayer", |
| "container_path_pattern": "model.layers", |
| "item_path_pattern": "model.layers.{i}", |
| "source": "module_tree_repeat_collapse" |
| }, |
| "repeated_class_name": "ASTLayer" |
| } |
| ], |
| "schema_version": "architecture-template-v0", |
| "templates": [ |
| { |
| "children": [ |
| "layer.attention", |
| "layer.layernorm_before", |
| "layer.layernorm_after", |
| "layer.mlp" |
| ], |
| "class_name": "ASTLayer", |
| "id": "layer", |
| "kind": "transformer_block", |
| "path_pattern": "model.layers.{i}" |
| }, |
| { |
| "attributes": { |
| "head_dim": 64, |
| "n_heads": 12, |
| "n_kv_heads": 12, |
| "pattern": "bidirectional", |
| "rope": false, |
| "variant": "MHA" |
| }, |
| "children": [ |
| "layer.attention.q_proj", |
| "layer.attention.k_proj", |
| "layer.attention.v_proj", |
| "layer.attention.o_proj" |
| ], |
| "class_name": "ASTAttention", |
| "id": "layer.attention", |
| "kind": "attention", |
| "path_pattern": "model.layers.{i}.attention" |
| }, |
| { |
| "attributes": { |
| "in_features": "config.hidden_size", |
| "out_features": "config.hidden_size" |
| }, |
| "class_name": "Linear", |
| "id": "layer.attention.q_proj", |
| "kind": "projection", |
| "path_pattern": "model.layers.{i}.attention.q_proj" |
| }, |
| { |
| "attributes": { |
| "in_features": "config.hidden_size", |
| "out_features": "config.hidden_size" |
| }, |
| "class_name": "Linear", |
| "id": "layer.attention.k_proj", |
| "kind": "projection", |
| "path_pattern": "model.layers.{i}.attention.k_proj" |
| }, |
| { |
| "attributes": { |
| "in_features": "config.hidden_size", |
| "out_features": "config.hidden_size" |
| }, |
| "class_name": "Linear", |
| "id": "layer.attention.v_proj", |
| "kind": "projection", |
| "path_pattern": "model.layers.{i}.attention.v_proj" |
| }, |
| { |
| "attributes": { |
| "in_features": "config.hidden_size", |
| "out_features": "config.hidden_size" |
| }, |
| "class_name": "Linear", |
| "id": "layer.attention.o_proj", |
| "kind": "projection", |
| "path_pattern": "model.layers.{i}.attention.o_proj" |
| }, |
| { |
| "attributes": { |
| "norm_type": "layer" |
| }, |
| "class_name": "LayerNorm", |
| "id": "layer.layernorm_before", |
| "kind": "normalization", |
| "path_pattern": "model.layers.{i}.layernorm_before" |
| }, |
| { |
| "attributes": { |
| "norm_type": "layer" |
| }, |
| "class_name": "LayerNorm", |
| "id": "layer.layernorm_after", |
| "kind": "normalization", |
| "path_pattern": "model.layers.{i}.layernorm_after" |
| }, |
| { |
| "attributes": { |
| "activation": "gelu", |
| "hidden_size": 768, |
| "intermediate_size": 3072 |
| }, |
| "children": [ |
| "layer.mlp.fc1", |
| "layer.mlp.fc2" |
| ], |
| "class_name": "ASTMLP", |
| "id": "layer.mlp", |
| "kind": "feed_forward", |
| "path_pattern": "model.layers.{i}.mlp" |
| }, |
| { |
| "attributes": { |
| "in_features": "config.hidden_size", |
| "out_features": "config.intermediate_size" |
| }, |
| "class_name": "Linear", |
| "id": "layer.mlp.fc1", |
| "kind": "projection", |
| "path_pattern": "model.layers.{i}.mlp.fc1" |
| }, |
| { |
| "attributes": { |
| "in_features": "config.intermediate_size", |
| "out_features": "config.hidden_size" |
| }, |
| "class_name": "Linear", |
| "id": "layer.mlp.fc2", |
| "kind": "projection", |
| "path_pattern": "model.layers.{i}.mlp.fc2" |
| } |
| ] |
| } |
|
|
