initial commit

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

config.json

@@ -0,0 +1,125 @@
+{
+  "auto_map": {
+    "AutoConfig": "configuration_seamless_m4t_v2_speech_encoder.SeamlessM4Tv2EncoderConfig",
+    "AutoModel": "modeling_seamless_m4t_v2_speech_encoder.SeamlessM4Tv2SpeechEncoder",
+    "AutoModelForSequenceClassification": "modeling_seamless_m4t_v2_speech_encoder.SeamlessM4Tv2ForAudioClassification",
+    "AutoModelForAudioClassification": "modeling_seamless_m4t_v2_speech_encoder.SeamlessM4Tv2ForAudioClassification"
+  },
+  "_name_or_path": "seamless-m4t-v2-large",
+  "activation_dropout": 0.0,
+  "activation_function": "relu",
+  "adaptor_dropout": 0.1,
+  "adaptor_kernel_size": 8,
+  "adaptor_stride": 8,
+  "add_adapter": true,
+  "architectures": [
+    "SeamlessM4TSpeechEncoder"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": 2,
+  "char_vocab_size": 10943,
+  "conv_depthwise_kernel_size": 31,
+  "decoder_attention_heads": 16,
+  "decoder_ffn_dim": 8192,
+  "decoder_layerdrop": 0.05,
+  "decoder_layers": 24,
+  "decoder_start_token_id": 3,
+  "dropout": 0.1,
+  "encoder_attention_heads": 16,
+  "encoder_ffn_dim": 8192,
+  "encoder_layerdrop": 0.05,
+  "encoder_layers": 24,
+  "eos_token_id": 3,
+  "feature_projection_input_dim": 160,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "is_encoder_decoder": true,
+  "lang_embed_dim": 256,
+  "layer_norm_eps": 1e-05,
+  "leaky_relu_slope": 0.1,
+  "left_max_position_embeddings": 64,
+  "max_new_tokens": 256,
+  "max_position_embeddings": 4096,
+  "model_type": "seamlessm4t-v2-large-speech_encoder",
+  "num_adapter_layers": 1,
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "pad_token_id": 0,
+  "position_embeddings_type": "relative_key",
+  "resblock_dilation_sizes": [
+    [
+      1,
+      3,
+      5
+    ],
+    [
+      1,
+      3,
+      5
+    ],
+    [
+      1,
+      3,
+      5
+    ]
+  ],
+  "resblock_kernel_sizes": [
+    3,
+    7,
+    11
+  ],
+  "right_max_position_embeddings": 8,
+  "sampling_rate": 16000,
+  "scale_embedding": true,
+  "speech_encoder_attention_heads": 16,
+  "speech_encoder_chunk_size": 20000,
+  "speech_encoder_dropout": 0.0,
+  "speech_encoder_hidden_act": "swish",
+  "speech_encoder_intermediate_size": 4096,
+  "speech_encoder_layerdrop": 0.1,
+  "speech_encoder_layers": 24,
+  "speech_encoder_left_chunk_num": 128,
+  "spkr_embed_dim": 256,
+  "t2u_bos_token_id": 0,
+  "t2u_decoder_attention_heads": 16,
+  "t2u_decoder_ffn_dim": 8192,
+  "t2u_decoder_layers": 6,
+  "t2u_encoder_attention_heads": 16,
+  "t2u_encoder_ffn_dim": 8192,
+  "t2u_encoder_layers": 6,
+  "t2u_eos_token_id": 2,
+  "t2u_max_position_embeddings": 4096,
+  "t2u_pad_token_id": 1,
+  "t2u_variance_pred_dropout": 0.5,
+  "t2u_variance_predictor_embed_dim": 1024,
+  "t2u_variance_predictor_hidden_dim": 256,
+  "t2u_variance_predictor_kernel_size": 3,
+  "t2u_vocab_size": 10082,
+  "torch_dtype": "float32",
+  "transformers_version": "4.45.2",
+  "unit_embed_dim": 1280,
+  "unit_hifi_gan_vocab_size": 10000,
+  "upsample_initial_channel": 512,
+  "upsample_kernel_sizes": [
+    11,
+    8,
+    8,
+    4,
+    4
+  ],
+  "upsample_rates": [
+    5,
+    4,
+    4,
+    2,
+    2
+  ],
+  "use_cache": true,
+  "var_pred_dropout": 0.5,
+  "variance_predictor_kernel_size": 3,
+  "vocab_size": 256102,
+  "vocoder_num_langs": 36,
+  "vocoder_num_spkrs": 200,
+  "vocoder_offset": 4
+}
+

configuration_seamless_m4t_v2_speech_encoder.py

@@ -0,0 +1,18 @@
+from transformers.models.seamless_m4t_v2.configuration_seamless_m4t_v2 import (
+    SeamlessM4Tv2Config,
+)
+from transformers.models.auto.configuration_auto import AutoConfig
+
+MODEL_TYPE = "seamlessm4t-v2-large-speech_encoder"
+
+
+class SeamlessM4Tv2EncoderConfig(SeamlessM4Tv2Config):
+    model_type = MODEL_TYPE
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+
+AutoConfig.register(MODEL_TYPE, SeamlessM4Tv2EncoderConfig)
+
+SeamlessM4Tv2EncoderConfig.register_for_auto_class()

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae3e6d6221ade043eaa54f3cafb2f37683617c209ad10d635b4fd73dcee2f591
+size 2540281584

modeling_seamless_m4t_v2_speech_encoder.py

@@ -0,0 +1,109 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers.models.seamless_m4t.modeling_seamless_m4t import (
+    _compute_new_attention_mask,
+)
+from transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2 import (
+    SeamlessM4Tv2SpeechEncoder,
+    SeamlessM4Tv2PreTrainedModel,
+)
+from .configuration_seamless_m4t_v2_speech_encoder import (
+    MODEL_TYPE,
+    SeamlessM4Tv2EncoderConfig,
+)
+from transformers.modeling_outputs import SequenceClassifierOutput
+
+from transformers.models.auto import AutoModel, AutoModelForAudioClassification, AutoModelForSequenceClassification
+
+
+class SeamlessM4Tv2SpeechEncoder(SeamlessM4Tv2SpeechEncoder):
+    model_type = MODEL_TYPE
+    config_class = SeamlessM4Tv2EncoderConfig
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def _compute_sub_sample_lengths_from_attention_mask(self, attention_mask):
+        pad = self.kernel_size // 2
+        seq_lens = attention_mask.size(1) - (1 - attention_mask.int()).sum(1)
+
+        seq_lens = ((seq_lens + 2 * pad - self.kernel_size) / self.stride) + 1
+
+        return seq_lens.floor()
+
+    @staticmethod
+    def mean_pooling(
+        hidden_states: torch.Tensor, attention_mask: torch.Tensor
+    ) -> torch.Tensor:
+        # hidden_states shape: (batch_size, sequence_length, hidden_size)
+        # attention_mask shape: (batch_size, sequence_length)
+
+        # Apply attention mask and avoid division by zero
+        input_mask_expanded = (
+            attention_mask.unsqueeze(-1).expand(hidden_states.size()).float()
+        )
+        sum_hidden_states = torch.sum(hidden_states * input_mask_expanded, 1)
+        sum_mask = input_mask_expanded.sum(1)
+
+        return sum_hidden_states / torch.clamp(sum_mask, min=1e-9)
+
+
+class SeamlessM4Tv2ForAudioClassification(SeamlessM4Tv2PreTrainedModel):
+    model_type = MODEL_TYPE
+    base_model_prefix = "model"
+    config_class = SeamlessM4Tv2EncoderConfig
+
+    def __init__(self, config, *args, **kwargs):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+
+        self.model = SeamlessM4Tv2SpeechEncoder(config)
+        self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False)
+
+    def forward(
+        self,
+        input_features: torch.Tensor,
+        attention_mask: torch.Tensor,
+        labels: None | torch.Tensor,
+        *args,
+        **kwargs,
+    ):
+        output_hidden_states = kwargs.pop("output_hidden_states", False)
+        outputs = self.model(
+            input_features,
+            attention_mask,
+            output_hidden_states=output_hidden_states,
+            *args,
+            **kwargs,
+        )
+        hidden_states = outputs.last_hidden_state
+        if attention_mask is not None:
+            sub_sampled_lengths = self._compute_sub_sample_lengths_from_attention_mask(
+                attention_mask
+            ).to(outputs.last_hidden_state.device)
+            attention_mask = _compute_new_attention_mask(
+                hidden_states=hidden_states, seq_lens=sub_sampled_lengths
+            )
+        hidden_states = self.model.mean_pooling(
+            outputs.last_hidden_state, attention_mask
+        )
+        logits = self.score(hidden_states)
+        if labels is not None:
+            loss = F.cross_entropy(logits, labels)
+        else:
+            loss = None
+        return SequenceClassifierOutput(
+            loss=loss,  # type: ignore
+            logits=logits,
+            hidden_states=outputs.hidden_states if output_hidden_states else None,
+        )
+
+
+AutoModel.register(SeamlessM4Tv2EncoderConfig, SeamlessM4Tv2SpeechEncoder)
+AutoModelForAudioClassification.register(
+    SeamlessM4Tv2EncoderConfig, SeamlessM4Tv2ForAudioClassification
+)
+AutoModelForSequenceClassification.register(
+    SeamlessM4Tv2EncoderConfig, SeamlessM4Tv2ForAudioClassification
+)

preprocessor_config.json

@@ -0,0 +1,111 @@
+{
+  "feature_extractor_type": "SeamlessM4TFeatureExtractor",
+  "feature_size": 80,
+  "language_code": [
+    "__afr__",
+    "__amh__",
+    "__arb__",
+    "__ary__",
+    "__arz__",
+    "__asm__",
+    "__azj__",
+    "__bel__",
+    "__ben__",
+    "__bos__",
+    "__bul__",
+    "__cat__",
+    "__ceb__",
+    "__ces__",
+    "__ckb__",
+    "__cmn__",
+    "__cmn_Hant__",
+    "__cym__",
+    "__dan__",
+    "__deu__",
+    "__ell__",
+    "__eng__",
+    "__est__",
+    "__eus__",
+    "__fin__",
+    "__fra__",
+    "__fuv__",
+    "__gaz__",
+    "__gle__",
+    "__glg__",
+    "__guj__",
+    "__heb__",
+    "__hin__",
+    "__hrv__",
+    "__hun__",
+    "__hye__",
+    "__ibo__",
+    "__ind__",
+    "__isl__",
+    "__ita__",
+    "__jav__",
+    "__jpn__",
+    "__kan__",
+    "__kat__",
+    "__kaz__",
+    "__khk__",
+    "__khm__",
+    "__kir__",
+    "__kor__",
+    "__lao__",
+    "__lit__",
+    "__lug__",
+    "__luo__",
+    "__lvs__",
+    "__mai__",
+    "__mal__",
+    "__mar__",
+    "__mkd__",
+    "__mlt__",
+    "__mni__",
+    "__mya__",
+    "__nld__",
+    "__nno__",
+    "__nob__",
+    "__npi__",
+    "__nya__",
+    "__ory__",
+    "__pan__",
+    "__pbt__",
+    "__pes__",
+    "__pol__",
+    "__por__",
+    "__ron__",
+    "__rus__",
+    "__sat__",
+    "__slk__",
+    "__slv__",
+    "__sna__",
+    "__snd__",
+    "__som__",
+    "__spa__",
+    "__srp__",
+    "__swe__",
+    "__swh__",
+    "__tam__",
+    "__tel__",
+    "__tgk__",
+    "__tgl__",
+    "__tha__",
+    "__tur__",
+    "__ukr__",
+    "__urd__",
+    "__uzn__",
+    "__vie__",
+    "__yor__",
+    "__yue__",
+    "__zlm__",
+    "__zul__"
+  ],
+  "num_mel_bins": 80,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "processor_class": "SeamlessM4TProcessor",
+  "return_attention_mask": true,
+  "sampling_rate": 16000,
+  "stride": 2
+}