trainer/AutoSSLoader.py

'''
A wrapper for loading the pretrained models from huggingface,
wav2vec2, hubert, wavlm are actually inherit from wav2vec2 class,
whisper is inherit from HFTransformersInterface class
---
NOTES:
For new SSL models, we suggesting using 
encoder_type==speechbrain.lobes.models.huggingface_transformers.wav2vec2.Wav2Vec2
as the encoder_type.
'''

from speechbrain.lobes.models.huggingface_transformers.wav2vec2 import Wav2Vec2
from speechbrain.lobes.models.huggingface_transformers.hubert import HuBERT
from speechbrain.lobes.models.huggingface_transformers.wavlm import WavLM
from speechbrain.lobes.models.huggingface_transformers.whisper import Whisper

from speechbrain.lobes.models.huggingface_transformers.mimi import Mimi
from trainer.CharsiuWav2Vec2Encoder import CharsiuWav2Vec2Encoder

pretrained_models={
    "wav2vec2_base": "facebook/wav2vec2-base", # 768
    "wav2vec2_base_jp": "rinna/japanese-wav2vec2-base", # 768
    "hubert_base": "facebook/hubert-base-ls960", # 768
    "wavlm_base": "microsoft/wavlm-base", # 768
    "wavlm_base_jp": "rinna/japanese-wavlm-base", # 768
    "wavlm_base_plus": "microsoft/wavlm-base-plus", # 768
    "hubert_multilingual": "utter-project/mHuBERT-147", # 768
    "clap" : "laion/clap-htsat-fused", # 768
    "data2vec_base": "facebook/data2vec-audio-base", # 768
    
    "wav2vec2_large": "facebook/wav2vec2-large", # 1024
    "wav2vec_large_xlsr_53": "facebook/wav2vec2-large-xlsr-53", # 1024
    "wav2vec2-xls-r-1b": "facebook/wav2vec2-xls-r-1b", # 1024
    "hubert_large": "facebook/hubert-large-ls960-ft", # 1024
    "hubert_large_ll60k": "facebook/hubert-large-ll60k", # 1024
    "wavlm_large": "microsoft/wavlm-large", # 1024
    "data2vec_large": "facebook/data2vec-audio-large", #1024
    "hubert_arabic": "omarxadel/hubert-large-arabic-egyptian", # 1024
    
    "whisper_medium": "openai/whisper-medium", # 1024
    "whisper_large_v3_turbo": "openai/whisper-large-v3-turbo", # 1280
    
    "mimi": "kyutai/mimi", # codec, 1

    # Charsiu frame-classification aligners. These expose the Charsiu-trained
    # wav2vec2 encoder states through CharsiuWav2Vec2Encoder.
    "charsiu_w2v2_fc_10ms": "/home/m64000/.cache/huggingface/hub/models--charsiu--en_w2v2_fc_10ms/snapshots/e9bf8dd314313fc57f6e4d0b5425bde4bbeac80f",
    "charsiu_w2v2_fc_20ms": "/home/m64000/.cache/huggingface/hub/models--charsiu--en_w2v2_fc_20ms/snapshots/41ae65b77e09407f8678700223b04d696c42e46f",
}

def AutoSSLLoader(model_name,
                freeze,
                freeze_feature_extractor,
                save_path,
                output_all_hiddens,
                encoder_type=None,
                encoder_only=False):
    """
    source: str, the name of the pretrained model e.g "hubert_multilingual", "clap", "data2vec_base", etc.
    freeze: bool, whether to freeze the model
    freeze_feature_extractor: bool, whether to freeze the feature extractor
    save_path: str, the path to save the model
    encoder_type: str, the type of the encoder
    """    

    if model_name == None:
        print(f"model_name for SSL is None, return None")
        return None
    else:
        model_id = pretrained_models.get(model_name, None)
    
    if model_id is None:
        raise ValueError(f"Unsupported model_name: {model_name}")
    
    else:
        model_id = model_id.lower()
        if str(model_name).startswith("charsiu_w2v2_fc_") or "charsiu/" in model_id:
            return CharsiuWav2Vec2Encoder(
                source=model_id,
                freeze=freeze,
                freeze_feature_extractor=freeze_feature_extractor,
                save_path=save_path,
                output_all_hiddens=output_all_hiddens,
            )
        elif "wav2vec2" in model_id:
            return Wav2Vec2(
                source=model_id,
                freeze=freeze,
                freeze_feature_extractor=freeze_feature_extractor,
                save_path=save_path,
                output_all_hiddens=output_all_hiddens
            )
        elif "hubert" in model_id:
            return HuBERT(
                source=model_id,
                freeze=freeze,
                freeze_feature_extractor=freeze_feature_extractor,
                save_path=save_path,
                output_all_hiddens=output_all_hiddens
            )
        elif "wavlm" in model_id:
            return WavLM(
                source=model_id,
                freeze=freeze,
                freeze_feature_extractor=freeze_feature_extractor,
                save_path=save_path,
                output_all_hiddens=output_all_hiddens
            )
        # TODO
        elif "whisper" in model_id:
            return Whisper(
                source=model_id,
                freeze=freeze,
                save_path=save_path,
                encoder_only=encoder_only,
            )
        elif "mimi" in model_id:
            return Mimi(
                source=model_id,
                freeze=freeze,
                save_path=save_path,
            )

        elif encoder_type:
            # use the give encoder 
            try:
                return encoder_type(
                    source=model_id,
                    freeze=freeze,
                    freeze_feature_extractor=freeze_feature_extractor,
                    save_path=save_path,
                    output_all_hiddens=output_all_hiddens
                )
            except:
                raise ValueError(f"Unsupported encoder type: {encoder_type}")
        else:
            raise ValueError(f"Unsupported model_name: {model_name}")