vbach_lib/hubert_manager.py

from pathlib import Path
from extra_utils import dw_file
from i18n import _i18n

BASE_DIR = Path(__file__).resolve().parent / "embedders"
BASE_DIR.mkdir(parents=True, exist_ok=True)
FAIRSEQ_DIR = BASE_DIR / "fairseq"
TRANSFORMERS_DIR = BASE_DIR / "transformers"

class HubertModelNotExist(Exception): pass

huberts_fairseq_dict = {
    "hubert_base": {
        "url": "https://huggingface.co/noblebarkrr/vbach_resources/resolve/main/fairseq/hubert_base.pt?download=true",
        "local_path": FAIRSEQ_DIR / "hubert_base.pt"
    },
    "contentvec_base": {
        "url": "https://huggingface.co/noblebarkrr/vbach_resources/resolve/main/fairseq/contentvec_base.pt?download=true",
        "local_path": FAIRSEQ_DIR / "contentvec_base.pt"
    },
    "korean_hubert_base": {
        "url": "https://huggingface.co/noblebarkrr/vbach_resources/resolve/main/fairseq/korean_hubert_base.pt?download=true",
        "local_path": FAIRSEQ_DIR / "korean_hubert_base.pt"
    },
    "chinese_hubert_base": {
        "url": "https://huggingface.co/noblebarkrr/vbach_resources/resolve/main/fairseq/chinese_hubert_base.pt?download=true",
        "local_path": FAIRSEQ_DIR / "chinese_hubert_base.pt"
    },
    "portuguese_hubert_base": {
        "url": "https://huggingface.co/noblebarkrr/vbach_resources/resolve/main/fairseq/portuguese_hubert_base.pt?download=true",
        "local_path": FAIRSEQ_DIR / "portuguese_hubert_base.pt"
    },
    "japanese_hubert_base": {
        "url": "https://huggingface.co/noblebarkrr/vbach_resources/resolve/main/fairseq/japanese_hubert_base.pt?download=true",
        "local_path": FAIRSEQ_DIR / "japanese_hubert_base.pt"
    }
}

huberts_transformers_dict = {
    "contentvec": {
        "base_dir": TRANSFORMERS_DIR / "contentvec",
        "url_bin": "https://huggingface.co/noblebarkrr/vbach_resources/resolve/main/transformers/contentvec/pytorch_model.bin?download=true",
        "url_json": "https://huggingface.co/noblebarkrr/vbach_resources/resolve/main/transformers/contentvec/config.json?download=true"
    },
    "spin": {
        "base_dir": TRANSFORMERS_DIR / "spin",
        "url_bin": "https://huggingface.co/noblebarkrr/vbach_resources/resolve/main/transformers/spin/pytorch_model.bin?download=true",
        "url_json": "https://huggingface.co/noblebarkrr/vbach_resources/resolve/main/transformers/spin/config.json?download=true"
    },
    "spin-v2": {
        "base_dir": TRANSFORMERS_DIR / "spinv2",
        "url_bin": "https://huggingface.co/noblebarkrr/vbach_resources/resolve/main/transformers/spinv2/pytorch_model.bin?download=true",
        "url_json": "https://huggingface.co/noblebarkrr/vbach_resources/resolve/main/transformers/spinv2/config.json?download=true"
    },
    "chinese-hubert-base": {
        "base_dir": TRANSFORMERS_DIR / "chinese_hubert_base",
        "url_bin": "https://huggingface.co/noblebarkrr/vbach_resources/resolve/main/transformers/chinese_hubert_base/pytorch_model.bin?download=true",
        "url_json": "https://huggingface.co/noblebarkrr/vbach_resources/resolve/main/transformers/chinese_hubert_base/config.json?download=true"
    },
    "japanese-hubert-base": {
        "base_dir": TRANSFORMERS_DIR / "japanese_hubert_base",
        "url_bin": "https://huggingface.co/noblebarkrr/vbach_resources/resolve/main/transformers/japanese_hubert_base/pytorch_model.bin?download=true",
        "url_json": "https://huggingface.co/noblebarkrr/vbach_resources/resolve/main/transformers/japanese_hubert_base/config.json?download=true"
    },
    "korean-hubert-base": {
        "base_dir": TRANSFORMERS_DIR / "korean_hubert_base",
        "url_bin": "https://huggingface.co/noblebarkrr/vbach_resources/resolve/main/transformers/korean_hubert_base/pytorch_model.bin?download=true",
        "url_json": "https://huggingface.co/noblebarkrr/vbach_resources/resolve/main/transformers/korean_hubert_base/config.json?download=true"
    },
}

huberts_fairseq = list(huberts_fairseq_dict.keys())
huberts_transformers = list(huberts_transformers_dict.keys())

def download_hubert(name: str, use_transformers: bool = False):
    if use_transformers:
        info = huberts_transformers_dict.get(name, {})
        if not info:
            raise HubertModelNotExist(_i18n("vbach_embedder_not"))
        base_dir = info["base_dir"]
        base_dir.mkdir(parents=True, exist_ok=True)
        
        urls = (info["url_bin"], info["url_json"])
        paths = (base_dir / "pytorch_model.bin", base_dir / "config.json")
        
        # Проверяем оба файла
        need_download = [not p.exists() for p in paths]
        if any(need_download):
            for url, path, need in zip(urls, paths, need_download):
                if need:
                    dw_file(url, path)
    else:
        info = huberts_fairseq_dict.get(name, {})
        if not info:
            HubertModelNotExist(_i18n("vbach_embedder_not"))
        url = info.get("url")
        path = info.get("local_path")
        if not path.exists():
            dw_file(url, path)

def get_hubert(name: str, use_transformers: bool = False):
    if use_transformers:
        info = huberts_transformers_dict.get(name, {})
        if not info:
            HubertModelNotExist(_i18n("vbach_embedder_not"))
        return info.get("base_dir")
    else:
        info = huberts_fairseq_dict.get(name, {})
        if not info:
            HubertModelNotExist(_i18n("vbach_embedder_not"))
        return info.get("local_path")