Add sox and libsox-dev to requirements.txt

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tabs/audios/modules/CosyVoice"]
+	path = tabs/audios/modules/CosyVoice
+	url = https://github.com/FunAudioLLM/CosyVoice.git

config.py

@@ -4,8 +4,10 @@ import json
 import torch
 
 
-# Setup Directories
-os.makedirs('.cache', exist_ok=True)
+# Setup Repo
+
+# Audios
+os.environ['PYTHONPATH'] = f'{os.path.dirname(__file__)}/modules/CosyVoice/third_party/Matcha-TTS:{os.environ.get("PYTHONPATH", "")}' # add tabs/audios/modules/CosyVoice/third_party/Matcha-TTS to PYTHONPATH
 
 
 css = """
@@ -32,9 +34,10 @@ body {
 class Config:
     # General
     SECRET_KEY = os.environ.get('SECRET_KEY', '12345678')
+    MODEL_DOWNLOAD_DIR = os.environ.get('HF_HOME', os.environ.get('HF_HUB_CACHE', '/.cache'))
+    os.makedirs(MODEL_DOWNLOAD_DIR, exist_ok=True)
     
     # Images
-    # IMAGE_MODELS = ["black-forest-labs/FLUX.1-dev", "stabilityai/stable-diffusion-xl-base-1.0"]
     IMAGES_MODELS = [{"repo_id": "black-forest-labs/FLUX.1-dev", "loader": "flux", "compute_type": torch.bfloat16,}, {"repo_id": "stabilityai/stable-diffusion-xl-base-1.0", "loader": "sdxl", "compute_type": torch.float16,}]
     with open('data/loras/sdxl.json') as f:
         IMAGES_LORAS_SDXL = json.load(f)
@@ -80,4 +83,4 @@ class Config:
 
 
     # Audios
-    AUDIOS_MODELS = [{"repo_id": "fal/AuraSR-v2"}]
+    AUDIOS_MODELS = []

packages.txt

@@ -1,2 +1,4 @@
 ffmpeg
 libgl1-mesa-glx
+sox
+libsox-dev

requirements.txt

@@ -1,4 +1,3 @@
---extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/
 spaces
 gradio
 torch
@@ -15,7 +14,6 @@ mediapipe
 controlnet_aux
 insightface
 omegaconf
-git+https://github.com/TencentARC/PhotoMaker.git
 torchao
 git+https://github.com/xhinker/sd_embed.git
 clip_interrogator
@@ -24,4 +22,28 @@ git+https://github.com/TencentARC/GFPGAN.git
 git+https://github.com/xinntao/Real-ESRGAN.git
 aura_sr
 deepfilternet
-styletts2
+conformer
+deepspeed
+gdown
+grpcio
+grpcio-tools
+hydra-core
+HyperPyYAML
+inflect
+librosa
+lightning
+matplotlib
+modelscope
+networkx
+onnx
+openai-whisper
+protobuf
+pydantic
+rich
+soundfile
+tensorboard
+WeTextProcessing
+wget
+fastapi-cli
+spacy
+spacy_langdetect

tabs/audios/events.py

@@ -1,3 +1,4 @@
+import re
 import os
 import gc
 import tempfile
@@ -5,11 +6,13 @@ from uuid import uuid4
 
 import spaces
 import gradio as gr
+import torchaudio
 import numpy as np
 from df.enhance import enhance, load_audio, save_audio
 
 from config import Config
 from .load_models import *
+from .modules.CosyVoice.cosyvoice.utils.file_utils import load_wav
 
 
 # Helper functions
@@ -17,6 +20,103 @@ def create_temp_file():
     return tempfile.NamedTemporaryFile(delete=False)
 
 
+
+def assign_language_tags(text):
+    # Process the text
+    # based on the language assign <|zh|><|en|><|jp|><|yue|><|ko|> for Chinese/English/Japanese/Cantonese/Korean
+    # at the start of the text for that language
+    # e.g. input: 你好 Hello こんにちは 你好 안녕하세요
+    # output: <|zh|>你好<|en|>Hello<|jp|>こんにちは<|yue|>你好<|ko|>안녕하세요
+    # Define language patterns
+    patterns = {
+        'zh': r'[\u4e00-\u9fff]+',  # Chinese characters
+        'en': r'[a-zA-Z]+',         # English letters
+        'jp': r'[\u3040-\u30ff\u31f0-\u31ff]+',  # Japanese characters
+        'ko': r'[\uac00-\ud7a3]+',  # Korean characters
+    }
+    
+    # Find all matches
+    matches = []
+    for lang, pattern in patterns.items():
+        for match in re.finditer(pattern, text):
+            matches.append((match.start(), match.end(), lang, match.group()))
+    
+    # Sort matches by start position
+    matches.sort(key=lambda x: x[0])
+    
+    # Build the result string
+    result = []
+    last_end = 0
+    zh_count = 0
+    for start, end, lang, content in matches:
+        if start > last_end:
+            result.append(text[last_end:start])
+        if lang == 'zh':
+            zh_count += 1
+            if zh_count > 1:
+                lang = 'yue'
+        result.append(f'<|{lang}|>{content}')
+        last_end = end
+    
+    if last_end < len(text):
+        result.append(text[last_end:])
+    
+    return ''.join(result)
+
+
+def update_mode(mode, sft_speaker, speaker_audio, voice_instructions):
+    if mode == 'SFT':
+        return (
+            gr.update( # sft_speaker
+                
+            ),
+            gr.update( # speaker_audio,
+                visible=False,
+            ),
+            gr.update( # voice_instructions,
+                visible=False,
+            ),
+        )
+    elif mode == 'VC':
+        return (
+            gr.update( # sft_speaker,
+                visible=False,
+            ),
+            gr.update( # speaker_audio,
+                visible=True,
+            ),
+            gr.update( # voice_instructions,
+                visible=True,
+            ),
+        )
+    elif mode == 'VC-CrossLingual':
+        return (
+            gr.update( # sft_speaker,
+                visible=False,
+            ),
+            gr.update( # speaker_audio,
+                visible=True,
+            ),
+            gr.update( # voice_instructions,
+                visible=False,
+            ),
+        )
+    elif mode == 'Instruct':
+        return (
+            gr.update( # sft_speaker,
+                visible=True,
+            ),
+            gr.update( # speaker_audio,
+                visible=False,
+            ),
+            gr.update( # voice_instructions,
+                visible=True,
+            ),
+        )
+    else:
+        raise gr.Error('Invalid mode')
+
+
 @spaces.GPU(duration=10)
 def clear_audio(audio: np.ndarray):
     # Save the audio file
@@ -36,30 +136,76 @@ def clear_audio(audio: np.ndarray):
 
 
 @spaces.GPU(duration=20)
-def gen_audio(
-    text,
-    language,
-    speaker_audio: np.ndarray,
-    tts_alpha,
-    tts_beta,
-    tts_diffusion_steps,
-    tts_embedding_scale,
-):
-    # Save the speaker audio file
-    speaker_audio_file = create_temp_file()
-    np.save(speaker_audio_file.name, speaker_audio)
+def gen_audio(text, mode, sft_speaker = None, speaker_audio = None, voice_instructions = None):
+    if mode == any(['VC', 'VC-CrossLingual']):
+        # Save the speaker audio file
+        speaker_audio_file = create_temp_file()
+        np.save(speaker_audio_file.name, speaker_audio)
+        prompt_speech_16k = load_wav('zero_shot_prompt.wav', 16000)
+    else:
+        speaker_audio_file = None
+        prompt_speech_16k = None
+
+    # Assign language tags
+    text = assign_language_tags(text)
     
     # Generate the audio
-    output = styletts2_model.inference(
-        text=text,
-        target_voice_path=speaker_audio_file.name,
-        output_wav_file=create_temp_file().name,
-        alpha=float(tts_alpha),
-        beta=float(tts_beta),
-        diffusion_steps=int(tts_diffusion_steps),
-        embedding_scale=int(tts_embedding_scale),
-    )
+    out_file = create_temp_file()
+    if mode == 'SFT':
+        if not sft_speaker:
+            raise gr.Error('Please select a speaker')
+        
+        for i, j in enumerate(cv_base.inference_sft(
+            tts_text=text,
+            spk_id=sft_speaker,
+        )):
+            torchaudio.save(
+                out_file.name.format(i),
+                j['tts_speech'],
+                22050,
+            )
+    elif mode == 'VC':
+        if not speaker_audio_file:
+            raise gr.Error('Please upload an audio')
+        
+        for i, j in enumerate(cv_sft.inference_zero_shot(
+            tts_text=text,
+            prompt_speech_16k=voice_instructions,
+            prompt_speech_16k=prompt_speech_16k,
+        )):
+            torchaudio.save(
+                out_file.name.format(i),
+                j['tts_speech'],
+                22050,
+            )
+    elif mode == 'VC-CrossLingual':
+        if not speaker_audio_file:
+            raise gr.Error('Please upload an audio')
+        
+        for i, j in enumerate(cv_sft.inference_cross_lingual(
+            tts_text=text,
+            prompt_speech_16k=prompt_speech_16k,
+        )):
+            torchaudio.save(
+                out_file.name.format(i),
+                j['tts_speech'],
+                22050,
+            )
+    elif mode == 'Instruct':
+        if not voice_instructions:
+            raise gr.Error('Please enter voice instructions')
+        
+        for i, j in enumerate(cv_instruct.inference_instruct(
+            tts_text=text,
+            spk_id=sft_speaker,
+            instruct_text=voice_instructions,
+        )):
+            torchaudio.save(
+                out_file.name.format(i),
+                j['tts_speech'],
+                22050,
+            )
     
     return gr.update( # output_audio
-        value=output,
+        value=out_file.name,
     )

tabs/audios/load_models.py

@@ -1,17 +1,38 @@
+import os
+
 import torch
 from df.enhance import init_df
-from styletts2 import tts
+from modelscope import snapshot_download
 
 from config import Config
+from .modules.CosyVoice.cosyvoice.cli.cosyvoice import CosyVoice
 
 
 def init_sys():
     device = "cuda" if torch.cuda.is_available() else "cpu"
     
+    # Load DeepFilterNet2
     df_model, df_state, _ = init_df()
     
-    styletts2_model = tts.StyleTTS2()
+    # Download CosyVoice models
+    snapshot_download('iic/CosyVoice-300M', local_dir=f'{Config.MODEL_DOWNLOAD_DIR}/audios/CosyVoice-300M')
+    snapshot_download('iic/CosyVoice-300M-SFT', local_dir=f'{Config.MODEL_DOWNLOAD_DIR}/audios/CosyVoice-300M-SFT')
+    snapshot_download('iic/CosyVoice-300M-Instruct', local_dir=f'{Config.MODEL_DOWNLOAD_DIR}/audios/CosyVoice-300M-Instruct')
+    snapshot_download('iic/CosyVoice-ttsfrd', local_dir=f'{Config.MODEL_DOWNLOAD_DIR}/audios/CosyVoice-ttsfrd')
+    
+    # Add `tabs/audios/modules/CosyVoice/third_party/Matcha-TTS` to your `PYTHONPATH`
+    os.environ['PYTHONPATH'] = f'{os.path.dirname(__file__)}/modules/CosyVoice/third_party/Matcha-TTS:{os.environ.get("PYTHONPATH", "")}'
+    
+    # Load CosyVoice TTS
+    cv_base = CosyVoice('pretrained_models/CosyVoice-300M')
+    
+    # Load CosyVoice SFT
+    cv_sft = CosyVoice('pretrained_models/CosyVoice-300M-SFT')
+    sft_speakers = cv_sft.list_avaliable_spks()
+    
+    # Load CosyVoice Instruct
+    cv_instruct = CosyVoice('pretrained_models/CosyVoice-300M-Instruct')
     
-    return device, df_model, df_state, styletts2_model
+    return device, df_model, df_state, cv_base, cv_sft, sft_speakers, cv_instruct
 
-device, df_model, df_state, styletts2_model = init_sys()
+device, df_model, df_state, cv_base, cv_sft, sft_speakers, cv_instruct = init_sys()

tabs/audios/modules/CosyVoice

@@ -0,0 +1 @@
+Subproject commit c901a12789e0a9d8cec54c3caf1bc304533bdf82

tabs/audios/ui.py

@@ -10,30 +10,19 @@ def audio_tab():
             with gr.Group():
                 with gr.Group():
                     text = gr.Textbox(lines=5, label="Enter text")
-                    language = gr.Dropdown(
-                        label="Language",
-                        choices=["en"],
-                        value="en",
-                    )
-                
-                with gr.Accordion('Voice Clone', open=True):
-                    speaker_audio = gr.Audio(label="Upload Audio", type='numpy')
+                    mode = gr.Radio(["SFT", "VC", "VC-CrossLingual", "Instruct"], label="Mode", value="SFT",) # automate with speech recognition pipeline
+                    sft_speaker = gr.Radio(sft_speakers, label="Select speaker")
+                with gr.Accordion('Voice Clone', open=False):
+                    speaker_audio = gr.Audio(label="Upload Audio", type='numpy', visible=False)
                     clear_speaker_audio = gr.Button(label="Clear Audio")
+                
+                with gr.Accordion('Instruct', open=False):
+                    voice_instructions = gr.Textbox(lines=5, label="Enter voice instructions", visible=False)
         
         with gr.Column():
             output_audio = gr.Audio(label="Output Audio", interactive=False, show_download_button=True)
             clear_output_audio = gr.Button(label="Clear Audio")
             generate_audio = gr.Button(label="Generate Audio")
-            
-            with gr.Accordion('Advance Settings', open=True):
-                settings = [
-                    ('Alpha', 'tts_alpha', 'float', 0.0, 1.0, 0.3, 0.1,),
-                    ('Beta', 'tts_beta', 'float', 0.0, 1.0, 0.7, 0.1,),
-                    ('Diffusion Steps', 'tts_diffusion_steps', 'int', 1, 100, 10, 1,),
-                    ('Embedding Scale', 'tts_embedding_scale', 'int', 0, 10, 1, 1,),
-                ]
-                for label, key, type_, min_, max_, value, step in settings:
-                    globals()[key] = gr.Slider(label=label, minimum=min_, maximum=max_, value=value, step=step)
     
     
     # Events
@@ -41,9 +30,16 @@ def audio_tab():
     clear_speaker_audio.click(clear_audio, speaker_audio, speaker_audio)
     clear_output_audio.click(clear_audio, output_audio, output_audio)
     
+    # Mode
+    mode.change(
+        update_mode,
+        [mode, sft_speaker, speaker_audio, voice_instructions],
+        [sft_speaker, speaker_audio, voice_instructions]
+    )
+    
     # Generate Audio
     generate_audio.click(
         gen_audio,
-        [text, language, speaker_audio, tts_alpha, tts_beta, tts_diffusion_steps, tts_embedding_scale], # type: ignore
+        [text, mode, sft_speaker, speaker_audio, voice_instructions],
         [output_audio]
     )