chore: Update TTS dependencies and add MeloTTS support

kitt/core/__init__.py

@@ -6,7 +6,7 @@ from typing import List
 
 import numpy as np
 import torch
-from TTS.api import TTS
+# from TTS.api import TTS
 
 os.environ["COQUI_TOS_AGREED"] = "1"
 
@@ -17,6 +17,9 @@ Voice = namedtuple("voice", ["name", "neutral", "angry", "speed"])
 file_full_path = pathlib.Path(os.path.realpath(__file__)).parent
 
 voices = [
+    Voice(
+        "Fast", neutral=None, angry=None, speed=1.0,
+    ),
     Voice(
         "Attenborough",
         neutral=f"{file_full_path}/audio/attenborough/neutral.wav",

kitt/core/tts.py

@@ -3,15 +3,21 @@ from replicate import Client
 from loguru import logger
 from kitt.skills.common import config
 import torch
-from parler_tts import ParlerTTSForConditionalGeneration
+
+# from parler_tts import ParlerTTSForConditionalGeneration
 from transformers import AutoTokenizer, set_seed
 import soundfile as sf
+from melo.api import TTS as MeloTTS
+
 
 replicate = Client(api_token=config.REPLICATE_API_KEY)
 
 Voice = namedtuple("voice", ["name", "neutral", "angry", "speed"])
 
 voices_replicate = [
+    Voice(
+        "Fast", neutral=None, angry=None, speed=1.0,
+    ),
     Voice(
         "Attenborough",
         neutral="https://zebel.ams3.digitaloceanspaces.com/xtts/short/attenborough-neutral.wav",
@@ -44,6 +50,7 @@ voices_replicate = [
     ),
 ]
 
+
 def voice_from_text(voice, voices):
     for v in voices:
         if voice == f"{v.name} - Neutral":
@@ -64,11 +71,7 @@ def speed_from_text(voice, voices):
 def run_tts_replicate(text: str, voice_character: str):
     voice = voice_from_text(voice_character, voices_replicate)
 
-    input = {
-        "text": text,
-        "speaker": voice,
-        "cleanup_voice": True
-    }
+    input = {"text": text, "speaker": voice, "cleanup_voice": True}
 
     output = replicate.run(
         # "afiaka87/tortoise-tts:e9658de4b325863c4fcdc12d94bb7c9b54cbfe351b7ca1b36860008172b91c71",
@@ -82,12 +85,13 @@ def run_tts_replicate(text: str, voice_character: str):
 def get_fast_tts():
     device = "cuda:0" if torch.cuda.is_available() else "cpu"
 
-    model = ParlerTTSForConditionalGeneration.from_pretrained("parler-tts/parler-tts-mini-expresso").to(device)
+    model = ParlerTTSForConditionalGeneration.from_pretrained(
+        "parler-tts/parler-tts-mini-expresso"
+    ).to(device)
     tokenizer = AutoTokenizer.from_pretrained("parler-tts/parler-tts-mini-expresso")
     return model, tokenizer, device
 
 
-
 fast_tts = get_fast_tts()
 
 
@@ -100,4 +104,20 @@ def run_tts_fast(text: str):
 
     generation = model.generate(input_ids=input_ids, prompt_input_ids=prompt_input_ids)
     audio_arr = generation.cpu().numpy().squeeze()
-    return model.config.sampling_rate, audio_arr, dict(text=text, voice="Thomas")
+    return (model.config.sampling_rate, audio_arr), dict(text=text, voice="Thomas")
+
+
+def load_melo_tts():
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    model = MeloTTS(language="EN", device=device)
+    return model
+
+
+melo_tts = load_melo_tts()
+
+
+def run_melo_tts(text: str, voice: str):
+    speed = 1.0
+    speaker_ids = melo_tts.hps.data.spk2id
+    audio = melo_tts.tts_to_file(text, speaker_ids["EN-Default"], None, speed=speed)
+    return melo_tts.hps.data.sampling_rate, audio

kitt/skills/weather.py

@@ -129,7 +129,7 @@ def get_forecast(city_name: str = "", when=0, **kwargs):
                 number_str = f"in {when-1} days"
 
             # Generate a sentence for the day's forecast
-            forecast_sentence = f"On {date} ({number_str}) in {city_name}, the weather will be {conditions} with a high of {max_temp_c}°C and a low of {min_temp_c}°C. There's a {chance_of_rain}% chance of rain. "
+            forecast_sentence = f"On {date} ({number_str}) in {city_name}, the weather will be {conditions} with a high of {max_temp_c}C and a low of {min_temp_c}C. There's a {chance_of_rain}% chance of rain. "
 
             # number = number + 1
             # Add the sentence to the result

main.py

@@ -8,7 +8,7 @@ import typer
 
 from kitt.skills.common import config, vehicle
 from kitt.skills.routing import calculate_route
-from kitt.core.tts import run_tts_replicate, run_tts_fast
+from kitt.core.tts import run_tts_replicate, run_tts_fast, run_melo_tts
 import ollama
 
 from langchain.tools.base import StructuredTool
@@ -196,7 +196,7 @@ def run_nexusraven_model(query, voice_character, state):
 
     if type(output_text) == tuple:
         output_text = output_text[0]
-    gr.Info(f"Output text: {output_text}, generating voice output...")
+    gr.Info(f"Output text: {output_text}\nGenerating voice output...")
     return (
         output_text,
         tts_gradio(output_text, voice_character, speaker_embedding_cache)[0],
@@ -216,11 +216,12 @@ def run_llama3_model(query, voice_character, state):
         functions=functions,
         backend=state["llm_backend"],
     )
-    gr.Info(f"Output text: {output_text}, generating voice output...")
+    gr.Info(f"Output text: {output_text}\nGenerating voice output...")
     voice_out = None
     if state["tts_enabled"]:
         # voice_out = run_tts_replicate(output_text, voice_character)
-        voice_out = run_tts_fast(output_text)[0]
+        # voice_out = run_tts_fast(output_text)[0]
+        voice_out = run_melo_tts(output_text, voice_character)    
         # voice_out = tts_gradio(output_text, voice_character, speaker_embedding_cache)[0]
     return (
         output_text,