simplify and update to current model

InferenceInterfaces/ControllableInterface.py

@@ -16,55 +16,24 @@ class ControllableInterface:
             os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
             os.environ["CUDA_VISIBLE_DEVICES"] = f"{gpu_id}"
         self.device = "cuda" if gpu_id != "cpu" else "cpu"
-        self.model = ToucanTTSInterface(device=self.device, tts_model_path="Meta")
-        self.wgan = GanWrapper(os.path.join(MODELS_DIR, "Embedding", "embedding_gan.pt"), device=self.device)
-        self.generated_speaker_embeds = list()
-        self.available_artificial_voices = available_artificial_voices
-        self.current_language = ""
-        self.current_accent = ""
+        self.model = ToucanTTSInterface(device=self.device, tts_model_path=None)
 
     def read(self,
              prompt,
-             reference_audio,
-             voice_seed,
-             prosody_creativity,
-             duration_scaling_factor,
-             pause_duration_scaling_factor,
-             pitch_variance_scale,
-             energy_variance_scale,
-             emb_slider_1,
-             emb_slider_2,
-             emb_slider_3,
-             emb_slider_4,
-             emb_slider_5,
-             emb_slider_6,
              loudness_in_db
              ):
-        if reference_audio is None:
-            self.wgan.set_latent(voice_seed)
-            controllability_vector = torch.tensor([emb_slider_1,
-                                                   emb_slider_2,
-                                                   emb_slider_3,
-                                                   emb_slider_4,
-                                                   emb_slider_5,
-                                                   emb_slider_6], dtype=torch.float32)
-            embedding = self.wgan.modify_embed(controllability_vector)
-            self.model.set_utterance_embedding(embedding=embedding)
-        else:
-            self.model.set_utterance_embedding(reference_audio)
-
         phones = self.model.text2phone.get_phone_string(prompt)
         if len(phones) > 1800:
             prompt = "Your input was too long. Please try either a shorter text or split it into several parts."
 
         print(prompt + "\n\n")
-        wav, sr, fig = self.model(prompt,
+        wav, sr = self.model(prompt,
                                   input_is_phones=False,
-                                  duration_scaling_factor=duration_scaling_factor,
-                                  pitch_variance_scale=pitch_variance_scale,
-                                  energy_variance_scale=energy_variance_scale,
-                                  pause_duration_scaling_factor=pause_duration_scaling_factor,
-                                  return_plot_as_filepath=True,
-                                  prosody_creativity=prosody_creativity,
+                                  duration_scaling_factor=1.0,
+                                  pitch_variance_scale=1.0,
+                                  energy_variance_scale=1.0,
+                                  pause_duration_scaling_factor=1.0,
+                                  return_plot_as_filepath=False,
+                                  prosody_creativity=0.5,
                                   loudness_in_db=loudness_in_db)
-        return sr, wav, fig
+        return sr, wav

InferenceInterfaces/ToucanTTSInterface.py

@@ -7,6 +7,7 @@ import pyloudnorm
 import sounddevice
 import soundfile
 import torch
+from huggingface_hub import hf_hub_download
 from speechbrain.pretrained import EncoderClassifier
 from torchaudio.transforms import Resample
 
@@ -24,16 +25,14 @@ class ToucanTTSInterface(torch.nn.Module):
 
     def __init__(self,
                  device="cpu",  # device that everything computes on. If a cuda device is available, this can speed things up by an order of magnitude.
-                 tts_model_path=os.path.join(MODELS_DIR, f"ToucanTTS_Meta", "best.pt"),  # path to the ToucanTTS checkpoint or just a shorthand if run standalone
-                 vocoder_model_path=os.path.join(MODELS_DIR, f"Vocoder", "best.pt"),  # path to the Vocoder checkpoint
+                 tts_model_path=None,  # path to the ToucanTTS checkpoint or just a shorthand if run standalone
+                 vocoder_model_path=None,  # path to the Vocoder checkpoint
                  language="eng",  # initial language of the model, can be changed later with the setter methods
                  ):
         super().__init__()
         self.device = device
-        if not tts_model_path.endswith(".pt"):
-            # default to shorthand system
-            tts_model_path = os.path.join(MODELS_DIR, f"ToucanTTS_{tts_model_path}", "best.pt")
-
+        tts_model_path = hf_hub_download(repo_id="Flux9665/ToucanTTS", filename="EnglishToucanTTS.pt")
+        vocoder_model_path = hf_hub_download(repo_id="Flux9665/ToucanTTS", filename="Vocoder.pt")
         ################################
         #   build text to phone        #
         ################################

README.md

@@ -1,10 +1,10 @@
 ---
-title: StochasticToucanTTS
+title: SimpleToucanTTS
 emoji: 🦜
 colorFrom: green
 colorTo: blue
 sdk: gradio
-sdk_version: 4.37.2
+sdk_version: 5.3
 app_file: app.py
 pinned: false
 license: mit

app.py

@@ -3,68 +3,27 @@ import torch.cuda
 
 from InferenceInterfaces.ControllableInterface import ControllableInterface
 from Utility.utils import float2pcm
-from Utility.utils import load_json_from_path
 
 
 class TTSWebUI:
 
-    def __init__(self, gpu_id="cpu", title="Stochastic Speech Synthesis with ToucanTTS", article="For a multilingual version, have a look at https://huggingface.co/spaces/Flux9665/MassivelyMultilingualTTS", available_artificial_voices=1000, path_to_iso_list="Preprocessing/multilinguality/iso_to_fullname.json"):
-        # iso_to_name = load_json_from_path(path_to_iso_list)
-        # accent_selection = [f"{iso_to_name[iso_code]} Accent ({iso_code})" for iso_code in iso_to_name]
-
-        self.controllable_ui = ControllableInterface(gpu_id=gpu_id,
-                                                     available_artificial_voices=available_artificial_voices)
+    def __init__(self, gpu_id="cpu", title="Simplistic Stochastic Speech Synthesis with ToucanTTS", article="For a multilingual version, have a look at https://huggingface.co/spaces/Flux9665/MassivelyMultilingualTTS"):
+        self.controllable_ui = ControllableInterface(gpu_id=gpu_id)
         self.iface = gr.Interface(fn=self.read,
                                   inputs=[gr.Textbox(lines=2,
                                                      placeholder="write what you want the synthesis to read here...",
                                                      value="What I cannot create, I do not understand.",
-                                                     label="Text input"),
-                                          gr.Audio(type="filepath", show_label=True, container=True, label="Voice to Clone (if left empty, will use an artificial voice instead)"),
-                                          gr.Slider(minimum=0, maximum=available_artificial_voices, step=1,
-                                                    value=279,
-                                                    label="Random Seed for the artificial Voice"),
-                                          gr.Slider(minimum=0.0, maximum=0.8, step=0.1, value=0.1, label="Prosody Creativity"),
-                                          gr.Slider(minimum=0.7, maximum=1.3, step=0.1, value=1.0, label="Duration Scale"),
-                                          # gr.Slider(minimum=0.5, maximum=1.5, step=0.1, value=1.0, label="Pitch Variance Scale"),
-                                          # gr.Slider(minimum=0.5, maximum=1.5, step=0.1, value=1.0, label="Energy Variance Scale"),
-                                          gr.Slider(minimum=-10.0, maximum=10.0, step=0.1, value=0.0, label="Femininity / Masculinity"),
-                                          gr.Slider(minimum=-10.0, maximum=10.0, step=0.1, value=0.0, label="Voice Depth")
-                                          ],
-                                  outputs=[gr.Audio(type="numpy", label="Speech"),
-                                           gr.Image(label="Visualization")],
+                                                     label="Text input")],
+                                  outputs=[gr.Audio(type="numpy", label="Speech")],
                                   title=title,
                                   theme="default",
                                   allow_flagging="never",
                                   article=article)
         self.iface.launch()
 
-    def read(self,
-             prompt,
-             reference_audio,
-             voice_seed,
-             prosody_creativity,
-             duration_scaling_factor,
-             # pitch_variance_scale,
-             # energy_variance_scale,
-             emb1,
-             emb2
-             ):
-        sr, wav, fig = self.controllable_ui.read(prompt,
-                                                 reference_audio,
-                                                 voice_seed,
-                                                 prosody_creativity,
-                                                 duration_scaling_factor,
-                                                 1.,
-                                                 1.0,
-                                                 1.0,
-                                                 emb1,
-                                                 emb2,
-                                                 0.,
-                                                 0.,
-                                                 0.,
-                                                 0.,
-                                                 -20.)
-        return (sr, float2pcm(wav)), fig
+    def read(self, prompt):
+        sr, wav = self.controllable_ui.read(prompt, -24.)
+        return sr, float2pcm(wav)
 
 
 if __name__ == '__main__':