Chithekitale commited on
Commit
3b84ba6
·
verified ·
1 Parent(s): 2e5ff17

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +192 -79
app.py CHANGED
@@ -1,109 +1,222 @@
1
  import gradio as gr
2
- import librosa
3
  import numpy as np
4
  import torch
5
  import tempfile
6
  import os
7
-
8
- from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan
9
-
10
-
 
 
 
 
 
 
11
  checkpoint = "Chithekitale/chichewa_tts_norules"
12
- processor = SpeechT5Processor.from_pretrained(checkpoint)
 
13
  model = SpeechT5ForTextToSpeech.from_pretrained(checkpoint)
14
  vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")
15
 
16
-
17
  speaker_embeddings = {
18
- "BDL": "spkemb/cmu_us_bdl_arctic-wav-arctic_a0009.npy",
19
- "CLB": "spkemb/cmu_us_clb_arctic-wav-arctic_a0144.npy",
20
- "KSP": "spkemb/cmu_us_ksp_arctic-wav-arctic_b0087.npy",
21
- "RMS": "spkemb/cmu_us_rms_arctic-wav-arctic_b0353.npy",
22
- "SLT": "spkemb/cmu_us_slt_arctic-wav-arctic_a0508.npy",
23
  }
24
 
 
 
 
 
 
 
 
25
 
26
- def predict(text, speaker):
27
- # Create downloadable text file
28
- tmp_txt = tempfile.NamedTemporaryFile(delete=False, suffix=".txt", mode="w", encoding="utf-8")
29
- tmp_txt.write(text if text else "")
30
- tmp_txt.close()
 
 
 
31
 
32
- if len(text.strip()) == 0:
33
- return (16000, np.zeros(0).astype(np.int16)), tmp_txt.name
34
 
35
- inputs = processor(text=text, return_tensors="pt")
 
 
 
 
 
36
 
37
- # limit input length
38
- input_ids = inputs["input_ids"]
39
- input_ids = input_ids[..., :model.config.max_text_positions]
40
 
41
- if speaker == "Surprise Me!":
42
- # load one of the provided speaker embeddings at random
43
- idx = np.random.randint(len(speaker_embeddings))
44
- key = list(speaker_embeddings.keys())[idx]
45
- speaker_embedding = np.load(speaker_embeddings[key])
46
 
47
- # randomly shuffle the elements
48
- np.random.shuffle(speaker_embedding)
49
 
50
- # randomly flip half the values
51
- x = (np.random.rand(512) >= 0.5) * 1.0
52
- x[x == 0] = -1.0
53
- speaker_embedding *= x
54
- else:
55
- speaker_embedding = np.load(speaker_embeddings[speaker[:3]])
56
 
57
- speaker_embedding = torch.tensor(speaker_embedding).unsqueeze(0)
 
58
 
59
- speech = model.generate_speech(input_ids, speaker_embedding, vocoder=vocoder)
60
 
61
- speech = (speech.numpy() * 32767).astype(np.int16)
62
- return (16000, speech), tmp_txt.name
 
 
 
63
 
 
64
 
65
- title = "Chichewa TTS no rules"
 
 
 
 
 
 
 
66
 
67
- description = """
 
 
 
 
 
 
68
 
69
- """
 
70
 
71
- article = """
 
 
 
72
 
73
- """
 
 
 
 
 
74
 
75
- examples = [
76
- ["It is not in the stars to hold our destiny but in ourselves.", "BDL (male)"],
77
- ["The octopus and Oliver went to the opera in October.", "CLB (female)"],
78
- ["She sells seashells by the seashore. I saw a kitten eating chicken in the kitchen.", "RMS (male)"],
79
- ["Brisk brave brigadiers brandished broad bright blades, blunderbusses, and bludgeons—balancing them badly.", "SLT (female)"],
80
- ["A synonym for cinnamon is a cinnamon synonym.", "BDL (male)"],
81
- ["How much wood would a woodchuck chuck if a woodchuck could chuck wood? He would chuck, he would, as much as he could, and chuck as much wood as a woodchuck would if a woodchuck could chuck wood.", "CLB (female)"],
82
- ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
83
 
84
- gr.Interface(
85
- fn=predict,
86
- inputs=[
87
- gr.Text(label="Input Text"),
88
- gr.Radio(
89
- label="Speaker",
90
- choices=[
91
- "BDL (male)",
92
- "CLB (female)",
93
- "KSP (male)",
94
- "RMS (male)",
95
- "SLT (female)",
96
- "Surprise Me!"
97
- ],
98
- value="BDL (male)"
99
- ),
100
- ],
101
- outputs=[
102
- gr.Audio(label="Generated Speech", type="numpy"),
103
- gr.File(label="Download Generated Text")
104
- ],
105
- title=title,
106
- description=description,
107
- article=article,
108
- examples=examples,
109
- ).launch()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  import gradio as gr
 
2
  import numpy as np
3
  import torch
4
  import tempfile
5
  import os
6
+ from scipy.io.wavfile import write
7
+ from transformers import (
8
+ SpeechT5Processor,
9
+ SpeechT5ForTextToSpeech,
10
+ SpeechT5HifiGan
11
+ )
12
+
13
+ # =========================
14
+ # Model loading
15
+ # =========================
16
  checkpoint = "Chithekitale/chichewa_tts_norules"
17
+
18
+ processor = SpeechT5Processor.from_pretrained("microsoft/speecht5_tts")
19
  model = SpeechT5ForTextToSpeech.from_pretrained(checkpoint)
20
  vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")
21
 
22
+ # Make all keys consistent
23
  speaker_embeddings = {
24
+ "SPK1": "spkemb/speaker_2.npy",
25
+ "SPK2": "spkemb/speaker_1.npy",
26
+ "SPK3": "spkemb/cmu_us_ksp_arctic-wav-arctic_b0087.npy",
27
+ "SPK4": "spkemb/cmu_us_rms_arctic-wav-arctic_b0353.npy",
28
+ "SPK5": "spkemb/cmu_us_slt_arctic-wav-arctic_a0508.npy",
29
  }
30
 
31
+ SPEAKER_CHOICES = [
32
+ "SPK1 (female)",
33
+ "SPK2 (male)",
34
+ "SPK3 (male)",
35
+ "SPK4 (male)",
36
+ "SPK5 (female)"
37
+ ]
38
 
39
+ EXAMPLES = [
40
+ ["Ndapita, koma ndibweranso pompano.", "SPK1 (female)"],
41
+ ["Koma apapa zikuoneka kuti ziyenda bwino.", "SPK2 (male)"],
42
+ ["Ineyo ndikuona kuti sizizasithanso.", "SPK3 (male)"],
43
+ ["Mwina kusogolo kuno anthu ena azalimba mtima, koma panopana ndakaika.", "SPK4 (male)"],
44
+ ["Simungasankhe munthu oti bola linamukana.", "SPK5 (female)"],
45
+ ["Kodi chimanga panopa chikugulisidwa zingati, kapena nanunso simukudziwa?", "SPK5 (female)"],
46
+ ]
47
 
48
+ SAMPLE_RATE = 16000
 
49
 
50
+ # =========================
51
+ # Helpers
52
+ # =========================
53
+ def get_speaker_key(speaker_label: str) -> str:
54
+ # "SPK1 (female)" -> "SPK1"
55
+ return speaker_label.split()[0]
56
 
57
+ def load_speaker_embedding(speaker: str) -> np.ndarray:
58
+ speaker_key = get_speaker_key(speaker)
 
59
 
60
+ if speaker_key not in speaker_embeddings:
61
+ raise ValueError(f"Unknown speaker key: {speaker_key}")
 
 
 
62
 
63
+ path = speaker_embeddings[speaker_key]
 
64
 
65
+ try:
66
+ speaker_embedding = np.load(path).astype(np.float32)
67
+ except Exception as e:
68
+ raise FileNotFoundError(
69
+ f"Could not load speaker embedding file: {path}. Error: {e}"
70
+ )
71
 
72
+ if speaker_embedding.ndim == 2:
73
+ speaker_embedding = speaker_embedding.mean(axis=0)
74
 
75
+ speaker_embedding = np.squeeze(speaker_embedding)
76
 
77
+ if speaker_embedding.shape != (512,):
78
+ raise ValueError(
79
+ f"Unexpected speaker embedding shape after processing: "
80
+ f"{speaker_embedding.shape}. Expected (512,)"
81
+ )
82
 
83
+ return speaker_embedding
84
 
85
+ def save_audio_to_wav(audio: np.ndarray, sample_rate: int = SAMPLE_RATE) -> str:
86
+ """
87
+ Save generated int16 audio to a temporary WAV file and return its path.
88
+ """
89
+ temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".wav")
90
+ temp_file.close()
91
+ write(temp_file.name, sample_rate, audio)
92
+ return temp_file.name
93
 
94
+ # =========================
95
+ # Inference
96
+ # =========================
97
+ def predict(text, speaker):
98
+ try:
99
+ if not text or len(text.strip()) == 0:
100
+ return None, None, "Please enter some Chichewa text."
101
 
102
+ inputs = processor(text=text, return_tensors="pt")
103
+ input_ids = inputs["input_ids"][..., :model.config.max_text_positions]
104
 
105
+ speaker_embedding = load_speaker_embedding(speaker)
106
+ speaker_embedding = torch.tensor(
107
+ speaker_embedding, dtype=torch.float32
108
+ ).unsqueeze(0)
109
 
110
+ with torch.no_grad():
111
+ speech = model.generate_speech(
112
+ input_ids,
113
+ speaker_embedding,
114
+ vocoder=vocoder
115
+ )
116
 
117
+ speech = speech.cpu().numpy()
118
+
119
+ # Normalize safely before int16 conversion
120
+ max_val = np.max(np.abs(speech))
121
+ if max_val > 0:
122
+ speech = speech / max_val
123
+
124
+ speech = (speech * 32767).astype(np.int16)
125
+
126
+ # Save WAV file for downloading
127
+ wav_path = save_audio_to_wav(speech, SAMPLE_RATE)
128
+
129
+ status = f"Generated speech successfully using speaker: {speaker}"
130
+ return (SAMPLE_RATE, speech), wav_path, status
131
+
132
+ except Exception as e:
133
+ return None, None, f"Error during generation: {str(e)}"
134
+
135
+ def clear_all():
136
+ return "", "SPK1 (female)", None, None, "Ready."
137
+
138
+ # =========================
139
+ # UI
140
+ # =========================
141
+ custom_css = """
142
+ .gradio-container {
143
+ max-width: 1100px !important;
144
+ margin: 0 auto;
145
+ }
146
+ .hero {
147
+ text-align: center;
148
+ padding: 10px 0 0 0;
149
+ }
150
+ .section-note {
151
+ font-size: 0.95rem;
152
+ opacity: 0.9;
153
+ }
154
+ """
155
 
156
+ with gr.Blocks(css=custom_css, title="Baseline Chichewa Speech Synthesis Demo") as demo:
157
+ gr.HTML(
158
+ """
159
+ <div class="hero">
160
+ <h1>Baseline Chichewa Synthesis</h1>
161
+ <p class="section-note">
162
+ Enter Chichewa text, choose a speaker voice, and generate speech audio.
163
+ </p>
164
+ </div>
165
+ """
166
+ )
167
+
168
+ with gr.Row():
169
+ with gr.Column(scale=5):
170
+ text_input = gr.Textbox(
171
+ label="Input Text",
172
+ placeholder="Type Chichewa text here...",
173
+ lines=6
174
+ )
175
+
176
+ speaker_input = gr.Radio(
177
+ label="Speaker Voice",
178
+ choices=SPEAKER_CHOICES,
179
+ value="SPK1 (female)"
180
+ )
181
+
182
+ with gr.Row():
183
+ generate_btn = gr.Button("Generate Speech", variant="primary")
184
+ clear_btn = gr.Button("Clear")
185
+
186
+ status_box = gr.Textbox(
187
+ label="System Status",
188
+ value="Ready.",
189
+ interactive=False
190
+ )
191
+
192
+ with gr.Column(scale=5):
193
+ audio_output = gr.Audio(
194
+ label="Generated Speech",
195
+ type="numpy",
196
+ autoplay=False
197
+ )
198
+
199
+ download_file = gr.File(
200
+ label="Download Audio File"
201
+ )
202
+
203
+ gr.Markdown("### Example Inputs")
204
+ gr.Examples(
205
+ examples=EXAMPLES,
206
+ inputs=[text_input, speaker_input]
207
+ )
208
+
209
+ generate_btn.click(
210
+ fn=predict,
211
+ inputs=[text_input, speaker_input],
212
+ outputs=[audio_output, download_file, status_box],
213
+ show_progress="full"
214
+ )
215
+
216
+ clear_btn.click(
217
+ fn=clear_all,
218
+ inputs=[],
219
+ outputs=[text_input, speaker_input, audio_output, download_file, status_box]
220
+ )
221
+
222
+ demo.launch()