cronos3k commited on
Commit
6da0f0b
Β·
verified Β·
1 Parent(s): e07131e

docs: add research/testing disclaimer to Gradio UI header

Browse files
Files changed (1) hide show
  1. app.py +724 -719
app.py CHANGED
@@ -1,719 +1,724 @@
1
- """
2
- LongCat-AudioDiT Enhanced – Gradio Web UI
3
-
4
- Primary workflow: Voice Cloning
5
- 1. Upload reference audio β†’ auto-transcribe with Whisper
6
- 2. Type text to synthesise in the cloned voice
7
- 3. Generate β†’ save to Voice Library with a name
8
- 4. Reuse any saved voice from the dropdown
9
-
10
- All actions are exposed as Gradio REST API endpoints.
11
-
12
- Usage:
13
- python app.py
14
- python app.py --port 7860 --share
15
- python app.py --device cpu
16
- """
17
-
18
- import argparse
19
- import logging
20
- import os
21
- import socket
22
- import time
23
- from pathlib import Path
24
-
25
- import gradio as gr
26
- import numpy as np
27
- import soundfile as sf
28
- import torch
29
- import torch.nn.functional as F
30
-
31
- from utils import normalize_text, load_audio, approx_duration_from_text
32
- from memory_manager import ModelMemoryManager
33
- from voice_library import get_library
34
- from download_models import (
35
- download_audiodit, download_whisper,
36
- _audiodit_present, _whisper_present,
37
- AUDIODIT_MODELS, WHISPER_MODELS,
38
- AUDIODIT_DIR, WHISPER_DIR,
39
- )
40
-
41
- logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
42
- logger = logging.getLogger(__name__)
43
-
44
- OUTPUT_DIR = Path(__file__).parent / "outputs"
45
- OUTPUT_DIR.mkdir(exist_ok=True)
46
-
47
- # ---------------------------------------------------------------------------
48
- # Memory manager
49
- # ---------------------------------------------------------------------------
50
- _mgr: ModelMemoryManager = None
51
-
52
- def get_manager(mode: str = "auto") -> ModelMemoryManager:
53
- global _mgr
54
- if _mgr is None or _mgr.mode.value != mode:
55
- if _mgr is not None:
56
- _mgr.release_all()
57
- _mgr = ModelMemoryManager(mode=mode)
58
- return _mgr
59
-
60
- # ---------------------------------------------------------------------------
61
- # Port helpers
62
- # ---------------------------------------------------------------------------
63
- def _port_free(port: int) -> bool:
64
- with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
65
- s.settimeout(1)
66
- return s.connect_ex(("127.0.0.1", port)) != 0
67
-
68
- def find_free_port(start: int = 7860, end: int = 7960) -> int:
69
- for p in range(start, end):
70
- if _port_free(p):
71
- return p
72
- raise RuntimeError(f"No free port found in {start}-{end}")
73
-
74
- # ---------------------------------------------------------------------------
75
- # Core: transcribe reference audio
76
- # ---------------------------------------------------------------------------
77
- def transcribe_reference(audio_path, whisper_size: str, language: str, memory_mode: str, device: str):
78
- """
79
- Transcribe a reference audio file with Whisper.
80
- Returns (transcription_text, status_msg).
81
- """
82
- if audio_path is None:
83
- return "", "Upload a reference audio file first."
84
-
85
- mgr = get_manager(memory_mode)
86
- try:
87
- whisper = mgr.get_whisper(whisper_size=whisper_size)
88
- except Exception as e:
89
- return "", f"Failed to load Whisper: {e}"
90
-
91
- lang_arg = language if language and language != "auto" else None
92
- try:
93
- text, detected = whisper.transcribe(str(audio_path), language=lang_arg)
94
- except Exception as e:
95
- return "", f"Transcription failed: {e}"
96
-
97
- return text, f"Transcribed [{detected}] β€” {len(text)} characters"
98
-
99
-
100
- # ---------------------------------------------------------------------------
101
- # Core: clone voice (reference audio + transcription β†’ new speech)
102
- # ---------------------------------------------------------------------------
103
- def clone_voice(
104
- text: str,
105
- ref_audio_path,
106
- ref_transcription: str,
107
- audiodit_size: str,
108
- nfe: int,
109
- guidance_strength: float,
110
- guidance_method: str,
111
- seed: int,
112
- memory_mode: str,
113
- device: str,
114
- ):
115
- """
116
- Synthesise `text` in the voice captured from `ref_audio_path`.
117
- Returns (output_audio_path, status_msg).
118
- """
119
- if not text or not text.strip():
120
- return None, "Enter text to synthesise."
121
- if ref_audio_path is None:
122
- return None, "Upload a reference audio file."
123
- if not ref_transcription or not ref_transcription.strip():
124
- return None, "Reference transcription is empty. Use 'Auto-Transcribe' first."
125
-
126
- mgr = get_manager(memory_mode)
127
- try:
128
- model, tokenizer = mgr.get_tts(audiodit_size=audiodit_size, device=device)
129
- except Exception as e:
130
- return None, f"Failed to load TTS model: {e}"
131
-
132
- torch.manual_seed(seed)
133
- if torch.cuda.is_available():
134
- torch.cuda.manual_seed(seed)
135
-
136
- sr = model.config.sampling_rate
137
- full_hop = model.config.latent_hop
138
- max_dur = model.config.max_wav_duration
139
-
140
- synth_text = normalize_text(text)
141
- ref_text = normalize_text(ref_transcription)
142
- full_text = f"{ref_text} {synth_text}"
143
-
144
- inputs = tokenizer([full_text], padding="longest", return_tensors="pt")
145
- inputs = {k: v.to(device) for k, v in inputs.items()}
146
-
147
- # Encode reference audio to get prompt duration
148
- try:
149
- off = 3
150
- pw = load_audio(str(ref_audio_path), sr)
151
- if pw.shape[-1] % full_hop != 0:
152
- pw = F.pad(pw, (0, full_hop - pw.shape[-1] % full_hop))
153
- pw_padded = F.pad(pw, (0, full_hop * off))
154
- with torch.no_grad():
155
- plt = model.vae.encode(pw_padded.unsqueeze(0).to(device))
156
- if off:
157
- plt = plt[..., :-off]
158
- prompt_dur = plt.shape[-1]
159
- prompt_wav = load_audio(str(ref_audio_path), sr).unsqueeze(0)
160
- except Exception as e:
161
- return None, f"Failed to process reference audio: {e}"
162
-
163
- prompt_time = prompt_dur * full_hop / sr
164
- dur_sec = approx_duration_from_text(synth_text, max_duration=max_dur - prompt_time)
165
- try:
166
- approx_pd = approx_duration_from_text(ref_text, max_duration=max_dur)
167
- ratio = np.clip(prompt_time / approx_pd, 1.0, 1.5)
168
- dur_sec = dur_sec * ratio
169
- except Exception:
170
- pass
171
-
172
- duration = int(dur_sec * sr // full_hop)
173
- duration = min(duration + prompt_dur, int(max_dur * sr // full_hop))
174
-
175
- try:
176
- with torch.no_grad():
177
- output = model(
178
- input_ids=inputs["input_ids"],
179
- attention_mask=inputs["attention_mask"],
180
- prompt_audio=prompt_wav,
181
- duration=duration,
182
- steps=nfe,
183
- cfg_strength=guidance_strength,
184
- guidance_method=guidance_method,
185
- )
186
- except Exception as e:
187
- return None, f"Generation failed: {e}"
188
-
189
- wav = output.waveform.squeeze().detach().cpu().numpy()
190
- out_path = OUTPUT_DIR / f"clone_{int(time.time())}.wav"
191
- sf.write(str(out_path), wav, sr)
192
-
193
- return str(out_path), f"Done β€” {len(wav)/sr:.2f}s generated"
194
-
195
-
196
- # ---------------------------------------------------------------------------
197
- # Core: plain TTS (no reference voice)
198
- # ---------------------------------------------------------------------------
199
- def plain_tts(
200
- text: str,
201
- audiodit_size: str,
202
- nfe: int,
203
- guidance_strength: float,
204
- guidance_method: str,
205
- seed: int,
206
- memory_mode: str,
207
- device: str,
208
- ):
209
- """Synthesise text with no voice reference (random voice)."""
210
- if not text or not text.strip():
211
- return None, "Enter text to synthesise."
212
-
213
- mgr = get_manager(memory_mode)
214
- try:
215
- model, tokenizer = mgr.get_tts(audiodit_size=audiodit_size, device=device)
216
- except Exception as e:
217
- return None, f"Failed to load TTS model: {e}"
218
-
219
- torch.manual_seed(seed)
220
- if torch.cuda.is_available():
221
- torch.cuda.manual_seed(seed)
222
-
223
- sr = model.config.sampling_rate
224
- full_hop = model.config.latent_hop
225
- max_dur = model.config.max_wav_duration
226
-
227
- t = normalize_text(text)
228
- inputs = tokenizer([t], padding="longest", return_tensors="pt")
229
- inputs = {k: v.to(device) for k, v in inputs.items()}
230
-
231
- dur_sec = approx_duration_from_text(t, max_duration=max_dur)
232
- duration = int(dur_sec * sr // full_hop)
233
- duration = min(duration, int(max_dur * sr // full_hop))
234
-
235
- try:
236
- with torch.no_grad():
237
- output = model(
238
- input_ids=inputs["input_ids"],
239
- attention_mask=inputs["attention_mask"],
240
- prompt_audio=None,
241
- duration=duration,
242
- steps=nfe,
243
- cfg_strength=guidance_strength,
244
- guidance_method=guidance_method,
245
- )
246
- except Exception as e:
247
- return None, f"Generation failed: {e}"
248
-
249
- wav = output.waveform.squeeze().detach().cpu().numpy()
250
- out_path = OUTPUT_DIR / f"tts_{int(time.time())}.wav"
251
- sf.write(str(out_path), wav, sr)
252
- return str(out_path), f"Done β€” {len(wav)/sr:.2f}s generated"
253
-
254
-
255
- # ---------------------------------------------------------------------------
256
- # Voice Library helpers (called from UI)
257
- # ---------------------------------------------------------------------------
258
- def library_names_with_placeholder() -> list[str]:
259
- lib = get_library()
260
- names = lib.names()
261
- return ["β€” select saved voice β€”"] + names
262
-
263
- def save_voice_to_library(name: str, audio_path, transcription: str):
264
- """Save a (audio, transcription) pair to the library. Returns (new_dropdown, status)."""
265
- name = (name or "").strip()
266
- if not name:
267
- return gr.update(), "Enter a name for this voice."
268
- if audio_path is None:
269
- return gr.update(), "No reference audio to save."
270
- if not transcription or not transcription.strip():
271
- return gr.update(), "Transcription is empty β€” auto-transcribe first."
272
- try:
273
- get_library().add(name, str(audio_path), transcription)
274
- except Exception as e:
275
- return gr.update(), f"Save failed: {e}"
276
- choices = library_names_with_placeholder()
277
- return gr.update(choices=choices, value=name), f"Saved '{name}' to voice library."
278
-
279
- def load_voice_from_library(name: str):
280
- """Load a saved voice. Returns (audio_path, transcription, status)."""
281
- if not name or name.startswith("β€”"):
282
- return None, "", ""
283
- entry = get_library().get(name)
284
- if entry is None:
285
- return None, "", f"Voice '{name}' not found."
286
- audio = entry["audio_path"]
287
- if not Path(audio).exists():
288
- return None, "", f"Audio file missing: {audio}"
289
- return audio, entry["transcription"], f"Loaded '{name}'"
290
-
291
- def delete_voice_from_library(name: str):
292
- """Delete a voice. Returns (new_dropdown_update, status)."""
293
- if not name or name.startswith("β€”"):
294
- return gr.update(), "Select a voice to delete."
295
- ok = get_library().remove(name)
296
- choices = library_names_with_placeholder()
297
- msg = f"Deleted '{name}'." if ok else f"Voice '{name}' not found."
298
- return gr.update(choices=choices, value=choices[0]), msg
299
-
300
- def refresh_library_dropdown():
301
- choices = library_names_with_placeholder()
302
- return gr.update(choices=choices)
303
-
304
- def library_summary():
305
- return get_library().summary_text()
306
-
307
-
308
- # ---------------------------------------------------------------------------
309
- # Status / unload
310
- # ---------------------------------------------------------------------------
311
- def get_status(memory_mode: str) -> str:
312
- return get_manager(memory_mode).status_str()
313
-
314
- def unload_all(memory_mode: str) -> str:
315
- mgr = get_manager(memory_mode)
316
- mgr.release_all()
317
- return "All models unloaded.\n" + mgr.status_str()
318
-
319
-
320
- # ---------------------------------------------------------------------------
321
- # Download helpers
322
- # ---------------------------------------------------------------------------
323
- def _model_inventory() -> str:
324
- lines = ["AudioDiT TTS models:"]
325
- for k, (repo, hint) in AUDIODIT_MODELS.items():
326
- st = "[downloaded]" if _audiodit_present(k) else "not downloaded"
327
- lines.append(f" AudioDiT-{k:<6} {hint:<8} {st}")
328
- lines.append("")
329
- lines.append("Whisper STT models:")
330
- for k, (repo, hint) in WHISPER_MODELS.items():
331
- st = "[downloaded]" if _whisper_present(k) else "not downloaded"
332
- lines.append(f" Whisper-{k:<10} {hint:<8} {st}")
333
- return "\n".join(lines)
334
-
335
- def download_with_progress(selected_models: list):
336
- if not selected_models:
337
- yield "Nothing selected."
338
- return
339
- log = []
340
- def emit(msg):
341
- log.append(msg)
342
- for label in selected_models:
343
- if label.startswith("AudioDiT-"):
344
- size = label.replace("AudioDiT-", "")
345
- _, hint = AUDIODIT_MODELS.get(size, ("", "?"))
346
- log.append(f"AudioDiT-{size} ({hint}): {'already downloaded' if _audiodit_present(size) else 'downloading...'}"); yield "\n".join(log)
347
- download_audiodit(size, callback=emit); yield "\n".join(log)
348
- elif label.startswith("Whisper-"):
349
- size = label.replace("Whisper-", "")
350
- _, hint = WHISPER_MODELS.get(size, ("", "?"))
351
- log.append(f"Whisper-{size} ({hint}): {'already downloaded' if _whisper_present(size) else 'downloading...'}"); yield "\n".join(log)
352
- download_whisper(size, callback=emit); yield "\n".join(log)
353
- log.extend(["", _model_inventory()])
354
- yield "\n".join(log)
355
-
356
-
357
- # ---------------------------------------------------------------------------
358
- # Gradio UI
359
- # ---------------------------------------------------------------------------
360
- def build_ui(default_device: str = "cuda"):
361
-
362
- AUDIODIT_CHOICES = ["1B", "3.5B"]
363
- WHISPER_CHOICES = ["turbo", "large-v3", "medium", "small"]
364
- MEMORY_MODES = ["auto", "simultaneous", "sequential"]
365
- GUIDANCE_METHODS = ["cfg", "apg"]
366
- LANGUAGE_CHOICES = [
367
- "auto", "en", "zh", "ja", "ko", "de", "fr", "es", "pt", "ru",
368
- "ar", "hi", "it", "nl", "pl", "tr", "uk", "vi", "id", "th",
369
- ]
370
-
371
- with gr.Blocks(title="LongCat-AudioDiT β€” Voice Cloning") as demo:
372
-
373
- gr.Markdown(
374
- "# LongCat-AudioDiT β€” Voice Cloning Studio\n"
375
- "State-of-the-art voice cloning: give it a reference audio, type your text, get the result."
376
- )
377
-
378
- # ── Global settings row ──────────────────────────────────────────
379
- with gr.Row():
380
- memory_mode_dd = gr.Dropdown(MEMORY_MODES, value="auto", label="Memory Mode", scale=1)
381
- device_dd = gr.Dropdown(["cuda", "cpu"], value=default_device, label="Device", scale=1)
382
- status_box = gr.Textbox(label="Model Status", lines=3, interactive=False, scale=3)
383
- with gr.Column(scale=1, min_width=160):
384
- btn_status = gr.Button("Refresh Status", size="sm")
385
- btn_unload = gr.Button("Unload All", size="sm", variant="stop")
386
-
387
- gr.Markdown("---")
388
-
389
- with gr.Tabs():
390
-
391
- # ================================================================
392
- # TAB 1 β€” Voice Cloning (primary workflow)
393
- # ================================================================
394
- with gr.Tab("Voice Cloning"):
395
-
396
- with gr.Row():
397
-
398
- # ── Left: reference voice ────────────────────────────
399
- with gr.Column(scale=2):
400
- gr.Markdown("### Reference Voice")
401
-
402
- with gr.Row():
403
- voice_dd = gr.Dropdown(
404
- choices=library_names_with_placeholder(),
405
- value="β€” select saved voice β€”",
406
- label="Saved Voices",
407
- scale=3,
408
- )
409
- btn_load_voice = gr.Button("Load", size="sm", scale=1)
410
- btn_refresh_lib = gr.Button("Refresh", size="sm", scale=1)
411
-
412
- ref_audio = gr.Audio(
413
- label="Reference Audio (upload or record)",
414
- type="filepath",
415
- )
416
-
417
- whisper_dd = gr.Dropdown(
418
- WHISPER_CHOICES, value="turbo",
419
- label="Whisper Model for Auto-Transcribe",
420
- )
421
- lang_dd = gr.Dropdown(
422
- LANGUAGE_CHOICES, value="auto", label="Language (auto=detect)"
423
- )
424
- btn_transcribe = gr.Button("Auto-Transcribe Reference", variant="secondary")
425
-
426
- ref_transcription = gr.Textbox(
427
- label="Reference Transcription (auto-filled or type manually)",
428
- lines=3,
429
- placeholder="What is being said in the reference audio?",
430
- )
431
-
432
- gr.Markdown("**Save this voice to library**")
433
- with gr.Row():
434
- voice_name_input = gr.Textbox(
435
- label="Voice Name", placeholder="e.g. Alice", scale=3
436
- )
437
- btn_save_voice = gr.Button("Save Voice", size="sm", scale=1, variant="primary")
438
- btn_delete_voice = gr.Button("Delete", size="sm", scale=1, variant="stop")
439
-
440
- lib_status = gr.Textbox(
441
- label="Library", lines=4, interactive=False,
442
- value=library_summary(),
443
- )
444
-
445
- # ── Right: synthesis ─────────────────────────────────
446
- with gr.Column(scale=3):
447
- gr.Markdown("### Text to Synthesise")
448
-
449
- synth_text = gr.Textbox(
450
- label="Text",
451
- lines=6,
452
- placeholder="Type what you want spoken in the reference voice…",
453
- )
454
-
455
- with gr.Row():
456
- audiodit_dd = gr.Dropdown(AUDIODIT_CHOICES, value="1B", label="AudioDiT Model")
457
- guidance_dd = gr.Dropdown(GUIDANCE_METHODS, value="cfg", label="Guidance")
458
-
459
- with gr.Accordion("Advanced", open=False):
460
- with gr.Row():
461
- nfe_sl = gr.Slider(4, 64, value=16, step=1, label="ODE Steps")
462
- strength_sl = gr.Slider(1.0, 10.0, value=4.0, step=0.5, label="Guidance Strength")
463
- seed_nb = gr.Number(value=1024, label="Seed", precision=0)
464
-
465
- btn_clone = gr.Button(
466
- "Generate β€” Clone Voice", variant="primary", size="lg"
467
- )
468
-
469
- clone_audio_out = gr.Audio(label="Output", type="filepath")
470
- clone_status = gr.Textbox(label="Status", lines=2, interactive=False)
471
-
472
- # ── Wire up Tab 1 ────────────────────────────────────────
473
-
474
- btn_transcribe.click(
475
- fn=transcribe_reference,
476
- inputs=[ref_audio, whisper_dd, lang_dd, memory_mode_dd, device_dd],
477
- outputs=[ref_transcription, clone_status],
478
- api_name="transcribe_reference",
479
- )
480
-
481
- btn_clone.click(
482
- fn=clone_voice,
483
- inputs=[
484
- synth_text, ref_audio, ref_transcription,
485
- audiodit_dd, nfe_sl, strength_sl, guidance_dd,
486
- seed_nb, memory_mode_dd, device_dd,
487
- ],
488
- outputs=[clone_audio_out, clone_status],
489
- api_name="clone_voice",
490
- )
491
-
492
- btn_save_voice.click(
493
- fn=save_voice_to_library,
494
- inputs=[voice_name_input, ref_audio, ref_transcription],
495
- outputs=[voice_dd, lib_status],
496
- api_name="save_voice",
497
- )
498
-
499
- btn_load_voice.click(
500
- fn=load_voice_from_library,
501
- inputs=[voice_dd],
502
- outputs=[ref_audio, ref_transcription, clone_status],
503
- api_name="load_voice",
504
- )
505
-
506
- btn_delete_voice.click(
507
- fn=delete_voice_from_library,
508
- inputs=[voice_dd],
509
- outputs=[voice_dd, lib_status],
510
- api_name="delete_voice",
511
- )
512
-
513
- btn_refresh_lib.click(
514
- fn=lambda: (refresh_library_dropdown(), library_summary()),
515
- inputs=[],
516
- outputs=[voice_dd, lib_status],
517
- api_name="list_voices",
518
- )
519
-
520
- # ================================================================
521
- # TAB 2 β€” Plain TTS (no reference voice)
522
- # ================================================================
523
- with gr.Tab("Plain TTS"):
524
- gr.Markdown(
525
- "Synthesise speech without a reference voice. "
526
- "The model picks a random voice β€” useful for testing or when you just need audio."
527
- )
528
- with gr.Row():
529
- with gr.Column(scale=3):
530
- tts_text = gr.Textbox(label="Text", lines=6, placeholder="Enter text here…")
531
- with gr.Row():
532
- tts_model_dd = gr.Dropdown(AUDIODIT_CHOICES, value="1B", label="Model")
533
- tts_guidance_dd = gr.Dropdown(GUIDANCE_METHODS, value="cfg", label="Guidance")
534
- with gr.Accordion("Advanced", open=False):
535
- with gr.Row():
536
- tts_nfe = gr.Slider(4, 64, value=16, step=1, label="ODE Steps")
537
- tts_guidance = gr.Slider(1.0, 10.0, value=4.0, step=0.5, label="Guidance Strength")
538
- tts_seed = gr.Number(value=1024, label="Seed", precision=0)
539
- tts_btn = gr.Button("Generate Speech", variant="primary", size="lg")
540
- with gr.Column(scale=2):
541
- tts_audio_out = gr.Audio(label="Output", type="filepath")
542
- tts_status = gr.Textbox(label="Status", lines=2, interactive=False)
543
-
544
- tts_btn.click(
545
- fn=plain_tts,
546
- inputs=[
547
- tts_text, tts_model_dd, tts_nfe, tts_guidance,
548
- tts_guidance_dd, tts_seed, memory_mode_dd, device_dd,
549
- ],
550
- outputs=[tts_audio_out, tts_status],
551
- api_name="plain_tts",
552
- )
553
-
554
- # ================================================================
555
- # TAB 3 β€” Transcribe Only
556
- # ================================================================
557
- with gr.Tab("Transcribe Audio"):
558
- gr.Markdown("Transcribe any audio file with Whisper β€” output is plain text.")
559
- with gr.Row():
560
- with gr.Column():
561
- stt_audio_in = gr.Audio(label="Audio", type="filepath")
562
- stt_model_dd = gr.Dropdown(WHISPER_CHOICES, value="turbo", label="Whisper Model")
563
- stt_lang_dd = gr.Dropdown(LANGUAGE_CHOICES, value="auto", label="Language")
564
- stt_btn = gr.Button("Transcribe", variant="primary", size="lg")
565
- with gr.Column():
566
- stt_text_out = gr.Textbox(label="Transcription", lines=10)
567
- stt_lang_out = gr.Textbox(label="Detected Language", scale=1)
568
- stt_status = gr.Textbox(label="Status", lines=2, interactive=False)
569
-
570
- stt_btn.click(
571
- fn=_stt_flat,
572
- inputs=[stt_audio_in, stt_model_dd, stt_lang_dd, memory_mode_dd, device_dd],
573
- outputs=[stt_text_out, stt_lang_out, stt_status],
574
- api_name="transcribe",
575
- )
576
-
577
- # ================================================================
578
- # TAB 4 β€” Download Models
579
- # ================================================================
580
- with gr.Tab("Download Models"):
581
- gr.Markdown(
582
- "**Download models before using them.** "
583
- "Select what you need, hit Download, watch the live log. "
584
- "Already-downloaded models are skipped automatically."
585
- )
586
-
587
- _dl_choices = (
588
- [f"AudioDiT-{k} ({hint})" for k, (_, hint) in AUDIODIT_MODELS.items()]
589
- + [f"Whisper-{k} ({hint})" for k, (_, hint) in WHISPER_MODELS.items()]
590
- )
591
- _dl_values = (
592
- [f"AudioDiT-{k}" for k in AUDIODIT_MODELS]
593
- + [f"Whisper-{k}" for k in WHISPER_MODELS]
594
- )
595
- _label_to_value = dict(zip(_dl_choices, _dl_values))
596
-
597
- dl_checkboxes = gr.CheckboxGroup(
598
- choices=_dl_choices,
599
- value=[_dl_choices[0], _dl_choices[2]],
600
- label="Models to Download",
601
- )
602
- with gr.Row():
603
- dl_btn = gr.Button("Download Selected", variant="primary", size="lg")
604
- dl_refresh = gr.Button("Refresh Status", size="lg")
605
-
606
- dl_log = gr.Textbox(
607
- label="Download Log", lines=16, interactive=False,
608
- value=_model_inventory(),
609
- )
610
-
611
- def _run_download(selected_labels):
612
- keys = [_label_to_value.get(lbl, lbl.split(" ")[0]) for lbl in selected_labels]
613
- yield from download_with_progress(keys)
614
-
615
- dl_btn.click(fn=_run_download, inputs=[dl_checkboxes], outputs=[dl_log])
616
- dl_refresh.click(fn=lambda: _model_inventory(), inputs=[], outputs=[dl_log])
617
-
618
- # ================================================================
619
- # TAB 5 β€” About
620
- # ================================================================
621
- with gr.Tab("About"):
622
- gr.Markdown("""
623
- ## LongCat-AudioDiT Enhanced
624
-
625
- Enhanced fork of [LongCat-AudioDiT](https://github.com/meituan-longcat/LongCat-AudioDiT) (Meituan) β€” Apache-2.0.
626
-
627
- ### API Endpoints (Gradio REST API)
628
- All actions are available as REST endpoints at `/api/`:
629
-
630
- | Endpoint | Description |
631
- |---|---|
632
- | `POST /api/clone_voice` | Clone a voice: text + reference audio + transcription β†’ audio |
633
- | `POST /api/transcribe_reference` | Transcribe reference audio with Whisper |
634
- | `POST /api/plain_tts` | Generate speech without a reference voice |
635
- | `POST /api/transcribe` | Transcribe any audio file |
636
- | `POST /api/save_voice` | Save a voice to the library |
637
- | `POST /api/load_voice` | Load a voice from the library by name |
638
- | `POST /api/delete_voice` | Delete a voice from the library |
639
- | `POST /api/list_voices` | List all saved voices |
640
-
641
- ### Models
642
- | Model | VRAM | Notes |
643
- |---|---|---|
644
- | AudioDiT-1B | ~4 GB | Fast, great quality |
645
- | AudioDiT-3.5B | ~10 GB | SOTA quality |
646
- | Whisper Turbo | ~1.6 GB | Fast transcription |
647
- | Whisper large-v3 | ~3 GB | Most accurate |
648
-
649
- ### Voice Library
650
- Voices are stored in `./voices/library.json` with audio files in `./voices/`.
651
- """)
652
-
653
- # ── Global callbacks ─────────────────────────────────────────────
654
- btn_status.click(fn=get_status, inputs=[memory_mode_dd], outputs=[status_box])
655
- btn_unload.click(fn=unload_all, inputs=[memory_mode_dd], outputs=[status_box])
656
- memory_mode_dd.change(fn=get_status, inputs=[memory_mode_dd], outputs=[status_box])
657
-
658
- return demo
659
-
660
-
661
- # ---------------------------------------------------------------------------
662
- # STT flat helper (avoids walrus-operator gymnastics in the lambda above)
663
- # ---------------------------------------------------------------------------
664
- def _stt_flat(audio_path, whisper_size, language, memory_mode, device):
665
- """Returns (transcription, detected_language, status_msg) β€” three separate values."""
666
- from memory_manager import ModelMemoryManager
667
- mgr = get_manager(memory_mode)
668
- try:
669
- whisper = mgr.get_whisper(whisper_size=whisper_size)
670
- except Exception as e:
671
- return "", "", f"Failed to load Whisper: {e}"
672
- if audio_path is None:
673
- return "", "", "Upload an audio file."
674
- lang_arg = language if language and language != "auto" else None
675
- try:
676
- text, detected = whisper.transcribe(str(audio_path), language=lang_arg)
677
- except Exception as e:
678
- return "", "", f"Transcription failed: {e}"
679
- return text, detected, f"Transcribed [{detected}] β€” {len(text)} chars"
680
-
681
-
682
- # ---------------------------------------------------------------------------
683
- # Entry point
684
- # ---------------------------------------------------------------------------
685
- def main():
686
- parser = argparse.ArgumentParser(description="LongCat-AudioDiT Voice Cloning Studio")
687
- parser.add_argument("--port", type=int, default=0)
688
- parser.add_argument("--host", type=str, default="0.0.0.0")
689
- parser.add_argument("--share", action="store_true")
690
- parser.add_argument("--device", type=str, default="auto")
691
- parser.add_argument("--mode", type=str, default="auto",
692
- choices=["auto", "simultaneous", "sequential"])
693
- args = parser.parse_args()
694
-
695
- device = "cuda" if (args.device == "auto" and torch.cuda.is_available()) else args.device
696
-
697
- if args.port == 0:
698
- port = find_free_port(7860, 7960)
699
- elif not _port_free(args.port):
700
- logger.warning("Port %d busy, searching…", args.port)
701
- port = find_free_port(args.port + 1, args.port + 100)
702
- else:
703
- port = args.port
704
-
705
- logger.info("Starting on %s:%d (device=%s, mode=%s)", args.host, port, device, args.mode)
706
- get_manager(args.mode)
707
-
708
- demo = build_ui(default_device=device)
709
- demo.launch(
710
- server_name=args.host,
711
- server_port=port,
712
- share=args.share,
713
- show_error=True,
714
- theme=gr.themes.Soft(),
715
- )
716
-
717
-
718
- if __name__ == "__main__":
719
- main()
 
 
 
 
 
 
1
+ """
2
+ LongCat-AudioDiT Enhanced – Gradio Web UI
3
+
4
+ Primary workflow: Voice Cloning
5
+ 1. Upload reference audio β†’ auto-transcribe with Whisper
6
+ 2. Type text to synthesise in the cloned voice
7
+ 3. Generate β†’ save to Voice Library with a name
8
+ 4. Reuse any saved voice from the dropdown
9
+
10
+ All actions are exposed as Gradio REST API endpoints.
11
+
12
+ Usage:
13
+ python app.py
14
+ python app.py --port 7860 --share
15
+ python app.py --device cpu
16
+ """
17
+
18
+ import argparse
19
+ import logging
20
+ import os
21
+ import socket
22
+ import time
23
+ from pathlib import Path
24
+
25
+ import gradio as gr
26
+ import numpy as np
27
+ import soundfile as sf
28
+ import torch
29
+ import torch.nn.functional as F
30
+
31
+ from utils import normalize_text, load_audio, approx_duration_from_text
32
+ from memory_manager import ModelMemoryManager
33
+ from voice_library import get_library
34
+ from download_models import (
35
+ download_audiodit, download_whisper,
36
+ _audiodit_present, _whisper_present,
37
+ AUDIODIT_MODELS, WHISPER_MODELS,
38
+ AUDIODIT_DIR, WHISPER_DIR,
39
+ )
40
+
41
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
42
+ logger = logging.getLogger(__name__)
43
+
44
+ OUTPUT_DIR = Path(__file__).parent / "outputs"
45
+ OUTPUT_DIR.mkdir(exist_ok=True)
46
+
47
+ # ---------------------------------------------------------------------------
48
+ # Memory manager
49
+ # ---------------------------------------------------------------------------
50
+ _mgr: ModelMemoryManager = None
51
+
52
+ def get_manager(mode: str = "auto") -> ModelMemoryManager:
53
+ global _mgr
54
+ if _mgr is None or _mgr.mode.value != mode:
55
+ if _mgr is not None:
56
+ _mgr.release_all()
57
+ _mgr = ModelMemoryManager(mode=mode)
58
+ return _mgr
59
+
60
+ # ---------------------------------------------------------------------------
61
+ # Port helpers
62
+ # ---------------------------------------------------------------------------
63
+ def _port_free(port: int) -> bool:
64
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
65
+ s.settimeout(1)
66
+ return s.connect_ex(("127.0.0.1", port)) != 0
67
+
68
+ def find_free_port(start: int = 7860, end: int = 7960) -> int:
69
+ for p in range(start, end):
70
+ if _port_free(p):
71
+ return p
72
+ raise RuntimeError(f"No free port found in {start}-{end}")
73
+
74
+ # ---------------------------------------------------------------------------
75
+ # Core: transcribe reference audio
76
+ # ---------------------------------------------------------------------------
77
+ def transcribe_reference(audio_path, whisper_size: str, language: str, memory_mode: str, device: str):
78
+ """
79
+ Transcribe a reference audio file with Whisper.
80
+ Returns (transcription_text, status_msg).
81
+ """
82
+ if audio_path is None:
83
+ return "", "Upload a reference audio file first."
84
+
85
+ mgr = get_manager(memory_mode)
86
+ try:
87
+ whisper = mgr.get_whisper(whisper_size=whisper_size)
88
+ except Exception as e:
89
+ return "", f"Failed to load Whisper: {e}"
90
+
91
+ lang_arg = language if language and language != "auto" else None
92
+ try:
93
+ text, detected = whisper.transcribe(str(audio_path), language=lang_arg)
94
+ except Exception as e:
95
+ return "", f"Transcription failed: {e}"
96
+
97
+ return text, f"Transcribed [{detected}] β€” {len(text)} characters"
98
+
99
+
100
+ # ---------------------------------------------------------------------------
101
+ # Core: clone voice (reference audio + transcription β†’ new speech)
102
+ # ---------------------------------------------------------------------------
103
+ def clone_voice(
104
+ text: str,
105
+ ref_audio_path,
106
+ ref_transcription: str,
107
+ audiodit_size: str,
108
+ nfe: int,
109
+ guidance_strength: float,
110
+ guidance_method: str,
111
+ seed: int,
112
+ memory_mode: str,
113
+ device: str,
114
+ ):
115
+ """
116
+ Synthesise `text` in the voice captured from `ref_audio_path`.
117
+ Returns (output_audio_path, status_msg).
118
+ """
119
+ if not text or not text.strip():
120
+ return None, "Enter text to synthesise."
121
+ if ref_audio_path is None:
122
+ return None, "Upload a reference audio file."
123
+ if not ref_transcription or not ref_transcription.strip():
124
+ return None, "Reference transcription is empty. Use 'Auto-Transcribe' first."
125
+
126
+ mgr = get_manager(memory_mode)
127
+ try:
128
+ model, tokenizer = mgr.get_tts(audiodit_size=audiodit_size, device=device)
129
+ except Exception as e:
130
+ return None, f"Failed to load TTS model: {e}"
131
+
132
+ torch.manual_seed(seed)
133
+ if torch.cuda.is_available():
134
+ torch.cuda.manual_seed(seed)
135
+
136
+ sr = model.config.sampling_rate
137
+ full_hop = model.config.latent_hop
138
+ max_dur = model.config.max_wav_duration
139
+
140
+ synth_text = normalize_text(text)
141
+ ref_text = normalize_text(ref_transcription)
142
+ full_text = f"{ref_text} {synth_text}"
143
+
144
+ inputs = tokenizer([full_text], padding="longest", return_tensors="pt")
145
+ inputs = {k: v.to(device) for k, v in inputs.items()}
146
+
147
+ # Encode reference audio to get prompt duration
148
+ try:
149
+ off = 3
150
+ pw = load_audio(str(ref_audio_path), sr)
151
+ if pw.shape[-1] % full_hop != 0:
152
+ pw = F.pad(pw, (0, full_hop - pw.shape[-1] % full_hop))
153
+ pw_padded = F.pad(pw, (0, full_hop * off))
154
+ with torch.no_grad():
155
+ plt = model.vae.encode(pw_padded.unsqueeze(0).to(device))
156
+ if off:
157
+ plt = plt[..., :-off]
158
+ prompt_dur = plt.shape[-1]
159
+ prompt_wav = load_audio(str(ref_audio_path), sr).unsqueeze(0)
160
+ except Exception as e:
161
+ return None, f"Failed to process reference audio: {e}"
162
+
163
+ prompt_time = prompt_dur * full_hop / sr
164
+ dur_sec = approx_duration_from_text(synth_text, max_duration=max_dur - prompt_time)
165
+ try:
166
+ approx_pd = approx_duration_from_text(ref_text, max_duration=max_dur)
167
+ ratio = np.clip(prompt_time / approx_pd, 1.0, 1.5)
168
+ dur_sec = dur_sec * ratio
169
+ except Exception:
170
+ pass
171
+
172
+ duration = int(dur_sec * sr // full_hop)
173
+ duration = min(duration + prompt_dur, int(max_dur * sr // full_hop))
174
+
175
+ try:
176
+ with torch.no_grad():
177
+ output = model(
178
+ input_ids=inputs["input_ids"],
179
+ attention_mask=inputs["attention_mask"],
180
+ prompt_audio=prompt_wav,
181
+ duration=duration,
182
+ steps=nfe,
183
+ cfg_strength=guidance_strength,
184
+ guidance_method=guidance_method,
185
+ )
186
+ except Exception as e:
187
+ return None, f"Generation failed: {e}"
188
+
189
+ wav = output.waveform.squeeze().detach().cpu().numpy()
190
+ out_path = OUTPUT_DIR / f"clone_{int(time.time())}.wav"
191
+ sf.write(str(out_path), wav, sr)
192
+
193
+ return str(out_path), f"Done β€” {len(wav)/sr:.2f}s generated"
194
+
195
+
196
+ # ---------------------------------------------------------------------------
197
+ # Core: plain TTS (no reference voice)
198
+ # ---------------------------------------------------------------------------
199
+ def plain_tts(
200
+ text: str,
201
+ audiodit_size: str,
202
+ nfe: int,
203
+ guidance_strength: float,
204
+ guidance_method: str,
205
+ seed: int,
206
+ memory_mode: str,
207
+ device: str,
208
+ ):
209
+ """Synthesise text with no voice reference (random voice)."""
210
+ if not text or not text.strip():
211
+ return None, "Enter text to synthesise."
212
+
213
+ mgr = get_manager(memory_mode)
214
+ try:
215
+ model, tokenizer = mgr.get_tts(audiodit_size=audiodit_size, device=device)
216
+ except Exception as e:
217
+ return None, f"Failed to load TTS model: {e}"
218
+
219
+ torch.manual_seed(seed)
220
+ if torch.cuda.is_available():
221
+ torch.cuda.manual_seed(seed)
222
+
223
+ sr = model.config.sampling_rate
224
+ full_hop = model.config.latent_hop
225
+ max_dur = model.config.max_wav_duration
226
+
227
+ t = normalize_text(text)
228
+ inputs = tokenizer([t], padding="longest", return_tensors="pt")
229
+ inputs = {k: v.to(device) for k, v in inputs.items()}
230
+
231
+ dur_sec = approx_duration_from_text(t, max_duration=max_dur)
232
+ duration = int(dur_sec * sr // full_hop)
233
+ duration = min(duration, int(max_dur * sr // full_hop))
234
+
235
+ try:
236
+ with torch.no_grad():
237
+ output = model(
238
+ input_ids=inputs["input_ids"],
239
+ attention_mask=inputs["attention_mask"],
240
+ prompt_audio=None,
241
+ duration=duration,
242
+ steps=nfe,
243
+ cfg_strength=guidance_strength,
244
+ guidance_method=guidance_method,
245
+ )
246
+ except Exception as e:
247
+ return None, f"Generation failed: {e}"
248
+
249
+ wav = output.waveform.squeeze().detach().cpu().numpy()
250
+ out_path = OUTPUT_DIR / f"tts_{int(time.time())}.wav"
251
+ sf.write(str(out_path), wav, sr)
252
+ return str(out_path), f"Done β€” {len(wav)/sr:.2f}s generated"
253
+
254
+
255
+ # ---------------------------------------------------------------------------
256
+ # Voice Library helpers (called from UI)
257
+ # ---------------------------------------------------------------------------
258
+ def library_names_with_placeholder() -> list[str]:
259
+ lib = get_library()
260
+ names = lib.names()
261
+ return ["β€” select saved voice β€”"] + names
262
+
263
+ def save_voice_to_library(name: str, audio_path, transcription: str):
264
+ """Save a (audio, transcription) pair to the library. Returns (new_dropdown, status)."""
265
+ name = (name or "").strip()
266
+ if not name:
267
+ return gr.update(), "Enter a name for this voice."
268
+ if audio_path is None:
269
+ return gr.update(), "No reference audio to save."
270
+ if not transcription or not transcription.strip():
271
+ return gr.update(), "Transcription is empty β€” auto-transcribe first."
272
+ try:
273
+ get_library().add(name, str(audio_path), transcription)
274
+ except Exception as e:
275
+ return gr.update(), f"Save failed: {e}"
276
+ choices = library_names_with_placeholder()
277
+ return gr.update(choices=choices, value=name), f"Saved '{name}' to voice library."
278
+
279
+ def load_voice_from_library(name: str):
280
+ """Load a saved voice. Returns (audio_path, transcription, status)."""
281
+ if not name or name.startswith("β€”"):
282
+ return None, "", ""
283
+ entry = get_library().get(name)
284
+ if entry is None:
285
+ return None, "", f"Voice '{name}' not found."
286
+ audio = entry["audio_path"]
287
+ if not Path(audio).exists():
288
+ return None, "", f"Audio file missing: {audio}"
289
+ return audio, entry["transcription"], f"Loaded '{name}'"
290
+
291
+ def delete_voice_from_library(name: str):
292
+ """Delete a voice. Returns (new_dropdown_update, status)."""
293
+ if not name or name.startswith("β€”"):
294
+ return gr.update(), "Select a voice to delete."
295
+ ok = get_library().remove(name)
296
+ choices = library_names_with_placeholder()
297
+ msg = f"Deleted '{name}'." if ok else f"Voice '{name}' not found."
298
+ return gr.update(choices=choices, value=choices[0]), msg
299
+
300
+ def refresh_library_dropdown():
301
+ choices = library_names_with_placeholder()
302
+ return gr.update(choices=choices)
303
+
304
+ def library_summary():
305
+ return get_library().summary_text()
306
+
307
+
308
+ # ---------------------------------------------------------------------------
309
+ # Status / unload
310
+ # ---------------------------------------------------------------------------
311
+ def get_status(memory_mode: str) -> str:
312
+ return get_manager(memory_mode).status_str()
313
+
314
+ def unload_all(memory_mode: str) -> str:
315
+ mgr = get_manager(memory_mode)
316
+ mgr.release_all()
317
+ return "All models unloaded.\n" + mgr.status_str()
318
+
319
+
320
+ # ---------------------------------------------------------------------------
321
+ # Download helpers
322
+ # ---------------------------------------------------------------------------
323
+ def _model_inventory() -> str:
324
+ lines = ["AudioDiT TTS models:"]
325
+ for k, (repo, hint) in AUDIODIT_MODELS.items():
326
+ st = "[downloaded]" if _audiodit_present(k) else "not downloaded"
327
+ lines.append(f" AudioDiT-{k:<6} {hint:<8} {st}")
328
+ lines.append("")
329
+ lines.append("Whisper STT models:")
330
+ for k, (repo, hint) in WHISPER_MODELS.items():
331
+ st = "[downloaded]" if _whisper_present(k) else "not downloaded"
332
+ lines.append(f" Whisper-{k:<10} {hint:<8} {st}")
333
+ return "\n".join(lines)
334
+
335
+ def download_with_progress(selected_models: list):
336
+ if not selected_models:
337
+ yield "Nothing selected."
338
+ return
339
+ log = []
340
+ def emit(msg):
341
+ log.append(msg)
342
+ for label in selected_models:
343
+ if label.startswith("AudioDiT-"):
344
+ size = label.replace("AudioDiT-", "")
345
+ _, hint = AUDIODIT_MODELS.get(size, ("", "?"))
346
+ log.append(f"AudioDiT-{size} ({hint}): {'already downloaded' if _audiodit_present(size) else 'downloading...'}"); yield "\n".join(log)
347
+ download_audiodit(size, callback=emit); yield "\n".join(log)
348
+ elif label.startswith("Whisper-"):
349
+ size = label.replace("Whisper-", "")
350
+ _, hint = WHISPER_MODELS.get(size, ("", "?"))
351
+ log.append(f"Whisper-{size} ({hint}): {'already downloaded' if _whisper_present(size) else 'downloading...'}"); yield "\n".join(log)
352
+ download_whisper(size, callback=emit); yield "\n".join(log)
353
+ log.extend(["", _model_inventory()])
354
+ yield "\n".join(log)
355
+
356
+
357
+ # ---------------------------------------------------------------------------
358
+ # Gradio UI
359
+ # ---------------------------------------------------------------------------
360
+ def build_ui(default_device: str = "cuda"):
361
+
362
+ AUDIODIT_CHOICES = ["1B", "3.5B"]
363
+ WHISPER_CHOICES = ["turbo", "large-v3", "medium", "small"]
364
+ MEMORY_MODES = ["auto", "simultaneous", "sequential"]
365
+ GUIDANCE_METHODS = ["cfg", "apg"]
366
+ LANGUAGE_CHOICES = [
367
+ "auto", "en", "zh", "ja", "ko", "de", "fr", "es", "pt", "ru",
368
+ "ar", "hi", "it", "nl", "pl", "tr", "uk", "vi", "id", "th",
369
+ ]
370
+
371
+ with gr.Blocks(title="LongCat-AudioDiT β€” Voice Cloning") as demo:
372
+
373
+ gr.Markdown(
374
+ "# LongCat-AudioDiT β€” Voice Cloning Studio\n"
375
+ "State-of-the-art voice cloning based on [LongCat-AudioDiT](https://github.com/meituan-longcat/LongCat-AudioDiT) by the Meituan LongCat Team. "
376
+ "Give it a reference audio, type your text, get the result.\n\n"
377
+ "> **Research & Testing Only.** This tool is provided strictly for research, educational, and personal experimentation purposes. "
378
+ "It is **not** intended for generating deceptive, misleading, or harmful content. "
379
+ "Do not use it to impersonate real individuals without their explicit consent, to create non-consensual deepfakes, "
380
+ "or for any activity that violates applicable laws. By using this tool you accept full responsibility for your use."
381
+ )
382
+
383
+ # ── Global settings row ──────────────────────────────────────────
384
+ with gr.Row():
385
+ memory_mode_dd = gr.Dropdown(MEMORY_MODES, value="auto", label="Memory Mode", scale=1)
386
+ device_dd = gr.Dropdown(["cuda", "cpu"], value=default_device, label="Device", scale=1)
387
+ status_box = gr.Textbox(label="Model Status", lines=3, interactive=False, scale=3)
388
+ with gr.Column(scale=1, min_width=160):
389
+ btn_status = gr.Button("Refresh Status", size="sm")
390
+ btn_unload = gr.Button("Unload All", size="sm", variant="stop")
391
+
392
+ gr.Markdown("---")
393
+
394
+ with gr.Tabs():
395
+
396
+ # ================================================================
397
+ # TAB 1 β€” Voice Cloning (primary workflow)
398
+ # ================================================================
399
+ with gr.Tab("Voice Cloning"):
400
+
401
+ with gr.Row():
402
+
403
+ # ── Left: reference voice ────────────────────────────
404
+ with gr.Column(scale=2):
405
+ gr.Markdown("### Reference Voice")
406
+
407
+ with gr.Row():
408
+ voice_dd = gr.Dropdown(
409
+ choices=library_names_with_placeholder(),
410
+ value="β€” select saved voice β€”",
411
+ label="Saved Voices",
412
+ scale=3,
413
+ )
414
+ btn_load_voice = gr.Button("Load", size="sm", scale=1)
415
+ btn_refresh_lib = gr.Button("Refresh", size="sm", scale=1)
416
+
417
+ ref_audio = gr.Audio(
418
+ label="Reference Audio (upload or record)",
419
+ type="filepath",
420
+ )
421
+
422
+ whisper_dd = gr.Dropdown(
423
+ WHISPER_CHOICES, value="turbo",
424
+ label="Whisper Model for Auto-Transcribe",
425
+ )
426
+ lang_dd = gr.Dropdown(
427
+ LANGUAGE_CHOICES, value="auto", label="Language (auto=detect)"
428
+ )
429
+ btn_transcribe = gr.Button("Auto-Transcribe Reference", variant="secondary")
430
+
431
+ ref_transcription = gr.Textbox(
432
+ label="Reference Transcription (auto-filled or type manually)",
433
+ lines=3,
434
+ placeholder="What is being said in the reference audio?",
435
+ )
436
+
437
+ gr.Markdown("**Save this voice to library**")
438
+ with gr.Row():
439
+ voice_name_input = gr.Textbox(
440
+ label="Voice Name", placeholder="e.g. Alice", scale=3
441
+ )
442
+ btn_save_voice = gr.Button("Save Voice", size="sm", scale=1, variant="primary")
443
+ btn_delete_voice = gr.Button("Delete", size="sm", scale=1, variant="stop")
444
+
445
+ lib_status = gr.Textbox(
446
+ label="Library", lines=4, interactive=False,
447
+ value=library_summary(),
448
+ )
449
+
450
+ # ── Right: synthesis ─────────────────────────────────
451
+ with gr.Column(scale=3):
452
+ gr.Markdown("### Text to Synthesise")
453
+
454
+ synth_text = gr.Textbox(
455
+ label="Text",
456
+ lines=6,
457
+ placeholder="Type what you want spoken in the reference voice…",
458
+ )
459
+
460
+ with gr.Row():
461
+ audiodit_dd = gr.Dropdown(AUDIODIT_CHOICES, value="1B", label="AudioDiT Model")
462
+ guidance_dd = gr.Dropdown(GUIDANCE_METHODS, value="cfg", label="Guidance")
463
+
464
+ with gr.Accordion("Advanced", open=False):
465
+ with gr.Row():
466
+ nfe_sl = gr.Slider(4, 64, value=16, step=1, label="ODE Steps")
467
+ strength_sl = gr.Slider(1.0, 10.0, value=4.0, step=0.5, label="Guidance Strength")
468
+ seed_nb = gr.Number(value=1024, label="Seed", precision=0)
469
+
470
+ btn_clone = gr.Button(
471
+ "Generate β€” Clone Voice", variant="primary", size="lg"
472
+ )
473
+
474
+ clone_audio_out = gr.Audio(label="Output", type="filepath")
475
+ clone_status = gr.Textbox(label="Status", lines=2, interactive=False)
476
+
477
+ # ── Wire up Tab 1 ────────────────────────────────────────
478
+
479
+ btn_transcribe.click(
480
+ fn=transcribe_reference,
481
+ inputs=[ref_audio, whisper_dd, lang_dd, memory_mode_dd, device_dd],
482
+ outputs=[ref_transcription, clone_status],
483
+ api_name="transcribe_reference",
484
+ )
485
+
486
+ btn_clone.click(
487
+ fn=clone_voice,
488
+ inputs=[
489
+ synth_text, ref_audio, ref_transcription,
490
+ audiodit_dd, nfe_sl, strength_sl, guidance_dd,
491
+ seed_nb, memory_mode_dd, device_dd,
492
+ ],
493
+ outputs=[clone_audio_out, clone_status],
494
+ api_name="clone_voice",
495
+ )
496
+
497
+ btn_save_voice.click(
498
+ fn=save_voice_to_library,
499
+ inputs=[voice_name_input, ref_audio, ref_transcription],
500
+ outputs=[voice_dd, lib_status],
501
+ api_name="save_voice",
502
+ )
503
+
504
+ btn_load_voice.click(
505
+ fn=load_voice_from_library,
506
+ inputs=[voice_dd],
507
+ outputs=[ref_audio, ref_transcription, clone_status],
508
+ api_name="load_voice",
509
+ )
510
+
511
+ btn_delete_voice.click(
512
+ fn=delete_voice_from_library,
513
+ inputs=[voice_dd],
514
+ outputs=[voice_dd, lib_status],
515
+ api_name="delete_voice",
516
+ )
517
+
518
+ btn_refresh_lib.click(
519
+ fn=lambda: (refresh_library_dropdown(), library_summary()),
520
+ inputs=[],
521
+ outputs=[voice_dd, lib_status],
522
+ api_name="list_voices",
523
+ )
524
+
525
+ # ================================================================
526
+ # TAB 2 β€” Plain TTS (no reference voice)
527
+ # ================================================================
528
+ with gr.Tab("Plain TTS"):
529
+ gr.Markdown(
530
+ "Synthesise speech without a reference voice. "
531
+ "The model picks a random voice β€” useful for testing or when you just need audio."
532
+ )
533
+ with gr.Row():
534
+ with gr.Column(scale=3):
535
+ tts_text = gr.Textbox(label="Text", lines=6, placeholder="Enter text here…")
536
+ with gr.Row():
537
+ tts_model_dd = gr.Dropdown(AUDIODIT_CHOICES, value="1B", label="Model")
538
+ tts_guidance_dd = gr.Dropdown(GUIDANCE_METHODS, value="cfg", label="Guidance")
539
+ with gr.Accordion("Advanced", open=False):
540
+ with gr.Row():
541
+ tts_nfe = gr.Slider(4, 64, value=16, step=1, label="ODE Steps")
542
+ tts_guidance = gr.Slider(1.0, 10.0, value=4.0, step=0.5, label="Guidance Strength")
543
+ tts_seed = gr.Number(value=1024, label="Seed", precision=0)
544
+ tts_btn = gr.Button("Generate Speech", variant="primary", size="lg")
545
+ with gr.Column(scale=2):
546
+ tts_audio_out = gr.Audio(label="Output", type="filepath")
547
+ tts_status = gr.Textbox(label="Status", lines=2, interactive=False)
548
+
549
+ tts_btn.click(
550
+ fn=plain_tts,
551
+ inputs=[
552
+ tts_text, tts_model_dd, tts_nfe, tts_guidance,
553
+ tts_guidance_dd, tts_seed, memory_mode_dd, device_dd,
554
+ ],
555
+ outputs=[tts_audio_out, tts_status],
556
+ api_name="plain_tts",
557
+ )
558
+
559
+ # ================================================================
560
+ # TAB 3 β€” Transcribe Only
561
+ # ================================================================
562
+ with gr.Tab("Transcribe Audio"):
563
+ gr.Markdown("Transcribe any audio file with Whisper β€” output is plain text.")
564
+ with gr.Row():
565
+ with gr.Column():
566
+ stt_audio_in = gr.Audio(label="Audio", type="filepath")
567
+ stt_model_dd = gr.Dropdown(WHISPER_CHOICES, value="turbo", label="Whisper Model")
568
+ stt_lang_dd = gr.Dropdown(LANGUAGE_CHOICES, value="auto", label="Language")
569
+ stt_btn = gr.Button("Transcribe", variant="primary", size="lg")
570
+ with gr.Column():
571
+ stt_text_out = gr.Textbox(label="Transcription", lines=10)
572
+ stt_lang_out = gr.Textbox(label="Detected Language", scale=1)
573
+ stt_status = gr.Textbox(label="Status", lines=2, interactive=False)
574
+
575
+ stt_btn.click(
576
+ fn=_stt_flat,
577
+ inputs=[stt_audio_in, stt_model_dd, stt_lang_dd, memory_mode_dd, device_dd],
578
+ outputs=[stt_text_out, stt_lang_out, stt_status],
579
+ api_name="transcribe",
580
+ )
581
+
582
+ # ================================================================
583
+ # TAB 4 β€” Download Models
584
+ # ================================================================
585
+ with gr.Tab("Download Models"):
586
+ gr.Markdown(
587
+ "**Download models before using them.** "
588
+ "Select what you need, hit Download, watch the live log. "
589
+ "Already-downloaded models are skipped automatically."
590
+ )
591
+
592
+ _dl_choices = (
593
+ [f"AudioDiT-{k} ({hint})" for k, (_, hint) in AUDIODIT_MODELS.items()]
594
+ + [f"Whisper-{k} ({hint})" for k, (_, hint) in WHISPER_MODELS.items()]
595
+ )
596
+ _dl_values = (
597
+ [f"AudioDiT-{k}" for k in AUDIODIT_MODELS]
598
+ + [f"Whisper-{k}" for k in WHISPER_MODELS]
599
+ )
600
+ _label_to_value = dict(zip(_dl_choices, _dl_values))
601
+
602
+ dl_checkboxes = gr.CheckboxGroup(
603
+ choices=_dl_choices,
604
+ value=[_dl_choices[0], _dl_choices[2]],
605
+ label="Models to Download",
606
+ )
607
+ with gr.Row():
608
+ dl_btn = gr.Button("Download Selected", variant="primary", size="lg")
609
+ dl_refresh = gr.Button("Refresh Status", size="lg")
610
+
611
+ dl_log = gr.Textbox(
612
+ label="Download Log", lines=16, interactive=False,
613
+ value=_model_inventory(),
614
+ )
615
+
616
+ def _run_download(selected_labels):
617
+ keys = [_label_to_value.get(lbl, lbl.split(" ")[0]) for lbl in selected_labels]
618
+ yield from download_with_progress(keys)
619
+
620
+ dl_btn.click(fn=_run_download, inputs=[dl_checkboxes], outputs=[dl_log])
621
+ dl_refresh.click(fn=lambda: _model_inventory(), inputs=[], outputs=[dl_log])
622
+
623
+ # ================================================================
624
+ # TAB 5 β€” About
625
+ # ================================================================
626
+ with gr.Tab("About"):
627
+ gr.Markdown("""
628
+ ## LongCat-AudioDiT Enhanced
629
+
630
+ Enhanced fork of [LongCat-AudioDiT](https://github.com/meituan-longcat/LongCat-AudioDiT) (Meituan) β€” Apache-2.0.
631
+
632
+ ### API Endpoints (Gradio REST API)
633
+ All actions are available as REST endpoints at `/api/`:
634
+
635
+ | Endpoint | Description |
636
+ |---|---|
637
+ | `POST /api/clone_voice` | Clone a voice: text + reference audio + transcription β†’ audio |
638
+ | `POST /api/transcribe_reference` | Transcribe reference audio with Whisper |
639
+ | `POST /api/plain_tts` | Generate speech without a reference voice |
640
+ | `POST /api/transcribe` | Transcribe any audio file |
641
+ | `POST /api/save_voice` | Save a voice to the library |
642
+ | `POST /api/load_voice` | Load a voice from the library by name |
643
+ | `POST /api/delete_voice` | Delete a voice from the library |
644
+ | `POST /api/list_voices` | List all saved voices |
645
+
646
+ ### Models
647
+ | Model | VRAM | Notes |
648
+ |---|---|---|
649
+ | AudioDiT-1B | ~4 GB | Fast, great quality |
650
+ | AudioDiT-3.5B | ~10 GB | SOTA quality |
651
+ | Whisper Turbo | ~1.6 GB | Fast transcription |
652
+ | Whisper large-v3 | ~3 GB | Most accurate |
653
+
654
+ ### Voice Library
655
+ Voices are stored in `./voices/library.json` with audio files in `./voices/`.
656
+ """)
657
+
658
+ # ── Global callbacks ─────────────────────────────────────────────
659
+ btn_status.click(fn=get_status, inputs=[memory_mode_dd], outputs=[status_box])
660
+ btn_unload.click(fn=unload_all, inputs=[memory_mode_dd], outputs=[status_box])
661
+ memory_mode_dd.change(fn=get_status, inputs=[memory_mode_dd], outputs=[status_box])
662
+
663
+ return demo
664
+
665
+
666
+ # ---------------------------------------------------------------------------
667
+ # STT flat helper (avoids walrus-operator gymnastics in the lambda above)
668
+ # ---------------------------------------------------------------------------
669
+ def _stt_flat(audio_path, whisper_size, language, memory_mode, device):
670
+ """Returns (transcription, detected_language, status_msg) β€” three separate values."""
671
+ from memory_manager import ModelMemoryManager
672
+ mgr = get_manager(memory_mode)
673
+ try:
674
+ whisper = mgr.get_whisper(whisper_size=whisper_size)
675
+ except Exception as e:
676
+ return "", "", f"Failed to load Whisper: {e}"
677
+ if audio_path is None:
678
+ return "", "", "Upload an audio file."
679
+ lang_arg = language if language and language != "auto" else None
680
+ try:
681
+ text, detected = whisper.transcribe(str(audio_path), language=lang_arg)
682
+ except Exception as e:
683
+ return "", "", f"Transcription failed: {e}"
684
+ return text, detected, f"Transcribed [{detected}] β€” {len(text)} chars"
685
+
686
+
687
+ # ---------------------------------------------------------------------------
688
+ # Entry point
689
+ # ---------------------------------------------------------------------------
690
+ def main():
691
+ parser = argparse.ArgumentParser(description="LongCat-AudioDiT Voice Cloning Studio")
692
+ parser.add_argument("--port", type=int, default=0)
693
+ parser.add_argument("--host", type=str, default="0.0.0.0")
694
+ parser.add_argument("--share", action="store_true")
695
+ parser.add_argument("--device", type=str, default="auto")
696
+ parser.add_argument("--mode", type=str, default="auto",
697
+ choices=["auto", "simultaneous", "sequential"])
698
+ args = parser.parse_args()
699
+
700
+ device = "cuda" if (args.device == "auto" and torch.cuda.is_available()) else args.device
701
+
702
+ if args.port == 0:
703
+ port = find_free_port(7860, 7960)
704
+ elif not _port_free(args.port):
705
+ logger.warning("Port %d busy, searching…", args.port)
706
+ port = find_free_port(args.port + 1, args.port + 100)
707
+ else:
708
+ port = args.port
709
+
710
+ logger.info("Starting on %s:%d (device=%s, mode=%s)", args.host, port, device, args.mode)
711
+ get_manager(args.mode)
712
+
713
+ demo = build_ui(default_device=device)
714
+ demo.launch(
715
+ server_name=args.host,
716
+ server_port=port,
717
+ share=args.share,
718
+ show_error=True,
719
+ theme=gr.themes.Soft(),
720
+ )
721
+
722
+
723
+ if __name__ == "__main__":
724
+ main()