Spaces:
Running on Zero
Running on Zero
| #!/usr/bin/env python3 | |
| """ | |
| OmniVoice Multilingual Voice Cloner | |
| Space: cosmichackerx/voice-cloner | |
| Zero-shot voice cloning & multilingual TTS powered by k2-fsa/OmniVoice. | |
| Crash-hardened: long-form, multi-speaker, voice profiles, BGM ducking, | |
| RVC-inspired polish (F0 / timbre / protect), OmniVoice advanced decoding. | |
| """ | |
| from __future__ import annotations | |
| import dataclasses | |
| import logging | |
| import os | |
| import re | |
| import tempfile | |
| import traceback | |
| from typing import Any | |
| import gradio as gr | |
| import numpy as np | |
| import spaces | |
| import torch | |
| import audio_enhance as ax | |
| logging.basicConfig( | |
| level=logging.INFO, | |
| format="%(asctime)s %(name)s %(levelname)s: %(message)s", | |
| ) | |
| logger = logging.getLogger("voice-cloner") | |
| CHECKPOINT = os.environ.get("OMNIVOICE_MODEL", "k2-fsa/OmniVoice") | |
| MAX_CHARS_PER_CHUNK = 280 | |
| MAX_TOTAL_CHARS = 8000 | |
| MAX_CHUNKS = 24 | |
| MAX_DIALOGUE_TURNS = 20 | |
| CROSSFADE_MS = 40 | |
| SILENCE_GAP_MS = 180 | |
| FEATURED = [ | |
| "Auto", | |
| "English", | |
| "Spanish", | |
| "French", | |
| "German", | |
| "Portuguese", | |
| "Italian", | |
| "Russian", | |
| "Chinese", | |
| "Japanese", | |
| "Korean", | |
| "Standard Arabic", | |
| "Hindi", | |
| "Turkish", | |
| "Indonesian", | |
| "Malay", | |
| "Vietnamese", | |
| "Thai", | |
| "Bengali", | |
| "Persian", | |
| "Urdu", | |
| "Dutch", | |
| "Polish", | |
| "Ukrainian", | |
| "Swahili", | |
| ] | |
| STYLE_PRESETS: dict[str, str] = { | |
| "Neutral": "", | |
| "Laughing intro": "[laughter] ", | |
| "Sigh then speak": "[sigh] ", | |
| "Surprised": "[surprise-oh] ", | |
| "Questioning tone": "[question-en] ", | |
| "Confirmation tone": "[confirmation-en] ", | |
| "Dissatisfied": "[dissatisfaction-hnn] ", | |
| } | |
| NONVERBAL_HELP = ( | |
| "Inline tags: [laughter] [sigh] [confirmation-en] [question-en] " | |
| "[question-ah] [question-oh] [surprise-ah] [surprise-oh] [dissatisfaction-hnn]" | |
| ) | |
| # One-click demo scripts — upload voice → click prompt → generate | |
| TRY_PROMPTS: dict[str, dict[str, str]] = { | |
| "YouTube intro": { | |
| "text": ( | |
| "Hey everyone, welcome back to the channel. Today I'm breaking down " | |
| "something that actually changed how I create content — stay until the end." | |
| ), | |
| "language": "English", | |
| "mode": "Voice clone", | |
| "style": "Neutral", | |
| "instruct": "", | |
| }, | |
| "Audiobook narrator": { | |
| "text": ( | |
| "The old lighthouse stood against the storm, its lamp cutting through fog " | |
| "like a promise. She climbed the spiral stairs, heart steady, knowing the " | |
| "letter in her coat would change everything." | |
| ), | |
| "language": "English", | |
| "mode": "Voice clone", | |
| "style": "Neutral", | |
| "instruct": "", | |
| }, | |
| "Podcast cold open": { | |
| "text": ( | |
| "You're listening to Signal Check. I'm your host, and in the next twenty minutes " | |
| "we'll unpack the story nobody expected — and what it means for all of us." | |
| ), | |
| "language": "English", | |
| "mode": "Voice clone", | |
| "style": "Neutral", | |
| "instruct": "", | |
| }, | |
| "Product ad read": { | |
| "text": ( | |
| "Meet Clarity Buds — all-day comfort, studio-clean sound, and a battery that " | |
| "keeps up with your life. Free shipping this week. Tap the link and hear the difference." | |
| ), | |
| "language": "English", | |
| "mode": "Voice clone", | |
| "style": "Confirmation tone", | |
| "instruct": "", | |
| }, | |
| "Meditation guide": { | |
| "text": ( | |
| "Take a slow breath in… and gently release. Soften your shoulders. " | |
| "There is nowhere else you need to be. Just this moment, just this breath." | |
| ), | |
| "language": "English", | |
| "mode": "Voice clone", | |
| "style": "Neutral", | |
| "instruct": "", | |
| }, | |
| "News bulletin": { | |
| "text": ( | |
| "Good evening. In tonight's top story, researchers released new findings on " | |
| "multilingual speech synthesis, marking a major step for accessible AI voice technology worldwide." | |
| ), | |
| "language": "English", | |
| "mode": "Voice clone", | |
| "style": "Neutral", | |
| "instruct": "", | |
| }, | |
| "Customer support": { | |
| "text": ( | |
| "Thanks for calling. I can help with that. Please confirm the email on your account, " | |
| "and I'll walk you through the next steps carefully." | |
| ), | |
| "language": "English", | |
| "mode": "Voice clone", | |
| "style": "Neutral", | |
| "instruct": "", | |
| }, | |
| "Gaming character": { | |
| "text": ( | |
| "[surprise-oh] The gate is open! Stay close — if we lose the map now, " | |
| "the whole quest falls apart. Move!" | |
| ), | |
| "language": "English", | |
| "mode": "Voice clone", | |
| "style": "Surprised", | |
| "instruct": "", | |
| }, | |
| "Laugh + punchline": { | |
| "text": ( | |
| "[laughter] Okay, that was not in the script. But seriously — if your AI voice " | |
| "can laugh with you, you're already winning." | |
| ), | |
| "language": "English", | |
| "mode": "Voice clone", | |
| "style": "Laughing intro", | |
| "instruct": "", | |
| }, | |
| "Spanish demo": { | |
| "text": ( | |
| "Hola, este es un demo de clonación de voz. Puedo hablar con naturalidad " | |
| "en español usando solo unos segundos de audio de referencia." | |
| ), | |
| "language": "Spanish", | |
| "mode": "Voice clone", | |
| "style": "Neutral", | |
| "instruct": "", | |
| }, | |
| "French demo": { | |
| "text": ( | |
| "Bonjour ! Voici une démonstration de clonage vocal. Avec quelques secondes " | |
| "d'audio, on peut générer une voix naturelle en français." | |
| ), | |
| "language": "French", | |
| "mode": "Voice clone", | |
| "style": "Neutral", | |
| "instruct": "", | |
| }, | |
| "Chinese demo": { | |
| "text": "大家好，这是零样本声音克隆演示。只需几秒参考音频，就能用中文自然地说出新的内容。", | |
| "language": "Chinese", | |
| "mode": "Voice clone", | |
| "style": "Neutral", | |
| "instruct": "", | |
| }, | |
| "Arabic demo": { | |
| "text": "مرحبًا، هذا عرض لنسخ الصوت بالذكاء الاصطناعي. يمكنني التحدث بالعربية بصوت قريب من المرجع.", | |
| "language": "Standard Arabic", | |
| "mode": "Voice clone", | |
| "style": "Neutral", | |
| "instruct": "", | |
| }, | |
| "Hindi demo": { | |
| "text": "नमस्ते! यह एक मुफ़्त एआई वॉइस क्लोन डेमो है। कुछ सेकंड के ऑडियो से नई हिंदी स्पीच बनाई जा सकती है।", | |
| "language": "Hindi", | |
| "mode": "Voice clone", | |
| "style": "Neutral", | |
| "instruct": "", | |
| }, | |
| "Japanese demo": { | |
| "text": "こんにちは。これはゼロショット音声クローンのデモです。短い参考音声から、自然な日本語を生成できます。", | |
| "language": "Japanese", | |
| "mode": "Voice clone", | |
| "style": "Neutral", | |
| "instruct": "", | |
| }, | |
| "Multi-speaker dialogue": { | |
| "text": ( | |
| "[Speaker 1]: Did you try the new free AI voice cloner yet?\n" | |
| "[Speaker 2]: Yes — I uploaded ten seconds of audio and it already sounds like me.\n" | |
| "[Speaker 1]: Perfect for podcasts and YouTube voiceovers." | |
| ), | |
| "language": "English", | |
| "mode": "Multi-speaker dialogue", | |
| "style": "Neutral", | |
| "instruct": "", | |
| }, | |
| "Voice design · British": { | |
| "text": ( | |
| "This voice was designed from text attributes only — no reference clip required. " | |
| "Ideal when you need a consistent narrator fast." | |
| ), | |
| "language": "English", | |
| "mode": "Voice design (no reference)", | |
| "style": "Neutral", | |
| "instruct": "female, young adult, low pitch, british accent", | |
| }, | |
| "Voice design · Whisper": { | |
| "text": "Come closer. Some secrets are meant to be whispered, not shouted.", | |
| "language": "English", | |
| "mode": "Voice design (no reference)", | |
| "style": "Neutral", | |
| "instruct": "male, middle-aged, low pitch, whisper", | |
| }, | |
| "Long-form sample": { | |
| "text": ( | |
| "Welcome to this long-form narration test. First, we set the scene with a calm introduction. " | |
| "Next, we explain why clean reference audio matters for voice cloning quality. " | |
| "Finally, we finish with a clear call to action: like the Space, try another prompt, " | |
| "and share your best clone with creators who need multilingual text to speech." | |
| ), | |
| "language": "English", | |
| "mode": "Voice clone", | |
| "style": "Neutral", | |
| "instruct": "", | |
| }, | |
| } | |
| SPEAKER_RE = re.compile( | |
| r"^\s*\[?\s*(speaker\s*[12]|s\s*[12]|spk\s*[12])\s*\]?\s*[:\-–—]\s*(.+)$", | |
| re.IGNORECASE, | |
| ) | |
| SENTENCE_SPLIT_RE = re.compile(r"(?<=[.!?。！？…])\s+|(?<=[\n])\s*") | |
| logger.info("Loading OmniVoice from %s ...", CHECKPOINT) | |
| from omnivoice import OmniVoice, OmniVoiceGenerationConfig, VoiceClonePrompt | |
| try: | |
| from omnivoice.utils.lang_map import LANG_NAMES, lang_display_name | |
| _ALL = sorted({lang_display_name(code) for code in LANG_NAMES}) | |
| except Exception: # noqa: BLE001 | |
| logger.warning("Language map unavailable; using featured list only.") | |
| _ALL = [] | |
| LANGUAGE_CHOICES = FEATURED + [name for name in _ALL if name not in FEATURED] | |
| model = OmniVoice.from_pretrained( | |
| CHECKPOINT, | |
| device_map="auto", | |
| dtype=torch.float16, | |
| load_asr=True, | |
| ) | |
| SAMPLE_RATE = int(model.sampling_rate) | |
| logger.info("Model ready (sample rate=%s).", SAMPLE_RATE) | |
| try: | |
| _GEN_FIELDS = {f.name for f in dataclasses.fields(OmniVoiceGenerationConfig)} | |
| except Exception: # noqa: BLE001 | |
| _GEN_FIELDS = { | |
| "num_step", | |
| "guidance_scale", | |
| "denoise", | |
| "preprocess_prompt", | |
| "postprocess_output", | |
| } | |
| def _safe_str(value: Any, default: str = "") -> str: | |
| try: | |
| if value is None: | |
| return default | |
| return str(value) | |
| except Exception: # noqa: BLE001 | |
| return default | |
| def _resolve_language(language: str | None) -> str | None: | |
| if not language or language == "Auto": | |
| return None | |
| return language | |
| def _to_gradio_audio(audio_np: np.ndarray) -> tuple[int, np.ndarray]: | |
| wav = ax.as_float_mono(audio_np) | |
| return SAMPLE_RATE, (wav * 32767.0).astype(np.int16) | |
| def _as_float_mono(audio_np: np.ndarray) -> np.ndarray: | |
| return ax.as_float_mono(audio_np) | |
| def _crossfade_concat(parts: list[np.ndarray], gap_ms: int = SILENCE_GAP_MS) -> np.ndarray: | |
| clean = [_as_float_mono(p) for p in parts if p is not None and np.asarray(p).size > 0] | |
| if not clean: | |
| return np.zeros(1, dtype=np.float32) | |
| if len(clean) == 1: | |
| return clean[0] | |
| gap = max(0, int(SAMPLE_RATE * gap_ms / 1000.0)) | |
| fade = max(0, int(SAMPLE_RATE * CROSSFADE_MS / 1000.0)) | |
| out = clean[0] | |
| for nxt in clean[1:]: | |
| if gap > 0: | |
| out = np.concatenate([out, np.zeros(gap, dtype=np.float32), nxt]) | |
| continue | |
| if fade <= 0 or out.size < fade or nxt.size < fade: | |
| out = np.concatenate([out, nxt]) | |
| continue | |
| fade_out = np.linspace(1.0, 0.0, fade, dtype=np.float32) | |
| fade_in = np.linspace(0.0, 1.0, fade, dtype=np.float32) | |
| mixed = out[-fade:] * fade_out + nxt[:fade] * fade_in | |
| out = np.concatenate([out[:-fade], mixed, nxt[fade:]]) | |
| return out | |
| def _chunk_text(text: str, max_chars: int = MAX_CHARS_PER_CHUNK) -> list[str]: | |
| text = re.sub(r"\s+", " ", text.strip()) | |
| if not text: | |
| return [] | |
| if len(text) <= max_chars: | |
| return [text] | |
| pieces = [p.strip() for p in SENTENCE_SPLIT_RE.split(text) if p and p.strip()] or [text] | |
| chunks: list[str] = [] | |
| buf = "" | |
| for piece in pieces: | |
| if len(piece) > max_chars: | |
| if buf: | |
| chunks.append(buf) | |
| buf = "" | |
| for i in range(0, len(piece), max_chars): | |
| chunks.append(piece[i : i + max_chars].strip()) | |
| continue | |
| candidate = f"{buf} {piece}".strip() if buf else piece | |
| if len(candidate) <= max_chars: | |
| buf = candidate | |
| else: | |
| if buf: | |
| chunks.append(buf) | |
| buf = piece | |
| if buf: | |
| chunks.append(buf) | |
| return [c for c in chunks if c][:MAX_CHUNKS] | |
| def _parse_dialogue(text: str) -> list[tuple[int, str]] | None: | |
| lines = [ln.strip() for ln in text.splitlines() if ln.strip()] | |
| if len(lines) < 2: | |
| return None | |
| turns: list[tuple[int, str]] = [] | |
| tagged = 0 | |
| for ln in lines: | |
| m = SPEAKER_RE.match(ln) | |
| if m: | |
| tagged += 1 | |
| label = re.sub(r"\s+", "", m.group(1).lower()) | |
| speaker = 2 if label[-1] == "2" else 1 | |
| turns.append((speaker, m.group(2).strip())) | |
| else: | |
| turns.append((turns[-1][0] if turns else 1, ln)) | |
| if tagged == 0: | |
| return None | |
| return turns[:MAX_DIALOGUE_TURNS] | |
| def _apply_style(text: str, style: str | None) -> str: | |
| prefix = STYLE_PRESETS.get(_safe_str(style, "Neutral"), "") | |
| if not prefix or text.lstrip().startswith("["): | |
| return text | |
| return prefix + text | |
| def _load_audio_file(path: str | None) -> np.ndarray | None: | |
| if not path or not os.path.isfile(path): | |
| return None | |
| try: | |
| import soundfile as sf | |
| data, sr = sf.read(path, always_2d=False) | |
| wav = np.asarray(data, dtype=np.float32) | |
| if wav.ndim > 1: | |
| wav = wav.mean(axis=-1) | |
| if int(sr) != SAMPLE_RATE and wav.size > 0: | |
| try: | |
| import librosa | |
| wav = librosa.resample(wav, orig_sr=int(sr), target_sr=SAMPLE_RATE) | |
| except Exception: # noqa: BLE001 | |
| duration = wav.size / float(sr) | |
| new_len = max(1, int(duration * SAMPLE_RATE)) | |
| x_old = np.linspace(0.0, 1.0, num=wav.size, endpoint=False) | |
| x_new = np.linspace(0.0, 1.0, num=new_len, endpoint=False) | |
| wav = np.interp(x_new, x_old, wav).astype(np.float32) | |
| return _as_float_mono(wav) | |
| except Exception: # noqa: BLE001 | |
| logger.exception("Failed to load audio: %s", path) | |
| return None | |
| def _mix_with_bgm( | |
| voice: np.ndarray, | |
| bgm_path: str | None, | |
| bgm_volume: float = 0.18, | |
| duck_db: float = 8.0, | |
| ) -> np.ndarray: | |
| try: | |
| voice = _as_float_mono(voice) | |
| if not bgm_path: | |
| return voice | |
| bgm = _load_audio_file(bgm_path) | |
| if bgm is None or bgm.size == 0: | |
| return voice | |
| bgm_volume = float(np.clip(bgm_volume, 0.0, 1.0)) | |
| duck_gain = 10 ** (-float(np.clip(duck_db, 0.0, 24.0)) / 20.0) | |
| if bgm.size < voice.size: | |
| bgm = np.tile(bgm, int(np.ceil(voice.size / bgm.size)))[: voice.size] | |
| else: | |
| bgm = bgm[: voice.size] | |
| frame = max(1, int(SAMPLE_RATE * 0.02)) | |
| energy = np.convolve(np.abs(voice), np.ones(frame) / frame, mode="same") | |
| thresh = max(0.02, float(np.percentile(energy, 55)) * 0.6) | |
| gain = np.where(energy > thresh, duck_gain, 1.0).astype(np.float32) | |
| win = max(1, int(SAMPLE_RATE * 0.05)) | |
| gain = np.convolve(gain, np.ones(win) / win, mode="same") | |
| mixed = voice + bgm * bgm_volume * gain | |
| peak = float(np.max(np.abs(mixed))) | |
| if peak > 0.98: | |
| mixed = mixed * (0.98 / peak) | |
| return mixed.astype(np.float32) | |
| except Exception: # noqa: BLE001 | |
| logger.exception("BGM mix failed; returning dry voice") | |
| return _as_float_mono(voice) | |
| def _build_gen_config( | |
| num_step: float, | |
| guidance_scale: float, | |
| denoise: bool, | |
| preprocess_prompt: bool, | |
| postprocess_output: bool, | |
| t_shift: float | None = 0.1, | |
| position_temperature: float | None = 5.0, | |
| class_temperature: float | None = 0.0, | |
| pad_duration: float | None = 0.1, | |
| fade_duration: float | None = 0.1, | |
| audio_chunk_duration: float | None = 15.0, | |
| audio_chunk_threshold: float | None = 30.0, | |
| use_native_longform: bool = False, | |
| ) -> OmniVoiceGenerationConfig: | |
| steps = max(8, min(64, int(num_step or 32))) | |
| cfg = max(0.5, min(3.5, float(guidance_scale if guidance_scale is not None else 2.0))) | |
| payload: dict[str, Any] = { | |
| "num_step": steps, | |
| "guidance_scale": cfg, | |
| "denoise": bool(denoise), | |
| "preprocess_prompt": bool(preprocess_prompt), | |
| "postprocess_output": bool(postprocess_output), | |
| "t_shift": float(t_shift if t_shift is not None else 0.1), | |
| "position_temperature": float(position_temperature if position_temperature is not None else 5.0), | |
| "class_temperature": float(class_temperature if class_temperature is not None else 0.0), | |
| "pad_duration": float(pad_duration if pad_duration is not None else 0.1), | |
| "fade_duration": float(fade_duration if fade_duration is not None else 0.1), | |
| } | |
| if use_native_longform: | |
| payload["audio_chunk_duration"] = float(audio_chunk_duration or 15.0) | |
| payload["audio_chunk_threshold"] = float(audio_chunk_threshold or 30.0) | |
| filtered = {k: v for k, v in payload.items() if k in _GEN_FIELDS} | |
| try: | |
| return OmniVoiceGenerationConfig(**filtered) | |
| except Exception: # noqa: BLE001 | |
| logger.exception("GenConfig advanced fields failed; using minimal config") | |
| return OmniVoiceGenerationConfig( | |
| num_step=steps, | |
| guidance_scale=cfg, | |
| denoise=bool(denoise), | |
| preprocess_prompt=bool(preprocess_prompt), | |
| postprocess_output=bool(postprocess_output), | |
| ) | |
| def _make_prompt(ref_audio: str | None, ref_text: str | None) -> VoiceClonePrompt | None: | |
| if not ref_audio: | |
| return None | |
| try: | |
| return model.create_voice_clone_prompt( | |
| ref_audio=ref_audio, | |
| ref_text=(ref_text.strip() if ref_text and str(ref_text).strip() else None), | |
| ) | |
| except Exception: # noqa: BLE001 | |
| logger.exception("Failed to create voice clone prompt") | |
| return None | |
| def _generate_one( | |
| text: str, | |
| language: str | None, | |
| gen_config: OmniVoiceGenerationConfig, | |
| prompt: VoiceClonePrompt | None, | |
| speed: float, | |
| duration: float | None, | |
| instruct: str | None, | |
| normalize_text: bool, | |
| blend_instruct: bool = False, | |
| ) -> np.ndarray | None: | |
| text = text.strip() | |
| if not text: | |
| return None | |
| kw: dict[str, Any] = { | |
| "text": text, | |
| "language": language, | |
| "generation_config": gen_config, | |
| } | |
| if prompt is not None: | |
| kw["voice_clone_prompt"] = prompt | |
| # OmniVoice tip: consistent instruct + ref improves dialect/style stability | |
| if blend_instruct and instruct and str(instruct).strip(): | |
| kw["instruct"] = str(instruct).strip() | |
| elif instruct and str(instruct).strip(): | |
| kw["instruct"] = str(instruct).strip() | |
| if speed is not None and abs(float(speed) - 1.0) > 1e-3: | |
| kw["speed"] = float(np.clip(speed, 0.7, 1.3)) | |
| if duration is not None and float(duration) > 0: | |
| kw["duration"] = float(duration) | |
| if normalize_text: | |
| kw["normalize_text"] = True | |
| try: | |
| audio = model.generate(**kw) | |
| if not audio: | |
| return None | |
| return _as_float_mono(audio[0]) | |
| except TypeError: | |
| for drop in ("normalize_text", "instruct"): | |
| kw.pop(drop, None) | |
| try: | |
| audio = model.generate(**kw) | |
| if audio: | |
| return _as_float_mono(audio[0]) | |
| except Exception: # noqa: BLE001 | |
| continue | |
| logger.exception("Generation failed for chunk") | |
| return None | |
| except Exception: # noqa: BLE001 | |
| logger.exception("Generation failed for chunk") | |
| return None | |
| def _export_audio(wav: np.ndarray, format_type: str = "WAV") -> str | None: | |
| try: | |
| import soundfile as sf | |
| import tempfile | |
| ext = f".{format_type.lower()}" | |
| tmp = tempfile.NamedTemporaryFile(delete=False, suffix=ext, prefix="clone_") | |
| tmp.close() | |
| fmt = format_type.upper() | |
| if fmt == "MP3": | |
| try: | |
| sf.write(tmp.name, wav, SAMPLE_RATE, format="MP3") | |
| except Exception: | |
| import scipy.io.wavfile as wavfile | |
| from pydub import AudioSegment | |
| import os | |
| tmp_wav = tmp.name + ".wav" | |
| wavfile.write(tmp_wav, SAMPLE_RATE, wav) | |
| audio = AudioSegment.from_wav(tmp_wav) | |
| audio.export(tmp.name, format="mp3") | |
| os.remove(tmp_wav) | |
| else: | |
| sf.write(tmp.name, wav, SAMPLE_RATE, format=fmt) | |
| return tmp.name | |
| except Exception as e: | |
| print(f"Export error: {e}") | |
| return None | |
| def update_export(audio_data, format_type): | |
| if audio_data is None: | |
| return None | |
| sr, y = audio_data | |
| return _export_audio(y, format_type) | |
| def _finalize( | |
| parts: list[np.ndarray], | |
| bgm_audio, | |
| bgm_volume, | |
| duck_db, | |
| ref_for_polish: np.ndarray | None, | |
| f0_semitones, | |
| index_rate, | |
| protect, | |
| loud_norm, | |
| enable_polish: bool, | |
| download_format: str = "WAV", | |
| ) -> tuple[np.ndarray, str, str | None]: | |
| voice = _crossfade_concat(parts) | |
| polish_note = "Polish: off" | |
| if enable_polish: | |
| voice, polish_note = ax.apply_rvc_inspired_polish( | |
| voice, | |
| ref_for_polish, | |
| SAMPLE_RATE, | |
| f0_semitones=float(f0_semitones or 0), | |
| index_rate=float(index_rate or 0), | |
| protect=float(protect or 0), | |
| loud_norm=bool(loud_norm), | |
| ) | |
| voice = _mix_with_bgm(voice, bgm_audio, float(bgm_volume or 0.18), float(duck_db or 8)) | |
| return voice, polish_note, _export_audio(voice, download_format) | |
| def _estimate_gpu_seconds( | |
| text, | |
| language, | |
| ref_audio, | |
| ref_text, | |
| num_step, | |
| guidance_scale, | |
| denoise, | |
| speed, | |
| duration, | |
| preprocess_prompt, | |
| postprocess_output, | |
| style, | |
| long_form, | |
| mode, | |
| ref_audio_2, | |
| ref_text_2, | |
| instruct, | |
| normalize_text, | |
| bgm_audio, | |
| bgm_volume, | |
| duck_db, | |
| *args, | |
| **kwargs, | |
| ) -> int: | |
| try: | |
| raw = _safe_str(text) | |
| n_chars = min(len(raw), MAX_TOTAL_CHARS) | |
| dialogue = _parse_dialogue(raw) if mode == "Multi-speaker dialogue" else None | |
| if dialogue: | |
| n_chunks = min(len(dialogue), MAX_DIALOGUE_TURNS) | |
| elif long_form or n_chars > 1200: | |
| n_chunks = max(1, min(MAX_CHUNKS, (n_chars // MAX_CHARS_PER_CHUNK) + 1)) | |
| else: | |
| n_chunks = 1 | |
| steps = int(num_step or 32) | |
| per = 8 + int(steps * 0.35) | |
| # Pitch/timbre polish is CPU — small buffer only | |
| total = 30 + n_chunks * per | |
| return int(min(280, max(45, total))) | |
| except Exception: # noqa: BLE001 | |
| return 120 | |
| # --------------------------------------------------------------------------- | |
| # Voice library | |
| # --------------------------------------------------------------------------- | |
| def save_voice_profile(name, ref_audio, ref_text, library): | |
| library = dict(library or {}) | |
| name = _safe_str(name).strip() | |
| if not name: | |
| return library, gr.update(), "Enter a profile name." | |
| if not ref_audio: | |
| return library, gr.update(), "Upload reference audio before saving a profile." | |
| if len(library) >= 12: | |
| return library, gr.update(choices=list(library.keys())), "Library full (max 12)." | |
| prompt = _make_prompt(ref_audio, ref_text) | |
| if prompt is None: | |
| return library, gr.update(choices=list(library.keys())), "Could not encode reference." | |
| try: | |
| tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".pt", prefix=f"voice_{name}_") | |
| tmp.close() | |
| prompt.save(tmp.name) | |
| library[name] = {"path": tmp.name, "ref_text": _safe_str(ref_text).strip(), "ref_audio": ref_audio} | |
| return library, gr.update(choices=list(library.keys()), value=name), f"Saved “{name}”." | |
| except Exception as exc: # noqa: BLE001 | |
| return library, gr.update(choices=list(library.keys())), f"Save failed: {exc}" | |
| def load_voice_profile(name, library): | |
| library = library or {} | |
| name = _safe_str(name).strip() | |
| if not name or name not in library: | |
| return None, "", None, "Select a saved profile." | |
| entry = library[name] | |
| path = entry.get("path") | |
| return entry.get("ref_audio"), entry.get("ref_text", ""), (path if path and os.path.isfile(path) else None), f"Loaded “{name}”." | |
| def import_voice_pt(pt_file, name, library): | |
| library = dict(library or {}) | |
| name = _safe_str(name).strip() or "Imported" | |
| if not pt_file or not os.path.isfile(pt_file): | |
| return library, gr.update(), "Upload a .pt voice prompt file." | |
| try: | |
| _ = VoiceClonePrompt.load(pt_file) | |
| tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".pt", prefix=f"voice_{name}_") | |
| tmp.close() | |
| with open(pt_file, "rb") as src, open(tmp.name, "wb") as dst: | |
| dst.write(src.read()) | |
| library[name] = {"path": tmp.name, "ref_text": "", "ref_audio": None} | |
| return library, gr.update(choices=list(library.keys()), value=name), f"Imported “{name}”." | |
| except Exception as exc: # noqa: BLE001 | |
| return library, gr.update(choices=list(library.keys())), f"Import failed: {exc}" | |
| def delete_voice_profile(name, library): | |
| library = dict(library or {}) | |
| name = _safe_str(name).strip() | |
| if name in library: | |
| try: | |
| path = library[name].get("path") | |
| if path and os.path.isfile(path): | |
| os.remove(path) | |
| except Exception: # noqa: BLE001 | |
| pass | |
| del library[name] | |
| return library, gr.update(choices=list(library.keys()), value=None), f"Deleted “{name}”." | |
| return library, gr.update(choices=list(library.keys())), "Nothing to delete." | |
| def _prompt_from_library_or_audio(profile_name, library, ref_audio, ref_text): | |
| library = library or {} | |
| name = _safe_str(profile_name).strip() | |
| if name and name in library: | |
| path = library[name].get("path") | |
| if path and os.path.isfile(path): | |
| try: | |
| return VoiceClonePrompt.load(path), f"Using saved profile “{name}”.", library[name].get("ref_audio") | |
| except Exception: # noqa: BLE001 | |
| logger.exception("Failed loading profile %s", name) | |
| if ref_audio: | |
| prompt = _make_prompt(ref_audio, ref_text) | |
| if prompt is not None: | |
| return prompt, "Using uploaded reference audio.", ref_audio | |
| return None, "Could not encode reference audio.", None | |
| return None, "No reference voice provided.", None | |
| def apply_quality_preset(preset: str): | |
| cfg = ax.QUALITY_PRESETS.get(preset, ax.QUALITY_PRESETS["Balanced"]) | |
| return cfg["num_step"], cfg["guidance_scale"], cfg["t_shift"] | |
| def analyze_ref_ui(ref_audio): | |
| return ax.analyze_reference(ref_audio, SAMPLE_RATE) | |
| def apply_try_prompt(prompt_name: str): | |
| """Fill text / language / mode / style / instruct from a curated try prompt.""" | |
| entry = TRY_PROMPTS.get(_safe_str(prompt_name)) | |
| if not entry: | |
| return gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), "Pick a try prompt." | |
| return ( | |
| entry["text"], | |
| entry.get("language", "Auto"), | |
| entry.get("mode", "Voice clone"), | |
| entry.get("style", "Neutral"), | |
| entry.get("instruct", ""), | |
| f"Loaded “{prompt_name}”. Upload/record your voice → enable consent → Generate.", | |
| ) | |
| # --------------------------------------------------------------------------- | |
| # Core generation | |
| # --------------------------------------------------------------------------- | |
| def clone_voice( | |
| text, | |
| language, | |
| ref_audio, | |
| ref_text, | |
| num_step, | |
| guidance_scale, | |
| denoise, | |
| speed, | |
| duration, | |
| preprocess_prompt, | |
| postprocess_output, | |
| style, | |
| long_form, | |
| mode, | |
| ref_audio_2, | |
| ref_text_2, | |
| instruct, | |
| normalize_text, | |
| bgm_audio, | |
| bgm_volume, | |
| duck_db, | |
| profile_name, | |
| library, | |
| consent, | |
| ref_prep, | |
| blend_instruct, | |
| use_native_longform, | |
| t_shift, | |
| position_temperature, | |
| class_temperature, | |
| pad_duration, | |
| fade_duration, | |
| audio_chunk_duration, | |
| audio_chunk_threshold, | |
| enable_polish, | |
| f0_semitones, | |
| index_rate, | |
| protect, | |
| loud_norm, | |
| download_format, | |
| progress=gr.Progress(track_tqdm=False), | |
| ): | |
| try: | |
| yield from _clone_voice_impl( | |
| text, language, ref_audio, ref_text, num_step, guidance_scale, denoise, | |
| speed, duration, preprocess_prompt, postprocess_output, style, long_form, | |
| mode, ref_audio_2, ref_text_2, instruct, normalize_text, bgm_audio, | |
| bgm_volume, duck_db, profile_name, library, consent, ref_prep, | |
| blend_instruct, use_native_longform, t_shift, position_temperature, | |
| class_temperature, pad_duration, fade_duration, audio_chunk_duration, | |
| audio_chunk_threshold, enable_polish, f0_semitones, index_rate, | |
| protect, loud_norm, download_format, progress, | |
| ) | |
| except Exception as exc: # noqa: BLE001 | |
| logger.error("Unhandled clone_voice error:\n%s", traceback.format_exc()) | |
| yield None, f"Unexpected error (recovered): {type(exc).__name__}: {exc}", None | |
| def _clone_voice_impl( | |
| text, language, ref_audio, ref_text, num_step, guidance_scale, denoise, | |
| speed, duration, preprocess_prompt, postprocess_output, style, long_form, | |
| mode, ref_audio_2, ref_text_2, instruct, normalize_text, bgm_audio, | |
| bgm_volume, duck_db, profile_name, library, consent, ref_prep, | |
| blend_instruct, use_native_longform, t_shift, position_temperature, | |
| class_temperature, pad_duration, fade_duration, audio_chunk_duration, | |
| audio_chunk_threshold, enable_polish, f0_semitones, index_rate, | |
| protect, loud_norm, download_format, progress, | |
| ): | |
| if not consent: | |
| yield None, "Please confirm you have rights/consent to clone this voice.", None | |
| return | |
| raw = _safe_str(text).strip() | |
| if not raw: | |
| yield None, "Enter text to speak in any of 600+ supported languages.", None | |
| return | |
| if len(raw) > MAX_TOTAL_CHARS: | |
| yield None, f"Text too long. Keep under {MAX_TOTAL_CHARS} characters.", None | |
| return | |
| mode = _safe_str(mode, "Voice clone") | |
| lang = _resolve_language(language) | |
| gen_config = _build_gen_config( | |
| num_step, guidance_scale, denoise, preprocess_prompt, postprocess_output, | |
| t_shift, position_temperature, class_temperature, pad_duration, fade_duration, | |
| audio_chunk_duration, audio_chunk_threshold, bool(use_native_longform), | |
| ) | |
| speed_f = float(speed) if speed is not None else 1.0 | |
| dur_f = float(duration) if duration is not None and float(duration or 0) > 0 else None | |
| use_duration = dur_f if (not long_form and mode != "Multi-speaker dialogue") else None | |
| blend = bool(blend_instruct) | |
| polish_on = bool(enable_polish) | |
| # Optional reference preparation (RVC/OmniVoice quality tip: clean short ref) | |
| prep_notes = [] | |
| if bool(ref_prep) and ref_audio: | |
| ref_audio, note = ax.prepare_reference(ref_audio, SAMPLE_RATE, enable=True, gate=True) | |
| prep_notes.append(note) | |
| if bool(ref_prep) and ref_audio_2: | |
| ref_audio_2, note2 = ax.prepare_reference(ref_audio_2, SAMPLE_RATE, enable=True, gate=True) | |
| prep_notes.append("S2: " + note2) | |
| def _done(parts, ref_path, msg): | |
| ref_wav = _load_audio_file(ref_path) if ref_path else None | |
| final, polish_note, wav_path = _finalize( | |
| parts, bgm_audio, bgm_volume, duck_db, ref_wav, | |
| f0_semitones, index_rate, protect, loud_norm, polish_on, download_format | |
| ) | |
| extra = " · ".join([x for x in prep_notes + [polish_note] if x]) | |
| if bgm_audio: | |
| msg += " · BGM ducked" | |
| if extra: | |
| msg += f" · {extra}" | |
| yield _to_gradio_audio(final), msg, wav_path | |
| # -------- Voice design -------- | |
| if mode == "Voice design (no reference)": | |
| instr = _safe_str(instruct).strip() | |
| if not instr: | |
| yield None, "Voice design needs instruct, e.g. “female, low pitch, british accent”.", None | |
| return | |
| styled = _apply_style(raw, style) | |
| # Prefer native OmniVoice chunking when enabled; else our splitter | |
| if use_native_longform and not long_form: | |
| chunks = [styled] | |
| else: | |
| chunks = _chunk_text(styled) if (long_form or len(styled) > 1200) else [styled] | |
| yield None, f"Voice design · {len(chunks)} chunk(s)…", None | |
| parts: list[np.ndarray] = [] | |
| for i, chunk in enumerate(chunks): | |
| progress((i + 1) / len(chunks), desc=f"Design {i + 1}/{len(chunks)}") | |
| wav = _generate_one( | |
| chunk, lang, gen_config, None, speed_f, | |
| use_duration if len(chunks) == 1 else None, instr, bool(normalize_text), False, | |
| ) | |
| if wav is None: | |
| if parts: | |
| yield from _done(parts, None, f"Stopped at chunk {i + 1} (partial kept).") | |
| else: | |
| yield None, f"Failed at chunk {i + 1}.", None | |
| return | |
| parts.append(wav) | |
| yield _to_gradio_audio(_crossfade_concat(parts)), f"Chunk {i + 1}/{len(chunks)}…", None | |
| yield from _done(parts, None, "Done — voice design ready.") | |
| return | |
| if mode == "Auto voice (random)": | |
| styled = _apply_style(raw, style) | |
| chunks = [styled] if (use_native_longform and not long_form) else ( | |
| _chunk_text(styled) if (long_form or len(styled) > 1200) else [styled] | |
| ) | |
| yield None, f"Auto voice · {len(chunks)} chunk(s)…", None | |
| parts = [] | |
| for i, chunk in enumerate(chunks): | |
| progress((i + 1) / len(chunks), desc=f"Auto {i + 1}/{len(chunks)}") | |
| wav = _generate_one( | |
| chunk, lang, gen_config, None, speed_f, | |
| use_duration if len(chunks) == 1 else None, None, bool(normalize_text), False, | |
| ) | |
| if wav is None: | |
| if parts: | |
| yield from _done(parts, None, f"Stopped at chunk {i + 1} (partial kept).") | |
| else: | |
| yield None, f"Failed at chunk {i + 1}.", None | |
| return | |
| parts.append(wav) | |
| yield _to_gradio_audio(_crossfade_concat(parts)), f"Chunk {i + 1}/{len(chunks)}…", None | |
| yield from _done(parts, None, "Done — auto voice ready.") | |
| return | |
| # -------- Multi-speaker -------- | |
| if mode == "Multi-speaker dialogue": | |
| turns = _parse_dialogue(raw) | |
| if not turns: | |
| yield None, "Use lines like:\n[Speaker 1]: Hello!\n[Speaker 2]: Hi!", None | |
| return | |
| if not ref_audio and not (profile_name and library and _safe_str(profile_name) in (library or {})): | |
| yield None, "Speaker 1 needs a reference clip (or saved profile).", None | |
| return | |
| if not ref_audio_2: | |
| yield None, "Speaker 2 needs a second reference clip.", None | |
| return | |
| p1, note1, ref1_path = _prompt_from_library_or_audio(profile_name, library, ref_audio, ref_text) | |
| p2 = _make_prompt(ref_audio_2, ref_text_2) | |
| if p1 is None or p2 is None: | |
| yield None, f"Could not encode speaker prompts. {note1}", None | |
| return | |
| yield None, f"Dialogue · {len(turns)} turns. {note1}", None | |
| parts = [] | |
| for i, (spk, line) in enumerate(turns): | |
| progress((i + 1) / len(turns), desc=f"Turn {i + 1}/{len(turns)}") | |
| prompt = p1 if spk == 1 else p2 | |
| wav = _generate_one( | |
| _apply_style(line, style), lang, gen_config, prompt, speed_f, None, | |
| instruct if blend else None, bool(normalize_text), blend, | |
| ) | |
| if wav is None: | |
| if parts: | |
| yield from _done(parts, ref1_path, f"Stopped at turn {i + 1} (partial kept).") | |
| else: | |
| yield None, f"Failed at turn {i + 1}.", None | |
| return | |
| parts.append(wav) | |
| yield _to_gradio_audio(_crossfade_concat(parts)), f"Turn {i + 1}/{len(turns)}…", None | |
| yield from _done(parts, ref1_path, "Done — multi-speaker dialogue ready.") | |
| return | |
| # -------- Standard / long-form clone -------- | |
| prompt, note, ref_path = _prompt_from_library_or_audio(profile_name, library, ref_audio, ref_text) | |
| if prompt is None: | |
| yield None, note or "Upload a clear 3–10 second reference voice clip.", None | |
| return | |
| styled = _apply_style(raw, style) | |
| do_long = bool(long_form) or (len(styled) > 1200 and not use_native_longform) | |
| if len(styled) > 1200 and not do_long and not use_native_longform: | |
| yield None, "Text over ~1200 chars — enable Long-form or Native OmniVoice chunking.", None | |
| return | |
| if use_native_longform and not do_long: | |
| chunks = [styled] # let OmniVoice split internally | |
| else: | |
| chunks = _chunk_text(styled) if do_long else [styled] | |
| if not chunks: | |
| yield None, "Nothing to speak after preprocessing.", None | |
| return | |
| yield None, f"{note} Generating {len(chunks)} chunk(s)…", None | |
| parts = [] | |
| for i, chunk in enumerate(chunks): | |
| progress((i + 1) / len(chunks), desc=f"Chunk {i + 1}/{len(chunks)}") | |
| wav = _generate_one( | |
| chunk, lang, gen_config, prompt, speed_f, | |
| use_duration if len(chunks) == 1 else None, | |
| instruct if blend else None, bool(normalize_text), blend, | |
| ) | |
| if wav is None: | |
| if parts: | |
| yield from _done(parts, ref_path, f"Stopped at chunk {i + 1} (partial kept).") | |
| else: | |
| yield None, f"Failed at chunk {i + 1}.", None | |
| return | |
| parts.append(wav) | |
| yield _to_gradio_audio(_crossfade_concat(parts)), f"Chunk {i + 1}/{len(chunks)}…", None | |
| msg = "Done — cloned speech ready." | |
| if len(chunks) > 1: | |
| msg = f"Done — long-form clone ({len(chunks)} chunks)." | |
| yield from _done(parts, ref_path, msg) | |
| # --------------------------------------------------------------------------- | |
| # UI | |
| # --------------------------------------------------------------------------- | |
| CUSTOM_CSS = """ | |
| /* Base Container */ | |
| .gradio-container { | |
| max-width: 95% !important; /* Expanded for landscape */ | |
| font-family: 'Inter', sans-serif !important; | |
| } | |
| /* Glassmorphism Panels */ | |
| .glass-panel { | |
| background: rgba(30, 41, 59, 0.95) !important; | |
| border: 1px solid rgba(255, 255, 255, 0.1) !important; | |
| border-radius: 16px !important; | |
| box-shadow: 0 8px 32px 0 rgba(0, 0, 0, 0.3) !important; | |
| padding: 32px !important; | |
| margin-bottom: 24px !important; | |
| } | |
| /* Checkbox & Input Polish */ | |
| label.checkbox { | |
| font-size: 1.2em !important; | |
| font-weight: 700 !important; | |
| color: #ffffff !important; | |
| letter-spacing: 0.5px !important; | |
| } | |
| label.checkbox input { | |
| transform: scale(1.5) !important; | |
| margin-right: 14px !important; | |
| cursor: pointer !important; | |
| } | |
| .gr-box, .gr-accordion, .gr-input, .gr-dropdown, .gr-radio { | |
| font-size: 1.05em !important; | |
| } | |
| #vertical-mode .wrap, #vertical-mode fieldset { | |
| flex-direction: column !important; | |
| align-items: flex-start !important; | |
| } | |
| #vertical-mode label { | |
| margin-bottom: 8px !important; | |
| } | |
| /* Typography */ | |
| h1 { | |
| font-weight: 800 !important; | |
| background: -webkit-linear-gradient(45deg, #4ade80, #3b82f6); | |
| -webkit-background-clip: text; | |
| -webkit-text-fill-color: transparent; | |
| text-align: center; | |
| margin-bottom: 10px !important; | |
| } | |
| .subtitle { | |
| text-align: center; | |
| color: #94a3b8 !important; | |
| font-size: 1.1em; | |
| margin-bottom: 30px !important; | |
| } | |
| /* Generate Button */ | |
| #clone-btn { | |
| background: linear-gradient(90deg, #10b981 0%, #3b82f6 100%) !important; | |
| border: none !important; | |
| color: white !important; | |
| font-weight: 700 !important; | |
| font-size: 1.2rem !important; | |
| border-radius: 12px !important; | |
| padding: 16px !important; | |
| transition: all 0.3s ease !important; | |
| box-shadow: 0 4px 15px rgba(16, 185, 129, 0.4) !important; | |
| } | |
| #clone-btn:hover { | |
| transform: translateY(-2px) !important; | |
| box-shadow: 0 6px 20px rgba(16, 185, 129, 0.6) !important; | |
| } | |
| /* Analyze Button */ | |
| #analyze-btn { | |
| background: linear-gradient(90deg, #3b82f6 0%, #6366f1 100%) !important; | |
| border: none !important; | |
| color: white !important; | |
| font-weight: 600 !important; | |
| border-radius: 8px !important; | |
| transition: all 0.3s ease !important; | |
| width: 100% !important; | |
| max-height: 42px !important; | |
| padding: 8px 16px !important; | |
| white-space: nowrap !important; | |
| display: flex !important; | |
| justify-content: center !important; | |
| align-items: center !important; | |
| line-height: 1 !important; | |
| } | |
| .align-center-row { | |
| align-items: center !important; | |
| } | |
| #analyze-btn:hover { | |
| transform: translateY(-1px) !important; | |
| box-shadow: 0 4px 15px rgba(59, 130, 246, 0.5) !important; | |
| } | |
| .gr-box { | |
| border-radius: 12px !important; | |
| } | |
| .gr-accordion { | |
| border: 1px solid rgba(255, 255, 255, 0.05) !important; | |
| border-radius: 12px !important; | |
| background: rgba(15, 23, 42, 0.6) !important; | |
| } | |
| """ | |
| with gr.Blocks( | |
| title="AI Voice Cloner — Free Zero-Shot TTS (600+ Languages) | OmniVoice", | |
| theme=gr.themes.Base( | |
| primary_hue="emerald", | |
| secondary_hue="blue", | |
| neutral_hue="slate", | |
| font=[gr.themes.GoogleFont("Inter"), "system-ui", "sans-serif"], | |
| ), | |
| css=CUSTOM_CSS, | |
| ) as demo: | |
| voice_library = gr.State({}) | |
| gr.HTML( | |
| """ | |
| <div class="subtitle"> | |
| <h1>AI Voice Cloner</h1> | |
| <p><b>Free online AI voice cloning</b> & <b>multilingual text-to-speech</b>. Upload <b>3–10 seconds</b> of audio → generate natural speech in <b>600+ languages</b>.<br> | |
| Zero-shot <b>voice clone</b> · cross-lingual TTS · long-form · multi-speaker · RVC-inspired polish · one-click <b>try prompts</b>.<br> | |
| Powered by <a href="https://huggingface.co/k2-fsa/OmniVoice" target="_blank" style="color: #60a5fa;">k2-fsa/OmniVoice</a> on <b>ZeroGPU</b> · Weights <b>CC-BY-NC</b> · ⭐ <b>Like this Space</b> to boost discovery</p> | |
| </div> | |
| """ | |
| ) | |
| with gr.Column(): | |
| with gr.Column(): | |
| with gr.Group(elem_classes="glass-panel"): | |
| gr.Markdown("### 🎙️ 1. Voice Identity") | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| mode = gr.Radio( | |
| label="Mode", | |
| choices=[ | |
| "Voice clone", | |
| "Multi-speaker dialogue", | |
| "Voice design (no reference)", | |
| "Auto voice (random)", | |
| ], | |
| value="Voice clone", | |
| elem_id="vertical-mode" | |
| ) | |
| consent = gr.Checkbox( | |
| value=False, | |
| label="I have rights / consent to use this voice (required)", | |
| ) | |
| with gr.Column(scale=2): | |
| ref_audio = gr.Audio( | |
| label="Reference voice — Speaker 1 (3–10 seconds) · upload or mic", | |
| type="filepath", | |
| sources=["upload", "microphone"], | |
| ) | |
| with gr.Row(equal_height=False, elem_classes="align-center-row"): | |
| analyze_btn = gr.Button("🔍 Analyze reference", variant="primary", elem_id="analyze-btn", scale=1) | |
| ref_prep = gr.Checkbox( | |
| value=True, | |
| label="Auto-prep ref (best 3–10s + soft gate)", | |
| info="Clean short refs beat long noisy ones", | |
| scale=1 | |
| ) | |
| ref_analysis = gr.Textbox(label="Reference QC", lines=2) | |
| ref_text = gr.Textbox( | |
| label="Reference transcript (optional)", | |
| lines=2, | |
| placeholder="Exact words in the reference clip. Empty = auto-transcribe.", | |
| ) | |
| with gr.Accordion("Speaker 2 (dialogue mode)", open=False): | |
| ref_audio_2 = gr.Audio( | |
| label="Reference voice — Speaker 2", | |
| type="filepath", | |
| sources=["upload", "microphone"], | |
| ) | |
| ref_text_2 = gr.Textbox(label="Speaker 2 transcript (optional)", lines=2) | |
| with gr.Accordion("📚 Voice Library (session + .pt)", open=False): | |
| profile_name = gr.Textbox(label="Profile name", placeholder="Narrator_A") | |
| profile_select = gr.Dropdown(label="Saved profiles", choices=[], value=None) | |
| with gr.Row(): | |
| save_profile_btn = gr.Button("Save", size="sm") | |
| load_profile_btn = gr.Button("Load", size="sm") | |
| delete_profile_btn = gr.Button("Delete", size="sm") | |
| import_pt = gr.File(label="Import .pt", file_types=[".pt"], type="filepath") | |
| import_btn = gr.Button("Import .pt", size="sm") | |
| profile_download = gr.File(label="Download profile (.pt)") | |
| library_status = gr.Textbox(label="Library status", lines=1) | |
| with gr.Group(elem_classes="glass-panel"): | |
| gr.Markdown("### 📝 2. Script & Generation") | |
| with gr.Accordion("✨ Try Prompts — click to fill (upload voice first)", open=False): | |
| gr.Markdown("1) Upload / record your voice · 2) Click a prompt · 3) Consent · 4) **Generate**") | |
| try_prompt = gr.Radio( | |
| label="Prompt pack", | |
| choices=list(TRY_PROMPTS.keys()), | |
| value="YouTube intro", | |
| ) | |
| load_prompt_btn = gr.Button("Load selected prompt into text box", variant="secondary") | |
| prompt_status = gr.Textbox(label="Prompt status", lines=1) | |
| text = gr.Textbox( | |
| label="Text to speak (any of 600+ languages)", | |
| lines=6, | |
| placeholder=( | |
| "Paste your script — or click a Try prompt above.\n\n" | |
| "[Speaker 1]: Hello!\n[Speaker 2]: Hi there!\n\n" | |
| f"{NONVERBAL_HELP}" | |
| ), | |
| ) | |
| with gr.Row(): | |
| language = gr.Dropdown( | |
| label="Language (Auto or pick from 600+)", | |
| choices=LANGUAGE_CHOICES, | |
| value="Auto", | |
| filterable=True, | |
| ) | |
| style = gr.Dropdown( | |
| label="Expression preset", | |
| choices=list(STYLE_PRESETS.keys()), | |
| value="Neutral", | |
| ) | |
| instruct = gr.Textbox( | |
| label="Instruct (voice design · optional clone blend)", | |
| lines=1, | |
| placeholder="female, low pitch, british accent | 四川话", | |
| info="Required in Voice design mode. For clone: enable blend when consistent with ref.", | |
| ) | |
| with gr.Row(): | |
| blend_instruct = gr.Checkbox( | |
| value=False, | |
| label="Blend instruct with clone (OmniVoice dialect/style tip)", | |
| ) | |
| long_form = gr.Checkbox( | |
| value=False, | |
| label="Long-form (app chunker)", | |
| info=f"Up to {MAX_TOTAL_CHARS} chars / {MAX_CHUNKS} chunks", | |
| ) | |
| with gr.Row(): | |
| use_native_longform = gr.Checkbox( | |
| value=False, | |
| label="Native OmniVoice long-form chunking", | |
| info="Uses audio_chunk_duration / threshold inside the model", | |
| ) | |
| normalize_text = gr.Checkbox(value=False, label="Normalize numbers → words (EN/ZH)") | |
| with gr.Group(elem_classes="glass-panel"): | |
| gr.Markdown("### ⚙️ 3. Polish & Advanced") | |
| quality_preset = gr.Radio( | |
| label="Quality preset", | |
| choices=list(ax.QUALITY_PRESETS.keys()), | |
| value="Balanced", | |
| ) | |
| with gr.Accordion("🎛️ RVC-inspired polish (crash-safe DSP · no .pth)", open=False): | |
| gr.Markdown("Concepts from [Annotated RVC](https://gudgud96.github.io/2024/09/26/annotated-rvc/) — lightweight spectral polish only (ZeroGPU-safe).") | |
| enable_polish = gr.Checkbox(value=True, label="Enable polish chain") | |
| f0_semitones = gr.Slider(-12, 12, value=0, step=1, label="F0 shift (semitones) ≈ RVC f0_up_key") | |
| index_rate = gr.Slider(0.0, 0.8, value=0.35, step=0.05, label="Timbre blend α ≈ RVC index_rate") | |
| protect = gr.Slider(0.0, 0.5, value=0.33, step=0.01, label="Protect consonants ≈ RVC protect") | |
| loud_norm = gr.Checkbox(value=True, label="Loudness normalize output") | |
| with gr.Accordion("🎵 Background Music", open=False): | |
| bgm_audio = gr.Audio(label="Backing track", type="filepath", sources=["upload"]) | |
| bgm_volume = gr.Slider(0.0, 0.6, value=0.18, step=0.02, label="BGM volume") | |
| duck_db = gr.Slider(0.0, 18.0, value=8.0, step=0.5, label="Duck (dB)") | |
| with gr.Accordion("🔬 Advanced OmniVoice decoding", open=False): | |
| speed = gr.Slider(0.7, 1.3, value=1.0, step=0.05, label="Speed") | |
| duration = gr.Number(value=None, label="Force duration (s, single-chunk)") | |
| num_step = gr.Slider(8, 64, value=32, step=1, label="Inference steps") | |
| guidance_scale = gr.Slider(0.5, 3.5, value=2.0, step=0.1, label="CFG guidance") | |
| t_shift = gr.Slider(0.01, 0.5, value=0.1, step=0.01, label="t_shift (noise schedule)") | |
| position_temperature = gr.Slider(0.0, 10.0, value=5.0, step=0.5, label="position_temperature") | |
| class_temperature = gr.Slider(0.0, 2.0, value=0.0, step=0.1, label="class_temperature") | |
| pad_duration = gr.Slider(0.0, 0.5, value=0.1, step=0.05, label="pad_duration (s)") | |
| fade_duration = gr.Slider(0.0, 0.5, value=0.1, step=0.05, label="fade_duration (s)") | |
| audio_chunk_duration = gr.Slider(5.0, 30.0, value=15.0, step=1.0, label="audio_chunk_duration") | |
| audio_chunk_threshold = gr.Slider(10.0, 60.0, value=30.0, step=1.0, label="audio_chunk_threshold") | |
| denoise = gr.Checkbox(value=True, label="Denoise output") | |
| preprocess_prompt = gr.Checkbox(value=True, label="Clean reference (model-side)") | |
| postprocess_output = gr.Checkbox(value=True, label="Trim output silences") | |
| with gr.Group(elem_classes="glass-panel"): | |
| gr.Markdown("### 🎧 Output") | |
| output_audio = gr.Audio( | |
| label="Cloned speech output (streams per chunk · downloadable)", | |
| type="numpy", | |
| interactive=False, | |
| ) | |
| with gr.Row(equal_height=False, elem_classes="align-center-row"): | |
| download_format = gr.Dropdown( | |
| choices=["WAV", "MP3", "FLAC", "OGG"], | |
| value="WAV", | |
| label="Format", | |
| scale=1, | |
| ) | |
| output_file = gr.File(label="Download Audio", scale=3) | |
| status = gr.Textbox(label="Status", lines=4) | |
| gr.Markdown( | |
| """ | |
| ### ⚡ Quick path to a viral demo | |
| 1. Upload a clean **6s** clip of *your* voice | |
| 2. Load **YouTube intro** or **Audiobook narrator** | |
| 3. Enable consent → **Generate** → share the WAV | |
| """ | |
| ) | |
| generate_btn = gr.Button( | |
| "🚀 Generate Cloned Voice", | |
| variant="primary", | |
| elem_id="clone-btn", | |
| ) | |
| quality_preset.change( | |
| fn=apply_quality_preset, | |
| inputs=[quality_preset], | |
| outputs=[num_step, guidance_scale, t_shift], | |
| ) | |
| analyze_btn.click(fn=analyze_ref_ui, inputs=[ref_audio], outputs=[ref_analysis]) | |
| load_prompt_btn.click( | |
| fn=apply_try_prompt, | |
| inputs=[try_prompt], | |
| outputs=[text, language, mode, style, instruct, prompt_status], | |
| api_name="load_try_prompt", | |
| ) | |
| # Also load when user changes the radio (faster UX) | |
| try_prompt.change( | |
| fn=apply_try_prompt, | |
| inputs=[try_prompt], | |
| outputs=[text, language, mode, style, instruct, prompt_status], | |
| ) | |
| gen_inputs = [ | |
| text, language, ref_audio, ref_text, num_step, guidance_scale, denoise, | |
| speed, duration, preprocess_prompt, postprocess_output, style, long_form, | |
| mode, ref_audio_2, ref_text_2, instruct, normalize_text, bgm_audio, | |
| bgm_volume, duck_db, profile_select, voice_library, consent, ref_prep, | |
| blend_instruct, use_native_longform, t_shift, position_temperature, | |
| class_temperature, pad_duration, fade_duration, audio_chunk_duration, | |
| audio_chunk_threshold, enable_polish, f0_semitones, index_rate, | |
| protect, loud_norm, download_format, | |
| ] | |
| generate_btn.click( | |
| fn=clone_voice, | |
| inputs=gen_inputs, | |
| outputs=[output_audio, status, output_file], | |
| api_name="clone_voice", | |
| ) | |
| save_profile_btn.click( | |
| fn=save_voice_profile, | |
| inputs=[profile_name, ref_audio, ref_text, voice_library], | |
| outputs=[voice_library, profile_select, library_status], | |
| api_name="save_voice_profile", | |
| ) | |
| load_profile_btn.click( | |
| fn=load_voice_profile, | |
| inputs=[profile_select, voice_library], | |
| outputs=[ref_audio, ref_text, profile_download, library_status], | |
| api_name="load_voice_profile", | |
| ) | |
| delete_profile_btn.click( | |
| fn=delete_voice_profile, | |
| inputs=[profile_select, voice_library], | |
| outputs=[voice_library, profile_select, library_status], | |
| api_name="delete_voice_profile", | |
| ) | |
| import_btn.click( | |
| fn=import_voice_pt, | |
| inputs=[import_pt, profile_name, voice_library], | |
| outputs=[voice_library, profile_select, library_status], | |
| api_name="import_voice_pt", | |
| ) | |
| download_format.change( | |
| fn=update_export, | |
| inputs=[output_audio, download_format], | |
| outputs=[output_file] | |
| ) | |
| demo.load( | |
| fn=lambda: apply_try_prompt("YouTube intro"), | |
| outputs=[text, language, mode, style, instruct, prompt_status], | |
| ) | |
| gr.Markdown( | |
| """ | |
| ### About — free AI voice cloner on Hugging Face | |
| [`cosmichackerx/voice-cloner`](https://huggingface.co/spaces/cosmichackerx/voice-cloner) · | |
| Model [`OmniVoice`](https://huggingface.co/k2-fsa/OmniVoice) · | |
| RVC notes [gudgud96](https://gudgud96.github.io/2024/09/26/annotated-rvc/) · | |
| API `/clone_voice` | |
| **Only clone voices you have consent to use.** ⭐ Like · share · duplicate to help this Space trend. | |
| """ | |
| ) | |
| if __name__ == "__main__": | |
| demo.queue(default_concurrency_limit=1).launch() | |