App. Yp

=============================

app.py — Gradio Space (Video Dubbing + Voice Cloning)

=============================

Features:

- Upload a video (mobile-friendly)

- Optional: upload a short reference voice (3–10 sec) for cloning

- Transcribe (Whisper tiny/base)

- Translate to target language (deep-translator → Google Translate backend)

- TTS (Coqui TTS XTTS v2). If reference voice provided → cloning; otherwise default speaker.

- Remux final dubbed audio onto original video with FFmpeg

- Lightweight fallback path (no cloning + small TTS) if XTTS model fails



Notes:

- CPU on Hugging Face Spaces can be slow for ML models. GPU is recommended.

- First build will download models — allow time.

- If you only have CPU, set MODEL_SIZE="tiny" for Whisper.

import os import tempfile import subprocess from pathlib import Path

import gradio as gr

-----------------------------

Optional dependencies are imported lazily inside functions

-----------------------------

---------- Utilities ----------

def run_ffmpeg(args): """Run ffmpeg with given arg list; raise on error.""" cmd = ["ffmpeg", "-y", *args] proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if proc.returncode != 0: raise RuntimeError(proc.stderr.decode("utf-8", errors="ignore"))

def extract_audio(video_path: str, out_wav: str, sr: int = 16000): run_ffmpeg(["-i", video_path, "-vn", "-ac", "1", "-ar", str(sr), out_wav])

def mux_audio_on_video(video_path: str, audio_path: str, out_path: str): # Keep original video; replace audio track; ensure duration matches shortest run_ffmpeg(["-i", video_path, "-i", audio_path, "-map", "0:v:0", "-map", "1:a:0", "-c:v", "copy", "-shortest", out_path])

---------- ML: Transcribe ----------

def transcribe_whisper(wav_path: str, model_size: str = "tiny") -> str: """Transcribe audio to text using OpenAI Whisper (open-source).""" import whisper  # pip install -U openai-whisper mdl = whisper.load_model(model_size) result = mdl.transcribe(wav_path) return result.get("text", "").strip()

---------- Translate ----------

def translate_text(text: str, target_lang_code: str) -> str: """Translate using deep-translator (Google Translate backend).""" from deep_translator import GoogleTranslator if not text.strip(): return "" trans = GoogleTranslator(source="auto", target=target_lang_code) return trans.translate(text)

---------- TTS (XTTS v2) ----------

def synth_xtts(text: str, lang: str, out_wav: str, ref_wav: str | None = None): """Synthesize speech via Coqui TTS XTTS v2. Optionally clone from ref_wav.""" from TTS.api import TTS  # pip install TTS

model_name = os.environ.get("XTTS_MODEL", "tts_models/multilingual/multi-dataset/xtts_v2")
tts = TTS(model_name)

# Map a few friendly codes to XTTS language tokens
# XTTS accepts: "en", "hi", "bn", "ta", "te", "ur", "es", "fr", etc.
lang_map = {
    "en": "en",
    "hi": "hi",
    "bn": "bn",
    "te": "te",
    "ta": "ta",
    "mr": "mr",
    "gu": "gu",
    "pa": "pa",
    "ur": "ur",
    "es": "es",
    "fr": "fr",
    "de": "de",
}
xtts_lang = lang_map.get(lang, lang)

if ref_wav and Path(ref_wav).exists():
    tts.tts_to_file(text=text, speaker_wav=ref_wav, language=xtts_lang, file_path=out_wav)
else:
    # Use one of the built-in speakers (multilingual)
    tts.tts_to_file(text=text, language=xtts_lang, file_path=out_wav)

---------- Lightweight fallback TTS (no cloning) ----------

def synth_fallback(text: str, lang: str, out_wav: str): """Very small TTS fallback using pyttsx3 (offline). Language support limited; no cloning.""" import pyttsx3  # pip install pyttsx3 engine = pyttsx3.init() # Try to set voice by language hint if available on the system # On Spaces, voices are limited; this is just a safety net. engine.save_to_file(text, out_wav) engine.runAndWait()

---------- Main pipeline ----------

def pipeline(video, target_lang, ref_voice=None, model_size="tiny"): try: with tempfile.TemporaryDirectory() as td: td = Path(td) in_video = td / "input.mp4" out_audio = td / "dub.wav" ref_wav = td / "ref.wav" src_wav = td / "src.wav" out_video = td / "dubbed.mp4"

# Save inputs
        if hasattr(video, "name"):
            video_path = Path(video.name)
        else:
            video_path = Path(str(video))
        # Gradio gives a temp path already; copy to local temp for safety
        Path(in_video).write_bytes(Path(video_path).read_bytes())

        if ref_voice is not None:
            # Normalize reference to 16k mono wav
            if hasattr(ref_voice, "name"):
                ref_p = Path(ref_voice.name)
            else:
                ref_p = Path(str(ref_voice))
            # Convert to wav
            run_ffmpeg(["-i", str(ref_p), "-ac", "1", "-ar", "16000", str(ref_wav)])
            ref_wav_path = str(ref_wav)
        else:
            ref_wav_path = None

        # 1) Extract original audio
        extract_audio(str(in_video), str(src_wav), sr=16000)

        # 2) Transcribe
        text = transcribe_whisper(str(src_wav), model_size=model_size)

        # 3) Translate
        translated = translate_text(text, target_lang)

        # 4) TTS (try XTTS; else fallback)
        try:
            synth_xtts(translated, target_lang, str(out_audio), ref_wav=ref_wav_path)
        except Exception as e:
            print("XTTS failed, using fallback TTS:", e)
            synth_fallback(translated, target_lang, str(out_audio))

        # 5) Remux audio back onto video
        mux_audio_on_video(str(in_video), str(out_audio), str(out_video))

        return str(out_video), text, translated
except Exception as e:
    import traceback
    return None, "", f"Error: {e}\n{traceback.format_exc()}"

---------- Gradio UI ----------

LANG_CHOICES = [ ("English (en)", "en"), ("Hindi (hi)", "hi"), ("Bengali (bn)", "bn"), ("Tamil (ta)", "ta"), ("Telugu (te)", "te"), ("Marathi (mr)", "mr"), ("Gujarati (gu)", "gu"), ("Punjabi (pa)", "pa"), ("Urdu (ur)", "ur"), ("Spanish (es)", "es"), ("French (fr)", "fr"), ("German (de)", "de"), ]

with gr.Blocks(css=".gr-button {font-weight:600}") as demo: gr.Markdown(""" # 🎬 Video Dubbing + Voice Cloning (Mobile Friendly) Upload a video → (optional) upload a short reference voice → choose target language → Get dubbed video.

**Notes**
- First run will download models (Whisper + XTTS). It may take time.
- For better speed/quality, use a GPU Space on Hugging Face.
- Reference voice: 3–10 seconds, clear voice, no music.
- If XTTS download fails, app will fall back to a tiny offline TTS (no cloning).
""")

with gr.Row():
    with gr.Column():
        in_video = gr.Video(label="Input Video (mp4/mov)")
        ref_audio = gr.Audio(sources=["upload", "microphone"], type="filepath", label="Optional: Reference Voice (3–10 sec)")
        tgt = gr.Dropdown(choices=[v for (_, v) in LANG_CHOICES], label="Target Language", value="hi")
        whisper_size = gr.Radio(["tiny", "base"], value="tiny", label="Whisper Model Size")
        btn = gr.Button("Dub Video", variant="primary")

    with gr.Column():
        out_video = gr.Video(label="Dubbed Video (preview)")
        src_text = gr.Textbox(label="Transcribed Text", lines=4)
        out_text = gr.Textbox(label="Translated Text", lines=4)

btn.click(fn=pipeline, inputs=[in_video, tgt, ref_audio, whisper_size], outputs=[out_video, src_text, out_text])

if name == "main": demo.launch()

=============================

requirements.txt (put this in a separate file in your Space)

=============================

gradio>=4.44.0

openai-whisper

torch

torchaudio

TTS

deep-translator

moviepy

ffmpeg-python

pyttsx3

Note: On HF Spaces, FFmpeg is preinstalled on most images. If not, use a Space with ffmpeg runtime or add a setup.sh to install it.

=============================

Optional: README.md (helpful instructions)

=============================

1) Create a new Hugging Face Space → SDK: Gradio → Hardware: GPU recommended

2) Add the following files:

- app.py (this file)

- requirements.txt (from the block above)

3) (Optional) In "Variables" set:

- XTTS_MODEL = tts_models/multilingual/multi-dataset/xtts_v2

4) First build will download Whisper + XTTS models (large downloads). Wait for the Space to finish building.

5) Usage:

- Upload a short video (mp4 preferred)

- (Optional) Upload a 3–10 sec clean voice clip to clone

- Choose target language → click "Dub Video"

6) Troubleshooting:

- If GPU is not available, use Whisper "tiny" and short videos (<30s) for testing

- If XTTS fails (RAM/GPU), app falls back to a simple offline TTS (no cloning)

- Audio desync? Ensure FPS is constant in source video; re-encode with ffmpeg if needed:

ffmpeg -i in.mp4 -pix_fmt yuv420p -r 30 -c:v libx264 -c:a aac fixed.mp4