""" Apni Awaaz 🎙️ — Dub English video into the Hindi people actually speak. Built for the Build Small Hackathon (June 2026). """ import gradio as gr import spaces import torch import edge_tts import asyncio import subprocess import tempfile import os from pathlib import Path from transformers import ( AutoModelForCausalLM, AutoTokenizer, pipeline, BitsAndBytesConfig, ) # ╔══════════════════════════════════════════════════════════════╗ # ║ THE PROMPT — this is the soul of the entire project ║ # ╚══════════════════════════════════════════════════════════════╝ SYSTEM_PROMPT = """You are a dubbing translator. You translate English dialogue into the Hindi that real people actually speak at home in North India — not the stiff, Sanskritized Hindi of Doordarshan or official dubs. RULES: 1. Use everyday Hindustani — the natural Hindi-Urdu mix people really speak. 2. NEVER use Sanskritized/शुद्ध words when a simpler one exists: - "प्राप्त करना" → "मिलना" / "पाना" - "आवश्यक" → "ज़रूरी" - "अत्यंत" → "बहुत" / "काफ़ी" - "उपयोग" → "इस्तेमाल" - "विचार करना" → "सोचना" - "संपन्न करना" → "करना" / "निपटाना" - "प्रतीक्षा" → "इंतज़ार" - "शीघ्र" → "जल्दी" - "अनुमति" → "इजाज़त" - "कृपया" → drop it or say "please" - "अवश्य" → "ज़रूर" - "उचित" → "सही" / "ठीक" 3. Keep English words Indians naturally keep: phone, office, meeting, tension, problem, time, chance, try, plan, sure, okay, sorry, thanks, bus, train, college, hospital, doctor, ticket, report, file. 4. Match the speaker's register. Casual stays casual, serious stays serious — but never sound like a newsreader. 5. Use natural fillers where they fit: "यार", "अरे", "बस", "ना", "वो", "मतलब", "basically". 6. Natural contractions: "कर लेंगे" not "कर लिया जाएगा", "हो जाएगा" not "संपन्न हो जाएगा". 7. Keep it CONCISE. Dubbed Hindi should be roughly the same length as the English. Don't pad. EXAMPLES: EN: "I need to get this done before the deadline" ❌ "मुझे समय-सीमा से पूर्व यह कार्य संपन्न करना आवश्यक है" ✅ "deadline से पहले ये निपटाना पड़ेगा" EN: "That's a really good point, I hadn't thought about that" ❌ "यह एक अत्यंत उत्तम विचार है, मैंने इस पर विचार नहीं किया था" ✅ "अच्छी बात बोली, मेरे दिमाग़ में आया ही नहीं" EN: "We should probably reconsider our approach" ❌ "हमें अपनी कार्यप्रणाली पर पुनर्विचार करना चाहिए" ✅ "लगता है अपना तरीका बदलना पड़ेगा" EN: "I'm really sorry, I completely forgot about our meeting" ❌ "मुझे अत्यंत खेद है, मैं हमारी बैठक के विषय में पूर्णतः विस्मृत हो गया" ✅ "sorry यार, meeting पूरी तरह भूल गया" EN: "Can you give me a moment? I need to think about this" ❌ "क्या आप मुझे कुछ क्षण प्रदान कर सकते हैं? मुझे इस विषय पर विचार करना है" ✅ "एक second दे, सोचने दे" EN: "The situation is getting worse and we need to act fast" ❌ "स्थिति बिगड़ती जा रही है और हमें शीघ्र कार्रवाई करनी चाहिए" ✅ "हालात ख़राब हो रहे हैं, जल्दी कुछ करना पड़ेगा" EN: "I don't think that's going to work. Let me try something else." ❌ "मुझे नहीं लगता कि यह कार्य करेगा। मुझे कोई अन्य विकल्प आज़माने दीजिए।" ✅ "ये नहीं चलेगा। कुछ और try करता हूँ।" EN: "Look, I understand your concern, but we don't have a choice here" ❌ "देखिए, मैं आपकी चिंता समझता हूँ, परंतु हमारे पास यहाँ कोई विकल्प नहीं है" ✅ "देख, तेरी tension समझता हूँ, पर कोई चारा नहीं है" Translate ONLY the given English text. Output ONLY the Hindi. No commentary.""" # ╔══════════════════════════════════════════════════════════════╗ # ║ MODEL LOADING ║ # ╚══════════════════════════════════════════════════════════════╝ # -- Globals (loaded once, reused) -- whisper_pipe = None llm_model = None llm_tokenizer = None def load_whisper(): """Load Whisper on CPU. ZeroGPU moves it when @spaces.GPU fires.""" global whisper_pipe if whisper_pipe is None: print("⏳ Loading Whisper...") whisper_pipe = pipeline( "automatic-speech-recognition", model="openai/whisper-medium", torch_dtype=torch.float16, device="cpu", ) print("✅ Whisper loaded (CPU, will move to GPU at runtime)") return whisper_pipe def load_llm(): """ Load Qwen 2.5 7B in 4-bit. Called inside @spaces.GPU so device_map="auto" lands on the A100. """ global llm_model, llm_tokenizer if llm_model is None: print("⏳ Loading Qwen 2.5 7B...") model_id = "Qwen/Qwen2.5-7B-Instruct" bnb_cfg = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_compute_dtype=torch.float16, bnb_4bit_quant_type="nf4", ) llm_tokenizer = AutoTokenizer.from_pretrained(model_id) llm_model = AutoModelForCausalLM.from_pretrained( model_id, quantization_config=bnb_cfg, device_map="auto", ) print("✅ Qwen loaded") return llm_model, llm_tokenizer # Pre-download weights at startup (stays on CPU, fast re-load later) load_whisper() # ╔══════════════════════════════════════════════════════════════╗ # ║ PIPELINE STEPS ║ # ╚══════════════════════════════════════════════════════════════╝ def extract_audio(video_path: str, out_path: str) -> str: subprocess.run( [ "ffmpeg", "-i", video_path, "-vn", "-acodec", "pcm_s16le", "-ar", "16000", "-ac", "1", out_path, "-y", ], check=True, capture_output=True, ) return out_path def get_duration(path: str) -> float: r = subprocess.run( ["ffprobe", "-v", "quiet", "-show_entries", "format=duration", "-of", "csv=p=0", path], capture_output=True, text=True, ) return float(r.stdout.strip()) def transcribe(audio_path: str) -> list[dict]: """→ [{"timestamp": (start, end), "text": "..."}]""" pipe = load_whisper() result = pipe( audio_path, return_timestamps=True, chunk_length_s=30, generate_kwargs={"language": "en"}, ) return result["chunks"] def translate_segment(text: str) -> str: model, tok = load_llm() messages = [ {"role": "system", "content": SYSTEM_PROMPT}, {"role": "user", "content": text}, ] prompt = tok.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) inputs = tok(prompt, return_tensors="pt").to(model.device) with torch.no_grad(): out = model.generate( **inputs, max_new_tokens=200, temperature=0.3, do_sample=True, top_p=0.9, ) resp = tok.decode(out[0][inputs.input_ids.shape[1]:], skip_special_tokens=True) return resp.strip().split("\n")[0] # first line only, no runaway generation async def _tts(text: str, path: str, voice: str): comm = edge_tts.Communicate(text, voice) await comm.save(path) def hindi_tts(text: str, path: str, voice: str = "hi-IN-MadhurNeural"): asyncio.run(_tts(text, path, voice)) return path def adjust_speed(in_path: str, out_path: str, target_sec: float) -> str: """Stretch/squeeze audio to fit the target duration (pitch-preserved).""" dur = get_duration(in_path) if dur <= 0 or target_sec <= 0: return in_path ratio = dur / target_sec ratio = max(0.5, min(2.0, ratio)) # atempo range subprocess.run( ["ffmpeg", "-i", in_path, "-filter:a", f"atempo={ratio:.4f}", "-y", out_path], check=True, capture_output=True, ) return out_path def stitch_and_merge( segments: list[dict], video_path: str, total_dur: float, tmpdir: str, ) -> str: """ Build the dubbed audio track and merge it back onto the video. Uses pydub for clean overlay at exact timestamps. """ from pydub import AudioSegment # silent canvas base = AudioSegment.silent(duration=int(total_dur * 1000), frame_rate=24000) for seg in segments: tts_file = seg["tts_path"] start_ms = int(seg["start"] * 1000) try: chunk = AudioSegment.from_file(tts_file) base = base.overlay(chunk, position=start_ms) except Exception as e: print(f"⚠️ overlay failed for segment at {seg['start']:.1f}s: {e}") dubbed_wav = os.path.join(tmpdir, "dubbed_track.wav") base.export(dubbed_wav, format="wav") # merge onto video (keep original video stream, replace audio) out_mp4 = os.path.join(tmpdir, "output.mp4") subprocess.run( [ "ffmpeg", "-i", video_path, "-i", dubbed_wav, "-c:v", "copy", "-map", "0:v:0", "-map", "1:a:0", "-shortest", "-y", out_mp4, ], check=True, capture_output=True, ) return out_mp4 # ╔══════════════════════════════════════════════════════════════╗ # ║ MAIN PIPELINE ║ # ╚══════════════════════════════════════════════════════════════╝ @spaces.GPU(duration=300) def dub_video(video_path: str, voice_gender: str, progress=gr.Progress()): if video_path is None: raise gr.Error("Upload a video first!") # ── move Whisper to the ZeroGPU A100 ── pipe = load_whisper() pipe.model.to("cuda") pipe.device = torch.device("cuda") # ── load LLM (first call downloads + quantises onto GPU) ── load_llm() voice = "hi-IN-MadhurNeural" if voice_gender == "Male" else "hi-IN-SwaraNeural" tmpdir = tempfile.mkdtemp(prefix="apni_") # 1 ── extract audio progress(0.05, desc="🎵 Extracting audio…") raw_audio = extract_audio(video_path, os.path.join(tmpdir, "raw.wav")) total_dur = get_duration(raw_audio) # safety: reject clips > 3 min to stay within GPU budget if total_dur > 180: raise gr.Error("Please keep clips under 3 minutes for now.") # 2 ── transcribe progress(0.15, desc="👂 Listening to English…") chunks = transcribe(raw_audio) if not chunks: raise gr.Error("Couldn't detect any speech. Try a clearer clip.") # 3 ── translate + TTS each segment translated = [] n = len(chunks) for i, ch in enumerate(chunks): frac = 0.2 + 0.6 * (i / n) progress(frac, desc=f"🗣️ Dubbing segment {i + 1}/{n}…") start, end = ch["timestamp"] if start is None or end is None: continue seg_dur = end - start if seg_dur <= 0: continue # translate hindi = translate_segment(ch["text"]) # TTS tts_raw = os.path.join(tmpdir, f"tts_{i}.mp3") hindi_tts(hindi, tts_raw, voice) # speed-adjust to fit original segment window tts_adj = os.path.join(tmpdir, f"tts_adj_{i}.wav") adjust_speed(tts_raw, tts_adj, seg_dur) translated.append({ "start": start, "end": end, "en": ch["text"], "hi": hindi, "tts_path": tts_adj, }) # 4 ── stitch + merge progress(0.85, desc="🎬 Stitching final video…") output_video = stitch_and_merge(translated, video_path, total_dur, tmpdir) # 5 ── build comparison log log_lines = [] for s in translated: log_lines.append( f"[{s['start']:.1f}s → {s['end']:.1f}s]\n" f" 🇬🇧 {s['en']}\n" f" 🇮🇳 {s['hi']}" ) log = "\n\n".join(log_lines) return output_video, log # ╔══════════════════════════════════════════════════════════════╗ # ║ GRADIO UI ║ # ╚══════════════════════════════════════════════════════════════╝ CSS = """ .main-title { text-align: center; margin-bottom: 0.2em; } .subtitle { text-align: center; opacity: 0.7; font-size: 1.05em; margin-top: 0; } .example-row { background: var(--block-background-fill); border-radius: 8px; padding: 12px 16px; margin: 6px 0; font-size: 0.92em; } footer { display: none !important; } """ with gr.Blocks(title="Apni Awaaz", css=CSS, theme=gr.themes.Soft()) as demo: gr.Markdown( "# 🎙️ Apni Awaaz\n" "#### Dub English video into the Hindi people actually speak", elem_classes="main-title", ) gr.Markdown( '_No more "मुझे यह कार्य संपन्न करना आवश्यक है"_ — ' '_just "ये करना पड़ेगा यार"_', elem_classes="subtitle", ) with gr.Row(equal_height=True): # ── left column: inputs ── with gr.Column(scale=1): vid_in = gr.Video(label="Upload an English clip (< 3 min)") voice_radio = gr.Radio( ["Male", "Female"], value="Male", label="Hindi voice", ) btn = gr.Button("🎬 Dub it in apni bhasha!", variant="primary", size="lg") # ── right column: outputs ── with gr.Column(scale=1): vid_out = gr.Video(label="Dubbed output") log_box = gr.Textbox( label="Translation log (EN → HI)", lines=12, interactive=False, show_copy_button=True, ) # ── "what it does" section ── with gr.Accordion("How is this different from normal dubbing?", open=False): gr.Markdown( "Most Hindi dubs use **शुद्ध हिंदी** — overly formal, Sanskritized language " "that nobody actually speaks at home.\n\n" "Apni Awaaz translates into **everyday Hindustani** — the natural mix of " "Hindi, Urdu, and English that your family actually uses at the dinner table.\n\n" "| Official dub | Apni Awaaz |\n" "|---|---|\n" '| "मुझे इस विषय पर विचार करने दीजिए" | "सोचने दे एक second" |\n' '| "यह अत्यंत मूल्यवान है" | "बहुत महँगा है यार" |\n' '| "कृपया मुझे अनुमति प्रदान करें" | "please, करने दे ना" |\n' ) btn.click( fn=dub_video, inputs=[vid_in, voice_radio], outputs=[vid_out, log_box], ) if __name__ == "__main__": demo.launch(show_api=False)