PlotWeaver
Live commentary translation platform — English to 40+ languages
ASR (Whisper) → MT (NLLB-200) → TTS (YourVoic + local models)
""" PlotWeaver — Live Commentary Translation Platform =================================================== Event management, multi-language dubbing, live streaming. """ import os import time import tempfile import numpy as np import re import soundfile as sf import gradio as gr import logging logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger(__name__) from languages import LANGUAGES, LANGUAGE_GROUPS, ALL_LANGUAGE_NAMES, QWEN_VOICES from tts_engine import synthesize_chunked from qwen_engine import dub_video_qwen, translate_chunk_qwen from pipeline import ( load_models, transcribe, translate_text, translate_sentence, split_into_sentences, extract_audio_from_video, get_media_duration, stretch_audio_to_duration, mux_video_audio, tts_pipe_local, ) import pipeline # Load all models at startup load_models() # ============================================================================= # Helper functions # ============================================================================= def get_voices_for_language(lang_name): """Get available voices for a language based on its engine.""" config = LANGUAGES.get(lang_name, {}) engine = config.get("tts_engine", "local") if engine == "qwen": return QWEN_VOICES elif engine == "yourvoic" and config.get("yourvoic_voices"): return config["yourvoic_voices"] elif engine == "local": return ["Default (local model)"] return ["Peter"] def full_pipeline_audio(audio_input, target_language): """Full pipeline: English audio → target language audio.""" if audio_input is None: return None, "Please upload or record audio." lang_config = LANGUAGES.get(target_language) if not lang_config: return None, f"Language '{target_language}' not configured." sample_rate, audio_array = audio_input audio_array = audio_array.astype(np.float32) if audio_array.ndim > 1: audio_array = audio_array.mean(axis=1) if audio_array.max() > 1.0 or audio_array.min() < -1.0: max_val = max(abs(audio_array.max()), abs(audio_array.min())) if max_val > 0: audio_array = audio_array / max_val log = [] total_start = time.time() # ASR t0 = time.time() english = transcribe(audio_array, sample_rate) log.append(f"**ASR** ({time.time()-t0:.2f}s)\n{english}") if not english: return None, "ASR returned empty text." # MT t0 = time.time() nllb_code = lang_config["nllb"] translated, en_sents, tgt_sents = translate_text(english, nllb_code, fast=False) log.append(f"\n**Translation** ({time.time()-t0:.2f}s)") for e, t in zip(en_sents, tgt_sents): log.append(f" EN: {e}\n {target_language.upper()}: {t}") if not translated: return None, "Translation returned empty." # TTS t0 = time.time() audio_out, sr_out = synthesize_chunked( translated, lang_config, tts_pipe=pipeline.tts_pipe_local ) log.append(f"\n**TTS** ({time.time()-t0:.2f}s) = {len(audio_out)/sr_out:.1f}s audio") total = time.time() - total_start log.append(f"\n**Total: {total:.2f}s**") return (sr_out, audio_out), "\n".join(log) def full_pipeline_text(english_text, target_language, voice_name): """Text-only pipeline: English text → target language audio.""" if not english_text or not english_text.strip(): return None, "Please enter English text." lang_config = LANGUAGES.get(target_language) if not lang_config: return None, f"Language '{target_language}' not configured." log = [] total_start = time.time() # MT t0 = time.time() nllb_code = lang_config["nllb"] translated, en_sents, tgt_sents = translate_text(english_text.strip(), nllb_code, fast=False) log.append(f"**Translation** ({time.time()-t0:.2f}s)") for e, t in zip(en_sents, tgt_sents): log.append(f" EN: {e}\n {target_language.upper()}: {t}") if not translated: return None, "Translation returned empty." # TTS t0 = time.time() audio_out, sr_out = synthesize_chunked( translated, lang_config, tts_pipe=pipeline.tts_pipe_local ) log.append(f"\n**TTS** ({time.time()-t0:.2f}s) = {len(audio_out)/sr_out:.1f}s audio") total = time.time() - total_start log.append(f"\n**Total: {total:.2f}s**") return (sr_out, audio_out), "\n".join(log) def dub_video(video_path, target_languages, dub_voice, chunk_seconds, progress=gr.Progress()): """ Dub a video into one or more target languages. Routes to Qwen Omni for global languages, local pipeline for African languages. """ if video_path is None: return None, "Please upload a video." if not target_languages: return None, "Please select at least one target language." results_log = [] output_videos = [] for lang_name in target_languages: lang_config = LANGUAGES.get(lang_name) if not lang_config: results_log.append(f"**{lang_name}**: not configured, skipped") continue engine = lang_config.get("tts_engine", "local") results_log.append(f"\n{'='*50}") results_log.append(f"**Dubbing: {lang_name}** (engine: {engine})") results_log.append(f"{'='*50}") try: if engine == "qwen": # Qwen Omni: end-to-end speech-to-speech (best for global languages) qwen_lang_name = lang_config.get("qwen_name", lang_name) voice = dub_voice if dub_voice in QWEN_VOICES else "Ethan" out_video, log_text = dub_video_qwen( video_path, qwen_lang_name, voice=voice, chunk_seconds=chunk_seconds, progress_fn=progress, ) results_log.append(log_text) if out_video: output_videos.append(out_video) else: # Local/YourVoic pipeline: ASR → NLLB → TTS work_dir = tempfile.mkdtemp(prefix=f"dub_{lang_name}_") extracted_audio = os.path.join(work_dir, "audio.wav") tgt_audio_raw = os.path.join(work_dir, "tgt_raw.wav") tgt_audio_aligned = os.path.join(work_dir, "tgt_aligned.wav") output_video = os.path.join(work_dir, f"dubbed_{lang_name}.mp4") progress(0.05, desc=f"{lang_name}: extracting audio...") extract_audio_from_video(video_path, extracted_audio) video_duration = get_media_duration(video_path) results_log.append(f"Video: {video_duration:.1f}s") audio_array, sr = sf.read(extracted_audio, dtype="float32") if audio_array.ndim > 1: audio_array = audio_array.mean(axis=1) progress(0.15, desc=f"{lang_name}: transcribing...") t0 = time.time() english = transcribe(audio_array, sr) results_log.append(f"ASR: {time.time()-t0:.1f}s") if not english: results_log.append("ASR empty — skipped") continue progress(0.4, desc=f"{lang_name}: translating...") t0 = time.time() nllb_code = lang_config["nllb"] translated, _, _ = translate_text(english, nllb_code, fast=True) results_log.append(f"MT: {time.time()-t0:.1f}s") if not translated: results_log.append("Translation empty — skipped") continue progress(0.65, desc=f"{lang_name}: synthesizing...") t0 = time.time() tgt_audio, tgt_sr = synthesize_chunked( translated, lang_config, tts_pipe=pipeline.tts_pipe_local ) sf.write(tgt_audio_raw, tgt_audio, tgt_sr) tgt_duration = len(tgt_audio) / tgt_sr results_log.append(f"TTS: {time.time()-t0:.1f}s ({tgt_duration:.1f}s audio)") progress(0.85, desc=f"{lang_name}: aligning...") MAX_STRETCH = 1.2 stretch_ratio = tgt_duration / video_duration if stretch_ratio <= MAX_STRETCH: if abs(stretch_ratio - 1.0) > 0.02: stretch_audio_to_duration(tgt_audio_raw, tgt_audio_aligned, video_duration) else: import shutil shutil.copy(tgt_audio_raw, tgt_audio_aligned) extend_video = False final_duration = video_duration else: import shutil shutil.copy(tgt_audio_raw, tgt_audio_aligned) extend_video = True final_duration = tgt_duration results_log.append(f"Audio longer ({stretch_ratio:.1f}x) — extending video") progress(0.95, desc=f"{lang_name}: combining...") mux_video_audio( video_path, tgt_audio_aligned, output_video, extend_video=extend_video, target_duration=final_duration ) output_videos.append(output_video) except Exception as e: logger.exception(f"Dubbing {lang_name} failed") results_log.append(f"Error: {str(e)}") progress(1.0, desc="Done!") final_video = output_videos[0] if output_videos else None return final_video, "\n".join(results_log) def update_voices(language): """Update voice dropdown when language changes.""" voices = get_voices_for_language(language) return gr.update(choices=voices, value=voices[0]) # ============================================================================= # Gradio UI # ============================================================================= EXAMPLES = [ "And it's a brilliant goal from the striker!", "The referee has shown a yellow card. Corner kick for the home team.", "What a save by the goalkeeper! The match is heading into injury time.", "He dribbles past two defenders and shoots! The ball hits the back of the net!", ] CSS = """ .main-header { text-align: center; margin-bottom: 0.5rem; } .main-header h1 { font-size: 1.8rem; font-weight: 700; margin: 0; } .main-header p { color: #666; font-size: 0.95rem; } .lang-group-label { font-weight: 600; font-size: 0.85rem; color: #888; text-transform: uppercase; letter-spacing: 0.05em; margin-top: 0.5rem; } """ with gr.Blocks( title="PlotWeaver — Live Commentary Translation", theme=gr.themes.Soft(), css=CSS, ) as demo: gr.HTML("""
Live commentary translation platform — English to 40+ languages
ASR (Whisper) → MT (NLLB-200) → TTS (YourVoic + local models)