Spaces:
Sleeping
Sleeping
| """ | |
| Karaoke Generator — two-step Gradio app | |
| Tab 1 · Transcribe | |
| Audio in → faster-whisper → timestamped JSON (download & edit externally). | |
| Tab 2 · Generate Video | |
| Audio + refined JSON + styling → karaoke MP4. | |
| JSON schema (one object per line/segment): | |
| [ | |
| { | |
| "start": 1.23, | |
| "end": 3.45, | |
| "text": "He Karunanidhi Swami", | |
| "words": [ | |
| {"word": "He", "start": 1.23, "end": 1.50}, | |
| {"word": "Karunanidhi", "start": 1.50, "end": 2.20}, | |
| {"word": "Swami", "start": 2.20, "end": 3.45} | |
| ] | |
| }, | |
| ... | |
| ] | |
| All models, cache and outputs live inside the project directory. | |
| """ | |
| import json | |
| import os | |
| import glob | |
| import queue | |
| import shutil | |
| import subprocess | |
| import threading | |
| from pathlib import Path | |
| import gradio as gr | |
| # --------------------------------------------------------------------------- # | |
| # Project-local storage | |
| # --------------------------------------------------------------------------- # | |
| PROJECT_DIR = Path(__file__).parent.resolve() | |
| # On HuggingFace Spaces the repo is read-only; use /tmp for all runtime files. | |
| # Locally, keep everything inside the project folder. | |
| ON_HF_SPACES = bool(os.environ.get("SPACE_ID")) | |
| _RUNTIME = Path("/tmp/karaoke") if ON_HF_SPACES else PROJECT_DIR | |
| WHISPER_DIR = _RUNTIME / "models" / "whisper" | |
| MODELS_DIR = _RUNTIME / "models" / "audio-separator" | |
| HF_MODELS_DIR = _RUNTIME / "models" / "hf" | |
| CACHE_DIR = _RUNTIME / "cache" | |
| OUTPUTS_DIR = _RUNTIME / "outputs" | |
| WORK_DIR = _RUNTIME / "work" | |
| for _d in (WHISPER_DIR, MODELS_DIR, HF_MODELS_DIR, CACHE_DIR, OUTPUTS_DIR, WORK_DIR): | |
| _d.mkdir(parents=True, exist_ok=True) | |
| os.environ["WHISPER_DOWNLOAD_ROOT"] = str(WHISPER_DIR) | |
| os.environ["TRANSFORMERS_CACHE"] = str(HF_MODELS_DIR) | |
| os.environ["HF_HOME"] = str(HF_MODELS_DIR) | |
| # --------------------------------------------------------------------------- # | |
| # Constants | |
| # --------------------------------------------------------------------------- # | |
| VIDEO_W, VIDEO_H = 1280, 720 | |
| FONT_CHOICES = [ | |
| "Arial", "Arial Black", "Verdana", "Tahoma", "Trebuchet MS", | |
| "Georgia", "Times New Roman", "Courier New", "Impact", "Comic Sans MS", | |
| ] | |
| WHISPER_MODELS = ["tiny", "base", "small", "medium", "large"] | |
| LANGUAGES = { | |
| "Auto-detect": None, | |
| "English": "en", | |
| "Hindi": "hi", | |
| "Gujarati": "gu", | |
| } | |
| # Fine-tuned HuggingFace models — keyed by (language_choice, model_name) | |
| HF_MODELS = { | |
| ("Gujarati", "small"): { | |
| "repo": "vasista22/whisper-gujarati-small", | |
| "lang": "gu", | |
| "label": "Gujarati fine-tuned (vasista22/whisper-gujarati-small)", | |
| }, | |
| } | |
| REMOVAL_METHODS = [ | |
| "Skip (use original audio — instant)", | |
| "Fast (ffmpeg phase cancel — seconds, stereo only)", | |
| "AI — MDX-Net (best quality, very slow on CPU)", | |
| ] | |
| # --------------------------------------------------------------------------- # | |
| # Color helper | |
| # --------------------------------------------------------------------------- # | |
| def hex_to_ass(color: str) -> str: | |
| color = (color or "").strip().lstrip("#") | |
| if len(color) == 3: | |
| color = "".join(c * 2 for c in color) | |
| if len(color) != 6: | |
| return "&H00FFFFFF" | |
| r, g, b = color[0:2], color[2:4], color[4:6] | |
| return f"&H00{b}{g}{r}".upper() | |
| # --------------------------------------------------------------------------- # | |
| # JSON ↔ internal segment format | |
| # --------------------------------------------------------------------------- # | |
| def segments_to_json(segments: list[dict]) -> str: | |
| """Convert internal segments to the user-facing JSON string.""" | |
| out = [] | |
| for seg in segments: | |
| out.append({ | |
| "start": round(seg["start"], 3), | |
| "end": round(seg["end"], 3), | |
| "text": " ".join(w[0] for w in seg["words"]).strip(), | |
| "words": [ | |
| {"word": w[0], "start": round(w[1], 3), "end": round(w[2], 3)} | |
| for w in seg["words"] | |
| ], | |
| }) | |
| return json.dumps(out, ensure_ascii=False, indent=2) | |
| def json_to_segments(json_str: str) -> list[dict]: | |
| """Parse user-edited JSON back into internal segment format.""" | |
| data = json.loads(json_str) | |
| segments = [] | |
| for item in data: | |
| words = [ | |
| [w["word"], float(w["start"]), float(w["end"])] | |
| for w in item.get("words", []) | |
| ] | |
| if not words: | |
| # plain text line — treat as single word | |
| words = [[item["text"], float(item["start"]), float(item["end"])]] | |
| segments.append({ | |
| "start": float(item["start"]), | |
| "end": float(item["end"]), | |
| "words": words, | |
| }) | |
| return segments | |
| # --------------------------------------------------------------------------- # | |
| # faster-whisper transcription | |
| # --------------------------------------------------------------------------- # | |
| def get_audio_duration(audio_path: str) -> float: | |
| """Return duration in seconds via ffprobe.""" | |
| r = subprocess.run( | |
| ["ffprobe", "-v", "error", "-show_entries", "format=duration", | |
| "-of", "default=noprint_wrappers=1:nokey=1", audio_path], | |
| capture_output=True, text=True, | |
| ) | |
| try: | |
| return float(r.stdout.strip()) | |
| except ValueError: | |
| return 0.0 | |
| def run_transcribe_hf(audio_path: str, hf_info: dict, | |
| segment_q: queue.Queue) -> None: | |
| """ | |
| Transcribe using a HuggingFace transformers pipeline (e.g. fine-tuned Whisper). | |
| Puts the same progress/done/error dicts as run_transcribe into segment_q. | |
| Words are grouped into line segments by silence gaps (>0.6 s) or max 8 words. | |
| """ | |
| try: | |
| import torch | |
| from transformers import pipeline as hf_pipeline | |
| repo = hf_info["repo"] | |
| lang = hf_info["lang"] | |
| cache = str(HF_MODELS_DIR) | |
| segment_q.put({"type": "progress", "pct": 0.02, | |
| "text": f"Loading {repo} (first run downloads ~500 MB)…"}) | |
| device = "cuda:0" if torch.cuda.is_available() else "cpu" | |
| pipe = hf_pipeline( | |
| task="automatic-speech-recognition", | |
| model=repo, | |
| chunk_length_s=30, | |
| device=device, | |
| model_kwargs={"cache_dir": cache}, | |
| ) | |
| pipe.model.config.forced_decoder_ids = ( | |
| pipe.tokenizer.get_decoder_prompt_ids(language=lang, task="transcribe") | |
| ) | |
| segment_q.put({"type": "progress", "pct": 0.10, | |
| "text": "Model loaded. Transcribing…"}) | |
| # Use chunk-level timestamps (return_timestamps=True) — works on all | |
| # fine-tuned Whisper models without needing alignment_heads in the config. | |
| # Word timestamps ("word" level) require alignment_heads which many | |
| # fine-tuned models don't export. | |
| result = pipe(str(audio_path), return_timestamps=True) | |
| chunks = result.get("chunks", []) | |
| segment_q.put({"type": "progress", "pct": 0.80, | |
| "text": f"Got {len(chunks)} chunks — distributing word timings…"}) | |
| # Each chunk has a (start, end) timestamp and a text string. | |
| # We split the text into words and distribute timings evenly within | |
| # the chunk. The user refines the JSON externally anyway. | |
| segments: list[dict] = [] | |
| for i, ch in enumerate(chunks): | |
| text = ch["text"].strip() | |
| if not text: | |
| continue | |
| ts = ch.get("timestamp") or (0.0, 0.0) | |
| c_s = float(ts[0] or 0.0) | |
| c_e = float(ts[1] or c_s + 1.0) | |
| words = text.split() | |
| if not words: | |
| continue | |
| dur = (c_e - c_s) / len(words) | |
| word_entries = [ | |
| [w, round(c_s + j * dur, 3), round(c_s + (j + 1) * dur, 3)] | |
| for j, w in enumerate(words) | |
| ] | |
| seg = {"start": c_s, "end": c_e, "words": word_entries} | |
| segments.append(seg) | |
| pct = min(0.80 + 0.15 * (i + 1) / max(len(chunks), 1), 0.95) | |
| segment_q.put({"type": "progress", "pct": pct, | |
| "text": f"[{c_s:.1f}s] {text}"}) | |
| segment_q.put({"type": "done", "segments": segments}) | |
| except Exception as e: | |
| segment_q.put({"type": "error", "msg": str(e)}) | |
| def run_transcribe_hf_with_fallback(audio_path: str, hf_info: dict, | |
| fallback_model: str, fallback_lang, | |
| segment_q: queue.Queue) -> None: | |
| """Try the HF fine-tuned model; on any error fall back to faster-whisper.""" | |
| # Collect everything from the HF attempt into a temp queue first. | |
| tmp_q: queue.Queue = queue.Queue() | |
| run_transcribe_hf(audio_path, hf_info, tmp_q) | |
| # Replay messages, watching for an error. | |
| hf_failed = False | |
| while not tmp_q.empty(): | |
| msg = tmp_q.get_nowait() | |
| if msg["type"] == "error": | |
| hf_failed = True | |
| segment_q.put({"type": "progress", "pct": 0.05, | |
| "text": f"Fine-tuned model failed ({msg['msg'][:120]}). " | |
| f"Falling back to faster-whisper '{fallback_model}'…"}) | |
| break | |
| segment_q.put(msg) | |
| if hf_failed: | |
| run_transcribe(audio_path, fallback_model, fallback_lang, segment_q) | |
| def run_transcribe(audio_path: str, model_name: str, language_code, | |
| segment_q: queue.Queue) -> list[dict]: | |
| """ | |
| Transcribes audio with faster-whisper. | |
| Puts progress dicts into segment_q as each segment arrives: | |
| {"type": "progress", "pct": float, "text": str} | |
| {"type": "done", "segments": list} | |
| {"type": "error", "msg": str} | |
| """ | |
| try: | |
| from faster_whisper import WhisperModel | |
| segment_q.put({"type": "progress", "pct": 0.02, | |
| "text": f"Loading faster-whisper '{model_name}' (int8, CPU)…"}) | |
| model = WhisperModel( | |
| model_name, | |
| device="cpu", | |
| compute_type="int8", | |
| download_root=str(WHISPER_DIR), | |
| ) | |
| duration = get_audio_duration(audio_path) | |
| lang_label = language_code or "auto" | |
| segment_q.put({"type": "progress", "pct": 0.05, | |
| "text": f"Model loaded. Transcribing (language={lang_label})…"}) | |
| segments_iter, info = model.transcribe( | |
| audio_path, | |
| language=language_code, | |
| word_timestamps=True, | |
| vad_filter=True, | |
| vad_parameters={"min_silence_duration_ms": 500}, | |
| ) | |
| detected = info.language if not language_code else language_code | |
| segment_q.put({"type": "progress", "pct": 0.08, | |
| "text": f"Language: {detected}. Processing audio…"}) | |
| segments = [] | |
| for seg in segments_iter: # ← work happens here, one segment at a time | |
| words = [] | |
| if seg.words: | |
| for w in seg.words: | |
| words.append([w.word.strip(), float(w.start), float(w.end)]) | |
| else: | |
| words.append([seg.text.strip(), float(seg.start), float(seg.end)]) | |
| if any(w[0] for w in words): | |
| segments.append({ | |
| "start": float(seg.start), | |
| "end": float(seg.end), | |
| "words": words, | |
| }) | |
| # stream each segment as it arrives | |
| pct = min(0.08 + 0.90 * (seg.end / duration), 0.97) if duration else 0.5 | |
| line_text = " ".join(w[0] for w in words).strip() | |
| segment_q.put({"type": "progress", "pct": pct, | |
| "text": f"[{seg.start:.1f}s] {line_text}"}) | |
| segment_q.put({"type": "done", "segments": segments}) | |
| except Exception as e: | |
| segment_q.put({"type": "error", "msg": str(e)}) | |
| # --------------------------------------------------------------------------- # | |
| # Vocal removal | |
| # --------------------------------------------------------------------------- # | |
| def remove_vocals_fast(src: str, dst: str): | |
| r = subprocess.run( | |
| ["ffmpeg", "-y", "-i", src, "-af", "pan=stereo|c0=c0-c1|c1=c1-c0", dst], | |
| capture_output=True, text=True, | |
| ) | |
| if r.returncode != 0 or not Path(dst).exists(): | |
| raise RuntimeError(f"ffmpeg vocal removal failed:\n{r.stderr[-600:]}") | |
| def remove_vocals_ai(src: str, out_dir: str, log_fn) -> str: | |
| from audio_separator.separator import Separator | |
| import logging | |
| log_fn("Loading MDX-Net model…") | |
| sep = Separator( | |
| log_level=logging.WARNING, | |
| model_file_dir=str(MODELS_DIR), | |
| output_dir=out_dir, | |
| output_format="wav", | |
| ) | |
| sep.load_model(model_filename="UVR_MDXNET_KARA_2.onnx") | |
| log_fn("Separating (CPU — slow)…") | |
| stems = sep.separate(src) | |
| if not stems or len(stems) < 2: | |
| raise RuntimeError("Separation failed — song may be too long for available RAM.") | |
| return stems[0] # instrumental | |
| # --------------------------------------------------------------------------- # | |
| # ASS subtitle builder | |
| # --------------------------------------------------------------------------- # | |
| def _ass_time(sec: float) -> str: | |
| sec = max(0.0, sec) | |
| h, rem = divmod(int(sec), 3600) | |
| m, s = divmod(rem, 60) | |
| cs = int(round((sec - int(sec)) * 100)) % 100 | |
| return f"{h}:{m:02d}:{s:02d}.{cs:02d}" | |
| def build_ass(segments, font_name, font_size, | |
| primary_hex, highlight_hex, outline_hex, outline_w): | |
| header = ( | |
| f"[Script Info]\nScriptType: v4.00+\n" | |
| f"PlayResX: {VIDEO_W}\nPlayResY: {VIDEO_H}\n" | |
| "WrapStyle: 2\nScaledBorderAndShadow: yes\n\n" | |
| "[V4+ Styles]\n" | |
| "Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, " | |
| "OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, " | |
| "ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, " | |
| "Alignment, MarginL, MarginR, MarginV, Encoding\n" | |
| f"Style: Default,{font_name},{font_size}," | |
| f"{hex_to_ass(highlight_hex)},{hex_to_ass(primary_hex)}," | |
| f"{hex_to_ass(outline_hex)},&H64000000," | |
| f"-1,0,0,0,100,100,0,0,1,{outline_w},1,2,60,60,80,1\n\n" | |
| "[Events]\n" | |
| "Format: Layer, Start, End, Style, MarginL, MarginR, MarginV, Effect, Text\n" | |
| ) | |
| events = [] | |
| for seg in segments: | |
| body = "".join( | |
| f"{{\\kf{max(1, int(round((we - ws) * 100)))}}}" | |
| f"{txt.replace('{','(').replace('}',')')} " | |
| for txt, ws, we in seg["words"] | |
| ).strip() | |
| events.append( | |
| f"Dialogue: 0,{_ass_time(seg['start'])},{_ass_time(seg['end'])}," | |
| f"Default,,0,0,0,,{body}" | |
| ) | |
| return header + "\n".join(events) + "\n" | |
| # --------------------------------------------------------------------------- # | |
| # Tab 1 — Transcribe | |
| # --------------------------------------------------------------------------- # | |
| def do_transcribe(audio_file, whisper_model, language_choice, progress=gr.Progress()): | |
| if audio_file is None: | |
| raise gr.Error("Please upload an audio file.") | |
| lang_code = LANGUAGES.get(language_choice) | |
| log_lines: list[str] = [] | |
| hf_info = HF_MODELS.get((language_choice, whisper_model)) | |
| seg_q: queue.Queue = queue.Queue() | |
| if hf_info: | |
| log_lines.append(f"Trying fine-tuned model: {hf_info['label']}") | |
| t = threading.Thread( | |
| target=run_transcribe_hf_with_fallback, | |
| args=(str(audio_file), hf_info, whisper_model, lang_code, seg_q), | |
| daemon=True, | |
| ) | |
| else: | |
| t = threading.Thread( | |
| target=run_transcribe, | |
| args=(str(audio_file), whisper_model, lang_code, seg_q), | |
| daemon=True, | |
| ) | |
| t.start() | |
| segments = None | |
| while True: | |
| try: | |
| msg = seg_q.get(timeout=0.5) | |
| except queue.Empty: | |
| if not t.is_alive(): | |
| break | |
| yield None, None, "\n".join(log_lines) | |
| continue | |
| if msg["type"] == "progress": | |
| log_lines.append(msg["text"]) | |
| progress(msg["pct"], desc=msg["text"][:60]) | |
| yield None, None, "\n".join(log_lines) | |
| elif msg["type"] == "done": | |
| segments = msg["segments"] | |
| break | |
| elif msg["type"] == "error": | |
| raise gr.Error(f"Transcription failed: {msg['msg']}") | |
| t.join() | |
| if segments is None: | |
| raise gr.Error("Transcription ended without results.") | |
| json_str = segments_to_json(segments) | |
| json_path = OUTPUTS_DIR / "lyrics.json" | |
| json_path.write_text(json_str, encoding="utf-8") | |
| log_lines.append(f"✓ Done — {len(segments)} segments saved to lyrics.json") | |
| progress(1.0, desc="Done!") | |
| yield json_str, str(json_path), "\n".join(log_lines) | |
| # --------------------------------------------------------------------------- # | |
| # Tab 2 — Generate Video | |
| # --------------------------------------------------------------------------- # | |
| def do_generate( | |
| audio_file, | |
| json_input, # either uploaded file path OR pasted text | |
| background_image, | |
| removal_method, | |
| font_file, | |
| font_name, | |
| font_size, | |
| primary_color, | |
| highlight_color, | |
| outline_color, | |
| outline_width, | |
| progress=gr.Progress(), | |
| ): | |
| if audio_file is None: | |
| raise gr.Error("Please upload an audio file.") | |
| if not json_input or not json_input.strip(): | |
| raise gr.Error("Please paste your lyrics JSON (from Tab 1, after editing).") | |
| if shutil.which("ffmpeg") is None: | |
| raise gr.Error("ffmpeg not found on PATH.") | |
| # Parse JSON | |
| try: | |
| segments = json_to_segments(json_input) | |
| except Exception as e: | |
| raise gr.Error(f"Invalid JSON: {e}") | |
| if not segments: | |
| raise gr.Error("JSON parsed but contains no segments.") | |
| job_dir = WORK_DIR / f"job_{os.getpid()}" | |
| job_dir.mkdir(parents=True, exist_ok=True) | |
| log_lines = [] | |
| def log(msg): log_lines.append(msg) | |
| def status(): return "\n".join(log_lines) | |
| method_key = removal_method.split("(")[0].strip().lower() | |
| # ---- vocal removal ---- # | |
| if method_key == "skip": | |
| backing = str(audio_file) | |
| log("Vocal removal: skipped.") | |
| progress(0.10, desc="Using original audio…") | |
| yield None, status() | |
| elif method_key == "fast": | |
| log("Vocal removal: ffmpeg phase cancel…") | |
| progress(0.05, desc="Removing vocals…") | |
| yield None, status() | |
| out_wav = str(job_dir / "instrumental.wav") | |
| try: | |
| remove_vocals_fast(str(audio_file), out_wav) | |
| except RuntimeError as e: | |
| raise gr.Error(str(e)) | |
| backing = out_wav | |
| log(" Done.") | |
| progress(0.20, desc="Vocal removal done.") | |
| yield None, status() | |
| else: # AI | |
| log("Vocal removal: MDX-Net (CPU — slow)…") | |
| progress(0.03, desc="AI vocal separation…") | |
| yield None, status() | |
| q2: queue.Queue = queue.Queue() | |
| res2, err2 = [], [] | |
| def _run_ai(): | |
| try: | |
| res2.append(remove_vocals_ai(str(audio_file), str(job_dir), | |
| lambda m: q2.put(m))) | |
| except Exception as e: | |
| err2.append(str(e)) | |
| t2 = threading.Thread(target=_run_ai, daemon=True) | |
| t2.start() | |
| pct = 0.03 | |
| while t2.is_alive(): | |
| try: | |
| log(f" {q2.get(timeout=2)}") | |
| except queue.Empty: | |
| pass | |
| pct = min(pct + 0.004, 0.45) | |
| progress(pct, desc="AI separation…") | |
| yield None, status() | |
| t2.join() | |
| if err2: | |
| raise gr.Error(f"AI separation failed: {err2[0]}") | |
| backing = res2[0] | |
| log(f" Instrumental: {Path(backing).name}") | |
| progress(0.50, desc="Separation done.") | |
| yield None, status() | |
| # ---- render ---- # | |
| log("Rendering karaoke video…") | |
| progress(0.80, desc="Rendering with ffmpeg…") | |
| yield None, status() | |
| fonts_dir, effective_font = None, font_name | |
| if font_file is not None: | |
| fonts_dir = job_dir / "fonts" | |
| fonts_dir.mkdir(exist_ok=True) | |
| shutil.copy(str(font_file), fonts_dir) | |
| effective_font = Path(font_file).stem | |
| ass_path = job_dir / "lyrics.ass" | |
| ass_path.write_text( | |
| build_ass(segments, effective_font, int(font_size), | |
| primary_color, highlight_color, outline_color, float(outline_width)), | |
| encoding="utf-8", | |
| ) | |
| bg_input = (["-loop", "1", "-i", str(background_image)] | |
| if background_image else | |
| ["-f", "lavfi", "-i", f"color=c=black:s={VIDEO_W}x{VIDEO_H}"]) | |
| scale = (f"scale={VIDEO_W}:{VIDEO_H}:force_original_aspect_ratio=increase," | |
| f"crop={VIDEO_W}:{VIDEO_H},setsar=1" | |
| if background_image else "setsar=1") | |
| ass_esc = str(ass_path).replace("\\", "/").replace(":", "\\:") | |
| vf = f"{scale},ass='{ass_esc}'" | |
| if fonts_dir: | |
| fd_esc = str(fonts_dir).replace("\\", "/").replace(":", "\\:") | |
| vf = f"{scale},ass='{ass_esc}':fontsdir='{fd_esc}'" | |
| out_video = OUTPUTS_DIR / f"karaoke_{os.getpid()}.mp4" | |
| r = subprocess.run([ | |
| "ffmpeg", "-y", *bg_input, "-i", backing, | |
| "-vf", vf, | |
| "-c:v", "libx264", "-tune", "stillimage", "-pix_fmt", "yuv420p", | |
| "-c:a", "aac", "-b:a", "192k", "-shortest", | |
| str(out_video), | |
| ], capture_output=True, text=True) | |
| if r.returncode != 0 or not out_video.exists(): | |
| raise gr.Error(f"ffmpeg failed:\n{r.stderr[-2000:]}") | |
| log(f"Saved: {out_video.name}") | |
| progress(1.0, desc="Done!") | |
| yield str(out_video), status() | |
| # --------------------------------------------------------------------------- # | |
| # UI | |
| # --------------------------------------------------------------------------- # | |
| def build_ui(): | |
| with gr.Blocks(title="Karaoke Generator") as demo: | |
| gr.Markdown( | |
| "# Karaoke Generator\n" | |
| f"> Files stored in: `{_RUNTIME}`" | |
| ) | |
| with gr.Tabs(): | |
| # ================================================================ # | |
| # TAB 1 — Transcribe | |
| # ================================================================ # | |
| with gr.Tab("Step 1 — Transcribe"): | |
| gr.Markdown( | |
| "Upload your audio, run Whisper to get timestamped lyrics, " | |
| "then **download the JSON and refine it** (fix wrong words, " | |
| "split/merge lines, correct timings) — paste it into Gemini or " | |
| "edit manually. Then go to **Step 2** to generate the video." | |
| ) | |
| with gr.Row(): | |
| with gr.Column(): | |
| t1_audio = gr.Audio(label="Audio file", type="filepath") | |
| with gr.Row(): | |
| t1_model = gr.Dropdown(WHISPER_MODELS, value="small", | |
| label="Whisper model") | |
| t1_lang = gr.Dropdown(list(LANGUAGES.keys()), | |
| value="Auto-detect", | |
| label="Language") | |
| t1_hf_notice = gr.Markdown(visible=False) | |
| gr.Markdown( | |
| "- **tiny** ~15 s · **small** ~1 min · **medium** ~3 min (all on CPU)\n" | |
| "- Pinning the language (Hindi/Gujarati) improves accuracy\n" | |
| "- Fine-tuned models auto-selected when available (e.g. Gujarati + small)" | |
| ) | |
| t1_run = gr.Button("Transcribe", variant="primary") | |
| with gr.Column(): | |
| t1_status = gr.Textbox(label="Status", lines=5, interactive=False) | |
| t1_download = gr.File(label="Download lyrics.json") | |
| t1_json = gr.Textbox( | |
| label="Lyrics JSON — review / copy to edit externally", | |
| lines=25, | |
| interactive=False, | |
| ) | |
| def update_hf_notice(lang, model): | |
| info = HF_MODELS.get((lang, model)) | |
| if info: | |
| return gr.Markdown( | |
| value=f"> ✨ **Fine-tuned model will be used:** `{info['repo']}` \n" | |
| f"> Trained specifically on Gujarati speech — better accuracy than generic Whisper.", | |
| visible=True, | |
| ) | |
| return gr.Markdown(visible=False) | |
| t1_lang.change(update_hf_notice, inputs=[t1_lang, t1_model], | |
| outputs=[t1_hf_notice]) | |
| t1_model.change(update_hf_notice, inputs=[t1_lang, t1_model], | |
| outputs=[t1_hf_notice]) | |
| t1_run.click( | |
| do_transcribe, | |
| inputs=[t1_audio, t1_model, t1_lang], | |
| outputs=[t1_json, t1_download, t1_status], | |
| ) | |
| # ================================================================ # | |
| # TAB 2 — Generate Video | |
| # ================================================================ # | |
| with gr.Tab("Step 2 — Generate Video"): | |
| gr.Markdown( | |
| "Paste your **edited JSON** below (or upload the file), " | |
| "choose vocal removal + styling, and render the karaoke video." | |
| ) | |
| with gr.Row(): | |
| # ---- left: inputs ---- # | |
| with gr.Column(): | |
| t2_audio = gr.Audio(label="Audio file", type="filepath") | |
| t2_json = gr.Textbox( | |
| label="Lyrics JSON (paste refined JSON here)", | |
| lines=14, | |
| interactive=True, | |
| placeholder='[{"start": 0.0, "end": 3.0, "text": "...", "words": [...]}]', | |
| ) | |
| t2_bg = gr.Image(label="Background image (optional)", | |
| type="filepath") | |
| gr.Markdown("### Vocal Removal") | |
| t2_removal = gr.Radio( | |
| REMOVAL_METHODS, value=REMOVAL_METHODS[1], | |
| label="Method", | |
| ) | |
| gr.Markdown("### Font & Style") | |
| with gr.Row(): | |
| t2_font_name = gr.Dropdown(FONT_CHOICES, value="Arial Black", | |
| label="Font family") | |
| t2_font_size = gr.Slider(24, 120, value=56, step=2, | |
| label="Font size") | |
| t2_font_file = gr.File(label="Custom font (.ttf/.otf)", | |
| file_types=[".ttf", ".otf"]) | |
| t2_outline = gr.Slider(0, 8, value=2.5, step=0.5, | |
| label="Outline thickness") | |
| with gr.Row(): | |
| t2_primary = gr.ColorPicker(value="#FFFFFF", | |
| label="Upcoming text") | |
| t2_highlight = gr.ColorPicker(value="#FFD400", | |
| label="Sung / highlight") | |
| t2_outline_c = gr.ColorPicker(value="#000000", | |
| label="Outline color") | |
| t2_run = gr.Button("Generate Karaoke Video", variant="primary") | |
| # ---- right: outputs ---- # | |
| with gr.Column(): | |
| t2_video = gr.Video(label="Karaoke video") | |
| t2_status = gr.Textbox(label="Status", lines=10, | |
| interactive=False) | |
| t2_run.click( | |
| do_generate, | |
| inputs=[ | |
| t2_audio, t2_json, t2_bg, t2_removal, | |
| t2_font_file, t2_font_name, t2_font_size, | |
| t2_primary, t2_highlight, t2_outline_c, t2_outline, | |
| ], | |
| outputs=[t2_video, t2_status], | |
| ) | |
| return demo | |
| if __name__ == "__main__": | |
| demo = build_ui() | |
| demo.queue() # required for streaming generators | |
| demo.launch( | |
| server_name="0.0.0.0", # needed for HF Spaces / Docker | |
| server_port=7860, | |
| ) | |