""" lipsync.py — Lip sync any video to any audio via LatentSync (fffiloni/LatentSync) Strategy: 1. Split input audio into ≤8s chunks (LatentSync hard limit) 2. Slice matching video segment per chunk (loops video if audio is longer) 3. Call fffiloni/LatentSync remotely for each chunk 4. Normalize each chunk to consistent codec/fps 5. Concatenate into one seamless final video """ import os import subprocess import tempfile from pathlib import Path # ───────────────────────────────────────────────────────────────────────────── # Utilities # ───────────────────────────────────────────────────────────────────────────── def get_duration(path: str) -> float: result = subprocess.run([ "ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", path ], capture_output=True, text=True, check=True) return float(result.stdout.strip()) def slice_video(video_path: str, start: float, duration: float, output_path: str): subprocess.run([ "ffmpeg", "-y", "-ss", str(start), "-i", video_path, "-t", str(duration), "-c:v", "libx264", "-c:a", "aac", "-r", "25", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", output_path ], check=True, capture_output=True) def slice_audio(audio_path: str, start: float, duration: float, output_path: str): subprocess.run([ "ffmpeg", "-y", "-ss", str(start), "-i", audio_path, "-t", str(duration), "-ar", "16000", "-ac", "1", output_path ], check=True, capture_output=True) def normalize_chunk(input_path: str, output_path: str): """Re-encode to consistent codec/fps/resolution for clean concat""" subprocess.run([ "ffmpeg", "-y", "-i", input_path, "-c:v", "libx264", "-c:a", "aac", "-r", "25", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-preset", "fast", output_path ], check=True, capture_output=True) def concat_videos(video_paths: list[str], output_path: str, tmp: str): list_file = os.path.join(tmp, "concat_list.txt") with open(list_file, "w") as f: for p in video_paths: f.write(f"file '{os.path.abspath(p)}'\n") subprocess.run([ "ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", list_file, "-c:v", "libx264", "-c:a", "aac", "-movflags", "+faststart", output_path ], check=True, capture_output=True) # ───────────────────────────────────────────────────────────────────────────── # Remote LatentSync call # ───────────────────────────────────────────────────────────────────────────── def run_latentsync_remote(video_chunk: str, audio_chunk: str, hf_token: str = None) -> str: import os from gradio_client import Client, handle_file # token passed in from user input client = Client("fffiloni/LatentSync", token=hf_token) # Print all available endpoints so we can find the right one info = client.view_api(return_format="dict") print("ENDPOINTS:", list(info["named_endpoints"].keys())) # Use the first available endpoint endpoint = list(info["named_endpoints"].keys())[0] print(f"Using endpoint: {endpoint}") result = client.predict( handle_file(video_chunk), handle_file(audio_chunk), api_name=endpoint, ) print(f" → LatentSync returned: {result}") return result # ───────────────────────────────────────────────────────────────────────────── # Main pipeline # ───────────────────────────────────────────────────────────────────────────── def run_lipsync_pipeline( original_video: str, dubbed_audio: str, output_path: str, chunk_duration: float = 8.0, hf_token: str = None, ) -> str: print(f"\nStarting lip sync pipeline...") video_dur = get_duration(original_video) audio_dur = get_duration(dubbed_audio) print(f" → Video: {video_dur:.1f}s | Audio: {audio_dur:.1f}s") with tempfile.TemporaryDirectory() as tmp: # Build chunks from audio length chunks = [] t = 0.0 while t < audio_dur: dur = min(chunk_duration, audio_dur - t) if dur < 0.5: break chunks.append((t, dur)) t += dur print(f" → {len(chunks)} chunks to process") normalized_chunks = [] for i, (start, dur) in enumerate(chunks): print(f" [{i+1}/{len(chunks)}] t={start:.1f}s dur={dur:.1f}s") # Slice audio chunk audio_chunk = os.path.join(tmp, f"audio_{i}.wav") slice_audio(dubbed_audio, start, dur, audio_chunk) # Slice video — loop if dubbed audio longer than original video video_start = start % video_dur safe_dur = min(dur + 0.5, video_dur - video_start) video_chunk = os.path.join(tmp, f"video_{i}.mp4") slice_video(original_video, video_start, safe_dur, video_chunk) try: synced_raw = run_latentsync_remote(video_chunk, audio_chunk, hf_token=hf_token) synced_norm = os.path.join(tmp, f"synced_norm_{i}.mp4") normalize_chunk(synced_raw, synced_norm) normalized_chunks.append(synced_norm) print(f" ✅ Chunk {i+1} done") except Exception as e: print(f" ⚠ Chunk {i+1} LatentSync failed: {e} — using raw video") fallback_norm = os.path.join(tmp, f"fallback_norm_{i}.mp4") normalize_chunk(video_chunk, fallback_norm) normalized_chunks.append(fallback_norm) if not normalized_chunks: raise RuntimeError("All chunks failed.") print(f" → Concatenating {len(normalized_chunks)} chunks...") concat_videos(normalized_chunks, output_path, tmp) size_mb = os.path.getsize(output_path) / (1024 * 1024) print(f" → Done: {output_path} ({size_mb:.1f} MB)") return output_path