Lipsync / lipsync.py
Okutsu1's picture
debug
57f1fab
Raw History Blame Contribute Delete
6.98 kB
"""
lipsync.py β€” Lip sync any video to any audio via LatentSync (fffiloni/LatentSync)
Strategy:
1. Split input audio into ≀8s chunks (LatentSync hard limit)
2. Slice matching video segment per chunk (loops video if audio is longer)
3. Call fffiloni/LatentSync remotely for each chunk
4. Normalize each chunk to consistent codec/fps
5. Concatenate into one seamless final video
"""
import os
import subprocess
import tempfile
from pathlib import Path
# ─────────────────────────────────────────────────────────────────────────────
# Utilities
# ─────────────────────────────────────────────────────────────────────────────
def get_duration(path: str) -> float:
result = subprocess.run([
"ffprobe", "-v", "error",
"-show_entries", "format=duration",
"-of", "default=noprint_wrappers=1:nokey=1",
path
], capture_output=True, text=True, check=True)
return float(result.stdout.strip())
def slice_video(video_path: str, start: float, duration: float, output_path: str):
subprocess.run([
"ffmpeg", "-y",
"-ss", str(start), "-i", video_path,
"-t", str(duration),
"-c:v", "libx264", "-c:a", "aac",
"-r", "25",
"-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2",
output_path
], check=True, capture_output=True)
def slice_audio(audio_path: str, start: float, duration: float, output_path: str):
subprocess.run([
"ffmpeg", "-y",
"-ss", str(start), "-i", audio_path,
"-t", str(duration),
"-ar", "16000", "-ac", "1",
output_path
], check=True, capture_output=True)
def normalize_chunk(input_path: str, output_path: str):
"""Re-encode to consistent codec/fps/resolution for clean concat"""
subprocess.run([
"ffmpeg", "-y", "-i", input_path,
"-c:v", "libx264", "-c:a", "aac",
"-r", "25",
"-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2",
"-preset", "fast",
output_path
], check=True, capture_output=True)
def concat_videos(video_paths: list[str], output_path: str, tmp: str):
list_file = os.path.join(tmp, "concat_list.txt")
with open(list_file, "w") as f:
for p in video_paths:
f.write(f"file '{os.path.abspath(p)}'\n")
subprocess.run([
"ffmpeg", "-y",
"-f", "concat", "-safe", "0",
"-i", list_file,
"-c:v", "libx264", "-c:a", "aac",
"-movflags", "+faststart",
output_path
], check=True, capture_output=True)
# ─────────────────────────────────────────────────────────────────────────────
# Remote LatentSync call
# ─────────────────────────────────────────────────────────────────────────────
def run_latentsync_remote(video_chunk: str, audio_chunk: str, hf_token: str = None) -> str:
import os
from gradio_client import Client, handle_file
# token passed in from user input
client = Client("fffiloni/LatentSync", token=hf_token)
# Print all available endpoints so we can find the right one
info = client.view_api(return_format="dict")
print("ENDPOINTS:", list(info["named_endpoints"].keys()))
# Use the first available endpoint
endpoint = list(info["named_endpoints"].keys())[0]
print(f"Using endpoint: {endpoint}")
result = client.predict(
handle_file(video_chunk),
handle_file(audio_chunk),
api_name=endpoint,
)
print(f" β†’ LatentSync returned: {result}")
return result
# ─────────────────────────────────────────────────────────────────────────────
# Main pipeline
# ─────────────────────────────────────────────────────────────────────────────
def run_lipsync_pipeline(
original_video: str,
dubbed_audio: str,
output_path: str,
chunk_duration: float = 8.0,
hf_token: str = None,
) -> str:
print(f"\nStarting lip sync pipeline...")
video_dur = get_duration(original_video)
audio_dur = get_duration(dubbed_audio)
print(f" β†’ Video: {video_dur:.1f}s | Audio: {audio_dur:.1f}s")
with tempfile.TemporaryDirectory() as tmp:
# Build chunks from audio length
chunks = []
t = 0.0
while t < audio_dur:
dur = min(chunk_duration, audio_dur - t)
if dur < 0.5:
break
chunks.append((t, dur))
t += dur
print(f" β†’ {len(chunks)} chunks to process")
normalized_chunks = []
for i, (start, dur) in enumerate(chunks):
print(f" [{i+1}/{len(chunks)}] t={start:.1f}s dur={dur:.1f}s")
# Slice audio chunk
audio_chunk = os.path.join(tmp, f"audio_{i}.wav")
slice_audio(dubbed_audio, start, dur, audio_chunk)
# Slice video β€” loop if dubbed audio longer than original video
video_start = start % video_dur
safe_dur = min(dur + 0.5, video_dur - video_start)
video_chunk = os.path.join(tmp, f"video_{i}.mp4")
slice_video(original_video, video_start, safe_dur, video_chunk)
try:
synced_raw = run_latentsync_remote(video_chunk, audio_chunk, hf_token=hf_token)
synced_norm = os.path.join(tmp, f"synced_norm_{i}.mp4")
normalize_chunk(synced_raw, synced_norm)
normalized_chunks.append(synced_norm)
print(f" βœ… Chunk {i+1} done")
except Exception as e:
print(f" ⚠ Chunk {i+1} LatentSync failed: {e} β€” using raw video")
fallback_norm = os.path.join(tmp, f"fallback_norm_{i}.mp4")
normalize_chunk(video_chunk, fallback_norm)
normalized_chunks.append(fallback_norm)
if not normalized_chunks:
raise RuntimeError("All chunks failed.")
print(f" β†’ Concatenating {len(normalized_chunks)} chunks...")
concat_videos(normalized_chunks, output_path, tmp)
size_mb = os.path.getsize(output_path) / (1024 * 1024)
print(f" β†’ Done: {output_path} ({size_mb:.1f} MB)")
return output_path