Latent audio lipsync

README.md

@@ -4,10 +4,9 @@ emoji: 👄
 colorFrom: blue
 colorTo: blue
 sdk: gradio
-sdk_version: 6.13.0
+sdk_version: 5.12.0
 app_file: app.py
 pinned: false
-disable_embedding: true
 short_description: Audio Conditioned LipSync with Latent Diffusion Models
 ---
 

app.py

@@ -1,33 +1,4 @@
-def _patch_asyncio_event_loop_del():
-    """
-    Patch a noisy asyncio teardown issue sometimes seen in Spaces environments.
-    In some runtime/container combinations, Python may try to close an already
-    invalid file descriptor when the event loop is garbage-collected. We silence
-    only that specific harmless case.
-    """
-    try:
-        import asyncio.base_events as base_events
-
-        original_del = getattr(base_events.BaseEventLoop, "__del__", None)
-        if original_del is None:
-            return
-
-        def patched_del(self):
-            try:
-                original_del(self)
-            except ValueError as e:
-                if "Invalid file descriptor" not in str(e):
-                    raise
-
-        base_events.BaseEventLoop.__del__ = patched_del
-    except Exception:
-        pass
-
-
-_patch_asyncio_event_loop_del()
-
 import gradio as gr
-import spaces
 import os
 import sys
 import shutil
@@ -40,7 +11,7 @@ from huggingface_hub import snapshot_download
 os.makedirs("checkpoints", exist_ok=True)
 
 snapshot_download(
-    repo_id = "ByteDance/LatentSync",
+    repo_id = "chunyu-li/LatentSync",
     local_dir = "./checkpoints"  
 )
 
@@ -108,26 +79,7 @@ from accelerate.utils import set_seed
 from latentsync.whisper.audio2feature import Audio2Feature
 
 
-@spaces.GPU(duration=180)
 def main(video_path, audio_path, progress=gr.Progress(track_tqdm=True)):
-    """
-    Perform lip-sync video generation using an input video and a separate audio track.
-    
-    This function takes an input video (usually a person speaking) and an audio file,
-    and synchronizes the video frames so that the lips of the speaker match the audio content.
-    It uses a latent diffusion model-based pipeline (LatentSync) for audio-conditioned lip synchronization.
-    
-    Args:
-        video_path (str): File path to the input video in MP4 format.
-        audio_path (str): File path to the input audio file (e.g., WAV or MP3).
-        progress (gr.Progress, optional): Gradio progress tracker for UI feedback (auto-injected).
-        
-    Returns:
-        str: File path to the generated output video with lip synchronization applied.
-    """
-
-    gr.Info("180 seconds will be used from your daily ZeroGPU time credits.")
-    
     inference_ckpt_path = "checkpoints/latentsync_unet.pt"
     unet_config_path = "configs/unet/second_stage.yaml"
     config = OmegaConf.load(unet_config_path)
@@ -171,12 +123,9 @@ def main(video_path, audio_path, progress=gr.Progress(track_tqdm=True)):
 
     unet = unet.to(dtype=torch.float16)
 
-    """
     # set xformers
-    
     if is_xformers_available():
         unet.enable_xformers_memory_efficient_attention()
-    """
 
     pipeline = LipsyncPipeline(
         vae=vae,
@@ -267,4 +216,4 @@ with gr.Blocks(css=css) as demo:
         outputs = [video_result]
     )
 
-demo.queue().launch(show_error=True, ssr_mode=False, mcp_server=True)
+demo.queue().launch(show_api=False, show_error=True)

latentsync/models/attention.py

@@ -9,10 +9,10 @@ import torch.nn.functional as F
 from torch import nn
 
 from diffusers.configuration_utils import ConfigMixin, register_to_config
-from diffusers.models.modeling_utils import ModelMixin
+from diffusers.modeling_utils import ModelMixin
 from diffusers.utils import BaseOutput
 from diffusers.utils.import_utils import is_xformers_available
-from diffusers.models.attention import Attention as CrossAttention, FeedForward, AdaLayerNorm
+from diffusers.models.attention import CrossAttention, FeedForward, AdaLayerNorm
 
 from einops import rearrange, repeat
 from .utils import zero_module

latentsync/models/motion_module.py

@@ -11,10 +11,10 @@ import torch.nn.functional as F
 from torch import nn
 
 from diffusers.configuration_utils import ConfigMixin, register_to_config
-from diffusers.models.modeling_utils import ModelMixin
+from diffusers.modeling_utils import ModelMixin
 from diffusers.utils import BaseOutput
 from diffusers.utils.import_utils import is_xformers_available
-from diffusers.models.attention import Attention as CrossAttention, FeedForward
+from diffusers.models.attention import CrossAttention, FeedForward
 
 from einops import rearrange, repeat
 import math

latentsync/models/unet.py

@@ -9,7 +9,7 @@ import torch.nn as nn
 import torch.utils.checkpoint
 
 from diffusers.configuration_utils import ConfigMixin, register_to_config
-from diffusers.models.modeling_utils import ModelMixin
+from diffusers.modeling_utils import ModelMixin
 from diffusers import UNet2DConditionModel
 from diffusers.utils import BaseOutput, logging
 from diffusers.models.embeddings import TimestepEmbedding, Timesteps

latentsync/pipelines/lipsync_pipeline.py

@@ -15,7 +15,7 @@ from packaging import version
 
 from diffusers.configuration_utils import FrozenDict
 from diffusers.models import AutoencoderKL
-from diffusers.pipelines.pipeline_utils import DiffusionPipeline
+from diffusers.pipeline_utils import DiffusionPipeline
 from diffusers.schedulers import (
     DDIMScheduler,
     DPMSolverMultistepScheduler,

requirements.txt

@@ -1,21 +1,21 @@
-torch==2.5.1
-torchvision==0.20.1
+torch==2.2.2
+torchvision==0.17.2
 --extra-index-url https://download.pytorch.org/whl/cu121
-xformers==0.0.29.post1
-triton==3.1.0
+xformers==0.0.26
+triton==2.2.0
 
-diffusers==0.33.1
-transformers==4.52.3
-huggingface-hub<1.0
+diffusers==0.11.1
+transformers==4.38.0
+huggingface-hub==0.25.2
 imageio==2.27.0
 decord==0.6.0
 accelerate==0.26.1
 einops==0.7.0
 omegaconf==2.3.0
-safetensors>=0.4.3
+safetensors==0.4.2
 opencv-python==4.9.0.80
 mediapipe==0.10.11
-av
+av==11.0.0
 torch-fidelity==0.3.0
 torchmetrics==1.3.1
 python_speech_features==0.6
@@ -27,8 +27,5 @@ face-alignment==1.4.1
 ninja==1.11.1.1
 pandas==2.0.3
 numpy==1.24.4
-pydub==0.25.1
-moviepy==1.0.3
-hf-xet==1.1.8
-spaces
-gradio[mcp]
+pydub
+moviepy==1.0.3