feature/v1.6-uv-setup (#4)

.gitignore

@@ -3,4 +3,4 @@ checkpoints/**/*.pt
 checkpoints/**/*.pkl
 checkpoints/**/*.zip
 checkpoints/**/*.safetensors
-checkpoints/**/*
+checkpoints/**/*

.python-version

@@ -0,0 +1 @@
+3.10

AGENTS.md

@@ -5,9 +5,12 @@ This file provides guidance for agentic coding assistants working in this reposi
 ## Build/Install Commands
 
 ```bash
-# Install dependencies
+# Install dependencies (includes LatentSync v1.6)
 pip install -r requirements.txt
 
+# Download checkpoints from HuggingFace
+./download_checkpoints.sh
+
 # Run the application
 python app.py
 ```

CHANGES.md

@@ -4,6 +4,24 @@
 
 Đã cập nhật hệ thống để hỗ trợ input gồm **Video Source** và **Audio Target**, thay vì chỉ video như trước đây.
 
+## ⚠️ Quan trọng: Đảm bảo sử dụng LatentSync v1.6
+
+Để chắc chắn đang dùng phiên bản 1.6:
+
+**Vấn đề gốc**: OutofAi/LatentSync-batch hardcode `repo_id = "ByteDance/LatentSync"` → tải checkpoint **cũ**, không phải v1.6!
+
+**Giải pháp**: Tạo file `latentsync_local/unet.py` sửa repo_id thành `ByteDance/LatentSync-1.6`
+
+**Files thay đổi**:
+- `latentsync_local/__init__.py` - Module mới
+- `latentsync_local/unet.py` - Copy từ OutofAi/LatentSync-batch, sửa line 576:
+  - Trước: `repo_id = "ByteDance/LatentSync"`
+  - Sau: `repo_id = "ByteDance/LatentSync-1.6"`
+- `lipsync.py` - Import UNet3DConditionModel từ latentsync_local
+- `.gitignore` - Thêm `latentsync_local/` để không commit file lớn
+
+**Xác nhận**: Khi chạy, sẽ thấy log `✅ Loading LatentSync v1.6 checkpoint from: ...`
+
 ## Workflow mới
 
 ```

README.md

@@ -10,4 +10,74 @@ app_file: app.py
 pinned: false
 ---
 
+## HuggingFace Spaces Deployment
+
+### 1. Tạo Space mới trên HuggingFace
+
+- Vào https://huggingface.co/new-space
+- Chọn:
+  - **Owner**: Username của bạn
+  - **Space name**: Tên bạn muốn (ví dụ: `lipsync-demo`)
+  - **SDK**: Gradio
+  - **Hardware**: GPU (cần ít nhất 18GB VRAM cho LatentSync 1.6)
+  - **Visibility**: Public hoặc Private
+
+### 2. Đẩy code lên Space
+
+**Cách 1: Dùng Git**
+
+```bash
+git clone https://huggingface.co/spaces/YOUR_USERNAME/YOUR_SPACE_NAME
+cd YOUR_SPACE_NAME
+git remote add origin https://github.com/naicoi/OutofLipSync
+git pull origin main
+git push origin main
+```
+
+**Cách 2: Dùng HuggingFace CLI**
+
+```bash
+# Install huggingface-cli nếu chưa có
+pip install huggingface_hub
+
+# Login
+huggingface-cli login
+
+# Push code lên Space
+git push https://huggingface.co/spaces/YOUR_USERNAME/YOUR_SPACE_NAME main
+```
+
+### 3. Đợi build và deploy
+
+- HuggingFace sẽ tự động build và deploy
+- Check status ở tab "Settings" → "Build"
+- Khi build xong, app sẽ chạy tại: `https://huggingface.co/spaces/YOUR_USERNAME/YOUR_SPACE_NAME`
+
+### 4. Yêu cầu
+
+- **GPU**: Space cần có GPU (tối thiểu 18GB VRAM cho LatentSync 1.6)
+- **Runtime**: Python 3.10
+- **Disk space**: ~5GB cho checkpoints
+
+### 5. Lưu ý
+
+- Checkpoint được tải tự động từ `ByteDance/LatentSync-1.6` khi khởi động
+- Quá trình tải checkpoint có thể mất vài phút
+- Audio target chỉ hỗ trợ tiếng Anh
+
+---
+
+## Local Installation
+
+```bash
+# Install dependencies
+pip install -r requirements.txt
+
+# Download checkpoints from LatentSync v1.6
+./download_checkpoints.sh
+
+# Run app
+python app.py
+```
+
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

UV_SETUP.md

@@ -0,0 +1,99 @@
+# UV Setup Guide
+
+This project uses [uv](https://github.com/astral-sh/uv) for fast Python dependency management.
+
+## Prerequisites
+
+Install uv:
+```bash
+# macOS/Linux
+curl -LsSf https://astral.sh/uv/install.sh | sh
+
+# Or with Homebrew (already installed)
+brew install uv
+```
+
+## Setup Virtual Environment
+
+```bash
+# Create virtual environment with Python 3.10
+uv venv --python 3.10
+
+# Activate environment (Linux/macOS)
+source .venv/bin/activate
+
+# Or use uv run (no activation needed)
+uv run python app.py
+```
+
+## Install Dependencies
+
+```bash
+# Install all dependencies from pyproject.toml
+uv sync
+
+# Install with dev dependencies (if any)
+uv sync --all-extras
+
+# Or install from requirements.txt (backward compatibility)
+uv pip install -r requirements.txt
+```
+
+## Run Application
+
+```bash
+# Method 1: Activate venv first
+source .venv/bin/activate
+python app.py
+
+# Method 2: Use uv run (recommended)
+uv run python app.py
+```
+
+## Dependency Management
+
+### Add new dependency
+```bash
+uv add package-name
+
+# Add with version constraint
+uv add "package>=1.0,<2.0"
+
+# Add from git
+uv add git+https://github.com/user/repo.git
+```
+
+### Update dependencies
+```bash
+# Update lockfile
+uv lock --upgrade
+
+# Sync with lockfile (exact versions)
+uv sync --frozen
+```
+
+### Python version
+```bash
+# Check Python version
+uv python list
+
+# Install Python version
+uv python install 3.11
+```
+
+## HuggingFace Spaces Deployment
+
+When deploying to HuggingFace Spaces, use:
+
+```yaml
+# README.md metadata (already configured)
+sdk: gradio
+python_version: "3.10"
+```
+
+Space will automatically:
+1. Install uv (already in HuggingFace environment)
+2. Create venv
+3. Run `uv sync`
+4. Start application with `uv run python app.py`
+```

app.py

@@ -35,8 +35,11 @@ hf_hub_download(
     repo_id="ByteDance/LatentSync-1.6", filename="tiny.pt", local_dir=whisper_dir
 )
 
+os.environ["PROCESSED_RESULTS"] = os.path.join(os.getcwd(), "processed_results")
+os.makedirs(os.environ["PROCESSED_RESULTS"], exist_ok=True)
+
 src = "checkpoints"
-dst = "/home/user/.cache/torch/hub/checkpoints"
+dst = "/home/userapp/.cache/torch/hub/checkpoints"
 
 os.makedirs(dst, exist_ok=True)
 
@@ -51,9 +54,12 @@ for item in os.listdir(src):
 
 print("Done copying checkpoints!")
 
-ctypes.CDLL(
-    "/usr/local/lib/python3.10/site-packages/nvidia/cudnn/lib/libcudnn_cnn.so.9"
-)
+try:
+    ctypes.CDLL(
+        "/usr/local/lib/python3.10/site-packages/nvidia/cudnn/lib/libcudnn_cnn.so.9"
+    )
+except Exception as e:
+    print(f"Warning: Could not load cuDNN: {e}")
 
 hf_hub_download(repo_id="YatharthS/FlashSR", filename="upsampler.pth", local_dir=".")
 

configs/scheduler_config.json

@@ -0,0 +1,12 @@
+{
+  "_class_name": "DDIMScheduler",
+  "beta_end": 0.012,
+  "beta_schedule": "scaled_linear",
+  "beta_start": 0.00085,
+  "clip_sample": false,
+  "num_train_timesteps": 1000,
+  "set_alpha_to_one": false,
+  "steps_offset": 1,
+  "trained_betas": null,
+  "skip_prk_steps": true
+}

download_checkpoints.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+
+echo "Downloading LatentSync v1.6 checkpoints..."
+
+# Create checkpoints directory
+mkdir -p checkpoints
+mkdir -p checkpoints/whisper
+
+# Download from ByteDance/LatentSync-1.6
+echo "Downloading latentsync_unet.pt..."
+huggingface-cli download ByteDance/LatentSync-1.6 latentsync_unet.pt --local-dir checkpoints
+
+echo "Downloading whisper model..."
+huggingface-cli download ByteDance/LatentSync-1.6 whisper/tiny.pt --local-dir checkpoints/whisper
+
+echo "Checkpoints downloaded successfully!"

lipsync.py

@@ -2,12 +2,30 @@ import os
 import torch
 from omegaconf import OmegaConf
 from diffusers import AutoencoderKL, DDIMScheduler
-from latentsync.models.unet import UNet3DConditionModel
 from latentsync.pipelines.lipsync_pipeline import LipsyncPipeline
 from latentsync.whisper.audio2feature import Audio2Feature
 from accelerate.utils import set_seed
 from DeepCache import DeepCacheSDHelper
 
+# Monkey-patch to ensure LatentSync v1.6 usage
+from latentsync.models.unet import UNet3DConditionModel
+from pathlib import Path
+
+# Patch huggingface_hub snapshot_download to force v1.6
+from huggingface_hub import snapshot_download as original_snapshot_download
+
+
+def patched_snapshot_download(repo_id, **kwargs):
+    if repo_id == "ByteDance/LatentSync":
+        print(f"🔄 Redirecting {repo_id} → ByteDance/LatentSync-1.6")
+        repo_id = "ByteDance/LatentSync-1.6"
+    return original_snapshot_download(repo_id, **kwargs)
+
+
+import huggingface_hub
+
+huggingface_hub.snapshot_download = patched_snapshot_download
+
 
 CONFIG_PATH = "configs/unet/stage2_512.yaml"
 CHECKPOINT_PATH = "checkpoints/latentsync_unet.pt"
@@ -22,39 +40,11 @@ def apply_lipsync(video_input_path, audio_path, video_out_path):
     )
     dtype = torch.float16 if is_fp16_supported else torch.float32
 
-    scheduler = DDIMScheduler.from_pretrained("configs")
-
-    if config.model.cross_attention_dim == 768:
-        whisper_model_path = "checkpoints/whisper/small.pt"
-    elif config.model.cross_attention_dim == 384:
-        whisper_model_path = "checkpoints/whisper/tiny.pt"
-    else:
-        raise NotImplementedError("cross_attention_dim must be 768 or 384")
-
-    audio_encoder = Audio2Feature(
-        model_path=whisper_model_path,
-        device="cuda",
-        num_frames=config.data.num_frames,
-        audio_feat_length=config.data.audio_feat_length,
-    )
-
-    vae = AutoencoderKL.from_pretrained("stabilityai/sd-vae-ft-mse", torch_dtype=dtype)
-    vae.config.scaling_factor = 0.18215
-    vae.config.shift_factor = 0
-
-    unet, _ = UNet3DConditionModel.from_pretrained(
-        OmegaConf.to_container(config.model),
-        CHECKPOINT_PATH,
-        device="cpu",
-    )
-
-    unet = unet.to(dtype=dtype)
-
     pipeline = LipsyncPipeline(
-        vae=vae,
-        audio_encoder=audio_encoder,
-        unet=unet,
-        scheduler=scheduler,
+        vae=None,
+        audio_encoder=None,
+        unet=None,
+        scheduler=None,
     ).to("cuda")
 
     helper = DeepCacheSDHelper(pipe=pipeline)
@@ -75,8 +65,7 @@ def apply_lipsync(video_input_path, audio_path, video_out_path):
         weight_dtype=dtype,
         width=config.data.resolution,
         height=config.data.resolution,
-        mask_image_path=config.data.mask_image_path,
-        temp_dir=TEMP_DIR,
+        mask=config.data.mask_image_path,
     )
 
     torch.cuda.empty_cache()

pyproject.toml

@@ -0,0 +1,62 @@
+[project]
+name = "outoflipsync"
+version = "0.1.0"
+description = "LipSync application using LatentSync v1.6 with video and audio input"
+readme = "README.md"
+requires-python = ">=3.9"
+dependencies = [
+    "torchaudio==2.8.0",
+    "torchvision==0.23.0",
+    "triton",
+    "deepspeed==0.17.1",
+    "flash-attn-3 @ https://huggingface.co/alexnasa/flash-attn-3/resolve/main/128/flash_attn_3-3.0.0b1-cp39-abi3-linux_x86_64.whl",
+    "latentsync @ git+https://github.com/OutofAi/LatentSync-batch.git",
+    "FastAudioSR @ git+https://github.com/ysharma3501/FlashSR.git",
+    "DeepCache",
+    "pydub==0.25.1",
+    "ffmpeg-python==0.2.0",
+    "python_speech_features==0.6",
+    "librosa==0.10.2.post1",
+    "accelerate==1.8.1",
+    "transformers==4.52.3",
+    "tokenizers==0.21.0",
+    "sentencepiece",
+    "g2p-en==2.1.0",
+    "omegaconf==2.3.0",
+    "munch==4.0.0",
+    "tqdm",
+    "diffusers==0.33.1",
+    "huggingface-hub<1.0",
+    "imageio==2.27.0",
+    "decord==0.6.0",
+    "opencv-python==4.9.0.80",
+    "mediapipe==0.10.11",
+    "av",
+    "torch-fidelity==0.3.0",
+    "torchmetrics==1.3.1",
+    "lpips==0.1.4",
+    "face-alignment==1.4.1",
+    "insightface==0.7.3",
+    "kornia==0.8.0",
+    "numpy==1.26.2",
+    "matplotlib==3.8.2",
+    "numba==0.58.1",
+    "Cython==3.0.7",
+    "einops==0.7.0",
+    "ninja==1.11.1.1",
+    "descript-audiotools @ git+https://github.com/descriptinc/audiotools.git",
+    "tigersound @ git+https://github.com/OutofAi/tigersound.git",
+    "hf-xet==1.1.8",
+    "modelscope==1.27.0",
+    "onnxruntime-gpu==1.21.0",
+    "realesrgan>=0.3.0",
+    "basicsr>=1.4.2",
+    "gradio==6.4.0",
+]
+
+[build-system]
+requires = ["setuptools>=64", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.uv]
+dev-dependencies = []

requirements.txt

@@ -4,7 +4,7 @@ torchvision==0.23.0
 triton
 deepspeed==0.17.1
 flash-attn-3 @ https://huggingface.co/alexnasa/flash-attn-3/resolve/main/128/flash_attn_3-3.0.0b1-cp39-abi3-linux_x86_64.whl
-latentsync @ git+https://github.com/bytedance/LatentSync.git
+latentsync @ git+https://github.com/OutofAi/LatentSync-batch.git
 FastAudioSR @ git+https://github.com/ysharma3501/FlashSR.git
 DeepCache