Spaces:
Running
Running
Create initial Gradio app for MuseTalk
Browse filesInitial setup of MuseTalk Space with Gradio interface for audio-driven lip sync
app.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import gradio as gr
|
| 2 |
+
import os
|
| 3 |
+
import subprocess
|
| 4 |
+
import sys
|
| 5 |
+
|
| 6 |
+
# MuseTalk Gradio Interface
|
| 7 |
+
def setup_musetalk():
|
| 8 |
+
"""Clone and setup MuseTalk repository"""
|
| 9 |
+
if not os.path.exists('MuseTalk'):
|
| 10 |
+
subprocess.run(['git', 'clone', 'https://github.com/TMElyralab/MuseTalk.git'], check=True)
|
| 11 |
+
os.chdir('MuseTalk')
|
| 12 |
+
subprocess.run([sys.executable, '-m', 'pip', 'install', '-r', 'requirements.txt'], check=True)
|
| 13 |
+
|
| 14 |
+
def inference(video_file, audio_file, bbox_shift=0):
|
| 15 |
+
"""Run MuseTalk inference"""
|
| 16 |
+
try:
|
| 17 |
+
# Run inference script
|
| 18 |
+
cmd = [
|
| 19 |
+
sys.executable,
|
| 20 |
+
'scripts/inference/gradio_demo.py',
|
| 21 |
+
'--video', video_file.name,
|
| 22 |
+
'--audio', audio_file.name,
|
| 23 |
+
'--bbox_shift', str(bbox_shift)
|
| 24 |
+
]
|
| 25 |
+
result = subprocess.run(cmd, capture_output=True, text=True)
|
| 26 |
+
|
| 27 |
+
# Return the generated video
|
| 28 |
+
output_path = 'results/output.mp4'
|
| 29 |
+
if os.path.exists(output_path):
|
| 30 |
+
return output_path
|
| 31 |
+
else:
|
| 32 |
+
return None
|
| 33 |
+
except Exception as e:
|
| 34 |
+
return f"Error: {str(e)}"
|
| 35 |
+
|
| 36 |
+
# Create Gradio interface
|
| 37 |
+
with gr.Blocks(title="MuseTalk - Real-time Audio-Driven Lip Sync") as demo:
|
| 38 |
+
gr.Markdown("""
|
| 39 |
+
# MuseTalk: Real-Time High-Quality Lip Synchronization
|
| 40 |
+
|
| 41 |
+
Upload a video and audio file to generate lip-synced output.
|
| 42 |
+
|
| 43 |
+
**Note:** First run will download required model weights (~2GB).
|
| 44 |
+
""")
|
| 45 |
+
|
| 46 |
+
with gr.Row():
|
| 47 |
+
with gr.Column():
|
| 48 |
+
video_input = gr.Video(label="Input Video")
|
| 49 |
+
audio_input = gr.Audio(label="Input Audio", type="filepath")
|
| 50 |
+
bbox_shift = gr.Slider(minimum=-10, maximum=10, value=0, step=1,
|
| 51 |
+
label="BBox Shift",
|
| 52 |
+
info="Adjust face bounding box position")
|
| 53 |
+
submit_btn = gr.Button("Generate", variant="primary")
|
| 54 |
+
|
| 55 |
+
with gr.Column():
|
| 56 |
+
video_output = gr.Video(label="Output Video")
|
| 57 |
+
|
| 58 |
+
submit_btn.click(
|
| 59 |
+
fn=inference,
|
| 60 |
+
inputs=[video_input, audio_input, bbox_shift],
|
| 61 |
+
outputs=video_output
|
| 62 |
+
)
|
| 63 |
+
|
| 64 |
+
gr.Markdown("""
|
| 65 |
+
## About MuseTalk
|
| 66 |
+
|
| 67 |
+
MuseTalk generates lip-synchronized videos from input video and audio files.
|
| 68 |
+
|
| 69 |
+
- [GitHub Repository](https://github.com/TMElyralab/MuseTalk)
|
| 70 |
+
- [Model Weights](https://huggingface.co/TMElyralab/MuseTalk)
|
| 71 |
+
""")
|
| 72 |
+
|
| 73 |
+
if __name__ == "__main__":
|
| 74 |
+
# Setup MuseTalk on first run
|
| 75 |
+
try:
|
| 76 |
+
setup_musetalk()
|
| 77 |
+
except Exception as e:
|
| 78 |
+
print(f"Setup warning: {e}")
|
| 79 |
+
|
| 80 |
+
demo.launch()
|