Spaces:
Paused
Paused
| import os | |
| os.environ["TOKENIZERS_PARALLELISM"] = "false" | |
| # ── spaces MUST be imported before any CUDA-related package ─────────────────── | |
| import spaces | |
| # ── Auto-upgrade torchao if version is too old (requires >= 0.16.0) ──────────── | |
| import subprocess, sys, re, pathlib, site as _site_mod | |
| # ── Patch 1: Fix diffusers torchao_quantizer.py — 'logger' not defined bug ─── | |
| try: | |
| _site_pkgs = pathlib.Path(_site_mod.getsitepackages()[0]) | |
| _quantizer_path = _site_pkgs / "diffusers/quantizers/torchao/torchao_quantizer.py" | |
| if _quantizer_path.exists(): | |
| _src = _quantizer_path.read_text(encoding="utf-8") | |
| if "logger = logging.getLogger" not in _src and "logger.warning" in _src: | |
| if "import logging" in _src: | |
| _src = _src.replace( | |
| "import logging\n", | |
| "import logging\nlogger = logging.getLogger(__name__) # patch: auto-injected\n", | |
| 1, | |
| ) | |
| else: | |
| _src = "import logging\nlogger = logging.getLogger(__name__) # patch: auto-injected\n" + _src | |
| _quantizer_path.write_text(_src, encoding="utf-8") | |
| print("[Patch] ✅ diffusers torchao_quantizer.py — logger injected successfully") | |
| else: | |
| print("[Patch] ✓ torchao_quantizer.py already has logger — no patch needed") | |
| else: | |
| print("[Patch] ⚠️ torchao_quantizer.py not found — skipping patch") | |
| except Exception as _pe: | |
| print(f"[Patch] ⚠️ Could not patch torchao_quantizer.py: {_pe}") | |
| # ── Patch 2: Ensure torchao >= 0.16.0 (required by diffusers Float8 quantizer) ─ | |
| try: | |
| import torchao as _tao | |
| from packaging.version import Version | |
| if Version(_tao.__version__) < Version("0.16.0"): | |
| print(f"[torchao] Found v{_tao.__version__} < 0.16.0 — upgrading in background…") | |
| subprocess.check_call([ | |
| sys.executable, "-m", "pip", "install", "-q", "--upgrade", "torchao>=0.16.0" | |
| ]) | |
| print("[torchao] ✅ Upgrade complete — will take effect on next restart") | |
| else: | |
| print(f"[torchao] ✅ Version OK: {_tao.__version__}") | |
| except Exception as _e: | |
| print(f"[torchao] version check failed: {_e}") | |
| import torch | |
| from diffusers.pipelines.wan.pipeline_wan_i2v import WanImageToVideoPipeline | |
| from diffusers.models.transformers.transformer_wan import WanTransformer3DModel | |
| from diffusers.utils.export_utils import export_to_video | |
| import gradio as gr | |
| import tempfile | |
| import numpy as np | |
| from PIL import Image | |
| import random | |
| import gc | |
| import json | |
| from datetime import datetime | |
| from torchao.quantization import quantize_ | |
| from torchao.quantization import Float8DynamicActivationFloat8WeightConfig | |
| from torchao.quantization import Int8WeightOnlyConfig | |
| import aoti | |
| # ── Anthropic for Prompt Analysis (Colorization / Music) ───────────────────── | |
| try: | |
| import anthropic | |
| _anthropic_key = os.environ.get("ANTHROPIC_API_KEY", "") | |
| ANTHROPIC_AVAILABLE = bool(_anthropic_key) | |
| if not ANTHROPIC_AVAILABLE: | |
| print("[Anthropic] ⚠️ ANTHROPIC_API_KEY not found in Secrets — Claude disabled") | |
| else: | |
| print("[Anthropic] ✅ API key found") | |
| except ImportError: | |
| ANTHROPIC_AVAILABLE = False | |
| print("[Anthropic] ⚠️ Library not installed") | |
| # ── MoviePy for audio mixing ─────────────────────────────────────────────────── | |
| try: | |
| from moviepy.editor import VideoFileClip, AudioFileClip | |
| MOVIEPY_AVAILABLE = True | |
| except ImportError: | |
| MOVIEPY_AVAILABLE = False | |
| print("[Music] moviepy not installed — audio features disabled") | |
| # ── MusicGen (Meta) ──────────────────────────────────────────────────────────── | |
| MUSICGEN_AVAILABLE = False | |
| _musicgen_pipe = None | |
| MUSICGEN_MODEL = "facebook/musicgen-small" | |
| MUSICGEN_SAMPLE_RATE = 32000 | |
| try: | |
| from transformers import pipeline as hf_pipeline | |
| import scipy.io.wavfile as _wavfile | |
| MUSICGEN_AVAILABLE = True | |
| print("[MusicGen] transformers available ✅") | |
| except ImportError: | |
| print("[MusicGen] transformers not installed — music generation disabled") | |
| def _load_musicgen(): | |
| global _musicgen_pipe | |
| if _musicgen_pipe is not None: | |
| return True | |
| if not MUSICGEN_AVAILABLE: | |
| return False | |
| try: | |
| print("[MusicGen] Loading model…") | |
| _musicgen_pipe = hf_pipeline( | |
| "text-to-audio", | |
| model=MUSICGEN_MODEL, | |
| device="cpu", | |
| torch_dtype=torch.float32, | |
| ) | |
| print("[MusicGen] ✅ Model ready") | |
| return True | |
| except Exception as e: | |
| print(f"[MusicGen] Load failed: {e}") | |
| return False | |
| def _prompt_to_music_description(video_prompt: str) -> str: | |
| if ANTHROPIC_AVAILABLE: | |
| try: | |
| client = anthropic.Anthropic() | |
| msg = client.messages.create( | |
| model="claude-opus-4-6", | |
| max_tokens=80, | |
| messages=[{ | |
| "role": "user", | |
| "content": ( | |
| "You are a professional music composer for films.\n" | |
| "Convert this video scene description into a short music generation prompt " | |
| "(max 20 words). Focus on: genre, instruments, tempo, mood.\n" | |
| "Examples:\n" | |
| "- 'cinematic orchestral, slow tempo, dramatic strings, epic brass'\n" | |
| "- 'ambient electronic, soft pads, 80 bpm, peaceful and dreamy'\n" | |
| "- 'upbeat jazz, piano and drums, 120 bpm, energetic and fun'\n\n" | |
| f"Video description: {video_prompt}\n\n" | |
| "Return ONLY the music prompt, nothing else." | |
| ), | |
| }], | |
| ) | |
| result = msg.content[0].text.strip() | |
| print(f"[MusicGen] Music description from Claude: {result}") | |
| return result | |
| except Exception as e: | |
| print(f"[MusicGen] Claude API error: {e}") | |
| prompt_lower = video_prompt.lower() | |
| if any(w in prompt_lower for w in ["action", "fast", "fight", "run", "dynamic"]): | |
| return "fast paced action music, energetic drums, 140 bpm, intense" | |
| if any(w in prompt_lower for w in ["nature", "forest", "ocean", "calm", "peaceful"]): | |
| return "peaceful ambient nature sounds, soft flute, 70 bpm, serene" | |
| if any(w in prompt_lower for w in ["magic", "fantasy", "dream", "ethereal", "sparkle"]): | |
| return "magical fantasy music, harp and strings, 90 bpm, whimsical" | |
| if any(w in prompt_lower for w in ["city", "urban", "night", "street"]): | |
| return "urban electronic music, synthesizer, 120 bpm, modern and cool" | |
| if any(w in prompt_lower for w in ["sad", "emotion", "cry", "nostalgic"]): | |
| return "emotional piano solo, slow tempo, 60 bpm, melancholic and tender" | |
| return "cinematic orchestral music, dramatic strings, 100 bpm, epic and powerful" | |
| def generate_music_for_video( | |
| video_path: str, | |
| video_prompt: str, | |
| music_duration: float, | |
| fade_in: float, | |
| fade_out: float, | |
| volume: float, | |
| progress=gr.Progress(track_tqdm=True), | |
| ) -> tuple: | |
| if video_path is None: | |
| return None, "", "⚠️ No video found — generate a video first." | |
| if not MOVIEPY_AVAILABLE: | |
| return video_path, "", "⚠️ moviepy is not installed." | |
| progress(0.05, desc="🤖 Claude is writing the music description…") | |
| music_prompt = _prompt_to_music_description(video_prompt) | |
| progress(0.15, desc="🎵 Loading MusicGen…") | |
| if not _load_musicgen(): | |
| return video_path, music_prompt, "❌ Failed to load MusicGen." | |
| try: | |
| tokens_per_sec = 50 | |
| max_tokens = max(128, min(1500, int(music_duration * tokens_per_sec))) | |
| progress(0.25, desc=f"🎼 Generating music: «{music_prompt[:50]}»…") | |
| audio_out = _musicgen_pipe( | |
| music_prompt, | |
| forward_params={ | |
| "do_sample": True, | |
| "max_new_tokens": max_tokens, | |
| }, | |
| ) | |
| audio_array = audio_out["audio"][0] | |
| sample_rate = audio_out["sampling_rate"] | |
| if audio_array.ndim == 2: | |
| audio_array = audio_array.mean(axis=0) | |
| audio_array = (audio_array / (np.abs(audio_array).max() + 1e-8) * 32767).astype(np.int16) | |
| with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp_wav: | |
| wav_path = tmp_wav.name | |
| _wavfile.write(wav_path, sample_rate, audio_array) | |
| progress(0.75, desc="🎬 Merging music with video…") | |
| video_clip = VideoFileClip(video_path) | |
| audio_clip = AudioFileClip(wav_path) | |
| audio_clip = audio_clip.subclip(0, min(audio_clip.duration, video_clip.duration)) | |
| if fade_in > 0: audio_clip = audio_clip.audio_fadein(fade_in) | |
| if fade_out > 0: audio_clip = audio_clip.audio_fadeout(fade_out) | |
| audio_clip = audio_clip.volumex(volume) | |
| final_video = video_clip.set_audio(audio_clip) | |
| with tempfile.NamedTemporaryFile(suffix="_musicgen.mp4", delete=False) as tmp_out: | |
| output_path = tmp_out.name | |
| final_video.write_videofile( | |
| output_path, | |
| codec="libx264", | |
| audio_codec="aac", | |
| temp_audiofile=output_path + "_tmp.m4a", | |
| remove_temp=True, | |
| verbose=False, | |
| logger=None, | |
| ) | |
| video_clip.close() | |
| audio_clip.close() | |
| final_video.close() | |
| os.unlink(wav_path) | |
| progress(1.0, desc="✅ Done!") | |
| status = ( | |
| f"✅ **AI music generated and merged successfully!**\n\n" | |
| f"🎼 **Music description:** {music_prompt}\n\n" | |
| f"⏱ Duration: {music_duration:.1f}s | 🔊 Volume: {int(volume*100)}% | " | |
| f"Fade In: {fade_in}s | Fade Out: {fade_out}s" | |
| ) | |
| return output_path, music_prompt, status | |
| except Exception as e: | |
| print(f"[MusicGen] Error: {e}") | |
| import traceback; traceback.print_exc() | |
| return video_path, music_prompt, f"❌ MusicGen error: {str(e)[:120]}" | |
| # ── Constants & Core Settings ───────────────────────────────────────────────── | |
| MODEL_ID = "Wan-AI/Wan2.2-I2V-A14B-Diffusers" | |
| SQUARE_DIM = 480 | |
| MAX_DIM = 832 | |
| MIN_DIM = 480 | |
| MULTIPLE_OF = 16 | |
| FIXED_FPS = 16 | |
| MIN_FRAMES_MODEL = 1 | |
| MAX_FRAMES_MODEL = 128 | |
| MIN_DURATION = 1.0 | |
| MAX_DURATION = 8.0 | |
| MAX_SEED = 2**32 - 1 | |
| HISTORY_FILE = "generation_history.json" | |
| default_prompt_i2v = "make this image come alive with cinematic motion" | |
| default_negative_prompt = ( | |
| "static, blurry, low quality, watermark, text, deformed, ugly" | |
| ) | |
| # ── 🎵 Built-in Music Library ───────────────────────────────────────────────── | |
| MUSIC_LIBRARY = { | |
| "🎬 Cinematic Epic": { | |
| "url": "https://www.soundhelix.com/examples/mp3/SoundHelix-Song-1.mp3", | |
| "mood": "dramatic, powerful, orchestral", | |
| "bpm": 120, | |
| }, | |
| "🌊 Ambient Flow": { | |
| "url": "https://www.soundhelix.com/examples/mp3/SoundHelix-Song-2.mp3", | |
| "mood": "calm, flowing, atmospheric", | |
| "bpm": 80, | |
| }, | |
| "⚡ Action Drive": { | |
| "url": "https://www.soundhelix.com/examples/mp3/SoundHelix-Song-3.mp3", | |
| "mood": "energetic, fast, intense", | |
| "bpm": 140, | |
| }, | |
| "🌿 Nature Serenity": { | |
| "url": "https://www.soundhelix.com/examples/mp3/SoundHelix-Song-4.mp3", | |
| "mood": "peaceful, soft, organic", | |
| "bpm": 70, | |
| }, | |
| "✨ Magical Wonder": { | |
| "url": "https://www.soundhelix.com/examples/mp3/SoundHelix-Song-5.mp3", | |
| "mood": "dreamy, magical, ethereal", | |
| "bpm": 95, | |
| }, | |
| "🎭 Dramatic Tension": { | |
| "url": "https://www.soundhelix.com/examples/mp3/SoundHelix-Song-6.mp3", | |
| "mood": "tense, suspenseful, dark", | |
| "bpm": 110, | |
| }, | |
| "🌅 Sunrise Journey": { | |
| "url": "https://www.soundhelix.com/examples/mp3/SoundHelix-Song-7.mp3", | |
| "mood": "uplifting, hopeful, warm", | |
| "bpm": 100, | |
| }, | |
| "🎸 Indie Vibes": { | |
| "url": "https://www.soundhelix.com/examples/mp3/SoundHelix-Song-8.mp3", | |
| "mood": "cool, modern, casual", | |
| "bpm": 125, | |
| }, | |
| "🎹 Piano Emotion": { | |
| "url": "https://www.soundhelix.com/examples/mp3/SoundHelix-Song-9.mp3", | |
| "mood": "emotional, intimate, classical", | |
| "bpm": 85, | |
| }, | |
| "🌃 Night City": { | |
| "url": "https://www.soundhelix.com/examples/mp3/SoundHelix-Song-10.mp3", | |
| "mood": "urban, electronic, night", | |
| "bpm": 130, | |
| }, | |
| "🚀 Futuristic": { | |
| "url": "https://www.soundhelix.com/examples/mp3/SoundHelix-Song-11.mp3", | |
| "mood": "sci-fi, electronic, futuristic", | |
| "bpm": 135, | |
| }, | |
| "🏔️ Epic Adventure": { | |
| "url": "https://www.soundhelix.com/examples/mp3/SoundHelix-Song-12.mp3", | |
| "mood": "heroic, adventurous, grand", | |
| "bpm": 115, | |
| }, | |
| } | |
| PRESET_TO_MUSIC = { | |
| "🌊 Flowing": "🌊 Ambient Flow", | |
| "🎥 Cinematic": "🎬 Cinematic Epic", | |
| "💨 Dynamic": "⚡ Action Drive", | |
| "🌿 Nature": "🌿 Nature Serenity", | |
| "✨ Magical": "✨ Magical Wonder", | |
| "🏃 Action": "⚡ Action Drive", | |
| "🌅 Timelapse": "🌅 Sunrise Journey", | |
| "🎭 Dramatic": "🎭 Dramatic Tension", | |
| } | |
| MOTION_PRESETS = { | |
| "🌊 Flowing": "gentle flowing motion, smooth camera drift", | |
| "🎥 Cinematic": "cinematic dolly shot, dramatic lighting, film grain", | |
| "💨 Dynamic": "fast dynamic motion, energetic movement", | |
| "🌿 Nature": "gentle breeze, leaves rustling, natural ambient motion", | |
| "✨ Magical": "magical sparkles, ethereal glow, fantastical transformation", | |
| "🏃 Action": "fast paced action, rapid movement, high energy", | |
| "🌅 Timelapse": "time lapse clouds, light changing, atmospheric mood", | |
| "🎭 Dramatic": "dramatic reveal, slow zoom in, tense atmosphere", | |
| } | |
| # ── Colorization support ────────────────────────────────────────────────────── | |
| import cv2, urllib.request as _urlreq | |
| OPENCV_COLOR_AVAILABLE = False | |
| _opencv_net = None | |
| _COLOR_CACHE = os.path.join(os.path.expanduser("~"), ".cache", "cv_colorizer") | |
| _MUSIC_CACHE = os.path.join(os.path.expanduser("~"), ".cache", "music_library") | |
| os.makedirs(_COLOR_CACHE, exist_ok=True) | |
| os.makedirs(_MUSIC_CACHE, exist_ok=True) | |
| _PROTOTXT_CONTENT = """ | |
| name: "colorization" | |
| layer { name: "data_l" type: "Input" top: "data_l" input_param { shape { dim: 1 dim: 1 dim: 224 dim: 224 } } } | |
| layer { name: "conv1_1" type: "Convolution" bottom: "data_l" top: "conv1_1" convolution_param { num_output: 64 kernel_size: 3 stride: 1 pad: 1 } } | |
| layer { name: "relu1_1" type: "ReLU" bottom: "conv1_1" top: "conv1_1" } | |
| layer { name: "conv1_2" type: "Convolution" bottom: "conv1_1" top: "conv1_2" convolution_param { num_output: 64 kernel_size: 3 stride: 1 pad: 1 } } | |
| layer { name: "relu1_2" type: "ReLU" bottom: "conv1_2" top: "conv1_2" } | |
| layer { name: "conv1_2norm" type: "BatchNorm" bottom: "conv1_2" top: "conv1_2norm" } | |
| layer { name: "pool1" type: "Pooling" bottom: "conv1_2norm" top: "pool1" pooling_param { pool: MAX kernel_size: 2 stride: 2 } } | |
| layer { name: "conv2_1" type: "Convolution" bottom: "pool1" top: "conv2_1" convolution_param { num_output: 128 kernel_size: 3 stride: 1 pad: 1 } } | |
| layer { name: "relu2_1" type: "ReLU" bottom: "conv2_1" top: "conv2_1" } | |
| layer { name: "conv2_2" type: "Convolution" bottom: "conv2_1" top: "conv2_2" convolution_param { num_output: 128 kernel_size: 3 stride: 1 pad: 1 } } | |
| layer { name: "relu2_2" type: "ReLU" bottom: "conv2_2" top: "conv2_2" } | |
| layer { name: "conv2_2norm" type: "BatchNorm" bottom: "conv2_2" top: "conv2_2norm" } | |
| layer { name: "pool2" type: "Pooling" bottom: "conv2_2norm" top: "pool2" pooling_param { pool: MAX kernel_size: 2 stride: 2 } } | |
| layer { name: "conv3_1" type: "Convolution" bottom: "pool2" top: "conv3_1" convolution_param { num_output: 256 kernel_size: 3 stride: 1 pad: 1 } } | |
| layer { name: "relu3_1" type: "ReLU" bottom: "conv3_1" top: "conv3_1" } | |
| layer { name: "conv3_2" type: "Convolution" bottom: "conv3_1" top: "conv3_2" convolution_param { num_output: 256 kernel_size: 3 stride: 1 pad: 1 } } | |
| layer { name: "relu3_2" type: "ReLU" bottom: "conv3_2" top: "conv3_2" } | |
| layer { name: "conv3_3" type: "Convolution" bottom: "conv3_2" top: "conv3_3" convolution_param { num_output: 256 kernel_size: 3 stride: 1 pad: 1 } } | |
| layer { name: "relu3_3" type: "ReLU" bottom: "conv3_3" top: "conv3_3" } | |
| layer { name: "conv3_3norm" type: "BatchNorm" bottom: "conv3_3" top: "conv3_3norm" } | |
| layer { name: "pool3" type: "Pooling" bottom: "conv3_3norm" top: "pool3" pooling_param { pool: MAX kernel_size: 2 stride: 2 } } | |
| layer { name: "conv4_1" type: "Convolution" bottom: "pool3" top: "conv4_1" convolution_param { num_output: 512 kernel_size: 3 stride: 1 pad: 1 dilation: 1 } } | |
| layer { name: "relu4_1" type: "ReLU" bottom: "conv4_1" top: "conv4_1" } | |
| layer { name: "conv4_2" type: "Convolution" bottom: "conv4_1" top: "conv4_2" convolution_param { num_output: 512 kernel_size: 3 stride: 1 pad: 1 dilation: 1 } } | |
| layer { name: "relu4_2" type: "ReLU" bottom: "conv4_2" top: "conv4_2" } | |
| layer { name: "conv4_3" type: "Convolution" bottom: "conv4_1" top: "conv4_3" convolution_param { num_output: 512 kernel_size: 3 stride: 1 pad: 1 dilation: 1 } } | |
| layer { name: "relu4_3" type: "ReLU" bottom: "conv4_3" top: "conv4_3" } | |
| layer { name: "conv4_3norm" type: "BatchNorm" bottom: "conv4_3" top: "conv4_3norm" } | |
| layer { name: "conv5_1" type: "Convolution" bottom: "conv4_3norm" top: "conv5_1" convolution_param { num_output: 512 kernel_size: 3 stride: 1 pad: 2 dilation: 2 } } | |
| layer { name: "relu5_1" type: "ReLU" bottom: "conv5_1" top: "conv5_1" } | |
| layer { name: "conv5_2" type: "Convolution" bottom: "conv5_1" top: "conv5_2" convolution_param { num_output: 512 kernel_size: 3 stride: 1 pad: 2 dilation: 2 } } | |
| layer { name: "relu5_2" type: "ReLU" bottom: "conv5_2" top: "conv5_2" } | |
| layer { name: "conv5_3" type: "Convolution" bottom: "conv5_2" top: "conv5_3" convolution_param { num_output: 512 kernel_size: 3 stride: 1 pad: 2 dilation: 2 } } | |
| layer { name: "relu5_3" type: "ReLU" bottom: "conv5_3" top: "conv5_3" } | |
| layer { name: "conv5_3norm" type: "BatchNorm" bottom: "conv5_3" top: "conv5_3norm" } | |
| layer { name: "conv6_1" type: "Convolution" bottom: "conv5_3norm" top: "conv6_1" convolution_param { num_output: 512 kernel_size: 3 stride: 1 pad: 2 dilation: 2 } } | |
| layer { name: "relu6_1" type: "ReLU" bottom: "conv6_1" top: "conv6_1" } | |
| layer { name: "conv6_2" type: "Convolution" bottom: "conv6_1" top: "conv6_2" convolution_param { num_output: 512 kernel_size: 3 stride: 1 pad: 2 dilation: 2 } } | |
| layer { name: "relu6_2" type: "ReLU" bottom: "conv6_2" top: "conv6_2" } | |
| layer { name: "conv6_3" type: "Convolution" bottom: "conv6_2" top: "conv6_3" convolution_param { num_output: 512 kernel_size: 3 stride: 1 pad: 2 dilation: 2 } } | |
| layer { name: "relu6_3" type: "ReLU" bottom: "conv6_3" top: "conv6_3" } | |
| layer { name: "conv6_3norm" type: "BatchNorm" bottom: "conv6_3" top: "conv6_3norm" } | |
| layer { name: "conv7_1" type: "Convolution" bottom: "conv6_3norm" top: "conv7_1" convolution_param { num_output: 256 kernel_size: 3 stride: 1 pad: 1 dilation: 1 } } | |
| layer { name: "relu7_1" type: "ReLU" bottom: "conv7_1" top: "conv7_1" } | |
| layer { name: "conv7_2" type: "Convolution" bottom: "conv7_1" top: "conv7_2" convolution_param { num_output: 256 kernel_size: 3 stride: 1 pad: 1 dilation: 1 } } | |
| layer { name: "relu7_2" type: "ReLU" bottom: "conv7_2" top: "conv7_2" } | |
| layer { name: "conv7_3" type: "Convolution" bottom: "conv7_2" top: "conv7_3" convolution_param { num_output: 256 kernel_size: 3 stride: 1 pad: 1 dilation: 1 } } | |
| layer { name: "relu7_3" type: "ReLU" bottom: "conv7_3" top: "conv7_3" } | |
| layer { name: "conv7_3norm" type: "BatchNorm" bottom: "conv7_3" top: "conv7_3norm" } | |
| layer { name: "conv8_1" type: "Deconvolution" bottom: "conv7_3norm" top: "conv8_1" convolution_param { num_output: 128 kernel_size: 4 stride: 2 pad: 1 } } | |
| layer { name: "relu8_1" type: "ReLU" bottom: "conv8_1" top: "conv8_1" } | |
| layer { name: "conv8_2" type: "Convolution" bottom: "conv8_1" top: "conv8_2" convolution_param { num_output: 128 kernel_size: 3 stride: 1 pad: 1 } } | |
| layer { name: "relu8_2" type: "ReLU" bottom: "conv8_2" top: "conv8_2" } | |
| layer { name: "conv8_3" type: "Convolution" bottom: "conv8_2" top: "conv8_3" convolution_param { num_output: 128 kernel_size: 3 stride: 1 pad: 1 } } | |
| layer { name: "relu8_3" type: "ReLU" bottom: "conv8_3" top: "conv8_3" } | |
| layer { name: "conv8_3norm" type: "BatchNorm" bottom: "conv8_3" top: "conv8_3norm" } | |
| layer { name: "conv8_313" type: "Convolution" bottom: "conv8_3norm" top: "conv8_313" convolution_param { num_output: 313 kernel_size: 1 stride: 1 pad: 0 } } | |
| layer { name: "class8_ab" type: "Convolution" bottom: "conv8_313" top: "class8_313_rh" convolution_param { num_output: 2 kernel_size: 1 bias_term: false } } | |
| layer { name: "conv8_313_rh" type: "Convolution" bottom: "class8_313_rh" top: "class8_ab" convolution_param { num_output: 2 kernel_size: 1 bias_term: false } } | |
| """ | |
| def _ensure_opencv_model() -> bool: | |
| global _opencv_net, OPENCV_COLOR_AVAILABLE | |
| if _opencv_net is not None: | |
| return True | |
| try: | |
| model_path = os.path.join(_COLOR_CACHE, "colorization_release_v2.caffemodel") | |
| pts_path = os.path.join(_COLOR_CACHE, "pts_in_hull.npy") | |
| proto_path = os.path.join(_COLOR_CACHE, "colorization_deploy_v2.prototxt") | |
| if not os.path.exists(proto_path): | |
| with open(proto_path, "w") as f: | |
| f.write(_PROTOTXT_CONTENT) | |
| if not os.path.exists(model_path): | |
| print("[Colorizer] Downloading caffemodel (~130 MB)…") | |
| _urlreq.urlretrieve( | |
| "https://eecs.berkeley.edu/~rich.zhang/projects/2016_colorization/files/demo_v2/colorization_release_v2.caffemodel", | |
| model_path, | |
| ) | |
| if not os.path.exists(pts_path): | |
| _urlreq.urlretrieve( | |
| "https://github.com/richzhang/colorization/raw/caffe/colorization/resources/pts_in_hull.npy", | |
| pts_path, | |
| ) | |
| net = cv2.dnn.readNetFromCaffe(proto_path, model_path) | |
| pts_arr = np.load(pts_path).transpose().reshape(2, 313, 1, 1).astype(np.float32) | |
| net.getLayer(net.getLayerId("class8_ab")).blobs = [pts_arr] | |
| net.getLayer(net.getLayerId("conv8_313_rh")).blobs = [ | |
| np.full([1, 313], 2.606, dtype=np.float32) | |
| ] | |
| _opencv_net = net | |
| OPENCV_COLOR_AVAILABLE = True | |
| return True | |
| except Exception as exc: | |
| print(f"[Colorizer] OpenCV DNN init failed: {exc}") | |
| return False | |
| def _opencv_colorize(img_pil: Image.Image) -> Image.Image: | |
| img_rgb = np.array(img_pil.convert("RGB")).astype(np.float32) / 255.0 | |
| img_lab = cv2.cvtColor(img_rgb, cv2.COLOR_RGB2Lab) | |
| l_chan = img_lab[:, :, 0] | |
| l_rs = cv2.resize(l_chan, (224, 224)) - 50.0 | |
| _opencv_net.setInput(cv2.dnn.blobFromImage(l_rs)) | |
| ab_dec = _opencv_net.forward()[0, :, :, :].transpose((1, 2, 0)) | |
| h, w = img_rgb.shape[:2] | |
| ab_up = cv2.resize(ab_dec, (w, h)) | |
| lab_out = np.concatenate([l_chan[:, :, np.newaxis], ab_up], axis=2) | |
| rgb_out = np.clip(cv2.cvtColor(lab_out.astype(np.float32), cv2.COLOR_Lab2RGB), 0, 1) | |
| return Image.fromarray((rgb_out * 255).astype(np.uint8)) | |
| import base64, io as _io, json as _json | |
| def _pil_to_b64(img: Image.Image, max_px: int = 512) -> str: | |
| img = img.convert("RGB") | |
| w, h = img.size | |
| if max(w, h) > max_px: | |
| ratio = max_px / max(w, h) | |
| img = img.resize((max(1, int(w*ratio)), max(1, int(h*ratio))), Image.LANCZOS) | |
| buf = _io.BytesIO() | |
| img.save(buf, format="JPEG", quality=82) | |
| return base64.b64encode(buf.getvalue()).decode() | |
| def _apply_palette(img_pil: Image.Image, palette: dict) -> Image.Image: | |
| img_rgb = np.array(img_pil.convert("RGB")) | |
| gray = cv2.cvtColor(img_rgb, cv2.COLOR_RGB2GRAY) | |
| h, w = gray.shape | |
| gf = gray.astype(np.float32) / 255.0 | |
| def gblur(m, k): | |
| k = max(3, int(k) | 1) | |
| return cv2.GaussianBlur(m.astype(np.float32), (k, k), 0) | |
| gray3 = cv2.cvtColor(gray, cv2.COLOR_GRAY2RGB) | |
| lab = cv2.cvtColor(gray3.astype(np.float32) / 255.0, cv2.COLOR_RGB2Lab) | |
| L = lab[:, :, 0].copy() | |
| face_mask = np.zeros((h, w), np.float32) | |
| try: | |
| det = cv2.CascadeClassifier(cv2.data.haarcascades + "haarcascade_frontalface_default.xml") | |
| faces = det.detectMultiScale(gray, scaleFactor=1.1, minNeighbors=4, minSize=(50, 50)) | |
| if len(faces) > 0: | |
| for (fx, fy, fw, fh) in faces: | |
| pad = int(min(fw, fh) * 0.18) | |
| face_mask[max(0,fy-pad):min(h,fy+fh+pad), max(0,fx-pad):min(w,fx+fw+pad)] = 1.0 | |
| face_mask = gblur(face_mask, min(w, h) // 7) | |
| else: | |
| cy, cx = h//2, w//2 | |
| Y, X = np.ogrid[:h, :w] | |
| ell = np.clip(1.0 - np.sqrt(((X-cx)/(w*0.38))**2 + ((Y-cy)/(h*0.48))**2), 0, 1) | |
| face_mask = gblur(ell**2, min(w,h)//5) | |
| except Exception: | |
| pass | |
| no_face = np.clip(1.0 - face_mask * 1.4, 0, 1) | |
| blur2 = gblur(gf, 5) | |
| texture = np.abs(gf - blur2) | |
| skin_raw = gblur(((gf > 0.22) & (gf < 0.92)).astype(np.float32) | |
| * np.clip(1.0 - texture*10.0, 0, 1), 23) | |
| skin_mask = gblur(np.clip(skin_raw * (face_mask*0.75 + 0.25), 0, 1), 29) | |
| hair_raw = gblur(np.clip((gf < 0.33).astype(np.float32) | |
| + (gf > 0.76).astype(np.float32)*0.55, 0, 1) * no_face, 17) | |
| dark_r = gblur((gf < 0.45).astype(np.float32), 9) | |
| lap = cv2.Laplacian(gray, cv2.CV_32F) | |
| bg_mask = gblur(np.clip(1.0 - np.abs(lap)/10.0, 0, 1) * no_face, 39) * 0.55 | |
| lz = np.zeros((h, w), np.float32) | |
| lz[int(h*0.60):int(h*0.76), int(w*0.30):int(w*0.70)] = 1.0 | |
| lip_mask = gblur(((gf > 0.26) & (gf < 0.70)).astype(np.float32) * lz * face_mask, 9) * 0.45 | |
| p = palette | |
| A_off = (bg_mask * p["bg_a"] | |
| + hair_raw * (dark_r * p["hd_a"] + (1-dark_r) * p["hl_a"]) | |
| + skin_mask * p["sk_a"] | |
| + lip_mask * p["lp_a"]) | |
| B_off = (bg_mask * p["bg_b"] | |
| + hair_raw * (dark_r * p["hd_b"] + (1-dark_r) * p["hl_b"]) | |
| + skin_mask * p["sk_b"] | |
| + lip_mask * p["lp_b"]) | |
| lab_out = np.stack([L, np.clip(A_off,-50,50), np.clip(B_off,-50,50)], axis=2).astype(np.float32) | |
| rgb_out = np.clip(cv2.cvtColor(lab_out, cv2.COLOR_Lab2RGB), 0, 1) | |
| hsv = cv2.cvtColor(rgb_out, cv2.COLOR_RGB2HSV) | |
| hsv[:,:,1] = np.clip(hsv[:,:,1] * 1.18, 0, 1) | |
| return Image.fromarray((np.clip(cv2.cvtColor(hsv, cv2.COLOR_HSV2RGB),0,1)*255).astype(np.uint8)) | |
| def _smart_colorize(img_pil: Image.Image, style: str = "natural") -> Image.Image: | |
| STYLES = { | |
| "natural": ( 6.0, 13.0, -1.0, -2.0, 3.0, 5.0, 4.0, 15.0, 11.0, 7.0), | |
| "cinematic": ( 5.0, 15.0, -3.0, -9.0, 2.0, 4.0, 4.0, 18.0, 13.0, 5.0), | |
| "warm": ( 8.0, 17.0, 2.0, 5.0, 4.0, 9.0, 6.0, 20.0, 14.0, 9.0), | |
| } | |
| sk_a,sk_b, bg_a,bg_b, hd_a,hd_b, hl_a,hl_b, lp_a,lp_b = STYLES.get(style, STYLES["natural"]) | |
| return _apply_palette(img_pil, dict(sk_a=sk_a,sk_b=sk_b,bg_a=bg_a,bg_b=bg_b, | |
| hd_a=hd_a,hd_b=hd_b,hl_a=hl_a,hl_b=hl_b, | |
| lp_a=lp_a,lp_b=lp_b)) | |
| def _claude_colorize(img_pil: Image.Image, style: str = "natural") -> tuple: | |
| style_desc = { | |
| "natural": "realistic true-to-life natural colors", | |
| "cinematic": "cinematic film look, warm skin, teal shadows, golden highlights", | |
| "warm": "warm golden-hour, amber tones, sun-kissed skin", | |
| }.get(style, "realistic natural colors") | |
| prompt = ( | |
| "You are a professional photo colorization AI.\n" | |
| "Analyze this black-and-white image carefully and return ONLY a valid JSON " | |
| "object with LAB color offset values for each region.\n" | |
| "Style requested: " + style_desc + "\n\n" | |
| "Return this exact JSON (numbers only, no comments, no markdown):\n" | |
| "{\n" | |
| ' "skin_a": <float -20 to 20>,\n' | |
| ' "skin_b": <float -5 to 25>,\n' | |
| ' "hair_a": <float -8 to 10>,\n' | |
| ' "hair_b": <float -5 to 20>,\n' | |
| ' "bg_a": <float -15 to 10>,\n' | |
| ' "bg_b": <float -20 to 15>,\n' | |
| ' "lip_a": <float 5 to 22>,\n' | |
| ' "lip_b": <float 0 to 15>,\n' | |
| ' "description": "<one sentence>"\n' | |
| "}\n\n" | |
| "LAB reference: A negative=green, A positive=red. B negative=blue, B positive=yellow.\n" | |
| "Return ONLY the JSON object." | |
| ) | |
| try: | |
| client = anthropic.Anthropic() | |
| b64 = _pil_to_b64(img_pil, max_px=480) | |
| msg = client.messages.create( | |
| model = "claude-opus-4-6", | |
| max_tokens = 300, | |
| messages = [{ | |
| "role": "user", | |
| "content": [ | |
| {"type": "image", "source": { | |
| "type": "base64", "media_type": "image/jpeg", "data": b64}}, | |
| {"type": "text", "text": prompt}, | |
| ], | |
| }], | |
| ) | |
| raw = msg.content[0].text.strip().replace("```json","").replace("```","").strip() | |
| vals = _json.loads(raw) | |
| desc = vals.pop("description", "") | |
| def gv(k, d): return float(vals.get(k, d)) | |
| palette = dict( | |
| sk_a=gv("skin_a",7), sk_b=gv("skin_b",14), | |
| bg_a=gv("bg_a",-2), bg_b=gv("bg_b",-4), | |
| hd_a=gv("hair_a",3), hd_b=gv("hair_b",6), | |
| hl_a=gv("hair_a",4), hl_b=gv("hair_b",15), | |
| lp_a=gv("lip_a",12), lp_b=gv("lip_b",8), | |
| ) | |
| result = _apply_palette(img_pil, palette) | |
| note = (" — " + desc) if desc else "" | |
| return result, "🤖 **Claude AI Colorized** — " + style_desc + note | |
| except Exception as exc: | |
| print("[Claude colorize] error:", exc) | |
| return None, str(exc) | |
| def _numpy_colorize(img_pil: Image.Image) -> Image.Image: | |
| return _smart_colorize(img_pil, style="natural") | |
| def colorize_image(input_img, style="natural", render_factor=35): | |
| if input_img is None: | |
| return None, "⚠️ Please upload an image first." | |
| if ANTHROPIC_AVAILABLE: | |
| result, status = _claude_colorize(input_img, style) | |
| if result is not None: | |
| return result, status | |
| if _ensure_opencv_model(): | |
| try: | |
| result = _opencv_colorize(input_img) | |
| return result, "✅ Colorized with **OpenCV DNN** — Zhang et al. (ECCV 2016)" | |
| except Exception as exc: | |
| print("[Colorizer] OpenCV DNN failed:", exc) | |
| result = _smart_colorize(input_img, style) | |
| labels = {"natural": "🌿 Natural", "cinematic": "🎬 Cinematic", "warm": "🌅 Warm"} | |
| return result, "✅ Colorized with **Smart Semantic Engine** — " + labels.get(style, style) | |
| # ── 🎵 Music Functions ───────────────────────────────────────────────────────── | |
| def _get_music_cache_path(track_name: str) -> str: | |
| safe_name = ( | |
| track_name | |
| .replace(" ", "_").replace("/", "_") | |
| .replace("🎬","").replace("🌊","").replace("⚡","").replace("🌿","") | |
| .replace("✨","").replace("🎭","").replace("🌅","").replace("🎸","") | |
| .replace("🎹","").replace("🌃","").replace("🚀","").replace("🏔️","") | |
| .strip() | |
| ) | |
| return os.path.join(_MUSIC_CACHE, f"{safe_name}.mp3") | |
| def _download_music(track_name: str) -> str | None: | |
| if track_name not in MUSIC_LIBRARY: | |
| return None | |
| cache_path = _get_music_cache_path(track_name) | |
| if os.path.exists(cache_path) and os.path.getsize(cache_path) > 10_000: | |
| return cache_path | |
| try: | |
| url = MUSIC_LIBRARY[track_name]["url"] | |
| print(f"[Music] Downloading: {track_name}") | |
| _urlreq.urlretrieve(url, cache_path) | |
| return cache_path | |
| except Exception as e: | |
| print(f"[Music] Download failed (network may be disabled): {e}") | |
| if os.path.exists(cache_path): | |
| os.unlink(cache_path) | |
| return None | |
| def add_music_to_video( | |
| video_path: str, | |
| music_source: str, | |
| selected_track: str, | |
| custom_audio_path: str, | |
| volume: float, | |
| fade_in: float, | |
| fade_out: float, | |
| loop_music: bool, | |
| ) -> tuple: | |
| if not MOVIEPY_AVAILABLE: | |
| return video_path, "⚠️ moviepy is not installed — video has no music." | |
| if music_source == "none" or video_path is None: | |
| return video_path, "🎬 Video without music." | |
| audio_file = None | |
| if music_source == "library": | |
| if not selected_track or selected_track not in MUSIC_LIBRARY: | |
| return video_path, "⚠️ Please select a track from the library first." | |
| audio_file = _download_music(selected_track) | |
| if audio_file is None: | |
| return video_path, ( | |
| "❌ Failed to download music.\n" | |
| "💡 **Solution:** Upload an audio file directly instead of using the library, " | |
| "or use the **🤖 MusicGen AI** tab to generate music without internet." | |
| ) | |
| elif music_source == "upload": | |
| if not custom_audio_path: | |
| return video_path, "⚠️ Please upload an audio file first." | |
| audio_file = custom_audio_path | |
| try: | |
| video_clip = VideoFileClip(video_path) | |
| video_dur = video_clip.duration | |
| audio_clip = AudioFileClip(audio_file) | |
| if loop_music and audio_clip.duration < video_dur: | |
| from moviepy.editor import concatenate_audioclips | |
| n_loops = int(np.ceil(video_dur / audio_clip.duration)) | |
| audio_clip = concatenate_audioclips([audio_clip] * n_loops) | |
| audio_clip = audio_clip.subclip(0, min(audio_clip.duration, video_dur)) | |
| if fade_in > 0: audio_clip = audio_clip.audio_fadein(fade_in) | |
| if fade_out > 0: audio_clip = audio_clip.audio_fadeout(fade_out) | |
| audio_clip = audio_clip.volumex(volume) | |
| final_video = video_clip.set_audio(audio_clip) | |
| with tempfile.NamedTemporaryFile(suffix="_music.mp4", delete=False) as tmp: | |
| output_path = tmp.name | |
| final_video.write_videofile( | |
| output_path, | |
| codec="libx264", | |
| audio_codec="aac", | |
| temp_audiofile=output_path + "_temp_audio.m4a", | |
| remove_temp=True, | |
| verbose=False, | |
| logger=None, | |
| ) | |
| video_clip.close() | |
| audio_clip.close() | |
| final_video.close() | |
| track_info = "" | |
| if music_source == "library" and selected_track in MUSIC_LIBRARY: | |
| mood = MUSIC_LIBRARY[selected_track]["mood"] | |
| track_info = f" | {selected_track} ({mood})" | |
| return output_path, ( | |
| f"✅ **Music added successfully!**{track_info}\n" | |
| f"🔊 Volume: {int(volume*100)}% | Fade In: {fade_in}s | Fade Out: {fade_out}s" | |
| ) | |
| except Exception as e: | |
| print(f"[Music] Error mixing audio: {e}") | |
| return video_path, f"❌ Error merging audio: {str(e)[:100]}" | |
| def suggest_music_for_preset(preset_name: str) -> str: | |
| return PRESET_TO_MUSIC.get(preset_name, "🎬 Cinematic Epic") | |
| def suggest_music_with_claude(prompt: str) -> tuple: | |
| if not ANTHROPIC_AVAILABLE: | |
| return "🎬 Cinematic Epic", "💡 Default selection (Claude not available)" | |
| track_list = "\n".join([ | |
| f"- {name}: {info['mood']} (BPM: {info['bpm']})" | |
| for name, info in MUSIC_LIBRARY.items() | |
| ]) | |
| try: | |
| client = anthropic.Anthropic() | |
| msg = client.messages.create( | |
| model="claude-opus-4-6", | |
| max_tokens=100, | |
| messages=[{ | |
| "role": "user", | |
| "content": ( | |
| "You are a music supervisor for video content.\n" | |
| f"Based on this video prompt: '{prompt}'\n\n" | |
| f"Choose the BEST matching music track from this list:\n{track_list}\n\n" | |
| "Reply with ONLY the exact track name (including emoji), nothing else." | |
| ), | |
| }], | |
| ) | |
| suggested = msg.content[0].text.strip() | |
| if suggested in MUSIC_LIBRARY: | |
| mood = MUSIC_LIBRARY[suggested]["mood"] | |
| return suggested, f"🤖 **Claude selected:** {suggested} — {mood}" | |
| return "🎬 Cinematic Epic", "💡 Default selection" | |
| except Exception as e: | |
| return "🎬 Cinematic Epic", f"💡 Default selection ({str(e)[:50]})" | |
| # ── Load pipeline — memory-efficient sequential loading ────────────────────── | |
| _BF16_REPO = 'cbensimon/Wan2.2-I2V-A14B-bf16-Diffusers' | |
| def _gc(): | |
| gc.collect() | |
| torch.cuda.empty_cache() | |
| torch.cuda.synchronize() | |
| print("[Load] Step 1/5 — loading transformer …") | |
| _transformer = WanTransformer3DModel.from_pretrained( | |
| _BF16_REPO, | |
| subfolder='transformer', | |
| torch_dtype=torch.bfloat16, | |
| device_map='cuda', | |
| low_cpu_mem_usage=True, | |
| ) | |
| _gc() | |
| print("[Load] Step 2/5 — loading transformer_2 …") | |
| _transformer_2 = WanTransformer3DModel.from_pretrained( | |
| _BF16_REPO, | |
| subfolder='transformer_2', | |
| torch_dtype=torch.bfloat16, | |
| device_map='cuda', | |
| low_cpu_mem_usage=True, | |
| ) | |
| _gc() | |
| print("[Load] Step 3/5 — loading full pipeline (text-encoder + VAE on CPU) …") | |
| pipe = WanImageToVideoPipeline.from_pretrained( | |
| MODEL_ID, | |
| transformer=_transformer, | |
| transformer_2=_transformer_2, | |
| torch_dtype=torch.bfloat16, | |
| low_cpu_mem_usage=True, | |
| ) | |
| del _transformer, _transformer_2 | |
| _gc() | |
| pipe.text_encoder.to('cpu') | |
| pipe.vae.to('cuda') | |
| pipe.transformer.to('cuda') | |
| pipe.transformer_2.to('cuda') | |
| # ── ARCHITECTURE FIX: Split CPU/GPU Routing Patch ───────────────────────────── | |
| WanImageToVideoPipeline._execution_device = property(lambda self: torch.device("cuda")) | |
| WanImageToVideoPipeline.device = property(lambda self: torch.device("cuda")) | |
| orig_te_forward = pipe.text_encoder.forward | |
| def patched_te_forward(*args, **kwargs): | |
| new_args = tuple(a.to("cpu") if isinstance(a, torch.Tensor) else a for a in args) | |
| new_kwargs = {k: v.to("cpu") if isinstance(v, torch.Tensor) else v for k, v in (kwargs or {}).items()} | |
| return orig_te_forward(*new_args, **new_kwargs) | |
| pipe.text_encoder.forward = patched_te_forward | |
| # ───────────────────────────────────────────────────────────────────────────── | |
| _gc() | |
| print("[Load] Step 4/5 — fusing LoRA weights …") | |
| pipe.load_lora_weights( | |
| "Kijai/WanVideo_comfy", | |
| weight_name="Lightx2v/lightx2v_I2V_14B_480p_cfg_step_distill_rank128_bf16.safetensors", | |
| adapter_name="lightx2v", | |
| ) | |
| pipe.load_lora_weights( | |
| "Kijai/WanVideo_comfy", | |
| weight_name="Lightx2v/lightx2v_I2V_14B_480p_cfg_step_distill_rank128_bf16.safetensors", | |
| adapter_name="lightx2v_2", | |
| load_into_transformer_2=True, | |
| ) | |
| pipe.set_adapters(["lightx2v", "lightx2v_2"], adapter_weights=[1.0, 1.0]) | |
| pipe.fuse_lora(adapter_names=["lightx2v"], lora_scale=3.0, components=["transformer"]) | |
| pipe.fuse_lora(adapter_names=["lightx2v_2"], lora_scale=1.0, components=["transformer_2"]) | |
| pipe.unload_lora_weights() | |
| _gc() | |
| print("[Load] Step 5/5 — quantising components …") | |
| quantize_(pipe.text_encoder, Int8WeightOnlyConfig()) | |
| _gc() | |
| quantize_(pipe.transformer, Float8DynamicActivationFloat8WeightConfig()) | |
| _gc() | |
| quantize_(pipe.transformer_2, Float8DynamicActivationFloat8WeightConfig()) | |
| _gc() | |
| aoti.aoti_blocks_load(pipe.transformer, 'zerogpu-aoti/Wan2', variant='fp8da') | |
| aoti.aoti_blocks_load(pipe.transformer_2, 'zerogpu-aoti/Wan2', variant='fp8da') | |
| _gc() | |
| print("[Load] ✅ Pipeline ready") | |
| # ── Helpers ──────────────────────────────────────────────────────────────────── | |
| def resize_image(image: Image.Image) -> Image.Image: | |
| width, height = image.size | |
| if width == height: | |
| return image.resize((SQUARE_DIM, SQUARE_DIM), Image.LANCZOS) | |
| aspect_ratio = width / height | |
| MAX_ASPECT_RATIO = MAX_DIM / MIN_DIM | |
| MIN_ASPECT_RATIO = MIN_DIM / MAX_DIM | |
| image_to_resize = image | |
| if aspect_ratio > MAX_ASPECT_RATIO: | |
| target_w, target_h = MAX_DIM, MIN_DIM | |
| crop_width = int(round(height * MAX_ASPECT_RATIO)) | |
| left = (width - crop_width) // 2 | |
| image_to_resize = image.crop((left, 0, left + crop_width, height)) | |
| elif aspect_ratio < MIN_ASPECT_RATIO: | |
| target_w, target_h = MIN_DIM, MAX_DIM | |
| crop_height = int(round(width / MIN_ASPECT_RATIO)) | |
| top = (height - crop_height) // 2 | |
| image_to_resize = image.crop((0, top, width, top + crop_height)) | |
| else: | |
| if width > height: | |
| target_w = MAX_DIM | |
| target_h = int(round(target_w / aspect_ratio)) | |
| else: | |
| target_h = MAX_DIM | |
| target_w = int(round(target_h * aspect_ratio)) | |
| final_w = max(MIN_DIM, min(MAX_DIM, round(target_w / MULTIPLE_OF) * MULTIPLE_OF)) | |
| final_h = max(MIN_DIM, min(MAX_DIM, round(target_h / MULTIPLE_OF) * MULTIPLE_OF)) | |
| return image_to_resize.resize((final_w, final_h), Image.LANCZOS) | |
| def get_num_frames(duration_seconds: float) -> int: | |
| raw = int(np.clip( | |
| int(round(duration_seconds * FIXED_FPS)), | |
| MIN_FRAMES_MODEL, | |
| MAX_FRAMES_MODEL, | |
| )) | |
| total = 1 + raw | |
| remainder = (total - 1) % 4 | |
| if remainder != 0: | |
| total += (4 - remainder) | |
| if total > MAX_FRAMES_MODEL: | |
| total -= 4 | |
| return total | |
| def get_duration(input_image, prompt, steps, negative_prompt, | |
| duration_seconds, guidance_scale, guidance_scale_2, | |
| seed, randomize_seed, progress): | |
| if input_image is None: | |
| return 60 | |
| BASE_FRAMES_HEIGHT_WIDTH = 81 * 832 * 624 | |
| BASE_STEP_DURATION = 15 | |
| if isinstance(input_image, str): | |
| input_image = Image.open(input_image).convert("RGB") | |
| elif isinstance(input_image, __import__("numpy").ndarray): | |
| input_image = Image.fromarray(input_image).convert("RGB") | |
| width, height = resize_image(input_image).size | |
| frames = get_num_frames(duration_seconds) | |
| factor = frames * width * height / BASE_FRAMES_HEIGHT_WIDTH | |
| step_duration = BASE_STEP_DURATION * factor ** 1.5 | |
| return 10 + int(steps) * step_duration | |
| def save_to_history(prompt: str, seed: int, duration: float): | |
| history = [] | |
| if os.path.exists(HISTORY_FILE): | |
| try: | |
| with open(HISTORY_FILE, 'r') as f: | |
| history = json.load(f) | |
| except Exception: | |
| history = [] | |
| history.append({ | |
| "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M"), | |
| "prompt": prompt[:120], | |
| "seed": seed, | |
| "duration": duration, | |
| }) | |
| with open(HISTORY_FILE, 'w') as f: | |
| json.dump(history[-100:], f, indent=2) | |
| def load_history_display(): | |
| if not os.path.exists(HISTORY_FILE): | |
| return "No generations yet." | |
| try: | |
| with open(HISTORY_FILE, 'r') as f: | |
| history = json.load(f) | |
| rows = [] | |
| for h in reversed(history[-20:]): | |
| rows.append( | |
| f"**{h['timestamp']}** | 🌱 {h['seed']} | ⏱ {h['duration']}s\n" | |
| f"> {h['prompt']}\n" | |
| ) | |
| return "\n---\n".join(rows) if rows else "No generations yet." | |
| except Exception: | |
| return "Could not load history." | |
| def apply_preset(preset_name: str) -> tuple: | |
| prompt = default_prompt_i2v | |
| suggested_music = "🎬 Cinematic Epic" | |
| if preset_name and preset_name in MOTION_PRESETS: | |
| prompt = f"make this image come alive, {MOTION_PRESETS[preset_name]}" | |
| suggested_music = PRESET_TO_MUSIC.get(preset_name, "🎬 Cinematic Epic") | |
| return prompt, suggested_music | |
| # ── Core generation ──────────────────────────────────────────────────────────── | |
| def generate_video( | |
| input_image, | |
| prompt, | |
| steps = 6, | |
| negative_prompt = default_negative_prompt, | |
| duration_seconds = 3.5, | |
| guidance_scale = 1.0, | |
| guidance_scale_2 = 1.0, | |
| seed = 42, | |
| randomize_seed = True, | |
| progress = gr.Progress(track_tqdm=True), | |
| ): | |
| if input_image is None: | |
| raise gr.Error("⚠️ Please upload an input image first!") | |
| if isinstance(input_image, str): | |
| input_image = Image.open(input_image).convert("RGB") | |
| elif isinstance(input_image, __import__("numpy").ndarray): | |
| input_image = Image.fromarray(input_image).convert("RGB") | |
| final_prompt = prompt | |
| num_frames = get_num_frames(duration_seconds) | |
| current_seed = random.randint(0, MAX_SEED) if randomize_seed else int(seed) | |
| resized = resize_image(input_image) | |
| torch.cuda.empty_cache() | |
| gc.collect() | |
| progress(0, desc="🎬 Generating video…") | |
| try: | |
| output_frames = pipe( | |
| image = resized, | |
| prompt = final_prompt, | |
| negative_prompt = negative_prompt, | |
| height = resized.height, | |
| width = resized.width, | |
| num_frames = num_frames, | |
| guidance_scale = float(guidance_scale), | |
| guidance_scale_2 = float(guidance_scale_2), | |
| num_inference_steps = int(steps), | |
| generator = torch.Generator(device="cuda").manual_seed(current_seed), | |
| ).frames[0] | |
| except torch.cuda.OutOfMemoryError: | |
| torch.cuda.empty_cache() | |
| gc.collect() | |
| raise gr.Error("🔴 GPU out of memory. Try reducing duration or resolution and retry.") | |
| with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as tmp: | |
| video_path = tmp.name | |
| export_to_video(output_frames, video_path, fps=FIXED_FPS) | |
| save_to_history(final_prompt, current_seed, duration_seconds) | |
| torch.cuda.empty_cache() | |
| gc.collect() | |
| return video_path, current_seed | |
| # ── CSS ──────────────────────────────────────────────────────────────────────── | |
| custom_css = """ | |
| @import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700&display=swap'); | |
| body, .gradio-container { | |
| font-family: 'Inter', sans-serif !important; | |
| background: #0f0f13 !important; | |
| color: #e2e2e8 !important; | |
| } | |
| .gradio-container .contain { | |
| max-width: 1280px !important; | |
| margin: 0 auto !important; | |
| padding: 0 16px !important; | |
| } | |
| .hero-header { | |
| text-align: center; | |
| padding: 32px 16px 16px; | |
| background: linear-gradient(135deg, #1a1a2e 0%, #16213e 50%, #0f3460 100%); | |
| border-radius: 16px; | |
| margin-bottom: 20px; | |
| border: 1px solid rgba(99,102,241,0.3); | |
| } | |
| .hero-header h1 { | |
| font-size: 2.2em !important; | |
| font-weight: 700 !important; | |
| background: linear-gradient(135deg, #818cf8, #c084fc, #f472b6) !important; | |
| -webkit-background-clip: text !important; | |
| -webkit-text-fill-color: transparent !important; | |
| margin-bottom: 8px !important; | |
| } | |
| .hero-header p { color: #94a3b8 !important; font-size: 1em !important; } | |
| .badge { | |
| display: inline-block; | |
| background: rgba(99,102,241,0.2); | |
| border: 1px solid rgba(99,102,241,0.5); | |
| border-radius: 20px; | |
| padding: 3px 12px; | |
| font-size: 0.78em; | |
| color: #818cf8; | |
| margin: 4px 3px; | |
| } | |
| .badge-music { | |
| display: inline-block; | |
| background: rgba(234,179,8,0.15); | |
| border: 1px solid rgba(234,179,8,0.4); | |
| border-radius: 20px; | |
| padding: 3px 12px; | |
| font-size: 0.78em; | |
| color: #fbbf24; | |
| margin: 4px 3px; | |
| } | |
| .tab-nav button { | |
| background: #1e1e2e !important; | |
| color: #94a3b8 !important; | |
| border: 1px solid #2d2d3f !important; | |
| border-radius: 8px 8px 0 0 !important; | |
| font-weight: 500 !important; | |
| font-size: 0.9em !important; | |
| padding: 10px 18px !important; | |
| margin-right: 4px !important; | |
| transition: all 0.2s !important; | |
| } | |
| .tab-nav button.selected { | |
| background: linear-gradient(135deg, #4f46e5, #7c3aed) !important; | |
| color: #ffffff !important; | |
| border-color: transparent !important; | |
| } | |
| .panel-card { | |
| background: #1a1a2e !important; | |
| border: 1px solid #2d2d3f !important; | |
| border-radius: 12px !important; | |
| padding: 16px !important; | |
| } | |
| .music-panel { | |
| background: linear-gradient(135deg, rgba(234,179,8,0.06), rgba(251,146,60,0.06)); | |
| border: 1px solid rgba(234,179,8,0.25); | |
| border-radius: 12px; | |
| padding: 16px; | |
| margin-top: 8px; | |
| } | |
| .music-info { | |
| background: linear-gradient(135deg, rgba(234,179,8,0.08), rgba(251,146,60,0.08)); | |
| border: 1px solid rgba(234,179,8,0.3); | |
| border-radius: 10px; | |
| padding: 12px 16px; | |
| font-size: 0.85em; | |
| color: #fde68a; | |
| margin-bottom: 12px; | |
| } | |
| .music-info b { color: #fbbf24; } | |
| .colorize-info { | |
| background: linear-gradient(135deg, rgba(16,185,129,0.08), rgba(6,182,212,0.08)); | |
| border: 1px solid rgba(16,185,129,0.25); | |
| border-radius: 10px; | |
| padding: 12px 16px; | |
| font-size: 0.85em; | |
| color: #6ee7b7; | |
| margin-bottom: 12px; | |
| } | |
| .colorize-info b { color: #34d399; } | |
| textarea, input[type="text"], input[type="number"] { | |
| background: #0f0f18 !important; | |
| border: 1px solid #3d3d5c !important; | |
| border-radius: 8px !important; | |
| color: #e2e2e8 !important; | |
| font-family: 'Inter', sans-serif !important; | |
| } | |
| textarea:focus, input:focus { | |
| border-color: #6366f1 !important; | |
| box-shadow: 0 0 0 3px rgba(99,102,241,0.15) !important; | |
| } | |
| .gr-slider input[type=range] { accent-color: #6366f1 !important; } | |
| .btn-generate { | |
| background: linear-gradient(135deg, #4f46e5 0%, #7c3aed 50%, #a21caf 100%) !important; | |
| border: none !important; | |
| border-radius: 10px !important; | |
| color: #ffffff !important; | |
| font-size: 1.05em !important; | |
| font-weight: 600 !important; | |
| padding: 14px 28px !important; | |
| width: 100% !important; | |
| cursor: pointer !important; | |
| transition: opacity 0.2s !important; | |
| box-shadow: 0 4px 20px rgba(99,102,241,0.4) !important; | |
| } | |
| .btn-generate:hover { opacity: 0.88 !important; } | |
| .btn-music { | |
| background: linear-gradient(135deg, #d97706 0%, #ea580c 100%) !important; | |
| border: none !important; | |
| border-radius: 10px !important; | |
| color: #ffffff !important; | |
| font-size: 1.05em !important; | |
| font-weight: 600 !important; | |
| padding: 14px 28px !important; | |
| width: 100% !important; | |
| cursor: pointer !important; | |
| transition: opacity 0.2s !important; | |
| box-shadow: 0 4px 20px rgba(217,119,6,0.4) !important; | |
| } | |
| .btn-music:hover { opacity: 0.88 !important; } | |
| .btn-colorize { | |
| background: linear-gradient(135deg, #059669 0%, #0891b2 100%) !important; | |
| border: none !important; | |
| border-radius: 10px !important; | |
| color: #ffffff !important; | |
| font-size: 1.05em !important; | |
| font-weight: 600 !important; | |
| padding: 14px 28px !important; | |
| width: 100% !important; | |
| cursor: pointer !important; | |
| transition: opacity 0.2s !important; | |
| box-shadow: 0 4px 20px rgba(16,185,129,0.35) !important; | |
| } | |
| .btn-colorize:hover { opacity: 0.88 !important; } | |
| .btn-send-to-generate { | |
| background: rgba(99,102,241,0.15) !important; | |
| border: 1px solid rgba(99,102,241,0.5) !important; | |
| border-radius: 8px !important; | |
| color: #a5b4fc !important; | |
| font-size: 0.9em !important; | |
| font-weight: 500 !important; | |
| padding: 9px 18px !important; | |
| cursor: pointer !important; | |
| transition: all 0.2s !important; | |
| width: 100% !important; | |
| } | |
| .btn-send-to-generate:hover { | |
| background: rgba(99,102,241,0.28) !important; | |
| color: #ffffff !important; | |
| } | |
| .btn-ai-music { | |
| background: rgba(234,179,8,0.15) !important; | |
| border: 1px solid rgba(234,179,8,0.5) !important; | |
| border-radius: 8px !important; | |
| color: #fbbf24 !important; | |
| font-size: 0.88em !important; | |
| font-weight: 500 !important; | |
| padding: 8px 16px !important; | |
| cursor: pointer !important; | |
| transition: all 0.2s !important; | |
| width: 100% !important; | |
| } | |
| .btn-ai-music:hover { background: rgba(234,179,8,0.25) !important; } | |
| .musicgen-panel { | |
| background: linear-gradient(135deg, rgba(139,92,246,0.08), rgba(236,72,153,0.06)); | |
| border: 1px solid rgba(139,92,246,0.3); | |
| border-radius: 12px; | |
| padding: 16px; | |
| margin-top: 8px; | |
| } | |
| .musicgen-info { | |
| background: linear-gradient(135deg, rgba(139,92,246,0.1), rgba(236,72,153,0.08)); | |
| border: 1px solid rgba(139,92,246,0.35); | |
| border-radius: 10px; | |
| padding: 14px 16px; | |
| font-size: 0.85em; | |
| color: #c4b5fd; | |
| margin-bottom: 14px; | |
| } | |
| .musicgen-info b { color: #a78bfa; } | |
| .btn-musicgen { | |
| background: linear-gradient(135deg, #7c3aed 0%, #db2777 100%) !important; | |
| border: none !important; | |
| border-radius: 10px !important; | |
| color: #ffffff !important; | |
| font-size: 1.05em !important; | |
| font-weight: 600 !important; | |
| padding: 14px 28px !important; | |
| width: 100% !important; | |
| cursor: pointer !important; | |
| transition: opacity 0.2s !important; | |
| box-shadow: 0 4px 20px rgba(124,58,237,0.45) !important; | |
| } | |
| .btn-musicgen:hover { opacity: 0.88 !important; } | |
| .music-prompt-box { | |
| background: rgba(139,92,246,0.06) !important; | |
| border: 1px solid rgba(139,92,246,0.25) !important; | |
| border-radius: 8px !important; | |
| color: #c4b5fd !important; | |
| font-size: 0.88em !important; | |
| } | |
| .preset-radio .wrap { gap: 8px !important; flex-wrap: wrap !important; } | |
| .preset-radio label { | |
| background: #1e1e2e !important; | |
| border: 1px solid #3d3d5c !important; | |
| border-radius: 20px !important; | |
| color: #a5b4fc !important; | |
| font-size: 0.85em !important; | |
| padding: 7px 16px !important; | |
| cursor: pointer !important; | |
| transition: all 0.2s !important; | |
| margin: 0 !important; | |
| } | |
| .preset-radio label:hover { | |
| background: rgba(99,102,241,0.2) !important; | |
| border-color: #6366f1 !important; | |
| color: #ffffff !important; | |
| } | |
| .preset-radio input[type="radio"]:checked + span, | |
| .preset-radio label.selected { | |
| background: linear-gradient(135deg,#4f46e5,#7c3aed) !important; | |
| border-color: transparent !important; | |
| color: #ffffff !important; | |
| } | |
| .preset-radio .gr-radio-row { display: none !important; } | |
| .preset-radio > .wrap > label > input { display: none !important; } | |
| video { | |
| border-radius: 12px !important; | |
| border: 2px solid #2d2d3f !important; | |
| } | |
| label > span { | |
| color: #a5b4fc !important; | |
| font-size: 0.85em !important; | |
| font-weight: 500 !important; | |
| text-transform: uppercase !important; | |
| letter-spacing: 0.04em !important; | |
| } | |
| .stats-bar { | |
| background: #12121c; | |
| border: 1px solid #2d2d3f; | |
| border-radius: 8px; | |
| padding: 8px 16px; | |
| font-size: 0.82em; | |
| color: #64748b; | |
| text-align: center; | |
| margin-top: 8px; | |
| } | |
| .history-box { | |
| background: #0f0f18 !important; | |
| border: 1px solid #2d2d3f !important; | |
| border-radius: 10px !important; | |
| padding: 12px !important; | |
| font-size: 0.84em !important; | |
| max-height: 400px !important; | |
| overflow-y: auto !important; | |
| } | |
| .gr-accordion { border: 1px solid #2d2d3f !important; border-radius: 10px !important; } | |
| footer { display: none !important; } | |
| .runpod-banner { | |
| background: linear-gradient(135deg, #1e1b4b 0%, #312e81 50%, #4f46e5 100%); | |
| border: 1px solid rgba(99, 102, 241, 0.4); | |
| border-radius: 12px; | |
| padding: 18px 24px; | |
| margin: 0 auto 20px auto; | |
| text-align: center; | |
| max-width: 900px; | |
| box-shadow: 0 4px 20px rgba(79, 70, 229, 0.25); | |
| transition: all 0.3s ease; | |
| text-decoration: none !important; | |
| display: block; | |
| } | |
| .runpod-banner:hover { | |
| transform: translateY(-3px); | |
| box-shadow: 0 8px 25px rgba(79, 70, 229, 0.4); | |
| border-color: rgba(99, 102, 241, 0.8); | |
| } | |
| .runpod-title { | |
| color: #ffffff; | |
| font-size: 1.15em; | |
| font-weight: 600; | |
| margin-bottom: 6px; | |
| } | |
| .runpod-subtitle { | |
| color: #c7d2fe; | |
| font-size: 0.95em; | |
| margin-bottom: 12px; | |
| } | |
| .runpod-highlight { | |
| color: #fbbf24; | |
| font-weight: 800; | |
| text-shadow: 0 0 10px rgba(251, 191, 36, 0.3); | |
| } | |
| .runpod-button { | |
| display: inline-block; | |
| background: #ffffff; | |
| color: #4f46e5 !important; | |
| padding: 8px 22px; | |
| border-radius: 8px; | |
| font-weight: 700; | |
| font-size: 0.9em; | |
| transition: background 0.2s; | |
| } | |
| .runpod-banner:hover .runpod-button { | |
| background: #fbbf24; | |
| color: #78350f !important; | |
| } | |
| """ | |
| # ── UI ───────────────────────────────────────────────────────────────────────── | |
| with gr.Blocks(theme=gr.themes.Base(), css=custom_css, title="Wan 2.2 I2V ⚡") as demo: | |
| gr.HTML(""" | |
| <div class="hero-header"> | |
| <h1>⚡ Wan 2.2 I2V — Lightning Video</h1> | |
| <p>Transform any image into a cinematic video in just 4–8 steps</p> | |
| <br> | |
| <span class="badge">🔥 14B Parameters</span> | |
| <span class="badge">⚡ Lightning LoRA</span> | |
| <span class="badge">🎯 FP8 Quantized</span> | |
| <span class="badge">🚀 AoT Compiled</span> | |
| <span class="badge">🎨 B&W Colorization</span> | |
| <span class="badge-music">🎵 Music Composer</span> | |
| <span class="badge-music">🤖 MusicGen AI</span> | |
| </div> | |
| <a href="https://console.runpod.io/deploy?template=adf0boho9x&ref=ev68fdmc" target="_blank" class="runpod-banner"> | |
| <div class="runpod-title"> | |
| 🚀 Skip the Queue & Run This Model Instantly on Your Own GPU! | |
| </div> | |
| <div class="runpod-subtitle"> | |
| Deploy your private instance on <b>RunPod</b> for lightning-fast generations | |
| and get a <span class="runpod-highlight">Bonus Credit up to $500!</span> | |
| <div style="font-size:0.75em;opacity:0.7;margin-top:6px;font-weight:normal;"> | |
| *Bonus is applied automatically after your first $10 account fund. | |
| </div> | |
| </div> | |
| <div class="runpod-button">Claim Your Bonus & Deploy Now ⚡</div> | |
| </a> | |
| """) | |
| with gr.Tabs(): | |
| # ── Tab 1: Generate ──────────────────────────────────────────────────── | |
| with gr.TabItem("🎬 Generate"): | |
| with gr.Row(equal_height=False): | |
| with gr.Column(scale=1, min_width=380): | |
| input_image_component = gr.Image( | |
| type="numpy", | |
| label="📸 Upload Image", | |
| height=280, | |
| elem_classes=["panel-card"], | |
| ) | |
| with gr.Group(elem_classes=["panel-card"]): | |
| prompt_input = gr.Textbox( | |
| label="✍️ Prompt", | |
| value=default_prompt_i2v, | |
| lines=3, | |
| placeholder="Describe the motion, camera, atmosphere…", | |
| ) | |
| gr.Markdown("**⚡ Motion Presets**", elem_classes=["panel-card"]) | |
| preset_radio = gr.Radio( | |
| choices=list(MOTION_PRESETS.keys()), | |
| value=None, | |
| label="", | |
| interactive=True, | |
| elem_classes=["preset-radio"], | |
| ) | |
| duration_seconds_input = gr.Slider( | |
| minimum=MIN_DURATION, maximum=MAX_DURATION, | |
| step=0.5, value=3.5, | |
| label="⏱️ Duration (seconds)", | |
| info=f"Model range: {MIN_FRAMES_MODEL}–{MAX_FRAMES_MODEL} frames @ {FIXED_FPS} fps", | |
| ) | |
| with gr.Accordion("⚙️ Advanced Settings", open=False): | |
| negative_prompt_input = gr.Textbox( | |
| label="🚫 Negative Prompt", | |
| value=default_negative_prompt, | |
| lines=3, | |
| ) | |
| with gr.Row(): | |
| seed_input = gr.Slider( | |
| label="🌱 Seed", | |
| minimum=0, maximum=MAX_SEED, step=1, value=42, | |
| ) | |
| randomize_seed_checkbox = gr.Checkbox( | |
| label="🎲 Randomize", | |
| value=True, | |
| ) | |
| steps_slider = gr.Slider( | |
| minimum=1, maximum=30, step=1, value=6, | |
| label="🔁 Inference Steps", | |
| info="4–8 steps recommended with Lightning LoRA", | |
| ) | |
| guidance_scale_input = gr.Slider( | |
| minimum=0.0, maximum=10.0, step=0.5, value=1.0, | |
| label="🎯 Guidance Scale (high-noise stage)", | |
| ) | |
| guidance_scale_2_input = gr.Slider( | |
| minimum=0.0, maximum=10.0, step=0.5, value=1.0, | |
| label="🎯 Guidance Scale 2 (low-noise stage)", | |
| ) | |
| generate_button = gr.Button( | |
| "🚀 Generate Video", | |
| variant="primary", | |
| size="lg", | |
| elem_classes=["btn-generate"], | |
| ) | |
| with gr.Column(scale=1, min_width=400): | |
| video_output = gr.Video( | |
| label="🎬 Generated Video", | |
| autoplay=True, | |
| interactive=False, | |
| height=360, | |
| ) | |
| with gr.Row(): | |
| seed_display = gr.Number( | |
| label="🌱 Used Seed", | |
| interactive=False, | |
| precision=0, | |
| ) | |
| # ── 🎵 Music Panel (inside Generate tab) ────────────────── | |
| with gr.Accordion("🎵 Add Music to Video", open=False, elem_classes=["music-panel"]): | |
| gr.HTML(""" | |
| <div class="music-info"> | |
| 🎵 <b>Music Composer</b> — Add cinematic music to your video! | |
| Choose from 12 ready-to-use royalty-free tracks or upload your own. | |
| <b>Claude AI</b> can automatically pick the best music for your prompt! | |
| </div> | |
| """) | |
| music_source_radio = gr.Radio( | |
| choices=["🎵 From Library", "📁 Upload File", "🔇 No Music"], | |
| value="🎵 From Library", | |
| label="Music Source", | |
| ) | |
| with gr.Group(visible=True) as music_library_group: | |
| music_track_dropdown = gr.Dropdown( | |
| choices=list(MUSIC_LIBRARY.keys()), | |
| value="🎬 Cinematic Epic", | |
| label="🎼 Select Track", | |
| info="12 Royalty-Free tracks ready to use", | |
| ) | |
| ai_music_suggest_btn = gr.Button( | |
| "🤖 Let Claude Pick the Best Music", | |
| elem_classes=["btn-ai-music"], | |
| size="sm", | |
| ) | |
| music_ai_status = gr.Markdown("", label="") | |
| with gr.Group(visible=False) as music_upload_group: | |
| custom_audio_upload = gr.Audio( | |
| label="📁 Upload Audio File (MP3/WAV/OGG)", | |
| type="filepath", | |
| sources=["upload"], | |
| ) | |
| with gr.Row(): | |
| music_volume = gr.Slider( | |
| minimum=0.0, maximum=1.0, step=0.05, value=0.6, | |
| label="🔊 Volume", | |
| ) | |
| with gr.Row(): | |
| music_fade_in = gr.Slider( | |
| minimum=0.0, maximum=3.0, step=0.5, value=1.0, | |
| label="🌅 Fade In (seconds)", | |
| ) | |
| music_fade_out = gr.Slider( | |
| minimum=0.0, maximum=3.0, step=0.5, value=1.5, | |
| label="🌇 Fade Out (seconds)", | |
| ) | |
| music_loop_checkbox = gr.Checkbox( | |
| label="🔁 Loop music if shorter than video", | |
| value=True, | |
| ) | |
| add_music_btn = gr.Button( | |
| "🎵 Merge Music with Video", | |
| elem_classes=["btn-music"], | |
| size="lg", | |
| ) | |
| music_status = gr.Markdown("", label="") | |
| video_with_music = gr.Video( | |
| label="🎬🎵 Video with Music", | |
| autoplay=True, | |
| interactive=False, | |
| height=300, | |
| visible=True, | |
| ) | |
| gr.HTML(""" | |
| <div class="stats-bar"> | |
| 💡 Tip: Use <b>4–6 steps</b> for fastest results | | |
| <b>Randomize seed</b> for variety | | |
| 🎵 <b>Add Music</b> for cinematic feel | | |
| 🎨 <b>Colorize B&W</b> images first! | |
| </div> | |
| """) | |
| # ── Tab 2: 🎵 Music Studio ───────────────────────────────────────────── | |
| with gr.TabItem("🎵 Music Studio"): | |
| gr.HTML(""" | |
| <div class="music-info"> | |
| 🎵 <b>Music Studio</b> — Add music to any existing video!<br> | |
| Upload a video and choose music from our library or from your own files. | |
| Claude AI automatically picks the most fitting music for you! | |
| </div> | |
| """) | |
| with gr.Row(equal_height=False): | |
| with gr.Column(scale=1, min_width=360): | |
| studio_video_input = gr.Video( | |
| label="📹 Upload Video", | |
| height=280, | |
| ) | |
| studio_music_source = gr.Radio( | |
| choices=["🎵 From Library", "📁 Upload File", "🔇 No Music"], | |
| value="🎵 From Library", | |
| label="Music Source", | |
| ) | |
| with gr.Group(visible=True) as studio_library_group: | |
| studio_track = gr.Dropdown( | |
| choices=list(MUSIC_LIBRARY.keys()), | |
| value="🎬 Cinematic Epic", | |
| label="🎼 Track", | |
| ) | |
| track_info_display = gr.Markdown( | |
| value="**Mood:** cinematic, dramatic | **BPM:** 120", | |
| label="", | |
| ) | |
| with gr.Group(visible=False) as studio_upload_group: | |
| studio_custom_audio = gr.Audio( | |
| label="📁 Upload Audio File", | |
| type="filepath", | |
| sources=["upload"], | |
| ) | |
| studio_volume = gr.Slider( | |
| minimum=0.0, maximum=1.0, step=0.05, value=0.65, | |
| label="🔊 Volume", | |
| ) | |
| with gr.Row(): | |
| studio_fade_in = gr.Slider( | |
| minimum=0.0, maximum=3.0, step=0.5, value=1.0, | |
| label="🌅 Fade In", | |
| ) | |
| studio_fade_out = gr.Slider( | |
| minimum=0.0, maximum=3.0, step=0.5, value=2.0, | |
| label="🌇 Fade Out", | |
| ) | |
| studio_loop = gr.Checkbox(label="🔁 Loop Music", value=True) | |
| studio_add_btn = gr.Button( | |
| "🎵 Merge Music", | |
| variant="primary", | |
| size="lg", | |
| elem_classes=["btn-music"], | |
| ) | |
| studio_status = gr.Markdown("") | |
| with gr.Column(scale=1, min_width=400): | |
| studio_output = gr.Video( | |
| label="🎬🎵 Result", | |
| autoplay=True, | |
| height=400, | |
| ) | |
| gr.Markdown("### 🎼 Music Library") | |
| music_grid_html = "<div style='display:grid;grid-template-columns:1fr 1fr;gap:8px;'>" | |
| for name, info in MUSIC_LIBRARY.items(): | |
| music_grid_html += f""" | |
| <div style='background:rgba(234,179,8,0.08);border:1px solid rgba(234,179,8,0.2); | |
| border-radius:8px;padding:10px;'> | |
| <div style='color:#fbbf24;font-weight:600;font-size:0.9em'>{name}</div> | |
| <div style='color:#94a3b8;font-size:0.78em;margin-top:4px'>{info['mood']}</div> | |
| <div style='color:#64748b;font-size:0.72em'>BPM: {info['bpm']}</div> | |
| </div>""" | |
| music_grid_html += "</div>" | |
| gr.HTML(music_grid_html) | |
| # ── Tab 3: 🤖 MusicGen AI ───────────────────────────────────────────── | |
| with gr.TabItem("🤖 MusicGen AI"): | |
| gr.HTML(""" | |
| <div class="musicgen-info"> | |
| 🤖 <b>MusicGen AI (Meta)</b> — Generates 100% original music custom-made for your video!<br><br> | |
| Workflow: <b>Claude</b> analyzes the prompt → writes a professional music description → | |
| <b>MusicGen</b> generates the music → automatically merged with your video.<br><br> | |
| ⚠️ <b>Note:</b> Generation takes 20–40 seconds on CPU — the quality is worth the wait! | |
| </div> | |
| """) | |
| with gr.Row(equal_height=False): | |
| with gr.Column(scale=1, min_width=360): | |
| musicgen_video_input = gr.Video( | |
| label="📹 Video to Add Music To", | |
| height=260, | |
| ) | |
| musicgen_prompt_input = gr.Textbox( | |
| label="✍️ Video Scene Description (for AI)", | |
| placeholder="e.g. A cat skateboarding on the beach in a playful and cheerful way…", | |
| lines=3, | |
| info="The more precise the description, the better the music fit", | |
| ) | |
| gr.HTML(""" | |
| <div style="background:rgba(139,92,246,0.06);border:1px solid rgba(139,92,246,0.2); | |
| border-radius:8px;padding:10px 14px;font-size:0.82em;color:#94a3b8;margin:4px 0"> | |
| 💡 <b>Tip:</b> You can copy the prompt directly from the Generate tab | |
| </div> | |
| """) | |
| with gr.Row(): | |
| musicgen_duration = gr.Slider( | |
| minimum=2.0, maximum=30.0, step=1.0, value=5.0, | |
| label="⏱️ Music Duration (seconds)", | |
| info="Will be automatically trimmed to video length", | |
| ) | |
| with gr.Row(): | |
| musicgen_volume = gr.Slider( | |
| minimum=0.1, maximum=1.0, step=0.05, value=0.7, | |
| label="🔊 Volume", | |
| ) | |
| with gr.Row(): | |
| musicgen_fade_in = gr.Slider( | |
| minimum=0.0, maximum=2.0, step=0.25, value=0.5, | |
| label="🌅 Fade In (seconds)", | |
| ) | |
| musicgen_fade_out = gr.Slider( | |
| minimum=0.0, maximum=2.0, step=0.25, value=1.0, | |
| label="🌇 Fade Out (seconds)", | |
| ) | |
| musicgen_generate_btn = gr.Button( | |
| "🤖 Generate AI Music & Merge", | |
| variant="primary", | |
| size="lg", | |
| elem_classes=["btn-musicgen"], | |
| ) | |
| copy_from_generate_btn = gr.Button( | |
| "📋 Transfer Video from Generate Tab", | |
| size="sm", | |
| elem_classes=["btn-send-to-generate"], | |
| ) | |
| with gr.Column(scale=1, min_width=400): | |
| musicgen_output_video = gr.Video( | |
| label="🎬🎵 Video with AI Music", | |
| autoplay=True, | |
| height=320, | |
| ) | |
| musicgen_music_prompt_display = gr.Textbox( | |
| label="🎼 Music Description Used by MusicGen", | |
| interactive=False, | |
| lines=2, | |
| elem_classes=["music-prompt-box"], | |
| placeholder="The music description will appear here after generation…", | |
| ) | |
| musicgen_status = gr.Markdown("", label="") | |
| gr.HTML(""" | |
| <div class="stats-bar"> | |
| 🤖 <b>MusicGen Small</b> (~300MB) loads automatically on first use | | |
| 🎼 <b>Claude</b> writes the music description | | |
| ✂️ Music is precisely trimmed to video length | |
| </div> | |
| """) | |
| with gr.Accordion("💡 Music Description Examples", open=False): | |
| gr.HTML(""" | |
| <div style="display:grid;grid-template-columns:1fr 1fr;gap:8px;margin-top:8px"> | |
| <div style="background:rgba(139,92,246,0.08);border:1px solid rgba(139,92,246,0.2); | |
| border-radius:8px;padding:10px"> | |
| <div style="color:#a78bfa;font-size:0.82em;font-weight:600">🌊 Nature Scene</div> | |
| <div style="color:#94a3b8;font-size:0.78em;margin-top:4px"> | |
| peaceful ambient, soft piano, nature sounds, 70 bpm | |
| </div> | |
| </div> | |
| <div style="background:rgba(139,92,246,0.08);border:1px solid rgba(139,92,246,0.2); | |
| border-radius:8px;padding:10px"> | |
| <div style="color:#a78bfa;font-size:0.82em;font-weight:600">⚡ Action Scene</div> | |
| <div style="color:#94a3b8;font-size:0.78em;margin-top:4px"> | |
| action rock, electric guitar, fast drums, 140 bpm | |
| </div> | |
| </div> | |
| <div style="background:rgba(139,92,246,0.08);border:1px solid rgba(139,92,246,0.2); | |
| border-radius:8px;padding:10px"> | |
| <div style="color:#a78bfa;font-size:0.82em;font-weight:600">🎬 Dramatic Scene</div> | |
| <div style="color:#94a3b8;font-size:0.78em;margin-top:4px"> | |
| cinematic orchestra, dramatic strings, 100 bpm, epic | |
| </div> | |
| </div> | |
| <div style="background:rgba(139,92,246,0.08);border:1px solid rgba(139,92,246,0.2); | |
| border-radius:8px;padding:10px"> | |
| <div style="color:#a78bfa;font-size:0.82em;font-weight:600">✨ Magical Scene</div> | |
| <div style="color:#94a3b8;font-size:0.78em;margin-top:4px"> | |
| magical fantasy, harp and bells, 85 bpm, whimsical | |
| </div> | |
| </div> | |
| </div> | |
| """) | |
| # ── Tab 4: Colorize B&W ──────────────────────────────────────────────── | |
| with gr.TabItem("🎨 Colorize B&W"): | |
| gr.HTML(""" | |
| <div class="colorize-info"> | |
| 🎨 <b>Deep-Learning Colorization</b> — Upload a black-and-white (or faded) image | |
| and get a vivid, AI-colorized version. Then send it directly to <b>Generate</b> | |
| to create a cinematic video from your newly colorized photo!<br><br> | |
| <b>Auto-selects best available engine:</b> | |
| 🥇 Claude AI Vision → | |
| 🥈 OpenCV DNN (auto-download) → | |
| 🥉 Smart Semantic (always works) | |
| </div> | |
| """) | |
| with gr.Row(equal_height=False): | |
| with gr.Column(scale=1, min_width=360): | |
| colorize_input = gr.Image( | |
| type="pil", label="🖼️ Upload B&W Image", height=300, | |
| elem_classes=["panel-card"], | |
| ) | |
| colorize_style = gr.Radio( | |
| choices=["natural", "cinematic", "warm"], | |
| value="natural", label="🎨 Color Style", | |
| ) | |
| render_factor_slider = gr.Slider( | |
| minimum=10, maximum=45, step=5, value=35, | |
| label="🔍 Render Factor", | |
| ) | |
| colorize_btn = gr.Button( | |
| "🎨 Colorize Image", variant="primary", size="lg", | |
| elem_classes=["btn-colorize"], | |
| ) | |
| colorize_status = gr.Markdown("", label="") | |
| with gr.Column(scale=1, min_width=400): | |
| colorize_output = gr.Image( | |
| type="pil", label="🌈 Colorized Result", height=380, | |
| interactive=False, elem_classes=["panel-card"], | |
| ) | |
| send_to_generate_btn = gr.Button( | |
| "🚀 Send to Generate Tab", size="lg", | |
| elem_classes=["btn-send-to-generate"], | |
| ) | |
| with gr.Accordion("⚖️ Before / After Comparison", open=False): | |
| with gr.Row(): | |
| compare_original = gr.Image(type="pil", label="Original (B&W)", interactive=False, height=220) | |
| compare_colorized = gr.Image(type="pil", label="Colorized", interactive=False, height=220) | |
| def _run_colorize(img, style, factor): | |
| colorized, status = colorize_image(img, style, factor) | |
| return colorized, status, img, colorized | |
| colorize_btn.click( | |
| fn=_run_colorize, | |
| inputs=[colorize_input, colorize_style, render_factor_slider], | |
| outputs=[colorize_output, colorize_status, compare_original, compare_colorized], | |
| ) | |
| send_to_generate_btn.click( | |
| fn=lambda img: img, | |
| inputs=[colorize_output], | |
| outputs=[input_image_component], | |
| ) | |
| # ── Tab 5: Examples ──────────────────────────────────────────────────── | |
| with gr.TabItem("📚 Examples"): | |
| gr.Markdown("### Ready-to-run examples") | |
| gr.Examples( | |
| examples=[ | |
| ["wan_i2v_input.JPG", "POV selfie video, white cat with sunglasses standing on surfboard, relaxed smile, tropical beach behind.", 4], | |
| ["wan22_input_2.jpg", "A sleek lunar vehicle glides, astronauts hop aboard. Aurora borealis ribbons dance across the star-filled sky.", 4], | |
| ["kill_bill.jpeg", "Uma Thurman's blade slowly warps and droops, molten silver flowing downward. Her expression shifts to bewilderment.", 6], | |
| ], | |
| inputs=[input_image_component, prompt_input, steps_slider], | |
| outputs=[video_output, seed_display], | |
| fn=generate_video, | |
| cache_examples=True, | |
| cache_mode="lazy", | |
| ) | |
| # ── Tab 6: History ───────────────────────────────────────────────────── | |
| with gr.TabItem("📊 History"): | |
| gr.Markdown("### Your last 20 generations") | |
| refresh_history_btn = gr.Button("🔄 Refresh", size="sm") | |
| history_display = gr.Markdown(value=load_history_display(), elem_classes=["history-box"]) | |
| refresh_history_btn.click(fn=load_history_display, inputs=[], outputs=[history_display]) | |
| # ── Tab 7: About ─────────────────────────────────────────────────────── | |
| with gr.TabItem("ℹ️ About"): | |
| gr.Markdown(""" | |
| ## About this Space | |
| | Feature | Detail | | |
| |---|---| | |
| | **Model** | Wan-AI/Wan2.2-I2V-A14B — 14 billion parameters | | |
| | **LoRA** | Lightning LoRA — 4–8 step generation | | |
| | **Quantisation** | FP8 dynamic + INT8 text encoder | | |
| | **🎨 Colorization** | Claude Vision + OpenCV DNN + Smart Semantic | | |
| | **🎵 Music** | 12 Royalty-Free tracks + custom upload + AI selection | | |
| ### 🎵 Music Composer | |
| - **12 ready-to-use tracks** Royalty-Free for all tastes | |
| - **Upload your own file** MP3/WAV/OGG | |
| - **Claude AI** selects the best music based on your prompt | |
| - **Fade In/Out** automatic control + volume adjustment | |
| - **Loop** music to cover the full video duration | |
| --- | |
| ### 🚀 Host It Yourself on RunPod | |
| Tired of waiting in the public queue? You can run this exact model and UI on your own private GPU using **RunPod**. | |
| * **Instant Access:** Get your own dedicated GPU in seconds and bypass all limits. | |
| * **Special Offer:** Use my [referral link](https://console.runpod.io/deploy?template=adf0boho9x&ref=ev68fdmc) to sign up. When you add your first $10 to fund your account, RunPod will instantly reward you with a **random bonus credit ranging from $5 to $500**! | |
| """) | |
| # ── Event Wiring ─────────────────────────────────────────────────────────── | |
| copy_from_generate_btn.click( | |
| fn=lambda v: v, | |
| inputs=[video_output], | |
| outputs=[musicgen_video_input], | |
| ) | |
| musicgen_generate_btn.click( | |
| fn=generate_music_for_video, | |
| inputs=[ | |
| musicgen_video_input, | |
| musicgen_prompt_input, | |
| musicgen_duration, | |
| musicgen_fade_in, | |
| musicgen_fade_out, | |
| musicgen_volume, | |
| ], | |
| outputs=[musicgen_output_video, musicgen_music_prompt_display, musicgen_status], | |
| ) | |
| ui_inputs = [ | |
| input_image_component, prompt_input, steps_slider, | |
| negative_prompt_input, duration_seconds_input, | |
| guidance_scale_input, guidance_scale_2_input, | |
| seed_input, randomize_seed_checkbox, | |
| ] | |
| generate_button.click( | |
| fn=generate_video, | |
| inputs=ui_inputs, | |
| outputs=[video_output, seed_display], | |
| ) | |
| preset_radio.change( | |
| fn=apply_preset, | |
| inputs=[preset_radio], | |
| outputs=[prompt_input, music_track_dropdown], | |
| ) | |
| def _toggle_music_source(source): | |
| is_lib = source == "🎵 From Library" | |
| is_upload = source == "📁 Upload File" | |
| return gr.update(visible=is_lib), gr.update(visible=is_upload) | |
| music_source_radio.change( | |
| fn=_toggle_music_source, | |
| inputs=[music_source_radio], | |
| outputs=[music_library_group, music_upload_group], | |
| ) | |
| studio_music_source.change( | |
| fn=_toggle_music_source, | |
| inputs=[studio_music_source], | |
| outputs=[studio_library_group, studio_upload_group], | |
| ) | |
| def _update_track_info(track_name): | |
| if track_name and track_name in MUSIC_LIBRARY: | |
| info = MUSIC_LIBRARY[track_name] | |
| return f"**Mood:** {info['mood']} | **BPM:** {info['bpm']}" | |
| return "" | |
| studio_track.change( | |
| fn=_update_track_info, | |
| inputs=[studio_track], | |
| outputs=[track_info_display], | |
| ) | |
| def _ai_suggest(prompt_text): | |
| track, status = suggest_music_with_claude(prompt_text) | |
| return track, status | |
| ai_music_suggest_btn.click( | |
| fn=_ai_suggest, | |
| inputs=[prompt_input], | |
| outputs=[music_track_dropdown, music_ai_status], | |
| ) | |
| def _add_music_generate( | |
| video_path, source, track, custom_audio, | |
| volume, fade_in, fade_out, loop | |
| ): | |
| src_map = { | |
| "🎵 From Library": "library", | |
| "📁 Upload File": "upload", | |
| "🔇 No Music": "none", | |
| } | |
| result_path, status = add_music_to_video( | |
| video_path, | |
| src_map.get(source, "none"), | |
| track, | |
| custom_audio, | |
| volume, fade_in, fade_out, loop, | |
| ) | |
| return result_path, status | |
| add_music_btn.click( | |
| fn=_add_music_generate, | |
| inputs=[ | |
| video_output, music_source_radio, music_track_dropdown, | |
| custom_audio_upload, music_volume, music_fade_in, music_fade_out, | |
| music_loop_checkbox, | |
| ], | |
| outputs=[video_with_music, music_status], | |
| ) | |
| def _studio_add_music(video, source, track, custom_audio, volume, fade_in, fade_out, loop): | |
| if video is None: | |
| return None, "⚠️ Please upload a video first." | |
| src_map = { | |
| "🎵 From Library": "library", | |
| "📁 Upload File": "upload", | |
| "🔇 No Music": "none", | |
| } | |
| result_path, status = add_music_to_video( | |
| video, | |
| src_map.get(source, "none"), | |
| track, | |
| custom_audio, | |
| volume, fade_in, fade_out, loop, | |
| ) | |
| return result_path, status | |
| studio_add_btn.click( | |
| fn=_studio_add_music, | |
| inputs=[ | |
| studio_video_input, studio_music_source, studio_track, | |
| studio_custom_audio, studio_volume, studio_fade_in, studio_fade_out, | |
| studio_loop, | |
| ], | |
| outputs=[studio_output, studio_status], | |
| ) | |
| # ── Launch ───────────────────────────────────────────────────────────────────── | |
| if __name__ == "__main__": | |
| demo.queue(max_size=10).launch(mcp_server=True) |