Quantumbraid's picture
Use bundled FFmpeg for chained videos
ee6f445 verified
Raw History Blame Contribute Delete
50.7 kB
import os import sys # Resolve bundled modules and downloaded assets from the app directory rather # than depending on whatever directory the Space launcher chose as cwd. APP_DIR = os.path.dirname(os.path.abspath(__file__)) if APP_DIR not in sys.path: sys.path.insert(0, APP_DIR) # Try importing spaces for Hugging Face ZeroGPU compatibility, fallback gracefully for private rented GPUs try: import spaces HAS_SPACES = True except Exception: HAS_SPACES = False class DummySpaces: def GPU(self, duration=None, size=None): def decorator(func): return func return decorator def aoti_load(self, *args, **kwargs): pass spaces = DummySpaces() import shutil import subprocess import copy import random import tempfile import warnings import time import gc import uuid from queue_manager import QueueManager from tqdm import tqdm import cv2 import imageio_ffmpeg import numpy as np import torch import torch._dynamo from torch.nn import functional as F from PIL import Image import gradio as gr from diffusers import ( FlowMatchEulerDiscreteScheduler, SASolverScheduler, DEISMultistepScheduler, DPMSolverMultistepInverseScheduler, UniPCMultistepScheduler, DPMSolverMultistepScheduler, DPMSolverSinglestepScheduler, ) from diffusers.pipelines.wan.pipeline_wan_i2v import WanImageToVideoPipeline from diffusers.utils.export_utils import export_to_video from torchao.quantization import quantize_, Float8DynamicActivationFloat8WeightConfig, Int8WeightOnlyConfig import aoti import lora_loader os.environ["TOKENIZERS_PARALLELISM"] = "true" warnings.filterwarnings("ignore") IS_ZERO_GPU = bool(os.getenv("SPACES_ZERO_GPU")) # if IS_ZERO_GPU: # print("Loading...") # subprocess.run("rm -rf /data-nvme/zerogpu-offload/*", env={}, shell=True) # --- FRAME EXTRACTION JS & LOGIC --- # JS to grab timestamp from the output video get_timestamp_js = """ function() { // Select the video element specifically inside the component with id 'generated-video' const video = document.querySelector('#generated-video video'); if (video) { console.log("Video found! Time: " + video.currentTime); return video.currentTime; } else { console.log("No video element found."); return 0; } } """ def extract_frame(video_path, timestamp): # Safety check: if no video is present if not video_path: return None print(f"Extracting frame at timestamp: {timestamp}") cap = cv2.VideoCapture(video_path) if not cap.isOpened(): return None # Calculate frame number fps = cap.get(cv2.CAP_PROP_FPS) target_frame_num = int(float(timestamp) * fps) # Cap total frames to prevent errors at the very end of video total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) if target_frame_num >= total_frames: target_frame_num = total_frames - 1 # Set position cap.set(cv2.CAP_PROP_POS_FRAMES, target_frame_num) ret, frame = cap.read() cap.release() if ret: rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) temp_dir = tempfile.gettempdir() extracted_path = os.path.join(temp_dir, f"extracted_frame_{int(time.time()*1000)}.png") Image.fromarray(rgb).save(extracted_path) return [extracted_path] return None # --- END FRAME EXTRACTION LOGIC --- def clear_vram(): gc.collect() torch.cuda.empty_cache() # RIFE RIFE_ZIP = os.path.join(APP_DIR, "RIFEv4.26_0921.zip") RIFE_DIR = os.path.join(APP_DIR, "train_log") if not os.path.exists(RIFE_ZIP): print("Downloading RIFE Model...") subprocess.run([ "wget", "-q", "https://huggingface.co/thornmaze/RIFE/resolve/main/RIFEv4.26_0921.zip", "-O", RIFE_ZIP ], check=True) subprocess.run(["unzip", "-o", RIFE_ZIP, "-d", APP_DIR], check=True) from train_log.RIFE_HDv3 import Model device = torch.device("cuda" if torch.cuda.is_available() else "cpu") rife_model = Model() rife_model.load_model(RIFE_DIR, -1) rife_model.eval() @torch.no_grad() def interpolate_bits(frames_np, multiplier=2, scale=1.0): """ Interpolation maintaining Numpy Float 0-1 format. Args: frames_np: Numpy Array (Time, Height, Width, Channels) - Float32 [0.0, 1.0] multiplier: int (2, 4, 8) Returns: List of Numpy Arrays (Height, Width, Channels) - Float32 [0.0, 1.0] """ # Handle input shape if isinstance(frames_np, list): # Convert list of arrays to one big array for easier shape handling if needed, # but here we just grab dims from first frame T = len(frames_np) H, W, C = frames_np[0].shape else: T, H, W, C = frames_np.shape # 1. No Interpolation Case if multiplier < 2: # Just convert 4D array to list of 3D arrays if isinstance(frames_np, np.ndarray): return list(frames_np) return frames_np n_interp = multiplier - 1 # Pre-calc padding for RIFE (requires dimensions divisible by 32/scale) tmp = max(128, int(128 / scale)) ph = ((H - 1) // tmp + 1) * tmp pw = ((W - 1) // tmp + 1) * tmp padding = (0, pw - W, 0, ph - H) # Helper: Numpy (H, W, C) Float -> Tensor (1, C, H, W) Half def to_tensor(frame_np): # frame_np is float32 0-1 t = torch.from_numpy(frame_np).to(device) # HWC -> CHW t = t.permute(2, 0, 1).unsqueeze(0) return F.pad(t, padding).half() # Helper: Tensor (1, C, H, W) Half -> Numpy (H, W, C) Float def from_tensor(tensor): # Crop padding t = tensor[0, :, :H, :W] # CHW -> HWC t = t.permute(1, 2, 0) # Keep as float32, range 0-1 return t.float().cpu().numpy() def make_inference(I0, I1, n): if rife_model.version >= 3.9: res = [] for i in range(n): res.append(rife_model.inference(I0, I1, (i+1) * 1. / (n+1), scale)) return res else: middle = rife_model.inference(I0, I1, scale) if n == 1: return [middle] first_half = make_inference(I0, middle, n=n//2) second_half = make_inference(middle, I1, n=n//2) if n % 2: return [*first_half, middle, *second_half] else: return [*first_half, *second_half] output_frames = [] # Process Frames # Load first frame into GPU I1 = to_tensor(frames_np[0]) total_steps = T - 1 with tqdm(total=total_steps, desc="Interpolating", unit="frame") as pbar: for i in range(total_steps): I0 = I1 # Add original frame to output output_frames.append(from_tensor(I0)) # Load next frame I1 = to_tensor(frames_np[i+1]) # Generate intermediate frames mid_tensors = make_inference(I0, I1, n_interp) # Append intermediate frames for mid in mid_tensors: output_frames.append(from_tensor(mid)) if (i + 1) % 50 == 0: pbar.update(50) pbar.update(total_steps % 50) # Add the very last frame output_frames.append(from_tensor(I1)) # Cleanup del I0, I1, mid_tensors torch.cuda.empty_cache() return output_frames # WAN MODEL_ID = "thornmaze/WAMU_v3_WAN2.2_I2V_LIGHTNING" LORA_MODELS = [] MAX_DIM = 832 MIN_DIM = 480 SQUARE_DIM = 640 MULTIPLE_OF = 16 MAX_SEED = np.iinfo(np.int32).max FIXED_FPS = 16 MIN_FRAMES_MODEL = 8 MAX_FRAMES_MODEL = 321 MIN_DURATION = round(MIN_FRAMES_MODEL / FIXED_FPS, 1) MAX_DURATION = round(MAX_FRAMES_MODEL / FIXED_FPS, 1) SCHEDULER_MAP = { "FlowMatchEulerDiscrete": FlowMatchEulerDiscreteScheduler, "SASolver": SASolverScheduler, "DEISMultistep": DEISMultistepScheduler, "DPMSolverMultistepInverse": DPMSolverMultistepInverseScheduler, "UniPCMultistep": UniPCMultistepScheduler, "DPMSolverMultistep": DPMSolverMultistepScheduler, "DPMSolverSinglestep": DPMSolverSinglestepScheduler, } pipe = WanImageToVideoPipeline.from_pretrained( MODEL_ID, torch_dtype=torch.bfloat16, ).to('cuda') original_scheduler = copy.deepcopy(pipe.scheduler) for i, lora in enumerate(LORA_MODELS): name_high_tr = lora["high_tr"].split(".")[0].split("/")[-1] + "Hh" name_low_tr = lora["low_tr"].split(".")[0].split("/")[-1] + "Ll" try: pipe.load_lora_weights( lora["repo_id"], weight_name=lora["high_tr"], adapter_name=name_high_tr ) kwargs_lora = {"load_into_transformer_2": True} pipe.load_lora_weights( lora["repo_id"], weight_name=lora["low_tr"], adapter_name=name_low_tr, **kwargs_lora ) pipe.set_adapters([name_high_tr, name_low_tr], adapter_weights=[1.0, 1.0]) pipe.fuse_lora(adapter_names=[name_high_tr], lora_scale=lora["high_scale"], components=["transformer"]) pipe.fuse_lora(adapter_names=[name_low_tr], lora_scale=lora["low_scale"], components=["transformer_2"]) pipe.unload_lora_weights() print(f"Applied: {lora['high_tr']}, hs={lora['high_scale']}/ls={lora['low_scale']}, {i+1}/{len(LORA_MODELS)}") except Exception as e: print("Error:", str(e)) print("Failed LoRA:", name_high_tr) pipe.unload_lora_weights() # if os.path.exists(CACHE_DIR): # shutil.rmtree(CACHE_DIR) # print("Deleted Hugging Face cache.") # else: # print("No hub cache found.") quantize_(pipe.text_encoder, Int8WeightOnlyConfig()) torch._dynamo.reset() quantize_(pipe.transformer, Float8DynamicActivationFloat8WeightConfig()) torch._dynamo.reset() quantize_(pipe.transformer_2, Float8DynamicActivationFloat8WeightConfig()) torch._dynamo.reset() if HAS_SPACES and IS_ZERO_GPU: try: spaces.aoti_load( module=pipe.transformer, repo_id='thornmaze/WanTransformer3DModel-sm120-cu130-raa', ) spaces.aoti_load( module=pipe.transformer_2, repo_id='thornmaze/WanTransformer3DModel-sm120-cu130-raa', ) except Exception as e: print(f"ZeroGPU AOTI load skipped/failed: {e}") queue_mgr = QueueManager() # pipe.vae.enable_slicing() # pipe.vae.enable_tiling() default_prompt_i2v = "make this image come alive, cinematic motion, smooth animation" default_negative_prompt = "色调艳丽, 过曝, 静态, 细节模糊不清, 字幕, 风格, 作品, 画作, 画面, 静止, 整体发灰, 最差质量, 低质量, JPEG压缩残留, 丑陋的, 残缺的, 多余的手指, 画得不好的手部, 画得不好的脸部, 畸形的, 毁容的, 形态畸形的肢体, 手指融合, 静止不动的画面, 杂乱的背景, 三条腿, 背景人很多, 倒着走" def model_title(): return "## Wan 2.2 I2V 14B Lightning — NSFW" def resize_image(image: Image.Image) -> Image.Image: width, height = image.size if width == height: return image.resize((SQUARE_DIM, SQUARE_DIM), Image.LANCZOS) aspect_ratio = width / height MAX_ASPECT_RATIO = MAX_DIM / MIN_DIM MIN_ASPECT_RATIO = MIN_DIM / MAX_DIM image_to_resize = image if aspect_ratio > MAX_ASPECT_RATIO: target_w, target_h = MAX_DIM, MIN_DIM crop_width = int(round(height * MAX_ASPECT_RATIO)) left = (width - crop_width) // 2 image_to_resize = image.crop((left, 0, left + crop_width, height)) elif aspect_ratio < MIN_ASPECT_RATIO: target_w, target_h = MIN_DIM, MAX_DIM crop_height = int(round(width / MIN_ASPECT_RATIO)) top = (height - crop_height) // 2 image_to_resize = image.crop((0, top, width, top + crop_height)) else: if width > height: target_w = MAX_DIM target_h = int(round(target_w / aspect_ratio)) else: target_h = MAX_DIM target_w = int(round(target_h * aspect_ratio)) final_w = round(target_w / MULTIPLE_OF) * MULTIPLE_OF final_h = round(target_h / MULTIPLE_OF) * MULTIPLE_OF final_w = max(MIN_DIM, min(MAX_DIM, final_w)) final_h = max(MIN_DIM, min(MAX_DIM, final_h)) return image_to_resize.resize((final_w, final_h), Image.LANCZOS) def resize_and_crop_to_match(target_image, reference_image): ref_width, ref_height = reference_image.size target_width, target_height = target_image.size scale = max(ref_width / target_width, ref_height / target_height) new_width, new_height = int(target_width * scale), int(target_height * scale) resized = target_image.resize((new_width, new_height), Image.Resampling.LANCZOS) left, top = (new_width - ref_width) // 2, (new_height - ref_height) // 2 return resized.crop((left, top, left + ref_width, top + ref_height)) def get_num_frames(duration_seconds: float): raw = int(round(duration_seconds * FIXED_FPS)) raw = max(MIN_FRAMES_MODEL, min(MAX_FRAMES_MODEL, raw)) return ((raw - 1) // 4) * 4 + 1 def get_inference_duration( resized_image, processed_last_image, prompt, steps, negative_prompt, num_frames, guidance_scale, guidance_scale_2, current_seed, scheduler_name, flow_shift, frame_multiplier, quality, duration_seconds, lora_groups, progress ): BASE_FRAMES_HEIGHT_WIDTH = 81 * 832 * 624 BASE_STEP_DURATION = 5. width, height = resized_image.size factor = num_frames * width * height / BASE_FRAMES_HEIGHT_WIDTH step_duration = BASE_STEP_DURATION * factor ** 1.5 gen_time = int(steps) * step_duration if guidance_scale > 1: gen_time = gen_time * 2.4 frame_factor = frame_multiplier // FIXED_FPS if frame_factor > 1: total_out_frames = (num_frames * frame_factor) - num_frames inter_time = (total_out_frames * 0.02) gen_time += inter_time return 15 + gen_time @spaces.GPU(duration=get_inference_duration, size='xlarge') def run_inference( resized_image, processed_last_image, prompt, steps, negative_prompt, num_frames, guidance_scale, guidance_scale_2, current_seed, scheduler_name, flow_shift, frame_multiplier, quality, duration_seconds, lora_groups=None, progress=None, ): scheduler_class = SCHEDULER_MAP.get(scheduler_name) if scheduler_class.__name__ != pipe.scheduler.config._class_name or flow_shift != pipe.scheduler.config.get("flow_shift", "shift"): config = copy.deepcopy(original_scheduler.config) if scheduler_class == FlowMatchEulerDiscreteScheduler: config['shift'] = flow_shift else: config['flow_shift'] = flow_shift pipe.scheduler = scheduler_class.from_config(config) clear_vram() task_name = str(uuid.uuid4())[:8] print(f"Generating {num_frames} frames, task: {task_name}, {duration_seconds}, {resized_image.size}, lora={lora_groups}") start = time.time() lora_loaded = False lora_adapter_names = [] if lora_groups: try: for idx, name in enumerate(lora_groups): if name and name != "(None)": lora_adapter_names += lora_loader.load_lora_to_pipe( pipe, name, adapter_name=f"lora_{idx}" ) # Activate ALL loaded adapters together (HIGH in transformer, LOW in # transformer_2); each load_lora_weights alone would leave only the # last adapter active. if lora_adapter_names: pipe.set_adapters(lora_adapter_names, adapter_weights=[1.0] * len(lora_adapter_names)) lora_loaded = True print(f"LoRA loaded: {lora_groups}") except Exception as e: print(f"LoRA warning: {e}") lora_loader.unload_lora(pipe) # LoRA prompt + user prompt combined — lora triggers first for activation try: additions = lora_loader.get_lora_prompt_additions(lora_groups) except Exception: additions = "" if additions: prompt = f"{additions} {prompt}" print(f"Combined prompt: {prompt}") result = pipe( image=resized_image, last_image=processed_last_image, prompt=prompt, negative_prompt=negative_prompt, height=resized_image.height, width=resized_image.width, num_frames=num_frames, guidance_scale=float(guidance_scale), guidance_scale_2=float(guidance_scale_2), num_inference_steps=int(steps), generator=torch.Generator(device="cuda").manual_seed(current_seed), output_type="np" ) if lora_loaded: lora_loader.unload_lora(pipe) print("gen time passed:", time.time() - start) raw_frames_np = result.frames[0] # Returns (T, H, W, C) float32 pipe.scheduler = original_scheduler frame_factor = frame_multiplier // FIXED_FPS if frame_factor > 1: start = time.time() print(f"Processing frames (RIFE Multiplier: {frame_factor}x)...") rife_model.device() rife_model.flownet = rife_model.flownet.half() final_frames = interpolate_bits(raw_frames_np, multiplier=int(frame_factor)) print("Interpolation time passed:", time.time() - start) else: final_frames = list(raw_frames_np) final_fps = FIXED_FPS * int(frame_factor) with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as tmpfile: video_path = tmpfile.name start = time.time() with tqdm(total=3, desc="Rendering Media", unit="clip") as pbar: pbar.update(2) export_to_video(final_frames, video_path, fps=final_fps, quality=quality) pbar.update(1) print(f"Export time passed, {final_fps} FPS:", time.time() - start) return video_path, task_name def last_frame_of_video(video_path): """Return the final decoded frame of a completed segment as a PIL image.""" capture = cv2.VideoCapture(video_path) try: frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT)) if frame_count <= 0: raise RuntimeError("rendered segment has no readable frames") capture.set(cv2.CAP_PROP_POS_FRAMES, frame_count - 1) ok, frame = capture.read() if not ok: raise RuntimeError("could not decode the final frame of a rendered segment") return Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) finally: capture.release() def _ffmpeg_executable(): """Use the system ffmpeg when present, otherwise imageio's bundled binary.""" configured = os.environ.get("FFMPEG_BINARY") candidates = [configured, shutil.which("ffmpeg")] try: candidates.append(imageio_ffmpeg.get_ffmpeg_exe()) except Exception as exc: print(f"imageio-ffmpeg lookup failed: {exc}") for candidate in candidates: if candidate and os.path.isfile(candidate): return candidate raise RuntimeError( "No FFmpeg executable is available. Install ffmpeg or keep imageio-ffmpeg in requirements.txt." ) def stitch_videos(segment_paths, artifact_dir=None): """Join equal-format render segments into one MP4 using ffmpeg's concat demuxer.""" if len(segment_paths) == 1: return segment_paths[0] if artifact_dir: os.makedirs(artifact_dir, exist_ok=True) with tempfile.NamedTemporaryFile( mode="w", suffix=".ffconcat", prefix="segments_", dir=artifact_dir, delete=False, encoding="utf-8", ) as concat_file: concat_path = concat_file.name for path in segment_paths: # ffconcat uses single-quoted paths; temporary filenames are local, # but escaping keeps this correct even if the system temp path varies. concat_file.write("file '{}'".format(os.path.abspath(path).replace("'", r"'\\''"))) concat_file.write(chr(10)) if artifact_dir: output_path = os.path.join(artifact_dir, "joined.mp4") else: with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as output_file: output_path = output_file.name ffmpeg = _ffmpeg_executable() copy_command = [ ffmpeg, "-y", "-f", "concat", "-safe", "0", "-i", concat_path, "-c", "copy", output_path, ] result = subprocess.run(copy_command, capture_output=True, text=True) if result.returncode != 0: # Segment encoders normally match, but fall back to a clean re-encode # when an MP4 container rejects stream-copy concatenation. reencode_command = [ ffmpeg, "-y", "-f", "concat", "-safe", "0", "-i", concat_path, "-c:v", "libx264", "-pix_fmt", "yuv420p", "-movflags", "+faststart", output_path, ] result = subprocess.run(reencode_command, capture_output=True, text=True) try: if result.returncode != 0 or not os.path.exists(output_path) or os.path.getsize(output_path) == 0: raise RuntimeError(f"ffmpeg could not join chained segments: {result.stderr[-1000:]}") return output_path finally: if os.path.exists(concat_path): os.unlink(concat_path) def generate_video( input_image, last_image, prompt, steps=4, negative_prompt=default_negative_prompt, duration_seconds=10.0, guidance_scale=1, guidance_scale_2=1, seed=42, randomize_seed=False, quality=5, scheduler="UniPCMultistep", flow_shift=6.0, frame_multiplier=16, sequential_renders=1, lora_groups=None, video_component=True, progress=None, artifact_dir=None, ): """ Generate a video from an input image using the Wan 2.2 14B I2V model with Lightning LoRA. This function takes an input image and generates a video animation based on the provided prompt and parameters. It uses an FP8 qunatized Wan 2.2 14B Image-to-Video model in with Lightning LoRA for fast generation in 4-8 steps. Args: input_image (PIL.Image): The input image to animate. Will be resized to target dimensions. last_image (PIL.Image, optional): The optional last image for the video. prompt (str): Text prompt describing the desired animation or motion. steps (int, optional): Number of inference steps. More steps = higher quality but slower. Defaults to 4. Range: 1-30. negative_prompt (str, optional): Negative prompt to avoid unwanted elements. Defaults to default_negative_prompt (contains unwanted visual artifacts). duration_seconds (float, optional): Duration of the generated video in seconds. Defaults to 2. Clamped between MIN_FRAMES_MODEL/FIXED_FPS and MAX_FRAMES_MODEL/FIXED_FPS. guidance_scale (float, optional): Controls adherence to the prompt. Higher values = more adherence. Defaults to 1.0. Range: 0.0-20.0. guidance_scale_2 (float, optional): Controls adherence to the prompt. Higher values = more adherence. Defaults to 1.0. Range: 0.0-20.0. seed (int, optional): Random seed for reproducible results. Defaults to 42. Range: 0 to MAX_SEED (2147483647). randomize_seed (bool, optional): Whether to use a random seed instead of the provided seed. Defaults to False. quality (float, optional): Video output quality. Default is 5. Uses variable bit rate. Highest quality is 10, lowest is 1. scheduler (str, optional): The name of the scheduler to use for inference. Defaults to "UniPCMultistep". flow_shift (float, optional): The flow shift value for compatible schedulers. Defaults to 6.0. frame_multiplier (int, optional): The int value for fps enhancer sequential_renders (int, optional): Number of independently rendered segments. Every following segment uses the prior segment's final frame as its starting image, then all segments are joined. video_component(bool, optional): Show video player in output. Defaults to True. progress (gr.Progress, optional): Gradio progress tracker. Defaults to gr.Progress(track_tqdm=True). Returns: tuple: A tuple containing: - video_path (str): Path for the video component. - video_path (str): Path for the file download component. Attempt to avoid reconversion in video component. - current_seed (int): The seed used for generation. Raises: gr.Error: If input_image is None (no image uploaded). Note: - Frame count is calculated as duration_seconds * FIXED_FPS (24) - Output dimensions are adjusted to be multiples of MOD_VALUE (32) - The function uses GPU acceleration via the @spaces.GPU decorator - Generation time varies based on steps and duration (see get_duration function) """ if input_image is None: raise gr.Error("Please upload an input image.") num_frames = get_num_frames(duration_seconds) current_seed = random.randint(0, MAX_SEED) if randomize_seed else int(seed) sequential_renders = max(1, min(12, int(sequential_renders or 1))) resized_image = resize_image(input_image) processed_last_image = None if last_image: processed_last_image = resize_and_crop_to_match(last_image, resized_image) if artifact_dir: artifact_dir = os.path.abspath(artifact_dir) os.makedirs(artifact_dir, exist_ok=True) segment_paths = [] current_input = resized_image current_last_image = processed_last_image try: for segment_index in range(sequential_renders): if progress: progress(segment_index / sequential_renders, desc=f"Rendering segment {segment_index + 1}/{sequential_renders}") # This remains the established GPU call boundary: one complete # inference/export per segment, never one long combined sampling run. video_path, task_n = run_inference( current_input, current_last_image, prompt, steps, negative_prompt, num_frames, guidance_scale, guidance_scale_2, current_seed + segment_index, scheduler, flow_shift, frame_multiplier, quality, duration_seconds, lora_groups, progress, ) if artifact_dir: durable_segment = os.path.join(artifact_dir, f"segment_{segment_index + 1:02d}.mp4") shutil.copy2(video_path, durable_segment) if os.path.abspath(video_path) != durable_segment and os.path.exists(video_path): os.unlink(video_path) video_path = durable_segment segment_paths.append(video_path) print(f"GPU complete: {task_n} ({segment_index + 1}/{sequential_renders})") if segment_index + 1 < sequential_renders: current_input = resize_image(last_frame_of_video(video_path)) if artifact_dir: current_input.save(os.path.join(artifact_dir, f"handoff_frame_{segment_index + 1:02d}.png")) # The optional user-selected end image applies to the first clip # only; chained clips are steered by their predecessor frame. current_last_image = None video_path = stitch_videos(segment_paths, artifact_dir=artifact_dir) if progress: progress(1, desc="Joining chained segments") finally: # Queued jobs keep every durable segment and handoff frame in /data so # a failed join can be retried or retrieved. Direct UI runs still use # temporary storage and retain the old cleanup behavior. if not artifact_dir: for segment_path in segment_paths: if segment_path != locals().get("video_path") and os.path.exists(segment_path): os.unlink(segment_path) return (video_path if video_component else None), video_path, current_seed def _render_selected_loras(lora_groups): selected = [str(name) for name in (lora_groups or []) if name and name != "(None)"] if not selected: return "<span style='color:#888'>Selected LoRAs: None</span>" return "<span style='color:#888'>Selected LoRAs: <b>" + "</b>, <b>".join(selected) + "</b></span>" def save_custom_lora(label, url, stage, selected_loras): """Save a direct safetensors URL to the mounted Bucket and refresh UI.""" try: message = lora_loader.add_custom_lora(label, url, stage) selected = [x for x in (selected_loras or []) if x in lora_loader.get_all_lora_choices()] return gr.update(choices=lora_loader.get_all_lora_choices(), value=selected), message, "" except Exception as exc: return gr.update(), f"Could not save LoRA: {exc}", url GLASS_CSS = """ /* ── Hidden timestamp (preserved) ── */ #hidden-timestamp { opacity: 0; height: 0px; width: 0px; margin: 0px; padding: 0px; overflow: hidden; position: absolute; pointer-events: none; } /* ── BACKGROUND + AMBIENT GLOW — radial-gradient on body ── */ body { background: #0a0e1a !important; background-image: radial-gradient(ellipse 700px 700px at -5% 30%, rgba(0, 201, 167, 0.38), transparent 70%), radial-gradient(ellipse 800px 800px at 98% 10%, rgba(168, 85, 247, 0.32), transparent 70%), radial-gradient(ellipse 500px 500px at 50% 80%, rgba(0, 132, 227, 0.12), transparent 70%) !important; background-attachment: fixed !important; } .gradio-container { background: transparent !important; min-height: 100vh !important; } /* ── TITLE / SUBTITLE ── */ .glass-title { font-size: 28px !important; font-weight: 700 !important; color: #ffffff !important; margin-bottom: 6px !important; line-height: 1.2 !important; } .glass-subtitle { font-size: 14px !important; color: rgba(255, 255, 255, 0.45) !important; margin-bottom: 24px !important; line-height: 1.5 !important; } /* ── SECTION LABELS (gr.HTML injected) ── */ .glass-label { font-size: 11px !important; font-weight: 700 !important; text-transform: uppercase !important; letter-spacing: 1.5px !important; color: #5eead4 !important; margin-bottom: 8px !important; display: block !important; } /* ── LORA PREVIEW ── */ #selected-lora-preview { color: rgba(255,255,255,0.5) !important; } #selected-lora-preview b { color: #5eead4 !important; } /* ── STATUS TEXT ── */ .glass-status { color: rgba(255,255,255,0.40) !important; font-size: 13px !important; text-align: center !important; margin-top: 12px !important; } .glass-status strong { color: rgba(255,255,255,0.70) !important; } /* ── INPUT PANEL glass ── */ #input-panel { background: rgba(15, 20, 35, 0.55) !important; border: 1px solid rgba(255, 255, 255, 0.10) !important; border-radius: 16px !important; box-shadow: 0 8px 32px rgba(0, 0, 0, 0.35) !important; padding: 28px !important; } /* ── QUEUE PANEL STYLES ── */ .queue-container { background: rgba(15, 20, 35, 0.55) !important; border: 1px solid rgba(255, 255, 255, 0.10) !important; border-radius: 16px !important; padding: 20px !important; max-height: 520px; overflow-y: auto; margin-bottom: 20px; } .queue-header-bar { display: flex; justify-content: space-between; padding-bottom: 10px; margin-bottom: 12px; border-bottom: 1px solid rgba(255, 255, 255, 0.1); color: #5eead4; font-weight: bold; font-size: 13px; } .queue-card { background: rgba(255, 255, 255, 0.04); border: 1px solid rgba(255, 255, 255, 0.08); border-radius: 12px; padding: 14px; margin-bottom: 10px; transition: all 0.2s ease; } .queue-card:hover { border-color: rgba(94, 234, 212, 0.4); background: rgba(255, 255, 255, 0.06); } .queue-card-header { display: flex; justify-content: space-between; align-items: center; margin-bottom: 6px; } .queue-job-id { font-family: monospace; font-size: 12px; color: rgba(255, 255, 255, 0.7); } .queue-badge { padding: 3px 10px; border-radius: 12px; font-size: 11px; font-weight: bold; text-transform: uppercase; } .badge-queued { background: rgba(255, 193, 7, 0.2); color: #ffc107; border: 1px solid rgba(255, 193, 7, 0.4); } .badge-running { background: rgba(13, 202, 240, 0.2); color: #0dcaf0; border: 1px solid rgba(13, 202, 240, 0.4); } .badge-completed { background: rgba(25, 135, 84, 0.2); color: #20c997; border: 1px solid rgba(25, 135, 84, 0.4); } .badge-failed { background: rgba(220, 53, 69, 0.2); color: #ea868f; border: 1px solid rgba(220, 53, 69, 0.4); } .queue-prompt { color: #ffffff; font-size: 13px; margin-bottom: 4px; word-break: break-word; } .queue-loras { color: #c4b5fd; font-size: 12px; margin-bottom: 4px; word-break: break-word; } .queue-meta { color: rgba(255, 255, 255, 0.45); font-size: 11px; } .btn-dl-link { display: inline-block; padding: 6px 14px; background: linear-gradient(135deg, #00b894, #0984e3); color: #ffffff !important; text-decoration: none !important; border-radius: 8px; font-size: 12px; font-weight: bold; } .btn-preview-link { display: inline-block; padding: 6px 14px; background: linear-gradient(135deg, #6c5ce7, #a855f7); color: #ffffff !important; border: none !important; border-radius: 8px; font-size: 12px; font-weight: bold; cursor: pointer; transition: all 0.2s ease; } .btn-preview-link:hover { opacity: 0.9; box-shadow: 0 2px 10px rgba(168, 85, 247, 0.4); } .btn-view-link { display: inline-block; padding: 6px 14px; background: rgba(255, 255, 255, 0.1); color: #ffffff !important; text-decoration: none !important; border-radius: 8px; font-size: 12px; font-weight: bold; } .btn-dl-link:hover { opacity: 0.9; } .btn-view-link:hover { background: rgba(255, 255, 255, 0.2); } .queue-processing { color: #0dcaf0; font-size: 12px; font-style: italic; margin-top: 6px; } .queue-pending { color: #ffc107; font-size: 12px; font-style: italic; margin-top: 6px; } .queue-error { color: #ea868f; font-size: 12px; margin-top: 6px; } .queue-empty { text-align: center; color: rgba(255,255,255,0.4); padding: 40px 20px; font-size: 14px; } /* ── HIDE footer ── */ .gradio-container footer { display: none !important; } """ GLASS_JS = """ <script> (function(){ function glassApply(){ var teal='rgba(94,234,212,0.85)'; var tealDark='rgba(94,234,212,0.95)'; /* Gradio 6 labels: <span data-testid="block-info"> */ document.querySelectorAll('[data-testid="block-info"]').forEach(function(el){ el.style.cssText+='text-transform:uppercase!important;font-size:11px!important;font-weight:700!important;letter-spacing:1px!important;color:'+teal+'!important'; }); /* Gradio 6 accordion */ document.querySelectorAll('button.label-wrap').forEach(function(el){ el.style.cssText+='text-transform:uppercase!important;color:'+teal+'!important;font-weight:600!important;letter-spacing:1px!important;font-size:12px!important;background:rgba(255,255,255,0.04)!important;border:1px solid rgba(255,255,255,0.08)!important;border-radius:12px!important;padding:12px 16px!important'; }); /* Gradio 6 checkbox labels */ document.querySelectorAll('[data-testid="block-label"]').forEach(function(el){ el.style.cssText+='text-transform:uppercase!important;font-size:11px!important;font-weight:700!important;letter-spacing:1px!important;color:'+tealDark+'!important'; }); /* Glass panels */ document.querySelectorAll('#input-panel').forEach(function(el){ el.style.cssText+='background:rgba(15,20,35,0.55)!important;border:1px solid rgba(255,255,255,0.10)!important;border-radius:16px!important;box-shadow:0 8px 32px rgba(0,0,0,0.35)!important;padding:28px!important'; }); } function checkAutoDownloads() { var pending = document.querySelectorAll('.pending-auto-download'); pending.forEach(function(el) { var url = el.getAttribute('data-download-url'); var filename = el.getAttribute('data-filename') || 'video.mp4'; var jobId = el.getAttribute('data-job-id'); var downloadedKey = 'downloaded_' + jobId; if (url && !sessionStorage.getItem(downloadedKey)) { sessionStorage.setItem(downloadedKey, 'true'); console.log('Auto-downloading video:', jobId, url); var a = document.createElement('a'); a.href = url; a.download = filename; document.body.appendChild(a); a.click(); document.body.removeChild(a); } }); } window.loadPreview = function(url) { console.log('Preview requested for:', url); var inputEl = document.querySelector('#preview-video-path textarea, #preview-video-path input'); var btn = document.querySelector('#trigger-preview-btn'); if (inputEl) { inputEl.value = url; inputEl.dispatchEvent(new Event('input', { bubbles: true })); inputEl.dispatchEvent(new Event('change', { bubbles: true })); setTimeout(function() { if (btn) btn.click(); }, 80); } }; if(document.readyState==='loading'){ document.addEventListener('DOMContentLoaded',function(){ setTimeout(glassApply,500); setInterval(checkAutoDownloads, 1500); }); } else { setTimeout(glassApply,500); setInterval(checkAutoDownloads, 1500); } setInterval(glassApply,2000); })(); </script> """ with gr.Blocks(delete_cache=(3600, 10800)) as demo: # Hidden bridge components for preview playback preview_video_path = gr.Textbox(visible=False, elem_id="preview-video-path") trigger_preview_btn = gr.Button("Trigger Preview", visible=False, elem_id="trigger-preview-btn") # ── TITLE & SUBTITLE ── gr.HTML("<div class='glass-title'>Wan 2.2 I2V 14B Lightning — Private GPU Workstation</div>") gr.HTML("<div class='glass-subtitle'>Run Wan 2.2 Image-to-Video generation with persistent background task queue & automatic downloads</div>") with gr.Row(equal_height=False): # ═══════════════ LEFT COLUMN — GLASS INPUT PANEL ═══════════════ with gr.Column(scale=4): gr.HTML("<div id='input-panel'><div class='glass-label'>Prompt</div>") prompt_input = gr.Textbox( label="", value=default_prompt_i2v, lines=4, show_label=False, placeholder="Describe the motion you want...", ) selected_lora_preview = gr.HTML( value=_render_selected_loras([]), elem_id="selected-lora-preview", ) gr.HTML("<div class='glass-label'>Input Images (Single or Multiple)</div>") input_image_component = gr.File( label="", file_count="multiple", file_types=["image"], show_label=False, ) duration_seconds_input = gr.Slider( minimum=MIN_DURATION, maximum=MAX_DURATION, step=0.1, value=10.0, label="Duration", ) sequential_renders_input = gr.Slider( minimum=1, maximum=12, step=1, value=1, label="Sequential Renders", info="Each render starts from the prior clip's final frame, then all clips are joined.", ) lora_dropdown = gr.Dropdown( choices=lora_loader.get_all_lora_choices(), label="LoRAs", multiselect=True, ) with gr.Accordion("Personal LoRA library", open=False): gr.Markdown( "Paste a CivitAI model page, a CivitAI download link, or a direct safetensors endpoint. " "It downloads once into the Space bucket and immediately joins the multi-select above. " "Name is optional for CivitAI; use the same name twice to attach HIGH and LOW files." ) custom_lora_name = gr.Textbox(label="LoRA name (optional for CivitAI)", placeholder="My character / motion LoRA") custom_lora_url = gr.Textbox(label="LoRA endpoint or CivitAI page", placeholder="https://civitai.red/models/... or https://.../model.safetensors") custom_lora_stage = gr.Dropdown( label="Wan stage", choices=[("Auto from filename", "auto"), ("High-noise transformer", "high"), ("Low-noise transformer", "low")], value="auto" ) save_custom_lora_button = gr.Button("Save to LoRA library", variant="secondary") custom_lora_status = gr.Markdown() with gr.Row(): add_queue_btn = gr.Button("Add to Queue", variant="primary") generate_button = gr.Button("Generate & Queue", variant="secondary") with gr.Accordion("Advanced Settings", open=False): last_image_component = gr.Image( type="pil", label="Last Image (Optional)", sources=["upload", "clipboard"], ) negative_prompt_input = gr.Textbox( label="Negative Prompt", value=default_negative_prompt, lines=2, ) with gr.Row(): quality_slider = gr.Slider(minimum=1, maximum=10, step=1, value=6, label="Quality") seed_input = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=42, interactive=True) randomize_seed_checkbox = gr.Checkbox(label="Randomize seed", value=True, interactive=True) with gr.Row(): steps_slider = gr.Slider(minimum=1, maximum=30, step=1, value=6, label="Steps") flow_shift_slider = gr.Slider(minimum=0.5, maximum=15.0, step=0.1, value=3.0, label="Flow Shift") guidance_scale_input = gr.Slider(minimum=0.0, maximum=10.0, step=0.5, value=1, label="Guidance Scale — high noise") guidance_scale_2_input = gr.Slider(minimum=0.0, maximum=10.0, step=0.5, value=1, label="Guidance Scale 2 — low noise") scheduler_dropdown = gr.Dropdown( label="Scheduler", choices=list(SCHEDULER_MAP.keys()), value="UniPCMultistep", ) frame_multi = gr.Dropdown( choices=[FIXED_FPS, FIXED_FPS*2, FIXED_FPS*4, FIXED_FPS*8], value=FIXED_FPS, label="Fluidity (FPS)", ) play_result_video = gr.Checkbox(label="Display result", value=True, interactive=True) gr.HTML("</div>") # close #input-panel # ═══════════════ RIGHT COLUMN — QUEUE & PREVIEW ═══════════════ with gr.Column(scale=6): gr.HTML("<div class='glass-label'>Queue Controls</div>") queue_status_text = gr.Markdown("Ready.") with gr.Row(): btn_add_que = gr.Button("Add to Queue", variant="primary") btn_start_que = gr.Button("Start Queue", variant="secondary") btn_check_que = gr.Button("Check Queue", variant="secondary") btn_stop_que = gr.Button("Stop Queue", variant="stop") with gr.Row(): btn_view_errors = gr.Button("View Error Log", variant="secondary") btn_restart_gradio = gr.Button("Restart Gradio", variant="secondary") btn_clear_queue = gr.Button("Clear Queue", variant="stop") error_log_output = gr.Textbox( label="Error Log", visible=False, lines=10, max_lines=20, interactive=False, elem_id="error-log-box" ) gr.HTML("<div class='glass-label' style='margin-top: 14px;'>Queue List</div>") queue_display_html = gr.HTML(value=queue_mgr.render_queue_html()) gr.HTML("<div class='glass-label'>Preview Player (On Demand)</div>") video_output = gr.Video( label="", autoplay=True, sources=["upload"], buttons=["download", "share"], interactive=True, elem_id="generated-video", ) with gr.Row(): grab_frame_btn = gr.Button("Use Current Frame as Input", variant="secondary") timestamp_box = gr.Number(value=0, label="Timestamp", visible=True, elem_id="hidden-timestamp") file_output = gr.File(label="Download File") # Periodic timer to update queue display every 3 seconds sync_timer = gr.Timer(3) sync_timer.tick( fn=queue_mgr.check_queue, inputs=[], outputs=[queue_status_text, queue_display_html] ) # Repopulate queue on browser refresh or initial load demo.load( fn=queue_mgr.check_queue, inputs=[], outputs=[queue_status_text, queue_display_html] ) # ═══════════════ EVENT WIRING ═══════════════ queue_inputs = [ input_image_component, last_image_component, prompt_input, steps_slider, negative_prompt_input, duration_seconds_input, sequential_renders_input, guidance_scale_input, guidance_scale_2_input, seed_input, randomize_seed_checkbox, quality_slider, scheduler_dropdown, flow_shift_slider, frame_multi, lora_dropdown ] def add_and_start_queue(*args): msg1, html = queue_mgr.add_job(*args) msg2, html = queue_mgr.start_queue(generate_video) return f"{msg1} {msg2}", html btn_add_que.click( fn=queue_mgr.add_job, inputs=queue_inputs, outputs=[queue_status_text, queue_display_html] ) add_queue_btn.click( fn=queue_mgr.add_job, inputs=queue_inputs, outputs=[queue_status_text, queue_display_html] ) generate_button.click( fn=add_and_start_queue, inputs=queue_inputs, outputs=[queue_status_text, queue_display_html] ) btn_start_que.click( fn=lambda: queue_mgr.start_queue(generate_video), inputs=[], outputs=[queue_status_text, queue_display_html] ) btn_stop_que.click( fn=queue_mgr.stop_queue, inputs=[], outputs=[queue_status_text, queue_display_html] ) btn_check_que.click( fn=queue_mgr.check_queue, inputs=[], outputs=[queue_status_text, queue_display_html] ) def toggle_error_log(): log_text = queue_mgr.get_error_log() return gr.update(value=log_text, visible=True) btn_view_errors.click( fn=toggle_error_log, inputs=[], outputs=[error_log_output] ) def restart_gradio(): import os, sys, threading def _restart(): import time; time.sleep(1) os.execv(sys.executable, [sys.executable] + sys.argv) threading.Thread(target=_restart, daemon=True).start() return "🔄 Restarting Gradio in 1 second…", queue_mgr.render_queue_html() btn_restart_gradio.click( fn=restart_gradio, inputs=[], outputs=[queue_status_text, queue_display_html] ) btn_clear_queue.click( fn=queue_mgr.clear_queue, inputs=[], outputs=[queue_status_text, queue_display_html] ) # Explicit preview trigger from Queue cards def load_preview_video(path): if not path: return None, None clean_path = path for prefix in ("/gradio_api/file=", "/file="): if clean_path.startswith(prefix): clean_path = clean_path[len(prefix):] break clean_path = os.path.abspath(clean_path) return clean_path, clean_path trigger_preview_btn.click( fn=load_preview_video, inputs=[preview_video_path], outputs=[video_output, file_output] ) lora_dropdown.change( fn=_render_selected_loras, inputs=[lora_dropdown], outputs=[selected_lora_preview], ) save_custom_lora_button.click( fn=save_custom_lora, inputs=[custom_lora_name, custom_lora_url, custom_lora_stage, lora_dropdown], outputs=[lora_dropdown, custom_lora_status, custom_lora_url], ) grab_frame_btn.click( fn=None, inputs=None, outputs=[timestamp_box], js=get_timestamp_js ) timestamp_box.change( fn=extract_frame, inputs=[video_output, timestamp_box], outputs=[input_image_component] ) if __name__ == "__main__": theme = gr.themes.Default().set( slider_color="#5eead4", slider_color_dark="#5eead4", checkbox_background_color_selected="#5eead4", checkbox_background_color_selected_dark="#5eead4", checkbox_border_color_selected="#5eead4", checkbox_border_color_focus="#5eead4", ) demo.queue().launch( server_name="0.0.0.0", server_port=7860, share=True, mcp_server=True, head=f"<style>{GLASS_CSS}</style>{GLASS_JS}", theme=theme, show_error=True, allowed_paths=["outputs", "/data/outputs"] )
import os
import sys
# Resolve bundled modules and downloaded assets from the app directory rather
# than depending on whatever directory the Space launcher chose as cwd.
APP_DIR = os.path.dirname(os.path.abspath(__file__))
if APP_DIR not in sys.path:
sys.path.insert(0, APP_DIR)
# Try importing spaces for Hugging Face ZeroGPU compatibility, fallback gracefully for private rented GPUs
try:
import spaces
HAS_SPACES = True
except Exception:
HAS_SPACES = False
class DummySpaces:
def GPU(self, duration=None, size=None):
def decorator(func):
return func
return decorator
def aoti_load(self, *args, **kwargs):
pass
spaces = DummySpaces()
import shutil
import subprocess
import copy
import random
import tempfile
import warnings
import time
import gc
import uuid
from queue_manager import QueueManager
from tqdm import tqdm
import cv2
import imageio_ffmpeg
import numpy as np
import torch
import torch._dynamo
from torch.nn import functional as F
from PIL import Image
import gradio as gr
from diffusers import (
FlowMatchEulerDiscreteScheduler,
SASolverScheduler,
DEISMultistepScheduler,
DPMSolverMultistepInverseScheduler,
UniPCMultistepScheduler,
DPMSolverMultistepScheduler,
DPMSolverSinglestepScheduler,
)
from diffusers.pipelines.wan.pipeline_wan_i2v import WanImageToVideoPipeline
from diffusers.utils.export_utils import export_to_video
from torchao.quantization import quantize_, Float8DynamicActivationFloat8WeightConfig, Int8WeightOnlyConfig
import aoti
import lora_loader
os.environ["TOKENIZERS_PARALLELISM"] = "true"
warnings.filterwarnings("ignore")
IS_ZERO_GPU = bool(os.getenv("SPACES_ZERO_GPU"))
# if IS_ZERO_GPU:
# print("Loading...")
# subprocess.run("rm -rf /data-nvme/zerogpu-offload/*", env={}, shell=True)
# --- FRAME EXTRACTION JS & LOGIC ---
# JS to grab timestamp from the output video
get_timestamp_js = """
function() {
// Select the video element specifically inside the component with id 'generated-video'
const video = document.querySelector('#generated-video video');
if (video) {
console.log("Video found! Time: " + video.currentTime);
return video.currentTime;
} else {
console.log("No video element found.");
return 0;
}
}
"""
def extract_frame(video_path, timestamp):
# Safety check: if no video is present
if not video_path:
return None
print(f"Extracting frame at timestamp: {timestamp}")
cap = cv2.VideoCapture(video_path)