Files changed (1) hide show
  1. app.py +732 -0
app.py ADDED
@@ -0,0 +1,732 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os; os.system('pip install --upgrade --no-deps spaces')
2
+ import spaces
3
+ import shutil
4
+ import subprocess
5
+ import sys
6
+ import copy
7
+ import random
8
+ import tempfile
9
+ import warnings
10
+ import time
11
+ import gc
12
+ import uuid
13
+ from tqdm import tqdm
14
+ import cv2
15
+ import numpy as np
16
+ import torch
17
+ import torch._dynamo
18
+ from huggingface_hub import list_models
19
+ from torch.nn import functional as F
20
+ from PIL import Image
21
+
22
+ import gradio as gr
23
+ from diffusers import (
24
+ FlowMatchEulerDiscreteScheduler,
25
+ SASolverScheduler,
26
+ DEISMultistepScheduler,
27
+ DPMSolverMultistepInverseScheduler,
28
+ UniPCMultistepScheduler,
29
+ DPMSolverMultistepScheduler,
30
+ DPMSolverSinglestepScheduler,
31
+ )
32
+ from diffusers.pipelines.wan.pipeline_wan_i2v import WanImageToVideoPipeline
33
+ from diffusers.utils.export_utils import export_to_video
34
+
35
+ from torchao.quantization import quantize_, Float8DynamicActivationFloat8WeightConfig, Int8WeightOnlyConfig
36
+ import aoti
37
+
38
+ os.environ["TOKENIZERS_PARALLELISM"] = "true"
39
+ warnings.filterwarnings("ignore")
40
+ IS_ZERO_GPU = bool(os.getenv("SPACES_ZERO_GPU"))
41
+
42
+ # if IS_ZERO_GPU:
43
+ # print("Loading...")
44
+ # subprocess.run("rm -rf /data-nvme/zerogpu-offload/*", env={}, shell=True)
45
+
46
+ # --- FRAME EXTRACTION JS & LOGIC ---
47
+
48
+ # JS to grab timestamp from the output video
49
+ get_timestamp_js = """
50
+ function() {
51
+ // Select the video element specifically inside the component with id 'generated-video'
52
+ const video = document.querySelector('#generated-video video');
53
+
54
+ if (video) {
55
+ console.log("Video found! Time: " + video.currentTime);
56
+ return video.currentTime;
57
+ } else {
58
+ console.log("No video element found.");
59
+ return 0;
60
+ }
61
+ }
62
+ """
63
+
64
+
65
+ def extract_frame(video_path, timestamp):
66
+ # Safety check: if no video is present
67
+ if not video_path:
68
+ return None
69
+
70
+ print(f"Extracting frame at timestamp: {timestamp}")
71
+
72
+ cap = cv2.VideoCapture(video_path)
73
+
74
+ if not cap.isOpened():
75
+ return None
76
+
77
+ # Calculate frame number
78
+ fps = cap.get(cv2.CAP_PROP_FPS)
79
+ target_frame_num = int(float(timestamp) * fps)
80
+
81
+ # Cap total frames to prevent errors at the very end of video
82
+ total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
83
+ if target_frame_num >= total_frames:
84
+ target_frame_num = total_frames - 1
85
+
86
+ # Set position
87
+ cap.set(cv2.CAP_PROP_POS_FRAMES, target_frame_num)
88
+ ret, frame = cap.read()
89
+ cap.release()
90
+
91
+ if ret:
92
+ # Convert from BGR (OpenCV) to RGB (Gradio)
93
+ # Gradio Image component handles Numpy array -> PIL conversion automatically
94
+ return cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
95
+
96
+ return None
97
+
98
+ # --- END FRAME EXTRACTION LOGIC ---
99
+
100
+
101
+ def clear_vram():
102
+ gc.collect()
103
+ torch.cuda.empty_cache()
104
+
105
+
106
+ # RIFE
107
+ if not os.path.exists("RIFEv4.26_0921.zip"):
108
+ print("Downloading RIFE Model...")
109
+ subprocess.run([
110
+ "wget", "-q",
111
+ "https://huggingface.co/r3gm/RIFE/resolve/main/RIFEv4.26_0921.zip",
112
+ "-O", "RIFEv4.26_0921.zip"
113
+ ], check=True)
114
+ subprocess.run(["unzip", "-o", "RIFEv4.26_0921.zip"], check=True)
115
+
116
+ # sys.path.append(os.getcwd())
117
+
118
+ from train_log.RIFE_HDv3 import Model
119
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
120
+ rife_model = Model()
121
+ rife_model.load_model("train_log", -1)
122
+ rife_model.eval()
123
+
124
+
125
+ @torch.no_grad()
126
+ def interpolate_bits(frames_np, multiplier=2, scale=1.0):
127
+ """
128
+ Interpolation maintaining Numpy Float 0-1 format.
129
+ Args:
130
+ frames_np: Numpy Array (Time, Height, Width, Channels) - Float32 [0.0, 1.0]
131
+ multiplier: int (2, 4, 8)
132
+ Returns:
133
+ List of Numpy Arrays (Height, Width, Channels) - Float32 [0.0, 1.0]
134
+ """
135
+
136
+ # Handle input shape
137
+ if isinstance(frames_np, list):
138
+ # Convert list of arrays to one big array for easier shape handling if needed,
139
+ # but here we just grab dims from first frame
140
+ T = len(frames_np)
141
+ H, W, C = frames_np[0].shape
142
+ else:
143
+ T, H, W, C = frames_np.shape
144
+
145
+ # 1. No Interpolation Case
146
+ if multiplier < 2:
147
+ # Just convert 4D array to list of 3D arrays
148
+ if isinstance(frames_np, np.ndarray):
149
+ return list(frames_np)
150
+ return frames_np
151
+
152
+ n_interp = multiplier - 1
153
+
154
+ # Pre-calc padding for RIFE (requires dimensions divisible by 32/scale)
155
+ tmp = max(128, int(128 / scale))
156
+ ph = ((H - 1) // tmp + 1) * tmp
157
+ pw = ((W - 1) // tmp + 1) * tmp
158
+ padding = (0, pw - W, 0, ph - H)
159
+
160
+ # Helper: Numpy (H, W, C) Float -> Tensor (1, C, H, W) Half
161
+ def to_tensor(frame_np):
162
+ # frame_np is float32 0-1
163
+ t = torch.from_numpy(frame_np).to(device)
164
+ # HWC -> CHW
165
+ t = t.permute(2, 0, 1).unsqueeze(0)
166
+ return F.pad(t, padding).half()
167
+
168
+ # Helper: Tensor (1, C, H, W) Half -> Numpy (H, W, C) Float
169
+ def from_tensor(tensor):
170
+ # Crop padding
171
+ t = tensor[0, :, :H, :W]
172
+ # CHW -> HWC
173
+ t = t.permute(1, 2, 0)
174
+ # Keep as float32, range 0-1
175
+ return t.float().cpu().numpy()
176
+
177
+ def make_inference(I0, I1, n):
178
+ if rife_model.version >= 3.9:
179
+ res = []
180
+ for i in range(n):
181
+ res.append(rife_model.inference(I0, I1, (i+1) * 1. / (n+1), scale))
182
+ return res
183
+ else:
184
+ middle = rife_model.inference(I0, I1, scale)
185
+ if n == 1:
186
+ return [middle]
187
+ first_half = make_inference(I0, middle, n=n//2)
188
+ second_half = make_inference(middle, I1, n=n//2)
189
+ if n % 2:
190
+ return [*first_half, middle, *second_half]
191
+ else:
192
+ return [*first_half, *second_half]
193
+
194
+ output_frames = []
195
+
196
+ # Process Frames
197
+ # Load first frame into GPU
198
+ I1 = to_tensor(frames_np[0])
199
+
200
+ total_steps = T - 1
201
+
202
+ with tqdm(total=total_steps, desc="Interpolating", unit="frame") as pbar:
203
+
204
+ for i in range(total_steps):
205
+ I0 = I1
206
+ # Add original frame to output
207
+ output_frames.append(from_tensor(I0))
208
+
209
+ # Load next frame
210
+ I1 = to_tensor(frames_np[i+1])
211
+
212
+ # Generate intermediate frames
213
+ mid_tensors = make_inference(I0, I1, n_interp)
214
+
215
+ # Append intermediate frames
216
+ for mid in mid_tensors:
217
+ output_frames.append(from_tensor(mid))
218
+
219
+ if (i + 1) % 50 == 0:
220
+ pbar.update(50)
221
+ pbar.update(total_steps % 50)
222
+
223
+ # Add the very last frame
224
+ output_frames.append(from_tensor(I1))
225
+
226
+ # Cleanup
227
+ del I0, I1, mid_tensors
228
+ torch.cuda.empty_cache()
229
+
230
+ return output_frames
231
+
232
+
233
+ # WAN
234
+
235
+ ORG_NAME = "TestOrganizationPleaseIgnore"
236
+ # MODEL_ID = "Wan-AI/Wan2.2-I2V-A14B-Diffusers"
237
+ MODEL_ID = os.getenv("REPO_ID") or random.choice(
238
+ list(list_models(author=ORG_NAME, filter='diffusers:WanImageToVideoPipeline'))
239
+ ).modelId
240
+ CACHE_DIR = os.path.expanduser("~/.cache/huggingface/")
241
+
242
+ LORA_MODELS = [
243
+ # {
244
+ # "repo_id": "exampleuser/example_lora_1",
245
+ # "high_tr": "example_lora_1_high.safetensors",
246
+ # "low_tr": "example_lora_1_low.safetensors",
247
+ # "high_scale": 0.5,
248
+ # "low_scale": 0.5
249
+ # },
250
+ # {
251
+ # "repo_id": "exampleuser/example_lora_2",
252
+ # "high_tr": "subfolder/example_lora_2_high.safetensors",
253
+ # "low_tr": "subfolder/example_lora_2_low.safetensors",
254
+ # "high_scale": 0.4,
255
+ # "low_scale": 0.4
256
+ # },
257
+ ]
258
+
259
+ MAX_DIM = 832
260
+ MIN_DIM = 480
261
+ SQUARE_DIM = 640
262
+ MULTIPLE_OF = 16
263
+ MAX_SEED = np.iinfo(np.int32).max
264
+
265
+ FIXED_FPS = 16
266
+ MIN_FRAMES_MODEL = 8
267
+ MAX_FRAMES_MODEL = 1280
268
+
269
+ MIN_DURATION = round(MIN_FRAMES_MODEL / FIXED_FPS, 1)
270
+ MAX_DURATION = round(MAX_FRAMES_MODEL / FIXED_FPS, 1)
271
+
272
+ SCHEDULER_MAP = {
273
+ "FlowMatchEulerDiscrete": FlowMatchEulerDiscreteScheduler,
274
+ "SASolver": SASolverScheduler,
275
+ "DEISMultistep": DEISMultistepScheduler,
276
+ "DPMSolverMultistepInverse": DPMSolverMultistepInverseScheduler,
277
+ "UniPCMultistep": UniPCMultistepScheduler,
278
+ "DPMSolverMultistep": DPMSolverMultistepScheduler,
279
+ "DPMSolverSinglestep": DPMSolverSinglestepScheduler,
280
+ }
281
+
282
+ pipe = WanImageToVideoPipeline.from_pretrained(
283
+ MODEL_ID,
284
+ torch_dtype=torch.bfloat16,
285
+ ).to('cuda')
286
+ original_scheduler = copy.deepcopy(pipe.scheduler)
287
+
288
+ for i, lora in enumerate(LORA_MODELS):
289
+ name_high_tr = lora["high_tr"].split(".")[0].split("/")[-1] + "Hh"
290
+ name_low_tr = lora["low_tr"].split(".")[0].split("/")[-1] + "Ll"
291
+
292
+ try:
293
+ pipe.load_lora_weights(
294
+ lora["repo_id"],
295
+ weight_name=lora["high_tr"],
296
+ adapter_name=name_high_tr
297
+ )
298
+
299
+ kwargs_lora = {"load_into_transformer_2": True}
300
+ pipe.load_lora_weights(
301
+ lora["repo_id"],
302
+ weight_name=lora["low_tr"],
303
+ adapter_name=name_low_tr,
304
+ **kwargs_lora
305
+ )
306
+
307
+ pipe.set_adapters([name_high_tr, name_low_tr], adapter_weights=[1.0, 1.0])
308
+
309
+ pipe.fuse_lora(adapter_names=[name_high_tr], lora_scale=lora["high_scale"], components=["transformer"])
310
+ pipe.fuse_lora(adapter_names=[name_low_tr], lora_scale=lora["low_scale"], components=["transformer_2"])
311
+
312
+ pipe.unload_lora_weights()
313
+
314
+ print(f"Applied: {lora['high_tr']}, hs={lora['high_scale']}/ls={lora['low_scale']}, {i+1}/{len(LORA_MODELS)}")
315
+ except Exception as e:
316
+ print("Error:", str(e))
317
+ print("Failed LoRA:", name_high_tr)
318
+ pipe.unload_lora_weights()
319
+
320
+ # if os.path.exists(CACHE_DIR):
321
+ # shutil.rmtree(CACHE_DIR)
322
+ # print("Deleted Hugging Face cache.")
323
+ # else:
324
+ # print("No hub cache found.")
325
+
326
+ quantize_(pipe.text_encoder, Int8WeightOnlyConfig())
327
+ torch._dynamo.reset()
328
+ quantize_(pipe.transformer, Float8DynamicActivationFloat8WeightConfig())
329
+ torch._dynamo.reset()
330
+ quantize_(pipe.transformer_2, Float8DynamicActivationFloat8WeightConfig())
331
+ torch._dynamo.reset()
332
+
333
+ spaces.aoti_load(
334
+ module=pipe.transformer,
335
+ repo_id='cbensimon/WanTransformer3DModel-sm120-cu130-raa',
336
+ )
337
+ spaces.aoti_load(
338
+ module=pipe.transformer_2,
339
+ repo_id='cbensimon/WanTransformer3DModel-sm120-cu130-raa',
340
+ )
341
+
342
+ # pipe.vae.enable_slicing()
343
+ # pipe.vae.enable_tiling()
344
+
345
+ default_prompt_i2v = "make this image come alive, cinematic motion, smooth animation"
346
+ default_negative_prompt = "色调艳丽, 过曝, 静态, 细节模糊不清, 字幕, 风格, 作品, 画作, 画面, 静止, 整体发灰, 最差质量, 低质量, JPEG压缩残留, 丑陋的, 残缺的, 多余的手指, 画得不好的手部, 画得不好的脸部, 畸形的, 毁容的, 形态畸形的肢体, 手指融合, 静止不动的画面, 杂乱的背景, 三条腿, 背景人很多, 倒着走"
347
+
348
+
349
+ def model_title():
350
+ repo_name = MODEL_ID.split('/')[-1].replace("_", " ")
351
+ url = f"https://huggingface.co/{MODEL_ID}"
352
+ return f"## This space is currently running [{repo_name}]({url}) 🐢"
353
+
354
+
355
+ def resize_image(image: Image.Image) -> Image.Image:
356
+ width, height = image.size
357
+ if width == height:
358
+ return image.resize((SQUARE_DIM, SQUARE_DIM), Image.LANCZOS)
359
+
360
+ aspect_ratio = width / height
361
+ MAX_ASPECT_RATIO = MAX_DIM / MIN_DIM
362
+ MIN_ASPECT_RATIO = MIN_DIM / MAX_DIM
363
+
364
+ image_to_resize = image
365
+ if aspect_ratio > MAX_ASPECT_RATIO:
366
+ target_w, target_h = MAX_DIM, MIN_DIM
367
+ crop_width = int(round(height * MAX_ASPECT_RATIO))
368
+ left = (width - crop_width) // 2
369
+ image_to_resize = image.crop((left, 0, left + crop_width, height))
370
+ elif aspect_ratio < MIN_ASPECT_RATIO:
371
+ target_w, target_h = MIN_DIM, MAX_DIM
372
+ crop_height = int(round(width / MIN_ASPECT_RATIO))
373
+ top = (height - crop_height) // 2
374
+ image_to_resize = image.crop((0, top, width, top + crop_height))
375
+ else:
376
+ if width > height:
377
+ target_w = MAX_DIM
378
+ target_h = int(round(target_w / aspect_ratio))
379
+ else:
380
+ target_h = MAX_DIM
381
+ target_w = int(round(target_h * aspect_ratio))
382
+
383
+ final_w = round(target_w / MULTIPLE_OF) * MULTIPLE_OF
384
+ final_h = round(target_h / MULTIPLE_OF) * MULTIPLE_OF
385
+ final_w = max(MIN_DIM, min(MAX_DIM, final_w))
386
+ final_h = max(MIN_DIM, min(MAX_DIM, final_h))
387
+ return image_to_resize.resize((final_w, final_h), Image.LANCZOS)
388
+
389
+
390
+ def resize_and_crop_to_match(target_image, reference_image):
391
+ ref_width, ref_height = reference_image.size
392
+ target_width, target_height = target_image.size
393
+ scale = max(ref_width / target_width, ref_height / target_height)
394
+ new_width, new_height = int(target_width * scale), int(target_height * scale)
395
+ resized = target_image.resize((new_width, new_height), Image.Resampling.LANCZOS)
396
+ left, top = (new_width - ref_width) // 2, (new_height - ref_height) // 2
397
+ return resized.crop((left, top, left + ref_width, top + ref_height))
398
+
399
+
400
+ def get_num_frames(duration_seconds: float):
401
+ return 1 + int(np.clip(
402
+ int(round(duration_seconds * FIXED_FPS)),
403
+ MIN_FRAMES_MODEL,
404
+ MAX_FRAMES_MODEL,
405
+ ))
406
+
407
+
408
+ def get_inference_duration(
409
+ resized_image,
410
+ processed_last_image,
411
+ prompt,
412
+ steps,
413
+ negative_prompt,
414
+ num_frames,
415
+ guidance_scale,
416
+ guidance_scale_2,
417
+ current_seed,
418
+ scheduler_name,
419
+ flow_shift,
420
+ frame_multiplier,
421
+ quality,
422
+ duration_seconds,
423
+ safe_mode,
424
+ progress
425
+ ):
426
+ BASE_FRAMES_HEIGHT_WIDTH = 81 * 832 * 624
427
+ BASE_STEP_DURATION = 5.
428
+ width, height = resized_image.size
429
+ factor = num_frames * width * height / BASE_FRAMES_HEIGHT_WIDTH
430
+ step_duration = BASE_STEP_DURATION * factor ** 1.5
431
+ gen_time = int(steps) * step_duration
432
+
433
+ if guidance_scale > 1:
434
+ gen_time = gen_time * 2.4
435
+
436
+ frame_factor = frame_multiplier // FIXED_FPS
437
+ if frame_factor > 1:
438
+ total_out_frames = (num_frames * frame_factor) - num_frames
439
+ inter_time = (total_out_frames * 0.02)
440
+ gen_time += inter_time
441
+
442
+ total_time = 15 + gen_time
443
+ if safe_mode:
444
+ total_time = total_time * 1.30
445
+
446
+ return total_time
447
+
448
+
449
+ @spaces.GPU(duration=get_inference_duration, size='xlarge')
450
+ def run_inference(
451
+ resized_image,
452
+ processed_last_image,
453
+ prompt,
454
+ steps,
455
+ negative_prompt,
456
+ num_frames,
457
+ guidance_scale,
458
+ guidance_scale_2,
459
+ current_seed,
460
+ scheduler_name,
461
+ flow_shift,
462
+ frame_multiplier,
463
+ quality,
464
+ duration_seconds,
465
+ safe_mode=False,
466
+ progress=gr.Progress(track_tqdm=True),
467
+ ):
468
+ scheduler_class = SCHEDULER_MAP.get(scheduler_name)
469
+ if scheduler_class.__name__ != pipe.scheduler.config._class_name or flow_shift != pipe.scheduler.config.get("flow_shift", "shift"):
470
+ config = copy.deepcopy(original_scheduler.config)
471
+ if scheduler_class == FlowMatchEulerDiscreteScheduler:
472
+ config['shift'] = flow_shift
473
+ else:
474
+ config['flow_shift'] = flow_shift
475
+ pipe.scheduler = scheduler_class.from_config(config)
476
+
477
+ clear_vram()
478
+
479
+ task_name = str(uuid.uuid4())[:8]
480
+ print(f"Generating {num_frames} frames, task: {task_name}, {duration_seconds}, {resized_image.size}")
481
+ start = time.time()
482
+ result = pipe(
483
+ image=resized_image,
484
+ last_image=processed_last_image,
485
+ prompt=prompt,
486
+ negative_prompt=negative_prompt,
487
+ height=resized_image.height,
488
+ width=resized_image.width,
489
+ num_frames=num_frames,
490
+ guidance_scale=float(guidance_scale),
491
+ guidance_scale_2=float(guidance_scale_2),
492
+ num_inference_steps=int(steps),
493
+ generator=torch.Generator(device="cuda").manual_seed(current_seed),
494
+ output_type="np"
495
+ )
496
+ print("gen time passed:", time.time() - start)
497
+
498
+ raw_frames_np = result.frames[0] # Returns (T, H, W, C) float32
499
+ pipe.scheduler = original_scheduler
500
+
501
+ frame_factor = frame_multiplier // FIXED_FPS
502
+ if frame_factor > 1:
503
+ start = time.time()
504
+ print(f"Processing frames (RIFE Multiplier: {frame_factor}x)...")
505
+ rife_model.device()
506
+ rife_model.flownet = rife_model.flownet.half()
507
+ final_frames = interpolate_bits(raw_frames_np, multiplier=int(frame_factor))
508
+ print("Interpolation time passed:", time.time() - start)
509
+ else:
510
+ final_frames = list(raw_frames_np)
511
+
512
+ final_fps = FIXED_FPS * int(frame_factor)
513
+
514
+ with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as tmpfile:
515
+ video_path = tmpfile.name
516
+
517
+ start = time.time()
518
+ with tqdm(total=3, desc="Rendering Media", unit="clip") as pbar:
519
+ pbar.update(2)
520
+ export_to_video(final_frames, video_path, fps=final_fps, quality=quality)
521
+ pbar.update(1)
522
+ print(f"Export time passed, {final_fps} FPS:", time.time() - start)
523
+
524
+ return video_path, task_name
525
+
526
+
527
+ def generate_video(
528
+ input_image,
529
+ last_image,
530
+ prompt,
531
+ steps=4,
532
+ negative_prompt=default_negative_prompt,
533
+ duration_seconds=MAX_DURATION,
534
+ guidance_scale=1,
535
+ guidance_scale_2=1,
536
+ seed=42,
537
+ randomize_seed=False,
538
+ quality=5,
539
+ scheduler="UniPCMultistep",
540
+ flow_shift=6.0,
541
+ frame_multiplier=16,
542
+ safe_mode=False,
543
+ video_component=True,
544
+ progress=gr.Progress(track_tqdm=True),
545
+ ):
546
+ """
547
+ Generate a video from an input image using the Wan 2.2 14B I2V model with Lightning LoRA.
548
+ This function takes an input image and generates a video animation based on the provided
549
+ prompt and parameters. It uses an FP8 qunatized Wan 2.2 14B Image-to-Video model in with Lightning LoRA
550
+ for fast generation in 4-8 steps.
551
+ Args:
552
+ input_image (PIL.Image): The input image to animate. Will be resized to target dimensions.
553
+ last_image (PIL.Image, optional): The optional last image for the video.
554
+ prompt (str): Text prompt describing the desired animation or motion.
555
+ steps (int, optional): Number of inference steps. More steps = higher quality but slower.
556
+ Defaults to 4. Range: 1-30.
557
+ negative_prompt (str, optional): Negative prompt to avoid unwanted elements.
558
+ Defaults to default_negative_prompt (contains unwanted visual artifacts).
559
+ duration_seconds (float, optional): Duration of the generated video in seconds.
560
+ Defaults to 2. Clamped between MIN_FRAMES_MODEL/FIXED_FPS and MAX_FRAMES_MODEL/FIXED_FPS.
561
+ guidance_scale (float, optional): Controls adherence to the prompt. Higher values = more adherence.
562
+ Defaults to 1.0. Range: 0.0-20.0.
563
+ guidance_scale_2 (float, optional): Controls adherence to the prompt. Higher values = more adherence.
564
+ Defaults to 1.0. Range: 0.0-20.0.
565
+ seed (int, optional): Random seed for reproducible results. Defaults to 42.
566
+ Range: 0 to MAX_SEED (2147483647).
567
+ randomize_seed (bool, optional): Whether to use a random seed instead of the provided seed.
568
+ Defaults to False.
569
+ quality (float, optional): Video output quality. Default is 5. Uses variable bit rate.
570
+ Highest quality is 10, lowest is 1.
571
+ scheduler (str, optional): The name of the scheduler to use for inference. Defaults to "UniPCMultistep".
572
+ flow_shift (float, optional): The flow shift value for compatible schedulers. Defaults to 6.0.
573
+ frame_multiplier (int, optional): The int value for fps enhancer
574
+ video_component(bool, optional): Show video player in output.
575
+ Defaults to True.
576
+ progress (gr.Progress, optional): Gradio progress tracker. Defaults to gr.Progress(track_tqdm=True).
577
+ Returns:
578
+ tuple: A tuple containing:
579
+ - video_path (str): Path for the video component.
580
+ - video_path (str): Path for the file download component. Attempt to avoid reconversion in video component.
581
+ - current_seed (int): The seed used for generation.
582
+ Raises:
583
+ gr.Error: If input_image is None (no image uploaded).
584
+ Note:
585
+ - Frame count is calculated as duration_seconds * FIXED_FPS (24)
586
+ - Output dimensions are adjusted to be multiples of MOD_VALUE (32)
587
+ - The function uses GPU acceleration via the @spaces.GPU decorator
588
+ - Generation time varies based on steps and duration (see get_duration function)
589
+ """
590
+
591
+ if input_image is None:
592
+ raise gr.Error("Please upload an input image.")
593
+
594
+ num_frames = get_num_frames(duration_seconds)
595
+ current_seed = random.randint(0, MAX_SEED) if randomize_seed else int(seed)
596
+ resized_image = resize_image(input_image)
597
+
598
+ processed_last_image = None
599
+ if last_image:
600
+ processed_last_image = resize_and_crop_to_match(last_image, resized_image)
601
+
602
+ video_path, task_n = run_inference(
603
+ resized_image,
604
+ processed_last_image,
605
+ prompt,
606
+ steps,
607
+ negative_prompt,
608
+ num_frames,
609
+ guidance_scale,
610
+ guidance_scale_2,
611
+ current_seed,
612
+ scheduler,
613
+ flow_shift,
614
+ frame_multiplier,
615
+ quality,
616
+ duration_seconds,
617
+ safe_mode,
618
+ progress,
619
+ )
620
+ print(f"GPU complete: {task_n}")
621
+
622
+ return (video_path if video_component else None), video_path, current_seed
623
+
624
+
625
+ CSS = """
626
+ #hidden-timestamp {
627
+ opacity: 0;
628
+ height: 0px;
629
+ width: 0px;
630
+ margin: 0px;
631
+ padding: 0px;
632
+ overflow: hidden;
633
+ position: absolute;
634
+ pointer-events: none;
635
+ }
636
+ """
637
+
638
+
639
+ with gr.Blocks(delete_cache=(3600, 10800)) as demo:
640
+ gr.Markdown(model_title())
641
+ gr.Markdown("Run Wan 2.2 in just 4-8 steps, fp8 quantization & AoT compilation - compatible with 🧨 diffusers and ZeroGPU")
642
+
643
+ with gr.Row():
644
+ with gr.Column():
645
+ input_image_component = gr.Image(type="pil", label="Input Image", sources=["upload", "clipboard"])
646
+ prompt_input = gr.Textbox(label="Prompt", value=default_prompt_i2v)
647
+ duration_seconds_input = gr.Slider(minimum=MIN_DURATION, maximum=MAX_DURATION, step=0.1, value=3.5, label="Duration (seconds)", info=f"Clamped to model's {MIN_FRAMES_MODEL}-{MAX_FRAMES_MODEL} frames at {FIXED_FPS}fps.")
648
+ frame_multi = gr.Dropdown(
649
+ choices=[FIXED_FPS, FIXED_FPS*2, FIXED_FPS*4, FIXED_FPS*8],
650
+ value=FIXED_FPS,
651
+ label="Video Fluidity (Frames per Second)",
652
+ info="Extra frames will be generated using flow estimation, which estimates motion between frames to make the video smoother."
653
+ )
654
+ safe_mode_checkbox = gr.Checkbox(
655
+ label="🛠️ Safe Mode",
656
+ value=True,
657
+ info="Requests 30% extra processing time to try to prevent unfinished tasks when the server is busy."
658
+ )
659
+ with gr.Accordion("Advanced Settings", open=False):
660
+ last_image_component = gr.Image(type="pil", label="Last Image (Optional)", sources=["upload", "clipboard"])
661
+ negative_prompt_input = gr.Textbox(label="Negative Prompt", value=default_negative_prompt, info="Used if any Guidance Scale > 1.", lines=3)
662
+ quality_slider = gr.Slider(minimum=1, maximum=10, step=1, value=6, label="Video Quality", info="If set to 10, the generated video may be too large and won't play in the Gradio preview.")
663
+ seed_input = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=42, interactive=True)
664
+ randomize_seed_checkbox = gr.Checkbox(label="Randomize seed", value=True, interactive=True)
665
+ steps_slider = gr.Slider(minimum=1, maximum=30, step=1, value=6, label="Inference Steps")
666
+ guidance_scale_input = gr.Slider(minimum=0.0, maximum=10.0, step=0.5, value=1, label="Guidance Scale - high noise stage", info="Values above 1 increase GPU usage and may take longer to process.")
667
+ guidance_scale_2_input = gr.Slider(minimum=0.0, maximum=10.0, step=0.5, value=1, label="Guidance Scale 2 - low noise stage")
668
+ scheduler_dropdown = gr.Dropdown(
669
+ label="Scheduler",
670
+ choices=list(SCHEDULER_MAP.keys()),
671
+ value="UniPCMultistep",
672
+ info="Select a custom scheduler."
673
+ )
674
+ flow_shift_slider = gr.Slider(minimum=0.5, maximum=15.0, step=0.1, value=3.0, label="Flow Shift")
675
+ play_result_video = gr.Checkbox(label="Display result", value=True, interactive=True)
676
+ gr.Markdown(f"[ZeroGPU help, tips and troubleshooting](https://huggingface.co/datasets/{ORG_NAME}/help/blob/main/gpu_help.md)")
677
+ gr.Markdown( # TestOrganizationPleaseIgnore/wamu-tools
678
+ "To use a different model, **duplicate this Space** first, then change the `REPO_ID` environment variable. "
679
+ "[See compatible models here](https://huggingface.co/models?other=diffusers:WanImageToVideoPipeline&sort=trending&search=WAN2.2_I2V_LIGHTNING)."
680
+ )
681
+
682
+ generate_button = gr.Button("Generate Video", variant="primary")
683
+
684
+ with gr.Column():
685
+ # ASSIGNED elem_id="generated-video" so JS can find it
686
+ video_output = gr.Video(label="Generated Video", autoplay=True, sources=["upload"], buttons=["download", "share"], interactive=True, elem_id="generated-video")
687
+
688
+ # --- Frame Grabbing UI ---
689
+ with gr.Row():
690
+ grab_frame_btn = gr.Button("📸 Use Current Frame as Input", variant="secondary")
691
+ timestamp_box = gr.Number(value=0, label="Timestamp", visible=True, elem_id="hidden-timestamp")
692
+ # -------------------------
693
+
694
+ file_output = gr.File(label="Download Video")
695
+
696
+ ui_inputs = [
697
+ input_image_component, last_image_component, prompt_input, steps_slider,
698
+ negative_prompt_input, duration_seconds_input,
699
+ guidance_scale_input, guidance_scale_2_input, seed_input, randomize_seed_checkbox,
700
+ quality_slider, scheduler_dropdown, flow_shift_slider, frame_multi,
701
+ safe_mode_checkbox,
702
+ play_result_video
703
+ ]
704
+
705
+ generate_button.click(
706
+ fn=generate_video,
707
+ inputs=ui_inputs,
708
+ outputs=[video_output, file_output, seed_input]
709
+ )
710
+
711
+ # --- Frame Grabbing Events ---
712
+ # 1. Click button -> JS runs -> puts time in hidden number box
713
+ grab_frame_btn.click(
714
+ fn=None,
715
+ inputs=None,
716
+ outputs=[timestamp_box],
717
+ js=get_timestamp_js
718
+ )
719
+
720
+ # 2. Hidden number box changes -> Python runs -> puts frame in Input Image
721
+ timestamp_box.change(
722
+ fn=extract_frame,
723
+ inputs=[video_output, timestamp_box],
724
+ outputs=[input_image_component]
725
+ )
726
+
727
+ if __name__ == "__main__":
728
+ demo.queue().launch(
729
+ mcp_server=True,
730
+ css=CSS,
731
+ show_error=True,
732
+ )