Spaces:
Running on Zero
Running on Zero
Update app.py
Browse files
app.py
CHANGED
|
@@ -31,10 +31,6 @@ from diffusers import (
|
|
| 31 |
from diffusers.pipelines.wan.pipeline_wan_i2v import WanImageToVideoPipeline
|
| 32 |
from diffusers.utils.export_utils import export_to_video
|
| 33 |
|
| 34 |
-
# Safety Checker Imports
|
| 35 |
-
from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker
|
| 36 |
-
from transformers import AutoImageProcessor
|
| 37 |
-
|
| 38 |
from torchao.quantization import quantize_, Float8DynamicActivationFloat8WeightConfig, Int8WeightOnlyConfig
|
| 39 |
import aoti
|
| 40 |
|
|
@@ -46,18 +42,6 @@ IS_ZERO_GPU = bool(os.getenv("SPACES_ZERO_GPU"))
|
|
| 46 |
# print("Loading...")
|
| 47 |
# subprocess.run("rm -rf /data-nvme/zerogpu-offload/*", env={}, shell=True)
|
| 48 |
|
| 49 |
-
# --- SAFETY CHECKER INITIALIZATION ---
|
| 50 |
-
safety_checker = StableDiffusionSafetyChecker.from_pretrained("CompVis/stable-diffusion-safety-checker").to("cuda")
|
| 51 |
-
feature_extractor = AutoImageProcessor.from_pretrained("CompVis/stable-diffusion-safety-checker")
|
| 52 |
-
|
| 53 |
-
def check_nsfw(frame_np):
|
| 54 |
-
"""Checks a single numpy frame (float32, 0-1) for NSFW content."""
|
| 55 |
-
# Convert back to uint8 PIL for the processor
|
| 56 |
-
img = Image.fromarray((frame_np * 255).astype(np.uint8))
|
| 57 |
-
inputs = feature_extractor(img, return_tensors="pt").to("cuda")
|
| 58 |
-
# Safety checker expects float16 on GPU
|
| 59 |
-
_, has_nsfw_concept = safety_checker(images=inputs.pixel_values, clip_input=inputs.pixel_values.to(torch.float16))
|
| 60 |
-
return has_nsfw_concept[0]
|
| 61 |
|
| 62 |
# --- FRAME EXTRACTION JS & LOGIC ---
|
| 63 |
|
|
@@ -373,7 +357,6 @@ def get_inference_duration(
|
|
| 373 |
quality,
|
| 374 |
duration_seconds,
|
| 375 |
safe_mode,
|
| 376 |
-
enable_safety_checker,
|
| 377 |
progress
|
| 378 |
):
|
| 379 |
BASE_FRAMES_HEIGHT_WIDTH = 81 * 832 * 624
|
|
@@ -417,7 +400,6 @@ def run_inference(
|
|
| 417 |
quality,
|
| 418 |
duration_seconds,
|
| 419 |
safe_mode,
|
| 420 |
-
enable_safety_checker,
|
| 421 |
progress=gr.Progress(track_tqdm=True),
|
| 422 |
):
|
| 423 |
scheduler_class = SCHEDULER_MAP.get(scheduler_name)
|
|
@@ -452,18 +434,6 @@ def run_inference(
|
|
| 452 |
raw_frames_np = result.frames[0] # Returns (T, H, W, C) float32
|
| 453 |
pipe.scheduler = original_scheduler
|
| 454 |
|
| 455 |
-
# --- SAFETY CHECKER LOGIC ---
|
| 456 |
-
is_nsfw = False
|
| 457 |
-
if enable_safety_checker:
|
| 458 |
-
# Implements an automated review process to check the results before making them publicly available.
|
| 459 |
-
# If the user did not supply a 'last_image', the final freely-generated frame is verified by
|
| 460 |
-
# the safety checker to ensure no unrequested explicit content was created.
|
| 461 |
-
if processed_last_image is None:
|
| 462 |
-
is_nsfw = check_nsfw(raw_frames_np[-1])
|
| 463 |
-
|
| 464 |
-
if is_nsfw:
|
| 465 |
-
return None, task_name, True
|
| 466 |
-
|
| 467 |
frame_factor = frame_multiplier // FIXED_FPS
|
| 468 |
if frame_factor > 1:
|
| 469 |
start = time.time()
|
|
@@ -484,7 +454,7 @@ def run_inference(
|
|
| 484 |
export_to_video(final_frames, video_path, fps=final_fps, quality=quality)
|
| 485 |
pbar.update(1)
|
| 486 |
|
| 487 |
-
return video_path, task_name
|
| 488 |
|
| 489 |
|
| 490 |
def generate_video(
|
|
@@ -504,54 +474,8 @@ def generate_video(
|
|
| 504 |
frame_multiplier=16,
|
| 505 |
video_component=True,
|
| 506 |
safe_mode=False,
|
| 507 |
-
enable_safety_checker=True,
|
| 508 |
progress=gr.Progress(track_tqdm=True),
|
| 509 |
):
|
| 510 |
-
"""
|
| 511 |
-
Generate a video from an input image using the Wan 2.2 14B I2V model with Lightning LoRA.
|
| 512 |
-
This function takes an input image and generates a video animation based on the provided
|
| 513 |
-
prompt and parameters. It uses an FP8 qunatized Wan 2.2 14B Image-to-Video model in with Lightning LoRA
|
| 514 |
-
for fast generation in 4-8 steps.
|
| 515 |
-
Args:
|
| 516 |
-
input_image (PIL.Image): The input image to animate. Will be resized to target dimensions.
|
| 517 |
-
last_image (PIL.Image, optional): The optional last image for the video.
|
| 518 |
-
prompt (str): Text prompt describing the desired animation or motion.
|
| 519 |
-
steps (int, optional): Number of inference steps. More steps = higher quality but slower.
|
| 520 |
-
Defaults to 4. Range: 1-30.
|
| 521 |
-
negative_prompt (str, optional): Negative prompt to avoid unwanted elements.
|
| 522 |
-
Defaults to default_negative_prompt (contains unwanted visual artifacts).
|
| 523 |
-
duration_seconds (float, optional): Duration of the generated video in seconds.
|
| 524 |
-
Defaults to 2. Clamped between MIN_FRAMES_MODEL/FIXED_FPS and MAX_FRAMES_MODEL/FIXED_FPS.
|
| 525 |
-
guidance_scale (float, optional): Controls adherence to the prompt. Higher values = more adherence.
|
| 526 |
-
Defaults to 1.0. Range: 0.0-20.0.
|
| 527 |
-
guidance_scale_2 (float, optional): Controls adherence to the prompt. Higher values = more adherence.
|
| 528 |
-
Defaults to 1.0. Range: 0.0-20.0.
|
| 529 |
-
seed (int, optional): Random seed for reproducible results. Defaults to 42.
|
| 530 |
-
Range: 0 to MAX_SEED (2147483647).
|
| 531 |
-
randomize_seed (bool, optional): Whether to use a random seed instead of the provided seed.
|
| 532 |
-
Defaults to False.
|
| 533 |
-
quality (float, optional): Video output quality. Default is 5. Uses variable bit rate.
|
| 534 |
-
Highest quality is 10, lowest is 1.
|
| 535 |
-
scheduler (str, optional): The name of the scheduler to use for inference. Defaults to "UniPCMultistep".
|
| 536 |
-
flow_shift (float, optional): The flow shift value for compatible schedulers. Defaults to 6.0.
|
| 537 |
-
frame_multiplier (int, optional): The int value for fps enhancer
|
| 538 |
-
video_component(bool, optional): Show video player in output.
|
| 539 |
-
Defaults to True.
|
| 540 |
-
enable_safety_checker(bool, optional): Enable NSFW filter.
|
| 541 |
-
progress (gr.Progress, optional): Gradio progress tracker. Defaults to gr.Progress(track_tqdm=True).
|
| 542 |
-
Returns:
|
| 543 |
-
tuple: A tuple containing:
|
| 544 |
-
- video_path (str): Path for the video component.
|
| 545 |
-
- video_path (str): Path for the file download component. Attempt to avoid reconversion in video component.
|
| 546 |
-
- current_seed (int): The seed used for generation.
|
| 547 |
-
Raises:
|
| 548 |
-
gr.Error: If input_image is None (no image uploaded).
|
| 549 |
-
Note:
|
| 550 |
-
- Frame count is calculated as duration_seconds * FIXED_FPS (24)
|
| 551 |
-
- Output dimensions are adjusted to be multiples of MOD_VALUE (32)
|
| 552 |
-
- The function uses GPU acceleration via the @spaces.GPU decorator
|
| 553 |
-
- Generation time varies based on steps and duration (see get_duration function)
|
| 554 |
-
"""
|
| 555 |
|
| 556 |
if input_image is None:
|
| 557 |
raise gr.Error("Please upload an input image.")
|
|
@@ -564,7 +488,7 @@ def generate_video(
|
|
| 564 |
if last_image:
|
| 565 |
processed_last_image = resize_and_crop_to_match(last_image, resized_image)
|
| 566 |
|
| 567 |
-
video_path, task_n
|
| 568 |
resized_image,
|
| 569 |
processed_last_image,
|
| 570 |
prompt,
|
|
@@ -580,13 +504,9 @@ def generate_video(
|
|
| 580 |
quality,
|
| 581 |
duration_seconds,
|
| 582 |
safe_mode,
|
| 583 |
-
enable_safety_checker,
|
| 584 |
progress,
|
| 585 |
)
|
| 586 |
|
| 587 |
-
if is_nsfw:
|
| 588 |
-
raise gr.Error("Generation blocked by guardrails: The resulting video may contain explicit content.")
|
| 589 |
-
|
| 590 |
print(f"GPU complete: {task_n}")
|
| 591 |
|
| 592 |
return (video_path if video_component else None), video_path, current_seed
|
|
@@ -643,7 +563,6 @@ with gr.Blocks(theme=gr.themes.Soft(), css=CSS, delete_cache=(3600, 3700)) as de
|
|
| 643 |
info="Select a custom scheduler."
|
| 644 |
)
|
| 645 |
flow_shift_slider = gr.Slider(minimum=0.5, maximum=15.0, step=0.1, value=3.0, label="Flow Shift")
|
| 646 |
-
safety_checker_input = gr.Checkbox(label="Enable Safety Filter", value=True, info="Prevents unrequested sensitive or explicit content.")
|
| 647 |
play_result_video = gr.Checkbox(label="Display result", value=True, interactive=True)
|
| 648 |
|
| 649 |
generate_button = gr.Button("Generate Video", variant="primary")
|
|
@@ -662,37 +581,4 @@ with gr.Blocks(theme=gr.themes.Soft(), css=CSS, delete_cache=(3600, 3700)) as de
|
|
| 662 |
|
| 663 |
ui_inputs =[
|
| 664 |
input_image_component, last_image_component, prompt_input, steps_slider,
|
| 665 |
-
|
| 666 |
-
guidance_scale_input, guidance_scale_2_input, seed_input, randomize_seed_checkbox,
|
| 667 |
-
quality_slider, scheduler_dropdown, flow_shift_slider, frame_multi,
|
| 668 |
-
play_result_video, safe_mode_checkbox, safety_checker_input
|
| 669 |
-
]
|
| 670 |
-
|
| 671 |
-
generate_button.click(
|
| 672 |
-
fn=generate_video,
|
| 673 |
-
inputs=ui_inputs,
|
| 674 |
-
outputs=[video_output, file_output, seed_input]
|
| 675 |
-
)
|
| 676 |
-
|
| 677 |
-
# --- Frame Grabbing Events ---
|
| 678 |
-
# 1. Click button -> JS runs -> puts time in hidden number box
|
| 679 |
-
grab_frame_btn.click(
|
| 680 |
-
fn=None,
|
| 681 |
-
inputs=None,
|
| 682 |
-
outputs=[timestamp_box],
|
| 683 |
-
js=get_timestamp_js
|
| 684 |
-
)
|
| 685 |
-
|
| 686 |
-
# 2. Hidden number box changes -> Python runs -> puts frame in Input Image
|
| 687 |
-
timestamp_box.change(
|
| 688 |
-
fn=extract_frame,
|
| 689 |
-
inputs=[video_output, timestamp_box],
|
| 690 |
-
outputs=[input_image_component]
|
| 691 |
-
)
|
| 692 |
-
|
| 693 |
-
if __name__ == "__main__":
|
| 694 |
-
demo.queue().launch(
|
| 695 |
-
mcp_server=True,
|
| 696 |
-
ssr_mode=False,
|
| 697 |
-
show_error=True,
|
| 698 |
-
)
|
|
|
|
| 31 |
from diffusers.pipelines.wan.pipeline_wan_i2v import WanImageToVideoPipeline
|
| 32 |
from diffusers.utils.export_utils import export_to_video
|
| 33 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
from torchao.quantization import quantize_, Float8DynamicActivationFloat8WeightConfig, Int8WeightOnlyConfig
|
| 35 |
import aoti
|
| 36 |
|
|
|
|
| 42 |
# print("Loading...")
|
| 43 |
# subprocess.run("rm -rf /data-nvme/zerogpu-offload/*", env={}, shell=True)
|
| 44 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
|
| 46 |
# --- FRAME EXTRACTION JS & LOGIC ---
|
| 47 |
|
|
|
|
| 357 |
quality,
|
| 358 |
duration_seconds,
|
| 359 |
safe_mode,
|
|
|
|
| 360 |
progress
|
| 361 |
):
|
| 362 |
BASE_FRAMES_HEIGHT_WIDTH = 81 * 832 * 624
|
|
|
|
| 400 |
quality,
|
| 401 |
duration_seconds,
|
| 402 |
safe_mode,
|
|
|
|
| 403 |
progress=gr.Progress(track_tqdm=True),
|
| 404 |
):
|
| 405 |
scheduler_class = SCHEDULER_MAP.get(scheduler_name)
|
|
|
|
| 434 |
raw_frames_np = result.frames[0] # Returns (T, H, W, C) float32
|
| 435 |
pipe.scheduler = original_scheduler
|
| 436 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 437 |
frame_factor = frame_multiplier // FIXED_FPS
|
| 438 |
if frame_factor > 1:
|
| 439 |
start = time.time()
|
|
|
|
| 454 |
export_to_video(final_frames, video_path, fps=final_fps, quality=quality)
|
| 455 |
pbar.update(1)
|
| 456 |
|
| 457 |
+
return video_path, task_name
|
| 458 |
|
| 459 |
|
| 460 |
def generate_video(
|
|
|
|
| 474 |
frame_multiplier=16,
|
| 475 |
video_component=True,
|
| 476 |
safe_mode=False,
|
|
|
|
| 477 |
progress=gr.Progress(track_tqdm=True),
|
| 478 |
):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 479 |
|
| 480 |
if input_image is None:
|
| 481 |
raise gr.Error("Please upload an input image.")
|
|
|
|
| 488 |
if last_image:
|
| 489 |
processed_last_image = resize_and_crop_to_match(last_image, resized_image)
|
| 490 |
|
| 491 |
+
video_path, task_n = run_inference(
|
| 492 |
resized_image,
|
| 493 |
processed_last_image,
|
| 494 |
prompt,
|
|
|
|
| 504 |
quality,
|
| 505 |
duration_seconds,
|
| 506 |
safe_mode,
|
|
|
|
| 507 |
progress,
|
| 508 |
)
|
| 509 |
|
|
|
|
|
|
|
|
|
|
| 510 |
print(f"GPU complete: {task_n}")
|
| 511 |
|
| 512 |
return (video_path if video_component else None), video_path, current_seed
|
|
|
|
| 563 |
info="Select a custom scheduler."
|
| 564 |
)
|
| 565 |
flow_shift_slider = gr.Slider(minimum=0.5, maximum=15.0, step=0.1, value=3.0, label="Flow Shift")
|
|
|
|
| 566 |
play_result_video = gr.Checkbox(label="Display result", value=True, interactive=True)
|
| 567 |
|
| 568 |
generate_button = gr.Button("Generate Video", variant="primary")
|
|
|
|
| 581 |
|
| 582 |
ui_inputs =[
|
| 583 |
input_image_component, last_image_component, prompt_input, steps_slider,
|
| 584 |
+
neg
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|