Upload 9 files

README.md

@@ -1,14 +1,12 @@
----
-title: FLUX.1 Kontext
-emoji: ⚡
-colorFrom: green
-colorTo: gray
-sdk: gradio
-sdk_version: 5.29.1
-app_file: app.py
-pinned: true
-license: mit
-short_description: 'Kontext image editing on FLUX[dev] '
----
-
+---
+title: Wan 2 2 First Last Frame
+emoji: 💻
+colorFrom: purple
+colorTo: gray
+sdk: gradio
+sdk_version: 5.29.1
+app_file: app.py
+pinned: false
+---
+
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,228 +1,512 @@
-# PyTorch 2.8 (temporary hack)
-import os
-os.system('pip install --upgrade --pre --extra-index-url https://download.pytorch.org/whl/nightly/cu126 "torch<2.9" spaces')
-
-# Actual demo code
-import gradio as gr
-import numpy as np
-import spaces
-import torch
-import random
-from PIL import Image
-
-from diffusers import FluxKontextPipeline
-from diffusers.utils import load_image
-
-from optimization import optimize_pipeline_
-
-MAX_SEED = np.iinfo(np.int32).max
-
-pipe = FluxKontextPipeline.from_pretrained("yuvraj108c/FLUX.1-Kontext-dev", torch_dtype=torch.bfloat16).to("cuda")
-optimize_pipeline_(pipe, image=Image.new("RGB", (512, 512)), prompt='prompt')
-
-input_image_debug_value = [None]
-prompt_debug_value = [None]
-
-@spaces.GPU(duration=40)
-def infer(input_image, prompt, seed=42, randomize_seed=False, guidance_scale=2.5, steps=28, progress=gr.Progress(track_tqdm=True)):
-    """
-    Perform image editing using the FLUX.1 Kontext pipeline.
-    
-    This function takes an input image and a text prompt to generate a modified version
-    of the image based on the provided instructions. It uses the FLUX.1 Kontext model
-    for contextual image editing tasks.
-    
-    Args:
-        input_image (PIL.Image.Image): The input image to be edited. Will be converted
-            to RGB format if not already in that format.
-        prompt (str): Text description of the desired edit to apply to the image.
-            Examples: "Remove glasses", "Add a hat", "Change background to beach".
-        seed (int, optional): Random seed for reproducible generation. Defaults to 42.
-            Must be between 0 and MAX_SEED (2^31 - 1).
-        randomize_seed (bool, optional): If True, generates a random seed instead of
-            using the provided seed value. Defaults to False.
-        guidance_scale (float, optional): Controls how closely the model follows the
-            prompt. Higher values mean stronger adherence to the prompt but may reduce
-            image quality. Range: 1.0-10.0. Defaults to 2.5.
-        steps (int, optional): Controls how many steps to run the diffusion model for.
-            Range: 1-30. Defaults to 28.
-        progress (gr.Progress, optional): Gradio progress tracker for monitoring
-            generation progress. Defaults to gr.Progress(track_tqdm=True).
-    
-    Returns:
-        tuple: A 3-tuple containing:
-            - PIL.Image.Image: The generated/edited image
-            - int: The seed value used for generation (useful when randomize_seed=True)
-            - gr.update: Gradio update object to make the reuse button visible
-    
-    Example:
-        >>> edited_image, used_seed, button_update = infer(
-        ...     input_image=my_image,
-        ...     prompt="Add sunglasses",
-        ...     seed=123,
-        ...     randomize_seed=False,
-        ...     guidance_scale=2.5
-        ... )
-    """
-    if randomize_seed:
-        seed = random.randint(0, MAX_SEED)
-    
-    if input_image:
-        input_image = input_image.convert("RGB")
-        image = pipe(
-            image=input_image, 
-            prompt=prompt,
-            guidance_scale=guidance_scale,
-            width = input_image.size[0],
-            height = input_image.size[1],
-            num_inference_steps=steps,
-            generator=torch.Generator().manual_seed(seed),
-        ).images[0]
-    else:
-        image = pipe(
-            prompt=prompt,
-            guidance_scale=guidance_scale,
-            num_inference_steps=steps,
-            generator=torch.Generator().manual_seed(seed),
-        ).images[0]
-    return image, seed, gr.Button(visible=True)
-
-def infer_example(input_image, prompt):
-    if input_image_debug_value[0] is not None or prompt_debug_value[0] is not None:
-        input_image=input_image_debug_value[0]
-        prompt=prompt_debug_value[0]
-        input_image_debug_value[0]=prompt_debug_value[0]=None
-    seed = random.randint(0, MAX_SEED)
-    image, seed, _ = infer(input_image, prompt, seed, True)
-    return image, seed
-
-css="""
-#col-container {
-    margin: 0 auto;
-    max-width: 960px;
-}
-"""
-
-with gr.Blocks(css=css) as demo:
-    
-    with gr.Column(elem_id="col-container"):
-        gr.Markdown(f"""# FLUX.1 Kontext [dev]
-Image editing and manipulation model guidance-distilled from FLUX.1 Kontext [pro], [[blog]](https://bfl.ai/announcements/flux-1-kontext-dev) [[model]](https://huggingface.co/black-forest-labs/FLUX.1-Kontext-dev)
-        """)
-        with gr.Row():
-            with gr.Column():
-                input_image = gr.Image(label="Upload the image for editing", type="pil")
-                with gr.Row():
-                    prompt = gr.Text(
-                        label="Prompt",
-                        show_label=False,
-                        max_lines=1,
-                        placeholder="Enter your prompt for editing (e.g., 'Remove glasses', 'Add a hat')",
-                        container=False,
-                    )
-                    run_button = gr.Button("Run", scale=0)
-                with gr.Accordion("Advanced Settings", open=False):
-                    
-                    seed = gr.Slider(
-                        label="Seed",
-                        minimum=0,
-                        maximum=MAX_SEED,
-                        step=1,
-                        value=0,
-                    )
-                    
-                    randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
-                    
-                    guidance_scale = gr.Slider(
-                        label="Guidance Scale",
-                        minimum=1,
-                        maximum=10,
-                        step=0.1,
-                        value=2.5,
-                    )       
-                    
-                    steps = gr.Slider(
-                        label="Steps",
-                        minimum=1,
-                        maximum=30,
-                        value=30,
-                        step=1
-                    )
-                    
-            with gr.Column():
-                result = gr.Image(label="Result", show_label=False, interactive=False)
-                reuse_button = gr.Button("Reuse this image", visible=False)
-        
-            
-
-        gr.Examples(
-            label = "Examples from demo",
-            examples=[
-                ["flowers.png", "turn the flowers into sunflowers"],
-                ["monster.png", "make this monster ride a skateboard on the beach"],
-                ["cat.png", "make this cat happy"]
-            ],
-            inputs=[input_image, prompt],
-            outputs=[result, seed],
-            fn=infer_example
-        )
-    
-    with gr.Row(visible=False):
-        input_image_debug=gr.Image(type="pil", label="Image Debug")
-        prompt_debug=gr.Textbox(label="Prompt Debug")
-        gr.Examples(
-            examples=[
-                ["cat.png", "make this cat happy"],
-                ["cat.png", "make this cat a dog"],
-                ["monster.png", "make this monster ride a skateboard on the beach"],
-                ["monster.png", "make this monster blue"],
-                ["monster.png", "make this monster orange"],
-                ["monster.png", "make this monster yellow"],
-                ["monster.png", "make this monster purple"],
-                ["monster.png", "make this monster brown"],
-                ["monster.png", "make this monster white"],
-                ["monster.png", "make this monster black"],
-                ["monster.png", "make this monster grey"],
-                ["monster.png", "make this monster pink"],
-                ["flowers.png", "turn the flowers into sunflowers"],
-                ["flowers.png", "turn the flowers into roses"],
-                ["flowers.png", "turn the flowers into umbrellas"]
-            ],
-            inputs=[input_image, prompt],
-            outputs=[result, seed],
-            fn=infer_example,
-            run_on_click=True,
-            cache_examples=True,
-            cache_mode='lazy'
-        )
-
-    def handle_field_debug_change(input_image_debug_data, prompt_debug_data):
-        input_image_debug_value[0] = input_image_debug_data
-        prompt_debug_value[0] = prompt_debug_data
-        return []
-
-    input_image_debug.upload(
-        fn=handle_field_debug_change,
-        inputs=[input_image_debug, prompt_debug],
-        outputs=[]
-    )
-    
-    prompt_debug.change(
-        fn=handle_field_debug_change,
-        inputs=[input_image_debug, prompt_debug],
-        outputs=[]
-    )
-            
-    gr.on(
-        triggers=[run_button.click, prompt.submit],
-        fn = infer,
-        inputs = [input_image, prompt, seed, randomize_seed, guidance_scale, steps],
-        outputs = [result, seed, reuse_button]
-    )
-    reuse_button.click(
-        fn = lambda image: image,
-        inputs = [result],
-        outputs = [input_image]
-    )
-
-demo.launch(mcp_server=True)
+import os
+# PyTorch 2.8 (temporary hack)
+os.system('pip install --upgrade --pre --extra-index-url https://download.pytorch.org/whl/nightly/cu126 "torch<2.9" spaces')
+
+# --- 1. Model Download and Setup (Diffusers Backend) ---
+try:
+    import spaces
+except:
+    class spaces():
+        def GPU(*args, **kwargs):
+            def decorator(function):
+                return lambda *dummy_args, **dummy_kwargs: function(*dummy_args, **dummy_kwargs)
+            return decorator
+
+import torch
+from diffusers import FlowMatchEulerDiscreteScheduler
+from diffusers.pipelines.wan.pipeline_wan_i2v import WanImageToVideoPipeline
+from diffusers.models.transformers.transformer_wan import WanTransformer3DModel
+from diffusers.utils.export_utils import export_to_video
+import gradio as gr
+import tempfile
+import time
+from datetime import datetime
+import numpy as np
+from PIL import Image
+import random
+import math
+import gc
+from gradio_client import Client, handle_file # Import for API call
+
+# Import the optimization function from the separate file
+from optimization import optimize_pipeline_
+
+# --- Constants and Model Loading ---
+MODEL_ID = "Wan-AI/Wan2.2-I2V-A14B-Diffusers"
+
+# --- NEW: Flexible Dimension Constants ---
+MAX_DIMENSION = 832
+MIN_DIMENSION = 480
+DIMENSION_MULTIPLE = 16
+SQUARE_SIZE = 480
+
+MAX_SEED = np.iinfo(np.int32).max
+
+FIXED_FPS = 24
+MIN_FRAMES_MODEL = 8
+MAX_FRAMES_MODEL = 81
+
+MIN_DURATION = round(MIN_FRAMES_MODEL/FIXED_FPS, 1)
+MAX_DURATION = round(MAX_FRAMES_MODEL/FIXED_FPS, 1)
+
+input_image_debug_value = [None]
+end_image_debug_value = [None]
+prompt_debug_value = [None]
+total_second_length_debug_value = [None]
+
+default_negative_prompt = "Vibrant colors, overexposure, static, blurred details, subtitles, error, style, artwork, painting, image, still, overall gray, worst quality, low quality, JPEG compression residue, ugly, mutilated, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, malformed limbs, fused fingers, still image, cluttered background, three legs, many people in the background, walking backwards, overexposure, jumpcut, crossfader, "
+
+print("Loading transformer...")
+
+transformer = WanTransformer3DModel.from_pretrained('cbensimon/Wan2.2-I2V-A14B-bf16-Diffusers',
+        subfolder='transformer',
+        torch_dtype=torch.bfloat16,
+        device_map='cuda',
+    )
+
+print("Loadingtransformer 2...")
+
+transformer_2 = WanTransformer3DModel.from_pretrained('cbensimon/Wan2.2-I2V-A14B-bf16-Diffusers',
+        subfolder='transformer_2',
+        torch_dtype=torch.bfloat16,
+        device_map='cuda',
+    )
+
+print("Loading models into memory. This may take a few minutes...")
+
+pipe = WanImageToVideoPipeline.from_pretrained(
+    MODEL_ID,
+    transformer = transformer,
+    transformer_2 = transformer_2,
+    torch_dtype=torch.bfloat16,
+)
+print("Loading scheduler...")
+pipe.scheduler = FlowMatchEulerDiscreteScheduler.from_config(pipe.scheduler.config, shift=8.0)
+pipe.to('cuda')
+
+print("Clean cache...")
+for i in range(3):
+    gc.collect()
+    torch.cuda.synchronize()
+    torch.cuda.empty_cache()
+
+print("Optimizing pipeline...")
+
+optimize_pipeline_(pipe,
+    image=Image.new('RGB', (MAX_DIMENSION, MIN_DIMENSION)), 
+    prompt='prompt',
+    height=MIN_DIMENSION,
+    width=MAX_DIMENSION,
+    num_frames=MAX_FRAMES_MODEL,
+)
+print("All models loaded and optimized. Gradio app is ready.")
+
+
+# --- 2. Image Processing and Application Logic ---
+def generate_end_frame(start_img, gen_prompt, progress=gr.Progress(track_tqdm=True)):
+    """Calls an external Gradio API to generate an image."""
+    if start_img is None:
+        raise gr.Error("Please provide a Start Frame first.")
+
+    hf_token = os.getenv("HF_TOKEN")
+    if not hf_token:
+        raise gr.Error("HF_TOKEN not found in environment variables. Please set it in your Space secrets.")
+
+    with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmpfile:
+        start_img.save(tmpfile.name)
+        tmp_path = tmpfile.name
+
+    progress(0.1, desc="Connecting to image generation API...")
+    client = Client("multimodalart/nano-banana-private")
+    
+    progress(0.5, desc=f"Generating with prompt: '{gen_prompt}'...")
+    try:
+        result = client.predict(
+            prompt=gen_prompt, 
+            images=[
+                {"image": handle_file(tmp_path)}
+            ],
+            manual_token=hf_token,
+            api_name="/unified_image_generator"
+        )
+    finally:
+        os.remove(tmp_path)
+
+    progress(1.0, desc="Done!")
+    print(result)
+    return result
+
+def switch_to_upload_tab():
+    """Returns a gr.Tabs update to switch to the first tab."""
+    return gr.Tabs(selected="upload_tab")
+
+
+def process_image_for_video(image: Image.Image) -> Image.Image:
+    """
+    Resizes an image based on the following rules for video generation:
+    1. The longest side will be scaled down to MAX_DIMENSION if it's larger.
+    2. The shortest side will be scaled up to MIN_DIMENSION if it's smaller.
+    3. The final dimensions will be rounded to the nearest multiple of DIMENSION_MULTIPLE.
+    4. Square images are resized to a fixed SQUARE_SIZE.
+    The aspect ratio is preserved as closely as possible.
+    """
+    width, height = image.size
+
+    # Rule 4: Handle square images
+    if width == height:
+        return image.resize((SQUARE_SIZE, SQUARE_SIZE), Image.Resampling.LANCZOS)
+
+    # Determine target dimensions while preserving aspect ratio
+    aspect_ratio = width / height
+    new_width, new_height = width, height
+
+    # Rule 1: Scale down if too large
+    if new_width > MAX_DIMENSION or new_height > MAX_DIMENSION:
+        if aspect_ratio > 1:  # Landscape
+            scale = MAX_DIMENSION / new_width
+        else:  # Portrait
+            scale = MAX_DIMENSION / new_height
+        new_width *= scale
+        new_height *= scale
+
+    # Rule 2: Scale up if too small
+    if new_width < MIN_DIMENSION or new_height < MIN_DIMENSION:
+        if aspect_ratio > 1:  # Landscape
+            scale = MIN_DIMENSION / new_height
+        else:  # Portrait
+            scale = MIN_DIMENSION / new_width
+        new_width *= scale
+        new_height *= scale
+
+    # Rule 3: Round to the nearest multiple of DIMENSION_MULTIPLE
+    final_width = int(round(new_width / DIMENSION_MULTIPLE) * DIMENSION_MULTIPLE)
+    final_height = int(round(new_height / DIMENSION_MULTIPLE) * DIMENSION_MULTIPLE)
+    
+    # Ensure final dimensions are at least the minimum
+    final_width = max(final_width, MIN_DIMENSION if aspect_ratio < 1 else SQUARE_SIZE)
+    final_height = max(final_height, MIN_DIMENSION if aspect_ratio > 1 else SQUARE_SIZE)
+
+
+    return image.resize((final_width, final_height), Image.Resampling.LANCZOS)
+
+def resize_and_crop_to_match(target_image, reference_image):
+    """Resizes and center-crops the target image to match the reference image's dimensions."""
+    ref_width, ref_height = reference_image.size
+    target_width, target_height = target_image.size
+    scale = max(ref_width / target_width, ref_height / target_height)
+    new_width, new_height = int(target_width * scale), int(target_height * scale)
+    resized = target_image.resize((new_width, new_height), Image.Resampling.LANCZOS)
+    left, top = (new_width - ref_width) // 2, (new_height - ref_height) // 2
+    return resized.crop((left, top, left + ref_width, top + ref_height))
+
+def init_view():
+    return gr.update(interactive = True)
+
+def generate_video(
+    start_image_pil,
+    end_image_pil,
+    prompt,
+    negative_prompt=default_negative_prompt,
+    duration_seconds=2.1,
+    steps=8,
+    guidance_scale=1,
+    guidance_scale_2=1,
+    seed=42,
+    randomize_seed=True,
+    progress=gr.Progress(track_tqdm=True)
+):
+    start = time.time()
+    allocation_time = 120
+    factor = 1
+    
+    if input_image_debug_value[0] is not None or end_image_debug_value[0] is not None or prompt_debug_value[0] is not None or total_second_length_debug_value[0] is not None:
+        start_image_pil = input_image_debug_value[0]
+        end_image_pil = end_image_debug_value[0]
+        prompt = prompt_debug_value[0]
+        duration_seconds = total_second_length_debug_value[0]
+        allocation_time = min(duration_seconds * 60 * 100, 10 * 60)
+        factor = 3.1
+
+    if start_image_pil is None or end_image_pil is None:
+        raise gr.Error("Please upload both a start and an end image.")
+
+    # Step 1: Process the start image to get our target dimensions based on the new rules.
+    processed_start_image = process_image_for_video(start_image_pil)
+    
+    # Step 2: Make the end image match the *exact* dimensions of the processed start image.
+    processed_end_image = resize_and_crop_to_match(end_image_pil, processed_start_image)
+    
+    target_height, target_width = processed_start_image.height, processed_start_image.width
+
+    # Handle seed and frame count
+    current_seed = random.randint(0, MAX_SEED) if randomize_seed else int(seed)
+    num_frames = np.clip(int(round(duration_seconds * FIXED_FPS)), MIN_FRAMES_MODEL, MAX_FRAMES_MODEL)
+
+    progress(0.2, desc=f"Generating {num_frames} frames at {target_width}x{target_height} (seed: {current_seed})...")
+
+    progress(0.1, desc="Preprocessing images...")
+    output_video, download_button, seed_input = generate_video_on_gpu(
+        start_image_pil,
+        end_image_pil,
+        prompt,
+        negative_prompt,
+        duration_seconds,
+        steps,
+        guidance_scale,
+        guidance_scale_2,
+        seed,
+        randomize_seed,
+        progress,
+        allocation_time,
+        factor,
+        target_height,
+        target_width,
+        current_seed,
+        num_frames,
+        processed_start_image,
+        processed_end_image
+    )
+    progress(1.0, desc="Done!")
+    end = time.time()
+    secondes = int(end - start)
+    minutes = math.floor(secondes / 60)
+    secondes = secondes - (minutes * 60)
+    hours = math.floor(minutes / 60)
+    minutes = minutes - (hours * 60)
+    information = ("Start the process again if you want a different result. " if randomize_seed else "") + \
+    "The video been generated in " + \
+    ((str(hours) + " h, ") if hours != 0 else "") + \
+    ((str(minutes) + " min, ") if hours != 0 or minutes != 0 else "") + \
+    str(secondes) + " sec. " + \
+    "The video resolution is " + str(target_width) + \
+    " pixels large and " + str(target_height) + \
+    " pixels high, so a resolution of " + f'{target_width * target_height:,}' + " pixels."
+    return [output_video, download_button, seed_input, gr.update(value = information, visible = True), gr.update(interactive = False)]
+
+def get_duration(
+    start_image_pil,
+    end_image_pil,
+    prompt,
+    negative_prompt,
+    duration_seconds,
+    steps,
+    guidance_scale,
+    guidance_scale_2,
+    seed,
+    randomize_seed,
+    progress,
+    allocation_time,
+    factor,
+    target_height,
+    target_width,
+    current_seed,
+    num_frames,
+    processed_start_image,
+    processed_end_image
+):
+    return allocation_time
+
+@spaces.GPU(duration=get_duration)
+def generate_video_on_gpu(
+    start_image_pil,
+    end_image_pil,
+    prompt,
+    negative_prompt,
+    duration_seconds,
+    steps,
+    guidance_scale,
+    guidance_scale_2,
+    seed,
+    randomize_seed,
+    progress,
+    allocation_time,
+    factor,
+    target_height,
+    target_width,
+    current_seed,
+    num_frames,
+    processed_start_image,
+    processed_end_image
+):
+    """
+    Generates a video by interpolating between a start and end image, guided by a text prompt,
+    using the diffusers Wan2.2 pipeline.
+    """
+    print("Generate a video with the prompt: " + prompt)
+
+    output_frames_list = pipe(
+        image=processed_start_image,
+        last_image=processed_end_image,
+        prompt=prompt,
+        negative_prompt=negative_prompt,
+        height=target_height,
+        width=target_width,
+        num_frames=int(num_frames * factor),
+        guidance_scale=float(guidance_scale),
+        guidance_scale_2=float(guidance_scale_2),
+        num_inference_steps=int(steps),
+        generator=torch.Generator(device="cuda").manual_seed(current_seed),
+    ).frames[0]
+
+    progress(0.9, desc="Encoding and saving video...")
+    
+    video_path = 'wan_' + datetime.now().strftime("%Y-%m-%d_%H-%M-%S.%f") + '.mp4'
+
+    export_to_video(output_frames_list, video_path, fps=FIXED_FPS)
+    print("Video exported: " + video_path)
+
+    return video_path, gr.update(value = video_path, visible = True), current_seed
+
+
+# --- 3. Gradio User Interface ---
+
+
+
+js = """
+function createGradioAnimation() {
+    window.addEventListener("beforeunload", function(e) {
+        if (document.getElementById('dummy_button_id') && !document.getElementById('dummy_button_id').disabled) {
+            var confirmationMessage = 'A process is still running. '
+                                    + 'If you leave before saving, your changes will be lost.';
+    
+            (e || window.event).returnValue = confirmationMessage;
+        }
+        return confirmationMessage;
+    });
+    return 'Animation created';
+}
+"""
+
+# Gradio interface
+with gr.Blocks(js=js) as app:
+    gr.Markdown("# Wan 2.2 First/Last Frame Video Fast")
+    gr.Markdown("Based on the [Wan 2.2 First/Last Frame workflow](https://www.reddit.com/r/StableDiffusion/comments/1me4306/psa_wan_22_does_first_frame_last_frame_out_of_the/), applied to 🧨 Diffusers + [lightx2v/Wan2.2-Lightning](https://huggingface.co/lightx2v/Wan2.2-Lightning) 8-step LoRA")
+
+    with gr.Row(elem_id="general_items"):
+        with gr.Column():
+            with gr.Group(elem_id="group_all"):
+                with gr.Row():
+                    start_image = gr.Image(type="pil", label="Start Frame", sources=["upload", "clipboard"])
+                    # Capture the Tabs component in a variable and assign IDs to tabs
+                    with gr.Tabs(elem_id="group_tabs") as tabs:
+                        with gr.TabItem("Upload", id="upload_tab"):
+                            end_image = gr.Image(type="pil", label="End Frame", sources=["upload", "clipboard"])
+                        with gr.TabItem("Generate", id="generate_tab"):
+                            generate_5seconds = gr.Button("Generate scene 5 seconds in the future", elem_id="fivesec")
+                            gr.Markdown("Generate a custom end-frame with an edit model like [Nano Banana](https://huggingface.co/spaces/multimodalart/nano-banana) or [Qwen Image Edit](https://huggingface.co/spaces/multimodalart/Qwen-Image-Edit-Fast)", elem_id="or_item")
+                prompt = gr.Textbox(label="Prompt", info="Describe the transition between the two images")
+
+                with gr.Accordion("Advanced Settings", open=False):
+                    duration_seconds_input = gr.Slider(minimum=MIN_DURATION, maximum=MAX_DURATION, step=0.1, value=2.1, label="Video Duration (seconds)", info=f"Clamped to model's {MIN_FRAMES_MODEL}-{MAX_FRAMES_MODEL} frames at {FIXED_FPS}fps.")
+                    negative_prompt_input = gr.Textbox(label="Negative Prompt", value=default_negative_prompt, lines=3)
+                    steps_slider = gr.Slider(minimum=1, maximum=30, step=1, value=8, label="Inference Steps")
+                    guidance_scale_input = gr.Slider(minimum=0.0, maximum=10.0, step=0.5, value=1.0, label="Guidance Scale - high noise")
+                    guidance_scale_2_input = gr.Slider(minimum=0.0, maximum=10.0, step=0.5, value=1.0, label="Guidance Scale - low noise")
+                    with gr.Row():
+                        seed_input = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=42)
+                        randomize_seed_checkbox = gr.Checkbox(label="Randomize seed", value=True)
+
+                generate_button = gr.Button("Generate Video", variant="primary")
+                dummy_button = gr.Button(elem_id = "dummy_button_id", visible = False, interactive = False)
+
+        with gr.Column():
+            output_video = gr.Video(label="Generated Video", autoplay = True, loop = True)
+            download_button = gr.DownloadButton(label="Download", visible = True)
+            video_information = gr.HTML(value = "", visible = True)
+
+    # Main video generation button
+    ui_inputs = [
+        start_image,
+        end_image,
+        prompt,
+        negative_prompt_input,
+        duration_seconds_input,
+        steps_slider,
+        guidance_scale_input,
+        guidance_scale_2_input,
+        seed_input,
+        randomize_seed_checkbox
+    ]
+    ui_outputs = [output_video, download_button, seed_input, video_information, dummy_button]
+
+    generate_button.click(fn = init_view, inputs = [], outputs = [dummy_button], queue = False, show_progress = False).success(
+        fn = generate_video,
+        inputs = ui_inputs,
+        outputs = ui_outputs
+    )
+
+    generate_5seconds.click(
+        fn=switch_to_upload_tab,
+        inputs=None,
+        outputs=[tabs]
+    ).then( 
+        fn=lambda img: generate_end_frame(img, "this image is a still frame from a movie. generate a new frame with what happens on this scene 5 seconds in the future"),
+        inputs=[start_image],
+        outputs=[end_image]
+    ).success(
+        fn=generate_video,
+        inputs=ui_inputs,
+        outputs=ui_outputs
+    )
+    
+    with gr.Row(visible=False):
+        prompt_debug=gr.Textbox(label="Prompt Debug")
+        input_image_debug=gr.Image(type="pil", label="Image Debug")
+        end_image_debug=gr.Image(type="pil", label="End Image Debug")
+        total_second_length_debug=gr.Slider(label="Additional Video Length to Generate (seconds) Debug", minimum=1, maximum=120, value=10, step=0.1)
+        gr.Examples(
+            examples=[["Schoolboy_without_backpack.webp", "Schoolboy_with_backpack.webp", "The schoolboy puts on his schoolbag."]],
+            inputs=[start_image, end_image, prompt],
+            outputs=ui_outputs,
+            fn=generate_video,
+            run_on_click=True,
+            cache_examples=True,
+            cache_mode='lazy',
+        )
+    
+    gr.Examples(
+        label = "Examples from demo",
+        examples = [
+            ["poli_tower.png", "tower_takes_off.png", "The man turns around."],
+            ["ugly_sonic.jpeg", "squatting_sonic.png", "पात्रं क्षेपणास्त्रं चकमाति।"],
+            ["Schoolboy_without_backpack.webp", "Schoolboy_with_backpack.webp", "The schoolboy puts on his schoolbag."],
+        ],
+        inputs = [start_image, end_image, prompt],
+        outputs = ui_outputs,
+        fn = generate_video,
+        cache_examples = False,
+    )
+
+    def handle_field_debug_change(input_image_debug_data, end_image_debug_data, prompt_debug_data, total_second_length_debug_data):
+        input_image_debug_value[0] = input_image_debug_data
+        end_image_debug_value[0] = end_image_debug_data
+        prompt_debug_value[0] = prompt_debug_data
+        total_second_length_debug_value[0] = total_second_length_debug_data
+        return []
+
+    input_image_debug.upload(
+        fn=handle_field_debug_change,
+        inputs=[input_image_debug, end_image_debug, prompt_debug, total_second_length_debug],
+        outputs=[]
+    )
+    
+    end_image_debug.upload(
+        fn=handle_field_debug_change,
+        inputs=[input_image_debug, end_image_debug, prompt_debug, total_second_length_debug],
+        outputs=[]
+    )
+    
+    prompt_debug.change(
+        fn=handle_field_debug_change,
+        inputs=[input_image_debug, end_image_debug, prompt_debug, total_second_length_debug],
+        outputs=[]
+    )
+    
+    total_second_length_debug.change(
+        fn=handle_field_debug_change,
+        inputs=[input_image_debug, end_image_debug, prompt_debug, total_second_length_debug],
+        outputs=[]
+    )
+
+if __name__ == "__main__":
+    app.launch(mcp_server=True, share=True)

optimization.py

@@ -8,21 +8,49 @@ from typing import ParamSpec
 import spaces
 import torch
 from torch.utils._pytree import tree_map_only
+from torchao.quantization import quantize_
+from torchao.quantization import Float8DynamicActivationFloat8WeightConfig
+from torchao.quantization import Int8WeightOnlyConfig
 
 from optimization_utils import capture_component_call
 from optimization_utils import aoti_compile
+from optimization_utils import drain_module_parameters
 
 
 P = ParamSpec('P')
 
+# --- CORRECTED DYNAMIC SHAPING ---
 
-TRANSFORMER_HIDDEN_DIM = torch.export.Dim('hidden', min=4096, max=8212)
+# VAE temporal scale factor is 1, latent_frames = num_frames. Range is [8, 81].
+LATENT_FRAMES_DIM = torch.export.Dim('num_latent_frames', min=8, max=81)
 
+# The transformer has a patch_size of (1, 2, 2), which means the input latent height and width
+# are effectively divided by 2. This creates constraints that fail if the symbolic tracer
+# assumes odd numbers are possible.
+#
+# To solve this, we define the dynamic dimension for the *patched* (i.e., post-division) size,
+# and then express the input shape as 2 * this dimension. This mathematically guarantees
+# to the compiler that the input latent dimensions are always even, satisfying the constraints.
+
+# App range for pixel dimensions: [480, 832]. VAE scale factor is 8.
+# Latent dimension range: [480/8, 832/8] = [60, 104].
+# Patched latent dimension range: [60/2, 104/2] = [30, 52].
+LATENT_PATCHED_HEIGHT_DIM = torch.export.Dim('latent_patched_height', min=30, max=52)
+LATENT_PATCHED_WIDTH_DIM = torch.export.Dim('latent_patched_width', min=30, max=52)
+
+# Now, we define the dynamic shapes for the transformer's `hidden_states` input,
+# which has the shape (batch_size, channels, num_frames, height, width).
 TRANSFORMER_DYNAMIC_SHAPES = {
-    'hidden_states': {1: TRANSFORMER_HIDDEN_DIM},
-    'img_ids': {0: TRANSFORMER_HIDDEN_DIM},
+    'hidden_states': {
+        2: LATENT_FRAMES_DIM,
+        3: 2 * LATENT_PATCHED_HEIGHT_DIM, # Guarantees even height
+        4: 2 * LATENT_PATCHED_WIDTH_DIM,  # Guarantees even width
+    },
 }
 
+# --- END OF CORRECTION ---
+
+
 INDUCTOR_CONFIGS = {
     'conv_1x1_as_mm': True,
     'epilogue_fusion': False,
@@ -37,24 +65,69 @@ def optimize_pipeline_(pipeline: Callable[P, Any], *args: P.args, **kwargs: P.kw
 
     @spaces.GPU(duration=1500)
     def compile_transformer():
-
+        
+        # This LoRA fusion part remains the same
+        pipeline.load_lora_weights(
+            "Kijai/WanVideo_comfy", 
+            weight_name="Lightx2v/lightx2v_I2V_14B_480p_cfg_step_distill_rank128_bf16.safetensors", 
+            adapter_name="lightx2v"
+        )
+        kwargs_lora = {}
+        kwargs_lora["load_into_transformer_2"] = True
+        pipeline.load_lora_weights(
+            "Kijai/WanVideo_comfy", 
+            weight_name="Lightx2v/lightx2v_I2V_14B_480p_cfg_step_distill_rank128_bf16.safetensors", 
+            adapter_name="lightx2v_2", **kwargs_lora
+        )
+        pipeline.set_adapters(["lightx2v", "lightx2v_2"], adapter_weights=[1., 1.])
+        pipeline.fuse_lora(adapter_names=["lightx2v"], lora_scale=3., components=["transformer"])
+        pipeline.fuse_lora(adapter_names=["lightx2v_2"], lora_scale=1., components=["transformer_2"])
+        pipeline.unload_lora_weights()
+        
+        # Capture a single call to get the args/kwargs structure
         with capture_component_call(pipeline, 'transformer') as call:
             pipeline(*args, **kwargs)
-
+        
         dynamic_shapes = tree_map_only((torch.Tensor, bool), lambda t: None, call.kwargs)
         dynamic_shapes |= TRANSFORMER_DYNAMIC_SHAPES
 
-        pipeline.transformer.fuse_qkv_projections()
-
-        exported = torch.export.export(
+        # Quantization remains the same
+        quantize_(pipeline.transformer, Float8DynamicActivationFloat8WeightConfig())
+        quantize_(pipeline.transformer_2, Float8DynamicActivationFloat8WeightConfig())
+        
+        # --- SIMPLIFIED COMPILATION ---
+        
+        exported_1 = torch.export.export(
             mod=pipeline.transformer,
             args=call.args,
             kwargs=call.kwargs,
             dynamic_shapes=dynamic_shapes,
         )
+        
+        exported_2 = torch.export.export(
+            mod=pipeline.transformer_2,
+            args=call.args,
+            kwargs=call.kwargs,
+            dynamic_shapes=dynamic_shapes,
+        )
+
+        compiled_1 = aoti_compile(exported_1, INDUCTOR_CONFIGS)
+        compiled_2 = aoti_compile(exported_2, INDUCTOR_CONFIGS)
+        
+        # Return the two compiled models
+        return compiled_1, compiled_2
+
+
+    # Quantize text encoder (same as before)
+    quantize_(pipeline.text_encoder, Int8WeightOnlyConfig())
+    
+    # Get the two dynamically-shaped compiled models
+    compiled_transformer_1, compiled_transformer_2 = compile_transformer()
 
-        return aoti_compile(exported, INDUCTOR_CONFIGS)
+    # --- SIMPLIFIED ASSIGNMENT ---
+    
+    pipeline.transformer.forward = compiled_transformer_1
+    drain_module_parameters(pipeline.transformer)
 
-    transformer_config = pipeline.transformer.config
-    pipeline.transformer = compile_transformer()
-    pipeline.transformer.config = transformer_config # pyright: ignore[reportAttributeAccessIssue]
+    pipeline.transformer_2.forward = compiled_transformer_2
+    drain_module_parameters(pipeline.transformer_2)

optimization_utils.py

@@ -10,7 +10,6 @@ from unittest.mock import patch
 import torch
 from torch._inductor.package.package import package_aoti
 from torch.export.pt2_archive._package import AOTICompiledModel
-from torch.export.pt2_archive._package_weights import TensorProperties
 from torch.export.pt2_archive._package_weights import Weights
 
 
@@ -21,31 +20,33 @@ INDUCTOR_CONFIGS_OVERRIDES = {
 }
 
 
+class ZeroGPUWeights:
+    def __init__(self, constants_map: dict[str, torch.Tensor], to_cuda: bool = False):
+        if to_cuda:
+            self.constants_map = {name: tensor.to('cuda') for name, tensor in constants_map.items()}
+        else:
+            self.constants_map = constants_map
+    def __reduce__(self):
+        constants_map: dict[str, torch.Tensor] = {}
+        for name, tensor in self.constants_map.items():
+            tensor_ = torch.empty_like(tensor, device='cpu').pin_memory()
+            constants_map[name] = tensor_.copy_(tensor).detach().share_memory_()
+        return ZeroGPUWeights, (constants_map, True)
+
+
 class ZeroGPUCompiledModel:
-    def __init__(self, archive_file: torch.types.FileLike, weights: Weights, cuda: bool = False):
+    def __init__(self, archive_file: torch.types.FileLike, weights: ZeroGPUWeights):
         self.archive_file = archive_file
         self.weights = weights
-        if cuda:
-            self.weights_to_cuda_()
         self.compiled_model: ContextVar[AOTICompiledModel | None] = ContextVar('compiled_model', default=None)
-    def weights_to_cuda_(self):
-        for name in self.weights:
-            tensor, properties = self.weights.get_weight(name)
-            self.weights[name] = (tensor.to('cuda'), properties)
     def __call__(self, *args, **kwargs):
         if (compiled_model := self.compiled_model.get()) is None:
-            constants_map = {name: value[0] for name, value in self.weights.items()}
             compiled_model = cast(AOTICompiledModel, torch._inductor.aoti_load_package(self.archive_file))
-            compiled_model.load_constants(constants_map, check_full_update=True, user_managed=True)
+            compiled_model.load_constants(self.weights.constants_map, check_full_update=True, user_managed=True)
             self.compiled_model.set(compiled_model)
         return compiled_model(*args, **kwargs)
     def __reduce__(self):
-        weight_dict: dict[str, tuple[torch.Tensor, TensorProperties]] = {}
-        for name in self.weights:
-            tensor, properties = self.weights.get_weight(name)
-            tensor_ = torch.empty_like(tensor, device='cpu').pin_memory()
-            weight_dict[name] = (tensor_.copy_(tensor).detach().share_memory_(), properties)
-        return ZeroGPUCompiledModel, (self.archive_file, Weights(weight_dict), True)
+        return ZeroGPUCompiledModel, (self.archive_file, self.weights)
 
 
 def aoti_compile(
@@ -61,7 +62,8 @@ def aoti_compile(
     files: list[str | Weights] = [file for file in artifacts if isinstance(file, str)]
     package_aoti(archive_file, files)
     weights, = (artifact for artifact in artifacts if isinstance(artifact, Weights))
-    return ZeroGPUCompiledModel(archive_file, weights)
+    zerogpu_weights = ZeroGPUWeights({name: weights.get_weight(name)[0] for name in weights})
+    return ZeroGPUCompiledModel(archive_file, zerogpu_weights)
 
 
 @contextlib.contextmanager
@@ -94,3 +96,12 @@ def capture_component_call(
         except CapturedCallException as e:
             captured_call.args = e.args
             captured_call.kwargs = e.kwargs
+
+
+def drain_module_parameters(module: torch.nn.Module):
+    state_dict_meta = {name: {'device': tensor.device, 'dtype': tensor.dtype} for name, tensor in module.state_dict().items()}
+    state_dict = {name: torch.nn.Parameter(torch.empty_like(tensor, device='cpu')) for name, tensor in module.state_dict().items()}
+    module.load_state_dict(state_dict, assign=True)
+    for name, param in state_dict.items():
+        meta = state_dict_meta[name]
+        param.data = torch.Tensor([]).to(**meta)

requirements.txt

@@ -1,5 +1,11 @@
-transformers
-git+https://github.com/huggingface/diffusers.git
-accelerate
-safetensors
-sentencepiece
+git+https://github.com/linoytsaban/diffusers.git@wan22-loras
+
+transformers==4.57.1
+accelerate==1.11.0
+safetensors==0.6.2
+sentencepiece==0.2.1
+peft==0.17.1
+ftfy==6.3.1
+imageio-ffmpeg==0.6.0
+opencv-python==4.12.0.88
+torchao==0.11.0