Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/bg_beach.jpg filter=lfs diff=lfs merge=lfs -text
+examples/bg_tent.jpg filter=lfs diff=lfs merge=lfs -text
+examples/obj_cake.jpg filter=lfs diff=lfs merge=lfs -text
+examples/obj_dog.jpg filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,13 +1,38 @@
 ---
-title: Direct Object Insertion
-emoji: 🏢
-colorFrom: purple
-colorTo: purple
+title: DIRECT 3D-Aware Object Insertion
+emoji: 🪑
+colorFrom: gray
+colorTo: red
 sdk: gradio
 sdk_version: 6.19.0
-python_version: '3.12'
 app_file: app.py
+short_description: 3D-aware object insertion with the DIRECT model (ICML 2026)
+python_version: "3.10"
+startup_duration_timeout: 1h
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# DIRECT: Direct 3D-Aware Object Insertion via Decomposed Visual Proxies
+
+Insert a reference object into a background scene with realistic, harmonized
+results using the [DIRECT](https://huggingface.co/superGong/DIRECT) model
+(ICML 2026), a FLUX.1-Fill-dev network guided by a decomposed visual proxy.
+
+Upload a background image and an object image (its background is removed
+automatically), choose where and how large to place the object, then click
+**Insert**.
+
+## Note on scope
+
+The full paper uses an interactive 3D viewer (TRELLIS + Viser) so users can pose
+a reconstructed 3D proxy of the object. That live 3D websocket viewer cannot run
+inside a single-port Hugging Face Space, so this demo drives the same DIRECT
+model with a simpler **2D placement** (position + scale) as its geometric
+guidance. The insertion / harmonization is performed by the real DIRECT weights.
+
+## Links
+
+- Paper: https://arxiv.org/abs/2606.06601
+- Project page: https://gong1130.github.io/DIRECT/
+- Code: https://github.com/Gong1130/DIRECT
+- Model weights: https://huggingface.co/superGong/DIRECT

app.py

@@ -0,0 +1,417 @@
+import os
+
+os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
+
+import spaces  # noqa: E402  (must come before torch / CUDA-touching imports)
+import math
+import time
+import random
+
+import numpy as np
+import torch
+import gradio as gr
+from PIL import Image
+
+from direct import DirectPipeline
+
+# ----------------------------------------------------------------------------
+# Config
+# ----------------------------------------------------------------------------
+MODEL_INPUT_RESOLUTION = 1024
+DIRECT_MODEL_PATH = "superGong/DIRECT"
+FLUX_MODEL_PATH = "black-forest-labs/FLUX.1-Fill-dev"
+SIGLIP_MODEL_PATH = "google/siglip2-so400m-patch14-384"
+
+HF_TOKEN = os.environ.get("HF_TOKEN")
+
+# ----------------------------------------------------------------------------
+# Load models at module scope (ZeroGPU packs weights to disk after this)
+# ----------------------------------------------------------------------------
+print("Loading DIRECT pipeline (FLUX.1-Fill-dev + SigLIP2 + DIRECT adapters)...")
+direct_pipeline = DirectPipeline.from_pretrained(
+    direct_model_path=DIRECT_MODEL_PATH,
+    flux_model_path=FLUX_MODEL_PATH,
+    siglip_model_path=SIGLIP_MODEL_PATH,
+    device=torch.device("cuda"),
+    torch_dtype=torch.bfloat16,
+    token=HF_TOKEN,
+)
+print("DIRECT pipeline loaded.")
+
+# Background remover for the object image (ungated). Loaded lazily/cheaply.
+_rembg_session = None
+
+
+def _get_rembg_session():
+    global _rembg_session
+    if _rembg_session is None:
+        from rembg import new_session
+
+        _rembg_session = new_session("u2net")
+    return _rembg_session
+
+
+# ----------------------------------------------------------------------------
+# Image-preparation helpers (2D proxy construction).
+#
+# The full DIRECT paper uses an interactive 3D viewer (TRELLIS + Viser) to let
+# users pose a reconstructed 3D proxy of the object. That live 3D websocket
+# viewer cannot run inside a single-port HF Space, so here we build the model's
+# geometric-guidance inputs from a simple 2D placement (position + scale). The
+# underlying DIRECT model (real weights) then performs the 3D-aware harmonized
+# insertion. See the notes in the UI for this limitation.
+# ----------------------------------------------------------------------------
+
+def segment_object(object_rgb: Image.Image) -> Image.Image:
+    """Return an RGBA image of the object with background removed."""
+    from rembg import remove
+
+    rgba = remove(object_rgb.convert("RGB"), session=_get_rembg_session())
+    return rgba.convert("RGBA")
+
+
+def _tight_crop_rgba(rgba: Image.Image) -> Image.Image:
+    alpha = np.array(rgba.split()[-1])
+    ys, xs = np.where(alpha > 10)
+    if ys.size == 0:
+        return rgba
+    y1, y2, x1, x2 = ys.min(), ys.max() + 1, xs.min(), xs.max() + 1
+    return rgba.crop((x1, y1, x2, y2))
+
+
+def center_reference(rgba: Image.Image, out_size: int = MODEL_INPUT_RESOLUTION) -> Image.Image:
+    """Object centered on black, square, with ~1.2 margin (model reference input)."""
+    obj = _tight_crop_rgba(rgba)
+    w, h = obj.size
+    side = max(int(math.ceil(max(w, h) * 1.2)), 1)
+    canvas = Image.new("RGB", (side, side), (0, 0, 0))
+    canvas.paste(obj, ((side - w) // 2, (side - h) // 2), obj)
+    return canvas.resize((out_size, out_size), Image.LANCZOS)
+
+
+def place_object(bg: Image.Image, obj_rgba: Image.Image, cx: float, cy: float, scale: float):
+    """Paste the (tight-cropped) object onto a copy of the background.
+
+    cx, cy in [0, 1] (center), scale in [0, 1] (object longest side as a
+    fraction of the background's longest side). Returns (placed_rgb, mask_L).
+    """
+    bg = bg.convert("RGB")
+    W, H = bg.size
+    obj = _tight_crop_rgba(obj_rgba)
+    ow, oh = obj.size
+    target_long = max(1, int(scale * max(W, H)))
+    ratio = target_long / max(ow, oh)
+    new_w = max(1, int(ow * ratio))
+    new_h = max(1, int(oh * ratio))
+    obj_r = obj.resize((new_w, new_h), Image.LANCZOS)
+
+    center_x = int(cx * W)
+    center_y = int(cy * H)
+    x0 = center_x - new_w // 2
+    y0 = center_y - new_h // 2
+
+    placed_rgb = bg.copy()
+    placed_rgb.paste(obj_r, (x0, y0), obj_r)
+
+    mask = Image.new("L", (W, H), 0)
+    obj_alpha = obj_r.split()[-1]
+    mask.paste(obj_alpha, (x0, y0), obj_alpha)
+
+    # Geometry proxy: the object RGB on a black canvas at its placed location.
+    geometry_full = Image.new("RGB", (W, H), (0, 0, 0))
+    geometry_full.paste(obj_r, (x0, y0), obj_r)
+
+    return placed_rgb, mask, geometry_full
+
+
+def get_mask_bbox(mask_pil, threshold=20):
+    arr = np.array(mask_pil)
+    ys, xs = np.where(arr > threshold)
+    if ys.size == 0:
+        return None
+    return (xs.min(), ys.min(), xs.max() + 1, ys.max() + 1)
+
+
+def get_smart_crop_bbox(mask_pil, min_ratio=0.02, max_ratio=0.3):
+    bbox = get_mask_bbox(mask_pil)
+    if bbox is None:
+        s = MODEL_INPUT_RESOLUTION
+        return (0, 0, s, s), s
+    min_x, min_y, max_x, max_y = bbox
+    mask_w, mask_h = max_x - min_x, max_y - min_y
+    area = mask_w * mask_h
+    side = int(math.sqrt(area / ((min_ratio + max_ratio) / 2.0)))
+    side = max(side, max(mask_w, mask_h) + 40)
+    cx = (min_x + max_x) // 2
+    cy = (min_y + max_y) // 2
+    half = side // 2
+    return (cx - half, cy - half, cx - half + side, cy - half + side), side
+
+
+def crop_and_pad(image, bbox, target_side):
+    x1, y1, x2, y2 = bbox
+    W, H = image.size
+    valid = image.crop((max(0, x1), max(0, y1), min(W, x2), min(H, y2)))
+    canvas = Image.new(image.mode, (target_side, target_side), 0)
+    canvas.paste(valid, (max(0, -x1), max(0, -y1)))
+    return canvas
+
+
+def dilate_mask(mask_np, radius=10):
+    import cv2
+
+    m = (mask_np > 0).astype(np.uint8) * 255
+    k = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (radius * 2 + 1, radius * 2 + 1))
+    return (cv2.dilate(m, k, iterations=1) > 0).astype(np.uint8)
+
+
+def refine_mask_holes(mask_bool, kernel_size=7):
+    import cv2
+
+    m = mask_bool.astype(np.uint8) * 255
+    k = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (kernel_size, kernel_size))
+    closed = cv2.morphologyEx(m, cv2.MORPH_CLOSE, k, iterations=2)
+    contours, _ = cv2.findContours(closed, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+    filled = np.zeros_like(closed)
+    cv2.drawContours(filled, contours, -1, 255, thickness=cv2.FILLED)
+    return filled > 127
+
+
+def adain_color_fix(target_pil, source_pil, mask_pil):
+    from torchvision.transforms import ToPILImage, ToTensor
+
+    to_tensor = ToTensor()
+    t = to_tensor(target_pil).unsqueeze(0)
+    s = to_tensor(source_pil).unsqueeze(0)
+    m = to_tensor(mask_pil).unsqueeze(0)
+    eps = 1e-5
+    res = t.clone()
+    for ch in range(3):
+        bg_idx = m[0, 0] < 0.1
+        if bg_idx.sum() < 10:
+            continue
+        s_pix = s[0, ch][bg_idx]
+        t_pix = t[0, ch][bg_idx]
+        s_mean, s_std = s_pix.mean(), s_pix.std() + eps
+        t_mean, t_std = t_pix.mean(), t_pix.std() + eps
+        res[0, ch] = (t[0, ch] - t_mean) * (s_std / t_std) + s_mean
+    return ToPILImage()(res.squeeze(0).clamp(0, 1))
+
+
+def build_inputs(bg_pil, composite_full, mask_full, reference_ref, geometry_full):
+    """Produce the model's 1024x1024 conditioning tensors from full-frame inputs."""
+    target_res = MODEL_INPUT_RESOLUTION
+
+    mask_np = np.array(mask_full)
+    dilated01 = dilate_mask(mask_np, radius=10)
+    dilated_pil = Image.fromarray(dilated01 * 255, mode="L")
+
+    # Context image: full background with the (dilated) insertion region blacked.
+    full_bg = np.array(bg_pil.convert("RGB"))
+    context_image = Image.fromarray((full_bg * (1 - dilated01[:, :, None])).astype(np.uint8))
+
+    ideal_bbox, target_side = get_smart_crop_bbox(dilated_pil)
+
+    patch_composite = crop_and_pad(composite_full, ideal_bbox, target_side)
+    patch_mask = crop_and_pad(dilated_pil, ideal_bbox, target_side)
+    patch_geometry = crop_and_pad(geometry_full, ideal_bbox, target_side)
+    patch_bg_ref = crop_and_pad(bg_pil.convert("RGB"), ideal_bbox, target_side)
+    patch_mask_orig = crop_and_pad(Image.fromarray(mask_np), ideal_bbox, target_side)
+
+    comp_arr = np.array(patch_composite)
+    mask_dilated_arr = np.array(patch_mask) > 127
+    mask_orig_arr = refine_mask_holes(np.array(patch_mask_orig) > 127, kernel_size=7)
+    diff_region = mask_dilated_arr & (~mask_orig_arr)
+    comp_arr[diff_region] = [0, 0, 0]
+    patch_composite = Image.fromarray(comp_arr)
+
+    composite_image = patch_composite.resize((target_res, target_res), Image.LANCZOS)
+    model_input_mask = Image.fromarray(np.array(patch_mask).astype(np.uint8)).resize(
+        (target_res, target_res), Image.NEAREST
+    )
+    geometry_image = patch_geometry.resize((target_res, target_res), Image.LANCZOS)
+    background_reference_image = patch_bg_ref.resize((target_res, target_res), Image.LANCZOS)
+
+    inpaint_mask = Image.fromarray(((np.array(model_input_mask) > 0) * 255).astype(np.uint8))
+
+    return {
+        "composite_image": composite_image,
+        "inpaint_mask": inpaint_mask,
+        "reference_image": reference_ref,
+        "geometry_image": geometry_image,
+        "context_image": context_image,
+        "model_input_mask": model_input_mask,
+        "background_reference_image": background_reference_image,
+        "ideal_bbox": ideal_bbox,
+        "target_side": target_side,
+    }
+
+
+def paste_back(bg_pil, generated_patch, inp):
+    fixed = adain_color_fix(
+        generated_patch, inp["background_reference_image"], inp["model_input_mask"]
+    )
+    fixed = fixed.resize((inp["target_side"], inp["target_side"]), Image.LANCZOS)
+    x1, y1, x2, y2 = inp["ideal_bbox"]
+    W, H = bg_pil.size
+    pad_left = max(0, -x1)
+    pad_top = max(0, -y1)
+    valid_w = min(W, x2) - max(0, x1)
+    valid_h = min(H, y2) - max(0, y1)
+    patch_valid = fixed.crop((pad_left, pad_top, pad_left + valid_w, pad_top + valid_h))
+    out = bg_pil.convert("RGB").copy()
+    out.paste(patch_valid, (max(0, x1), max(0, y1)))
+    return out
+
+
+# ----------------------------------------------------------------------------
+# Inference
+# ----------------------------------------------------------------------------
+
+def _estimate_duration(bg, obj, cx, cy, scale, seed, ref_scale, steps, *a, **k):
+    try:
+        steps = int(steps)
+    except Exception:
+        steps = 28
+    return min(230, 60 + int(steps * 4.5))
+
+
+@spaces.GPU(duration=_estimate_duration)
+def insert_object(
+    bg: Image.Image,
+    obj: Image.Image,
+    cx: float,
+    cy: float,
+    scale: float,
+    seed: int,
+    ref_scale: float,
+    steps: int,
+    progress=gr.Progress(track_tqdm=True),
+):
+    """Insert a reference object into a background image with 3D-aware harmonization.
+
+    Args:
+        bg: Background scene image.
+        obj: Reference object image (background is removed automatically).
+        cx: Horizontal placement of the object center (0=left, 1=right).
+        cy: Vertical placement of the object center (0=top, 1=bottom).
+        scale: Object size as a fraction of the background's longest side.
+        seed: Random seed for reproducibility.
+        ref_scale: Reference guidance scale (identity preservation strength).
+        steps: Number of inference steps.
+
+    Returns:
+        The composited image with the object inserted, and a preview of the raw
+        2D placement used as geometric guidance.
+    """
+    if bg is None:
+        raise gr.Error("Please provide a background image.")
+    if obj is None:
+        raise gr.Error("Please provide an object image.")
+
+    t0 = time.perf_counter()
+    bg = bg.convert("RGB")
+    obj_rgba = segment_object(obj)
+
+    reference_ref = center_reference(obj_rgba, out_size=MODEL_INPUT_RESOLUTION)
+    placed_rgb, mask_full, geometry_full = place_object(bg, obj_rgba, cx, cy, scale)
+
+    inp = build_inputs(bg, placed_rgb, mask_full, reference_ref, geometry_full)
+
+    seed = int(seed)
+    final_images = direct_pipeline(
+        composite_image=inp["composite_image"],
+        inpaint_mask=inp["inpaint_mask"],
+        reference_image=inp["reference_image"],
+        geometry_image=inp["geometry_image"],
+        context_image=inp["context_image"],
+        seed=seed,
+        guidance_scale=30,
+        num_inference_steps=int(steps),
+        height=MODEL_INPUT_RESOLUTION,
+        width=MODEL_INPUT_RESOLUTION,
+        use_autocast=True,
+        reference_guidance_scale=float(ref_scale),
+    )
+    generated_patch = final_images[0]
+    result = paste_back(bg, generated_patch, inp)
+    print(f"[insert_object] done in {time.perf_counter() - t0:.1f}s (steps={steps})")
+    return result, placed_rgb
+
+
+def randomize_seed():
+    return random.randint(0, 2**31 - 1)
+
+
+# ----------------------------------------------------------------------------
+# UI
+# ----------------------------------------------------------------------------
+CSS = """
+#col-container { max-width: 1200px; margin: 0 auto; }
+.dark .gradio-container { color: var(--body-text-color); }
+"""
+
+INTRO = """
+# DIRECT: 3D-Aware Object Insertion
+
+Insert a reference **object** into a **background** scene with realistic,
+harmonized results, powered by the [DIRECT](https://huggingface.co/superGong/DIRECT)
+model (ICML 2026) — a FLUX.1-Fill-dev network guided by a decomposed visual proxy.
+
+**How to use:** upload a background and an object image (its background is
+removed automatically), choose *where* and *how big* to place it, then click **Insert**.
+
+> **Note.** The full paper uses an interactive 3D viewer (TRELLIS + Viser) to pose a
+> reconstructed 3D proxy of the object. That live 3D viewer cannot run inside a
+> single-port Space, so this demo drives the same DIRECT model with a simpler
+> **2D placement** (position + scale) as its geometric guidance.
+
+[Paper](https://arxiv.org/abs/2606.06601) · [Project page](https://gong1130.github.io/DIRECT/) · [Code](https://github.com/Gong1130/DIRECT)
+"""
+
+with gr.Blocks(theme=gr.themes.Citrus(), css=CSS) as demo:
+    with gr.Column(elem_id="col-container"):
+        gr.Markdown(INTRO)
+        with gr.Row():
+            with gr.Column(scale=1):
+                bg_input = gr.Image(label="Background image", type="pil", height=300)
+                obj_input = gr.Image(label="Object image", type="pil", height=300)
+                run_btn = gr.Button("Insert", variant="primary")
+            with gr.Column(scale=1):
+                out_result = gr.Image(label="Inserted result", type="pil", height=360)
+                out_preview = gr.Image(label="2D placement (geometric guidance)", type="pil", height=240)
+
+        with gr.Accordion("Placement & advanced settings", open=True):
+            with gr.Row():
+                cx = gr.Slider(0.0, 1.0, value=0.5, step=0.01, label="Horizontal position")
+                cy = gr.Slider(0.0, 1.0, value=0.6, step=0.01, label="Vertical position")
+                scale = gr.Slider(0.05, 0.9, value=0.35, step=0.01, label="Object size")
+            with gr.Row():
+                ref_scale = gr.Slider(1.0, 5.0, value=2.0, step=0.1, label="Reference guidance scale")
+                steps = gr.Slider(15, 50, value=28, step=1, label="Inference steps")
+                seed = gr.Number(label="Seed", value=42, precision=0)
+            rand_btn = gr.Button("🎲 Randomize seed")
+
+        gr.Examples(
+            examples=[
+                ["examples/bg_landscape.jpg", "examples/obj_ducks.jpg", 0.55, 0.70, 0.28, 42, 2.0, 28],
+                ["examples/bg_tent.jpg", "examples/obj_dog.jpg", 0.45, 0.68, 0.30, 7, 2.0, 28],
+                ["examples/bg_beach.jpg", "examples/obj_cake.jpg", 0.50, 0.72, 0.22, 123, 2.5, 30],
+            ],
+            inputs=[bg_input, obj_input, cx, cy, scale, seed, ref_scale, steps],
+            outputs=[out_result, out_preview],
+            fn=insert_object,
+            cache_examples=True,
+            cache_mode="lazy",
+        )
+
+    rand_btn.click(fn=randomize_seed, outputs=seed)
+    run_btn.click(
+        fn=insert_object,
+        inputs=[bg_input, obj_input, cx, cy, scale, seed, ref_scale, steps],
+        outputs=[out_result, out_preview],
+        api_name="insert",
+    )
+
+if __name__ == "__main__":
+    demo.launch(mcp_server=True)

direct/__init__.py

@@ -0,0 +1,3 @@
+from .pipeline import DirectPipeline
+
+__all__ = ["DirectPipeline"]

direct/layers.py

@@ -0,0 +1,410 @@
+from typing import Optional, Union
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+from diffusers.models.attention_processor import Attention
+
+
+class LoRALinearLayer(nn.Module):
+    def __init__(
+        self,
+        in_features: int,
+        out_features: int,
+        rank: int = 4,
+        network_alpha: Optional[float] = None,
+        device: Optional[Union[torch.device, str]] = None,
+        dtype: Optional[torch.dtype] = None,
+        number=0,
+        n_loras=1,
+    ):
+        super().__init__()
+        self.down = nn.Linear(in_features, rank, bias=False, device=device, dtype=dtype)
+        self.up = nn.Linear(rank, out_features, bias=False, device=device, dtype=dtype)
+        # This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script.
+        # See https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning
+        self.network_alpha = network_alpha
+        self.rank = rank
+        self.out_features = out_features
+        self.in_features = in_features
+
+        nn.init.normal_(self.down.weight, std=1 / rank)
+        nn.init.zeros_(self.up.weight)
+
+        self.number = number
+        self.n_loras = n_loras
+
+    def forward(self, hidden_states: torch.Tensor, cond_seq_len: int = None) -> torch.Tensor:
+        orig_dtype = hidden_states.dtype
+        dtype = self.down.weight.dtype
+
+        batch_size = hidden_states.shape[0]
+        cond_size = cond_seq_len
+
+        block_size = hidden_states.shape[1] - cond_size * self.n_loras
+        shape = (batch_size, hidden_states.shape[1], 3072)
+        mask = torch.ones(shape, device=hidden_states.device, dtype=dtype)
+        mask[:, : block_size + self.number * cond_size, :] = 0
+        mask[:, block_size + (self.number + 1) * cond_size :, :] = 0
+        hidden_states = mask * hidden_states
+
+        down_hidden_states = self.down(hidden_states.to(dtype))
+        up_hidden_states = self.up(down_hidden_states)
+
+        if self.network_alpha is not None:
+            up_hidden_states *= self.network_alpha / self.rank
+
+        return up_hidden_states.to(orig_dtype)
+
+
+class TextLoRALinearLayer(nn.Module):
+    def __init__(
+        self,
+        in_features: int,
+        out_features: int,
+        rank: int = 4,
+        network_alpha: Optional[float] = None,
+        device: Optional[Union[torch.device, str]] = None,
+        dtype: Optional[torch.dtype] = None,
+        token_length=512,
+    ):
+        super().__init__()
+        self.down = nn.Linear(in_features, rank, bias=False, device=device, dtype=dtype)
+        self.up = nn.Linear(rank, out_features, bias=False, device=device, dtype=dtype)
+        # This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script.
+        # See https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning
+        self.network_alpha = network_alpha
+        self.rank = rank
+        self.out_features = out_features
+        self.in_features = in_features
+
+        nn.init.normal_(self.down.weight, std=1 / rank)
+        nn.init.zeros_(self.up.weight)
+
+        self.token_length = token_length
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        orig_dtype = hidden_states.dtype
+        dtype = self.down.weight.dtype
+
+        batch_size, seq_len, feature_dim = hidden_states.shape
+        if seq_len > self.token_length:
+            mask = torch.ones((batch_size, seq_len, feature_dim), device=hidden_states.device, dtype=dtype)
+            mask[:, self.token_length :, :] = 0
+            hidden_states = mask * hidden_states
+
+        down_hidden_states = self.down(hidden_states.to(dtype))
+        up_hidden_states = self.up(down_hidden_states)
+
+        if self.network_alpha is not None:
+            up_hidden_states *= self.network_alpha / self.rank
+
+        return up_hidden_states.to(orig_dtype)
+
+
+class MultiSingleStreamBlockLoraProcessor(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        ranks=[],
+        lora_weights=[],
+        network_alphas=[],
+        device=None,
+        dtype=None,
+        n_loras=1,
+        text_lora_config=None,
+    ):
+        super().__init__()
+        self.n_loras = n_loras
+        if text_lora_config is not None:
+            self.text_len = text_lora_config.get("token_length", 512)
+        else:
+            self.text_len = 512
+
+        self.q_loras = nn.ModuleList(
+            [
+                LoRALinearLayer(
+                    dim, dim, ranks[i], network_alphas[i], device=device, dtype=dtype, number=i, n_loras=n_loras
+                )
+                for i in range(n_loras)
+            ]
+        )
+        self.k_loras = nn.ModuleList(
+            [
+                LoRALinearLayer(
+                    dim, dim, ranks[i], network_alphas[i], device=device, dtype=dtype, number=i, n_loras=n_loras
+                )
+                for i in range(n_loras)
+            ]
+        )
+        self.v_loras = nn.ModuleList(
+            [
+                LoRALinearLayer(
+                    dim, dim, ranks[i], network_alphas[i], device=device, dtype=dtype, number=i, n_loras=n_loras
+                )
+                for i in range(n_loras)
+            ]
+        )
+        self.lora_weights = lora_weights
+
+        if text_lora_config is not None:
+            t_rank = text_lora_config.get("rank", 4)
+            t_alpha = text_lora_config.get("alpha", None)
+
+            self.text_q_lora = TextLoRALinearLayer(
+                dim, dim, t_rank, t_alpha, device=device, dtype=dtype, token_length=self.text_len
+            )
+            self.text_k_lora = TextLoRALinearLayer(
+                dim, dim, t_rank, t_alpha, device=device, dtype=dtype, token_length=self.text_len
+            )
+            self.text_v_lora = TextLoRALinearLayer(
+                dim, dim, t_rank, t_alpha, device=device, dtype=dtype, token_length=self.text_len
+            )
+
+    def __call__(
+        self,
+        attn: Attention,
+        hidden_states: torch.FloatTensor,
+        encoder_hidden_states: torch.FloatTensor = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        image_rotary_emb: Optional[torch.Tensor] = None,
+        use_cond=False,
+    ) -> torch.FloatTensor:
+        batch_size, seq_len, _ = hidden_states.shape
+
+        total_img_seq_len = seq_len - self.text_len
+        assert total_img_seq_len % (1 + self.n_loras) == 0, (
+            f"total_img_seq_len:{total_img_seq_len}, n_loras:{self.n_loras}, "
+            f"seq_len:{seq_len}, text_len:{self.text_len}"
+        )
+        cond_seq_len = total_img_seq_len // (1 + self.n_loras)
+
+        query = attn.to_q(hidden_states)
+        key = attn.to_k(hidden_states)
+        value = attn.to_v(hidden_states)
+
+        for i in range(self.n_loras):
+            query = query + self.lora_weights[i] * self.q_loras[i](hidden_states, cond_seq_len=cond_seq_len)
+            key = key + self.lora_weights[i] * self.k_loras[i](hidden_states, cond_seq_len=cond_seq_len)
+            value = value + self.lora_weights[i] * self.v_loras[i](hidden_states, cond_seq_len=cond_seq_len)
+
+        if getattr(self, "text_q_lora", None) is not None:
+            query = query + self.text_q_lora(hidden_states)
+        if getattr(self, "text_k_lora", None) is not None:
+            key = key + self.text_k_lora(hidden_states)
+        if getattr(self, "text_v_lora", None) is not None:
+            value = value + self.text_v_lora(hidden_states)
+
+        inner_dim = key.shape[-1]
+        head_dim = inner_dim // attn.heads
+
+        query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+        key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+        value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+
+        if attn.norm_q is not None:
+            query = attn.norm_q(query)
+        if attn.norm_k is not None:
+            key = attn.norm_k(key)
+
+        if image_rotary_emb is not None:
+            from diffusers.models.embeddings import apply_rotary_emb
+
+            query = apply_rotary_emb(query, image_rotary_emb)
+            key = apply_rotary_emb(key, image_rotary_emb)
+
+        cond_size = cond_seq_len
+        block_size = hidden_states.shape[1] - cond_size * self.n_loras
+
+        hidden_states = F.scaled_dot_product_attention(query, key, value, dropout_p=0.0, is_causal=False)
+
+        hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
+        hidden_states = hidden_states.to(query.dtype)
+
+        cond_hidden_states = hidden_states[:, block_size:, :]
+        hidden_states = hidden_states[:, :block_size, :]
+
+        return hidden_states if not use_cond else (hidden_states, cond_hidden_states)
+
+
+class MultiDoubleStreamBlockLoraProcessor(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        ranks=[],
+        lora_weights=[],
+        network_alphas=[],
+        device=None,
+        dtype=None,
+        n_loras=1,
+        text_lora_config=None,
+    ):
+        super().__init__()
+        self.n_loras = n_loras
+        self.q_loras = nn.ModuleList(
+            [
+                LoRALinearLayer(
+                    dim, dim, ranks[i], network_alphas[i], device=device, dtype=dtype, number=i, n_loras=n_loras
+                )
+                for i in range(n_loras)
+            ]
+        )
+        self.k_loras = nn.ModuleList(
+            [
+                LoRALinearLayer(
+                    dim, dim, ranks[i], network_alphas[i], device=device, dtype=dtype, number=i, n_loras=n_loras
+                )
+                for i in range(n_loras)
+            ]
+        )
+        self.v_loras = nn.ModuleList(
+            [
+                LoRALinearLayer(
+                    dim, dim, ranks[i], network_alphas[i], device=device, dtype=dtype, number=i, n_loras=n_loras
+                )
+                for i in range(n_loras)
+            ]
+        )
+        self.proj_loras = nn.ModuleList(
+            [
+                LoRALinearLayer(
+                    dim, dim, ranks[i], network_alphas[i], device=device, dtype=dtype, number=i, n_loras=n_loras
+                )
+                for i in range(n_loras)
+            ]
+        )
+        self.lora_weights = lora_weights
+        if text_lora_config is not None:
+            t_rank = text_lora_config.get("rank", 4)
+            t_alpha = text_lora_config.get("alpha", None)
+            t_len = text_lora_config.get("token_length", 512)
+
+            self.text_q_lora = TextLoRALinearLayer(
+                dim, dim, t_rank, t_alpha, device=device, dtype=dtype, token_length=t_len
+            )
+            self.text_k_lora = TextLoRALinearLayer(
+                dim, dim, t_rank, t_alpha, device=device, dtype=dtype, token_length=t_len
+            )
+            self.text_v_lora = TextLoRALinearLayer(
+                dim, dim, t_rank, t_alpha, device=device, dtype=dtype, token_length=t_len
+            )
+            self.text_proj_lora = TextLoRALinearLayer(
+                dim, dim, t_rank, t_alpha, device=device, dtype=dtype, token_length=t_len
+            )
+
+    def __call__(
+        self,
+        attn: Attention,
+        hidden_states: torch.FloatTensor,
+        encoder_hidden_states: torch.FloatTensor = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        image_rotary_emb: Optional[torch.Tensor] = None,
+        use_cond=False,
+    ) -> torch.FloatTensor:
+        batch_size, total_img_seq_len, _ = hidden_states.shape
+
+        # `context` projections.
+        inner_dim = 3072
+        head_dim = inner_dim // attn.heads
+        encoder_hidden_states_query_proj = attn.add_q_proj(encoder_hidden_states)
+        encoder_hidden_states_key_proj = attn.add_k_proj(encoder_hidden_states)
+        encoder_hidden_states_value_proj = attn.add_v_proj(encoder_hidden_states)
+
+        if getattr(self, "text_q_lora", None) is not None:
+            encoder_hidden_states_query_proj = encoder_hidden_states_query_proj + self.text_q_lora(
+                encoder_hidden_states
+            )
+        if getattr(self, "text_k_lora", None) is not None:
+            encoder_hidden_states_key_proj = encoder_hidden_states_key_proj + self.text_k_lora(encoder_hidden_states)
+        if getattr(self, "text_v_lora", None) is not None:
+            encoder_hidden_states_value_proj = encoder_hidden_states_value_proj + self.text_v_lora(
+                encoder_hidden_states
+            )
+
+        encoder_hidden_states_query_proj = encoder_hidden_states_query_proj.view(
+            batch_size, -1, attn.heads, head_dim
+        ).transpose(1, 2)
+        encoder_hidden_states_key_proj = encoder_hidden_states_key_proj.view(
+            batch_size, -1, attn.heads, head_dim
+        ).transpose(1, 2)
+        encoder_hidden_states_value_proj = encoder_hidden_states_value_proj.view(
+            batch_size, -1, attn.heads, head_dim
+        ).transpose(1, 2)
+
+        if attn.norm_added_q is not None:
+            encoder_hidden_states_query_proj = attn.norm_added_q(encoder_hidden_states_query_proj)
+        if attn.norm_added_k is not None:
+            encoder_hidden_states_key_proj = attn.norm_added_k(encoder_hidden_states_key_proj)
+
+        assert total_img_seq_len % (1 + self.n_loras) == 0, (
+            f"total_img_seq_len:{total_img_seq_len}, n_loras:{self.n_loras}"
+        )
+        cond_seq_len = total_img_seq_len // (1 + self.n_loras)
+
+        query = attn.to_q(hidden_states)
+        key = attn.to_k(hidden_states)
+        value = attn.to_v(hidden_states)
+
+        for i in range(self.n_loras):
+            query = query + self.lora_weights[i] * self.q_loras[i](hidden_states, cond_seq_len=cond_seq_len)
+            key = key + self.lora_weights[i] * self.k_loras[i](hidden_states, cond_seq_len=cond_seq_len)
+            value = value + self.lora_weights[i] * self.v_loras[i](hidden_states, cond_seq_len=cond_seq_len)
+
+        inner_dim = key.shape[-1]
+        head_dim = inner_dim // attn.heads
+        query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+        key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+        value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+
+        if attn.norm_q is not None:
+            query = attn.norm_q(query)
+        if attn.norm_k is not None:
+            key = attn.norm_k(key)
+
+        # attention
+        query = torch.cat([encoder_hidden_states_query_proj, query], dim=2)
+        key = torch.cat([encoder_hidden_states_key_proj, key], dim=2)
+        value = torch.cat([encoder_hidden_states_value_proj, value], dim=2)
+
+        if image_rotary_emb is not None:
+            from diffusers.models.embeddings import apply_rotary_emb
+
+            query = apply_rotary_emb(query, image_rotary_emb)
+            key = apply_rotary_emb(key, image_rotary_emb)
+
+        cond_size = cond_seq_len
+        block_size = hidden_states.shape[1] - cond_size * self.n_loras
+
+        hidden_states = F.scaled_dot_product_attention(query, key, value, dropout_p=0.0, is_causal=False)
+
+        hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
+        hidden_states = hidden_states.to(query.dtype)
+
+        encoder_hidden_states, hidden_states = (
+            hidden_states[:, : encoder_hidden_states.shape[1]],
+            hidden_states[:, encoder_hidden_states.shape[1] :],
+        )
+
+        hidden_states_input = hidden_states
+        hidden_states = attn.to_out[0](hidden_states)
+        for i in range(self.n_loras):
+            hidden_states = hidden_states + self.lora_weights[i] * self.proj_loras[i](
+                hidden_states_input, cond_seq_len=cond_seq_len
+            )
+
+        hidden_states = attn.to_out[1](hidden_states)
+
+        encoder_input = encoder_hidden_states
+        encoder_hidden_states = attn.to_add_out(encoder_hidden_states)
+        if getattr(self, "text_proj_lora", None) is not None:
+            encoder_hidden_states = encoder_hidden_states + self.text_proj_lora(encoder_input)
+
+        cond_hidden_states = hidden_states[:, block_size:, :]
+        hidden_states = hidden_states[:, :block_size, :]
+
+        return (
+            (hidden_states, encoder_hidden_states, cond_hidden_states)
+            if use_cond
+            else (encoder_hidden_states, hidden_states)
+        )

direct/pipeline.py

@@ -0,0 +1,1507 @@
+import inspect
+import json
+import os
+import re
+from typing import Any, Callable, Dict, List, Optional, Union
+
+import numpy as np
+import torch
+from PIL import Image
+from transformers import (
+    AutoModel,
+    AutoProcessor,
+    CLIPTextModel,
+    CLIPTokenizer,
+    T5EncoderModel,
+    T5TokenizerFast,
+)
+
+from diffusers.image_processor import VaeImageProcessor
+from diffusers.loaders import FluxLoraLoaderMixin, FromSingleFileMixin, TextualInversionLoaderMixin
+from diffusers.models import AutoencoderKL
+from diffusers.schedulers import FlowMatchEulerDiscreteScheduler
+from diffusers.utils import (
+    USE_PEFT_BACKEND,
+    is_torch_xla_available,
+    logging,
+    scale_lora_layers,
+    unscale_lora_layers,
+)
+from diffusers.utils.torch_utils import randn_tensor
+from diffusers.pipelines.pipeline_utils import DiffusionPipeline
+from diffusers.pipelines.flux.pipeline_output import FluxPipelineOutput
+from contextlib import nullcontext
+from huggingface_hub import hf_hub_download
+from safetensors.torch import load_file
+
+from .layers import MultiDoubleStreamBlockLoraProcessor, MultiSingleStreamBlockLoraProcessor
+from .transformer_flux import FluxTransformer2DModelwithcond
+
+
+if is_torch_xla_available():
+    import torch_xla.core.xla_model as xm
+
+    XLA_AVAILABLE = True
+else:
+    XLA_AVAILABLE = False
+
+
+logger = logging.get_logger(__name__)  # pylint: disable=invalid-name
+
+
+# Copied from diffusers.pipelines.flux.pipeline_flux.calculate_shift
+def calculate_shift(
+    image_seq_len,
+    base_seq_len: int = 256,
+    max_seq_len: int = 4096,
+    base_shift: float = 0.5,
+    max_shift: float = 1.15,
+):
+    m = (max_shift - base_shift) / (max_seq_len - base_seq_len)
+    b = base_shift - m * base_seq_len
+    mu = image_seq_len * m + b
+    return mu
+
+
+# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps
+def retrieve_timesteps(
+    scheduler,
+    num_inference_steps: Optional[int] = None,
+    device: Optional[Union[str, torch.device]] = None,
+    timesteps: Optional[List[int]] = None,
+    sigmas: Optional[List[float]] = None,
+    **kwargs,
+):
+    r"""
+    Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles
+    custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.
+
+    Args:
+        scheduler (`SchedulerMixin`):
+            The scheduler to get timesteps from.
+        num_inference_steps (`int`):
+            The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`
+            must be `None`.
+        device (`str` or `torch.device`, *optional*):
+            The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
+        timesteps (`List[int]`, *optional*):
+            Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,
+            `num_inference_steps` and `sigmas` must be `None`.
+        sigmas (`List[float]`, *optional*):
+            Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,
+            `num_inference_steps` and `timesteps` must be `None`.
+
+    Returns:
+        `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the
+        second element is the number of inference steps.
+    """
+    if timesteps is not None and sigmas is not None:
+        raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values")
+    if timesteps is not None:
+        accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
+        if not accepts_timesteps:
+            raise ValueError(
+                f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
+                f" timestep schedules. Please check whether you are using the correct scheduler."
+            )
+        scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)
+        timesteps = scheduler.timesteps
+        num_inference_steps = len(timesteps)
+    elif sigmas is not None:
+        accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
+        if not accept_sigmas:
+            raise ValueError(
+                f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
+                f" sigmas schedules. Please check whether you are using the correct scheduler."
+            )
+        scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)
+        timesteps = scheduler.timesteps
+        num_inference_steps = len(timesteps)
+    else:
+        scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)
+        timesteps = scheduler.timesteps
+    return timesteps, num_inference_steps
+
+
+# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents
+def retrieve_latents(
+    encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample"
+):
+    if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":
+        return encoder_output.latent_dist.sample(generator)
+    elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":
+        return encoder_output.latent_dist.mode()
+    elif hasattr(encoder_output, "latents"):
+        return encoder_output.latents
+    else:
+        raise AttributeError("Could not access latents of provided encoder_output")
+
+def to_tensor_0_1(image_input: Union[Image.Image, np.ndarray, torch.Tensor]) -> torch.Tensor:
+    """
+    Converts a PIL Image or NumPy array image into a PyTorch tensor
+    with shape (1, C, H, W), float32 dtype, and values scaled to [0.0, 1.0].
+
+    Tensor inputs are assumed to already use CHW or BCHW layout and are returned unchanged
+    except for adding a batch dimension to CHW tensors.
+    """
+    if isinstance(image_input, torch.Tensor):
+        if image_input.ndim == 3:
+            return image_input.unsqueeze(0)
+        if image_input.ndim == 4:
+            return image_input
+        raise ValueError(f"Input tensor has unexpected dimensions: {image_input.ndim}. Expected 3 or 4.")
+
+    if isinstance(image_input, Image.Image):
+        pil_image = image_input
+
+        if pil_image.mode == 'RGB':
+            image_np = np.array(pil_image)
+        elif pil_image.mode == 'L':
+            image_np = np.expand_dims(np.array(pil_image), axis=-1)
+        elif pil_image.mode == 'RGBA':
+            pil_image = pil_image.convert('RGB')
+            image_np = np.array(pil_image)
+        else:
+            raise ValueError(f"Unsupported PIL image mode: {pil_image.mode}. Expected RGB, L, or RGBA.")
+
+    elif isinstance(image_input, np.ndarray):
+        image_np = image_input.copy()
+
+        if image_np.ndim == 2:
+            image_np = np.expand_dims(image_np, axis=-1)
+
+        if image_np.ndim != 3:
+            raise ValueError(f"Input NumPy array has unexpected dimensions: {image_np.ndim}. Expected 2 or 3.")
+
+        num_channels = image_np.shape[-1]
+        if num_channels == 4:
+            image_np = image_np[:, :, :3]
+        elif num_channels not in [1, 3]:
+            raise ValueError(f"Input NumPy array has {num_channels} channels, expected 1 (Grayscale) or 3/4 (RGB/RGBA).")
+
+    else:
+        raise TypeError(f"Input must be PIL Image, NumPy array, or torch.Tensor, got {type(image_input)}")
+
+    if image_np.dtype == np.uint8:
+        image_np_float = image_np.astype(np.float32) / 255.0
+    elif np.issubdtype(image_np.dtype, np.floating):
+        if image_np.max() > 1.0 + 1e-3:
+            print("Warning: Input float NumPy array has values > 1.0. Assuming range [0, 255] and scaling.")
+            image_np_float = image_np.astype(np.float32) / 255.0
+        else:
+            image_np_float = image_np.astype(np.float32)
+        image_np_float = np.clip(image_np_float, 0.0, 1.0)
+    else:
+        raise TypeError(f"Unsupported NumPy array dtype: {image_np.dtype}. Expected uint8 or float.")
+
+    tensor = torch.from_numpy(image_np_float.transpose((2, 0, 1)))
+    return tensor.unsqueeze(0)
+
+class _DirectFluxFillPipeline(
+    DiffusionPipeline,
+    FluxLoraLoaderMixin,
+    FromSingleFileMixin,
+    TextualInversionLoaderMixin,
+):
+    r"""
+    The Flux Fill pipeline for image inpainting/outpainting.
+
+    Reference: https://blackforestlabs.ai/flux-1-tools/
+
+    Args:
+        transformer ([`FluxTransformer2DModelwithcond`]):
+            Conditional Transformer (MMDiT) architecture to denoise the encoded image latents.
+        scheduler ([`FlowMatchEulerDiscreteScheduler`]):
+            A scheduler to be used in combination with `transformer` to denoise the encoded image latents.
+        vae ([`AutoencoderKL`]):
+            Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
+        text_encoder ([`CLIPTextModel`]):
+            [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically
+            the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.
+        text_encoder_2 ([`T5EncoderModel`]):
+            [T5](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5EncoderModel), specifically
+            the [google/t5-v1_1-xxl](https://huggingface.co/google/t5-v1_1-xxl) variant.
+        tokenizer (`CLIPTokenizer`):
+            Tokenizer of class
+            [CLIPTokenizer](https://huggingface.co/docs/transformers/en/model_doc/clip#transformers.CLIPTokenizer).
+        tokenizer_2 (`T5TokenizerFast`):
+            Second Tokenizer of class
+            [T5TokenizerFast](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5TokenizerFast).
+    """
+
+    model_cpu_offload_seq = "text_encoder->text_encoder_2->transformer->vae"
+    _optional_components = []
+    _callback_tensor_inputs = ["latents", "prompt_embeds"]
+
+    def __init__(
+        self,
+        scheduler: FlowMatchEulerDiscreteScheduler,
+        vae: AutoencoderKL,
+        text_encoder: CLIPTextModel,
+        tokenizer: CLIPTokenizer,
+        text_encoder_2: T5EncoderModel,
+        tokenizer_2: T5TokenizerFast,
+        transformer: FluxTransformer2DModelwithcond,
+    ):
+        super().__init__()
+
+        self.register_modules(
+            vae=vae,
+            text_encoder=text_encoder,
+            text_encoder_2=text_encoder_2,
+            tokenizer=tokenizer,
+            tokenizer_2=tokenizer_2,
+            transformer=transformer,
+            scheduler=scheduler,
+        )
+        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8
+        # Flux latents are turned into 2x2 patches and packed. This means the latent width and height has to be divisible
+        # by the patch size. So the vae scale factor is multiplied by the patch size to account for this
+        self.latent_channels = self.vae.config.latent_channels if getattr(self, "vae", None) else 16
+        self.image_processor = VaeImageProcessor(
+            vae_scale_factor=self.vae_scale_factor * 2, vae_latent_channels=self.latent_channels
+        )
+        self.mask_processor = VaeImageProcessor(
+            vae_scale_factor=self.vae_scale_factor * 2,
+            vae_latent_channels=self.latent_channels,
+            do_normalize=False,
+            do_binarize=False,
+            do_convert_grayscale=True,
+        )
+        self.tokenizer_max_length = (
+            self.tokenizer.model_max_length if hasattr(self, "tokenizer") and self.tokenizer is not None else 77
+        )
+        self.default_sample_size = 128
+
+    # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_t5_prompt_embeds
+    def _get_t5_prompt_embeds(
+        self,
+        prompt: Union[str, List[str]] = None,
+        num_images_per_prompt: int = 1,
+        max_sequence_length: int = 512,
+        device: Optional[torch.device] = None,
+        dtype: Optional[torch.dtype] = None,
+    ):
+        device = device or self._execution_device
+        dtype = dtype or self.text_encoder.dtype
+
+        prompt = [prompt] if isinstance(prompt, str) else prompt
+        batch_size = len(prompt)
+
+        if isinstance(self, TextualInversionLoaderMixin):
+            prompt = self.maybe_convert_prompt(prompt, self.tokenizer_2)
+
+        text_inputs = self.tokenizer_2(
+            prompt,
+            padding="max_length",
+            max_length=max_sequence_length,
+            truncation=True,
+            return_length=False,
+            return_overflowing_tokens=False,
+            return_tensors="pt",
+        )
+        text_input_ids = text_inputs.input_ids
+        untruncated_ids = self.tokenizer_2(prompt, padding="longest", return_tensors="pt").input_ids
+
+        if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids):
+            removed_text = self.tokenizer_2.batch_decode(untruncated_ids[:, self.tokenizer_max_length - 1 : -1])
+            logger.warning(
+                "The following part of your input was truncated because `max_sequence_length` is set to "
+                f" {max_sequence_length} tokens: {removed_text}"
+            )
+
+        prompt_embeds = self.text_encoder_2(text_input_ids.to(device), output_hidden_states=False)[0]
+
+        dtype = self.text_encoder_2.dtype
+        prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
+
+        _, seq_len, _ = prompt_embeds.shape
+
+        # duplicate text embeddings and attention mask for each generation per prompt, using mps friendly method
+        prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
+        prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)
+
+        return prompt_embeds
+
+    # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._get_clip_prompt_embeds
+    def _get_clip_prompt_embeds(
+        self,
+        prompt: Union[str, List[str]],
+        num_images_per_prompt: int = 1,
+        device: Optional[torch.device] = None,
+    ):
+        device = device or self._execution_device
+
+        prompt = [prompt] if isinstance(prompt, str) else prompt
+        batch_size = len(prompt)
+
+        if isinstance(self, TextualInversionLoaderMixin):
+            prompt = self.maybe_convert_prompt(prompt, self.tokenizer)
+
+        text_inputs = self.tokenizer(
+            prompt,
+            padding="max_length",
+            max_length=self.tokenizer_max_length,
+            truncation=True,
+            return_overflowing_tokens=False,
+            return_length=False,
+            return_tensors="pt",
+        )
+
+        text_input_ids = text_inputs.input_ids
+        untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids
+        if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids):
+            removed_text = self.tokenizer.batch_decode(untruncated_ids[:, self.tokenizer_max_length - 1 : -1])
+            logger.warning(
+                "The following part of your input was truncated because CLIP can only handle sequences up to"
+                f" {self.tokenizer_max_length} tokens: {removed_text}"
+            )
+        prompt_embeds = self.text_encoder(text_input_ids.to(device), output_hidden_states=False)
+
+        # Use pooled output of CLIPTextModel
+        prompt_embeds = prompt_embeds.pooler_output
+        prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
+
+        # duplicate text embeddings for each generation per prompt, using mps friendly method
+        prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt)
+        prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, -1)
+
+        return prompt_embeds
+
+    def prepare_mask_latents(
+        self,
+        mask,
+        masked_image,
+        batch_size,
+        num_channels_latents,
+        num_images_per_prompt,
+        height,
+        width,
+        dtype,
+        device,
+        generator,
+    ):
+        # 1. calculate the height and width of the latents
+        # VAE applies 8x compression on images but we must also account for packing which requires
+        # latent height and width to be divisible by 2.
+        height = 2 * (int(height) // (self.vae_scale_factor * 2))
+        width = 2 * (int(width) // (self.vae_scale_factor * 2))
+
+        # 2. encode the masked image
+        if masked_image.shape[1] == num_channels_latents:
+            masked_image_latents = masked_image
+        else:
+            masked_image_latents = retrieve_latents(self.vae.encode(masked_image), generator=generator)
+
+        masked_image_latents = (masked_image_latents - self.vae.config.shift_factor) * self.vae.config.scaling_factor
+        masked_image_latents = masked_image_latents.to(device=device, dtype=dtype)
+
+        # 3. duplicate mask and masked_image_latents for each generation per prompt, using mps friendly method
+        batch_size = batch_size * num_images_per_prompt
+        if mask.shape[0] < batch_size:
+            if not batch_size % mask.shape[0] == 0:
+                raise ValueError(
+                    "The passed mask and the required batch size don't match. Masks are supposed to be duplicated to"
+                    f" a total batch size of {batch_size}, but {mask.shape[0]} masks were passed. Make sure the number"
+                    " of masks that you pass is divisible by the total requested batch size."
+                )
+            mask = mask.repeat(batch_size // mask.shape[0], 1, 1, 1)
+        if masked_image_latents.shape[0] < batch_size:
+            if not batch_size % masked_image_latents.shape[0] == 0:
+                raise ValueError(
+                    "The passed images and the required batch size don't match. Images are supposed to be duplicated"
+                    f" to a total batch size of {batch_size}, but {masked_image_latents.shape[0]} images were passed."
+                    " Make sure the number of images that you pass is divisible by the total requested batch size."
+                )
+            masked_image_latents = masked_image_latents.repeat(batch_size // masked_image_latents.shape[0], 1, 1, 1)
+
+        # 4. pack the masked_image_latents
+        # batch_size, num_channels_latents, height, width -> batch_size, height//2 * width//2 , num_channels_latents*4
+        masked_image_latents = self._pack_latents(
+            masked_image_latents,
+            batch_size,
+            num_channels_latents,
+            height,
+            width,
+        )
+
+        # 5.resize mask to latents shape we we concatenate the mask to the latents
+        mask = mask[:, 0, :, :]  # batch_size, 8 * height, 8 * width (mask has not been 8x compressed)
+        mask = mask.view(
+            batch_size, height, self.vae_scale_factor, width, self.vae_scale_factor
+        )  # batch_size, height, 8, width, 8
+        mask = mask.permute(0, 2, 4, 1, 3)  # batch_size, 8, 8, height, width
+        mask = mask.reshape(
+            batch_size, self.vae_scale_factor * self.vae_scale_factor, height, width
+        )  # batch_size, 8*8, height, width
+
+        # 6. pack the mask:
+        # batch_size, 64, height, width -> batch_size, height//2 * width//2 , 64*2*2
+        mask = self._pack_latents(
+            mask,
+            batch_size,
+            self.vae_scale_factor * self.vae_scale_factor,
+            height,
+            width,
+        )
+        mask = mask.to(device=device, dtype=dtype)
+
+        return mask, masked_image_latents
+
+    # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline.encode_prompt
+    def encode_prompt(
+        self,
+        prompt: Union[str, List[str]],
+        prompt_2: Optional[Union[str, List[str]]] = None,
+        device: Optional[torch.device] = None,
+        num_images_per_prompt: int = 1,
+        prompt_embeds: Optional[torch.FloatTensor] = None,
+        pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
+        max_sequence_length: int = 512,
+        lora_scale: Optional[float] = None,
+    ):
+        r"""
+
+        Args:
+            prompt (`str` or `List[str]`, *optional*):
+                prompt to be encoded
+            prompt_2 (`str` or `List[str]`, *optional*):
+                The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
+                used in all text-encoders
+            device: (`torch.device`):
+                torch device
+            num_images_per_prompt (`int`):
+                number of images that should be generated per prompt
+            prompt_embeds (`torch.FloatTensor`, *optional*):
+                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
+                provided, text embeddings will be generated from `prompt` input argument.
+            pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
+                Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
+                If not provided, pooled text embeddings will be generated from `prompt` input argument.
+            lora_scale (`float`, *optional*):
+                A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.
+        """
+        device = device or self._execution_device
+
+        # set lora scale so that monkey patched LoRA
+        # function of text encoder can correctly access it
+        if lora_scale is not None and isinstance(self, FluxLoraLoaderMixin):
+            self._lora_scale = lora_scale
+
+            # dynamically adjust the LoRA scale
+            if self.text_encoder is not None and USE_PEFT_BACKEND:
+                scale_lora_layers(self.text_encoder, lora_scale)
+            if self.text_encoder_2 is not None and USE_PEFT_BACKEND:
+                scale_lora_layers(self.text_encoder_2, lora_scale)
+
+        prompt = [prompt] if isinstance(prompt, str) else prompt
+
+        if prompt_embeds is None:
+            prompt_2 = prompt_2 or prompt
+            prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2
+
+            # We only use the pooled prompt output from the CLIPTextModel
+            pooled_prompt_embeds = self._get_clip_prompt_embeds(
+                prompt=prompt,
+                device=device,
+                num_images_per_prompt=num_images_per_prompt,
+            )
+            prompt_embeds = self._get_t5_prompt_embeds(
+                prompt=prompt_2,
+                num_images_per_prompt=num_images_per_prompt,
+                max_sequence_length=max_sequence_length,
+                device=device,
+            )
+
+        if self.text_encoder is not None:
+            if isinstance(self, FluxLoraLoaderMixin) and USE_PEFT_BACKEND:
+                # Retrieve the original scale by scaling back the LoRA layers
+                unscale_lora_layers(self.text_encoder, lora_scale)
+
+        if self.text_encoder_2 is not None:
+            if isinstance(self, FluxLoraLoaderMixin) and USE_PEFT_BACKEND:
+                # Retrieve the original scale by scaling back the LoRA layers
+                unscale_lora_layers(self.text_encoder_2, lora_scale)
+
+        dtype = self.text_encoder.dtype if self.text_encoder is not None else self.transformer.dtype
+        text_ids = torch.zeros(prompt_embeds.shape[1], 3).to(device=device, dtype=dtype)
+
+        return prompt_embeds, pooled_prompt_embeds, text_ids
+
+    # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3_inpaint.StableDiffusion3InpaintPipeline._encode_vae_image
+    def _encode_vae_image(self, image: torch.Tensor, generator: torch.Generator):
+        if isinstance(generator, list):
+            image_latents = [
+                retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i])
+                for i in range(image.shape[0])
+            ]
+            image_latents = torch.cat(image_latents, dim=0)
+        else:
+            image_latents = retrieve_latents(self.vae.encode(image), generator=generator)
+
+        image_latents = (image_latents - self.vae.config.shift_factor) * self.vae.config.scaling_factor
+
+        return image_latents
+
+    # Copied from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3_img2img.StableDiffusion3Img2ImgPipeline.get_timesteps
+    def get_timesteps(self, num_inference_steps, strength, device):
+        # get the original timestep using init_timestep
+        init_timestep = min(num_inference_steps * strength, num_inference_steps)
+
+        t_start = int(max(num_inference_steps - init_timestep, 0))
+        timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :]
+        if hasattr(self.scheduler, "set_begin_index"):
+            self.scheduler.set_begin_index(t_start * self.scheduler.order)
+
+        return timesteps, num_inference_steps - t_start
+
+    def check_inputs(
+        self,
+        prompt,
+        prompt_2,
+        strength,
+        height,
+        width,
+        prompt_embeds=None,
+        pooled_prompt_embeds=None,
+        callback_on_step_end_tensor_inputs=None,
+        max_sequence_length=None,
+        image=None,
+        mask_image=None,
+        masked_image_latents=None,
+    ):
+        if strength < 0 or strength > 1:
+            raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}")
+
+        if height % (self.vae_scale_factor * 2) != 0 or width % (self.vae_scale_factor * 2) != 0:
+            logger.warning(
+                f"`height` and `width` have to be divisible by {self.vae_scale_factor * 2} but are {height} and {width}. Dimensions will be resized accordingly"
+            )
+
+        if callback_on_step_end_tensor_inputs is not None and not all(
+            k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
+        ):
+            raise ValueError(
+                f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
+            )
+
+        if prompt is not None and prompt_embeds is not None:
+            raise ValueError(
+                f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
+                " only forward one of the two."
+            )
+        elif prompt_2 is not None and prompt_embeds is not None:
+            raise ValueError(
+                f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
+                " only forward one of the two."
+            )
+        elif prompt is None and prompt_embeds is None:
+            raise ValueError(
+                "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
+            )
+        elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
+            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
+        elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)):
+            raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}")
+
+        if prompt_embeds is not None and pooled_prompt_embeds is None:
+            raise ValueError(
+                "If `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`."
+            )
+
+        if max_sequence_length is not None and max_sequence_length > 512:
+            raise ValueError(f"`max_sequence_length` cannot be greater than 512 but is {max_sequence_length}")
+
+        if image is not None and masked_image_latents is not None:
+            raise ValueError(
+                "Please provide either  `image` or `masked_image_latents`, `masked_image_latents` should not be passed."
+            )
+
+        if image is not None and mask_image is None:
+            raise ValueError("Please provide `mask_image` when passing `image`.")
+
+    @staticmethod
+    # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._prepare_latent_image_ids
+    def _prepare_latent_image_ids(batch_size, height, width, device, dtype):
+        latent_image_ids = torch.zeros(height, width, 3)
+        latent_image_ids[..., 1] = latent_image_ids[..., 1] + torch.arange(height)[:, None]
+        latent_image_ids[..., 2] = latent_image_ids[..., 2] + torch.arange(width)[None, :]
+
+        latent_image_id_height, latent_image_id_width, latent_image_id_channels = latent_image_ids.shape
+
+        latent_image_ids = latent_image_ids.reshape(
+            latent_image_id_height * latent_image_id_width, latent_image_id_channels
+        )
+
+        return latent_image_ids.to(device=device, dtype=dtype)
+
+    @staticmethod
+    # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._pack_latents
+    def _pack_latents(latents, batch_size, num_channels_latents, height, width):
+        latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2)
+        latents = latents.permute(0, 2, 4, 1, 3, 5)
+        latents = latents.reshape(batch_size, (height // 2) * (width // 2), num_channels_latents * 4)
+
+        return latents
+
+    @staticmethod
+    # Copied from diffusers.pipelines.flux.pipeline_flux.FluxPipeline._unpack_latents
+    def _unpack_latents(latents, height, width, vae_scale_factor):
+        batch_size, num_patches, channels = latents.shape
+
+        # VAE applies 8x compression on images but we must also account for packing which requires
+        # latent height and width to be divisible by 2.
+        height = 2 * (int(height) // (vae_scale_factor * 2))
+        width = 2 * (int(width) // (vae_scale_factor * 2))
+
+        latents = latents.view(batch_size, height // 2, width // 2, channels // 4, 2, 2)
+        latents = latents.permute(0, 3, 1, 4, 2, 5)
+
+        latents = latents.reshape(batch_size, channels // (2 * 2), height, width)
+
+        return latents
+
+    def enable_vae_slicing(self):
+        r"""
+        Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to
+        compute decoding in several steps. This is useful to save some memory and allow larger batch sizes.
+        """
+        self.vae.enable_slicing()
+
+    def disable_vae_slicing(self):
+        r"""
+        Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to
+        computing decoding in one step.
+        """
+        self.vae.disable_slicing()
+
+    def enable_vae_tiling(self):
+        r"""
+        Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to
+        compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow
+        processing larger images.
+        """
+        self.vae.enable_tiling()
+
+    def disable_vae_tiling(self):
+        r"""
+        Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to
+        computing decoding in one step.
+        """
+        self.vae.disable_tiling()
+
+    # Copied from diffusers.pipelines.flux.pipeline_flux_img2img.FluxImg2ImgPipeline.prepare_latents
+    def prepare_latents(
+        self,
+        image,
+        reference_image,
+        geometry_image,
+        timestep,
+        batch_size,
+        num_channels_latents,
+        height,
+        width,
+        dtype,
+        device,
+        generator,
+        latents=None,
+    ):
+        if isinstance(generator, list) and len(generator) != batch_size:
+            raise ValueError(
+                f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
+                f" size of {batch_size}. Make sure the batch size matches the length of the generators."
+            )
+
+        # VAE applies 8x compression on images but we must also account for packing which requires
+        # latent height and width to be divisible by 2.
+        height = 2 * (int(height) // (self.vae_scale_factor * 2))
+        width = 2 * (int(width) // (self.vae_scale_factor * 2))
+        shape = (batch_size, num_channels_latents, height, width)
+
+        def _process_image_input(img: Optional[torch.Tensor], type_id: int):
+            """
+            Returns (packed_image_latents or None, image_ids or None).
+            type_id: integer marker to set in ids[..., 0]
+            """
+            img = img.to(device=device, dtype=dtype)
+            # if channels != latent_channels, assume it's pixel image -> encode
+            if img.shape[1] != self.latent_channels:
+                img_latents = self._encode_vae_image(image=img, generator=generator)
+            else:
+                img_latents = img
+            if batch_size > img_latents.shape[0]:
+                if batch_size % img_latents.shape[0] == 0:
+                    additional = batch_size // img_latents.shape[0]
+                    img_latents = torch.cat([img_latents] * additional, dim=0)
+                else:
+                    raise ValueError(
+                        f"Cannot duplicate `image` of batch size {img_latents.shape[0]} to {batch_size} text prompts."
+                    )
+            else:
+                img_latents = torch.cat([img_latents], dim=0)
+
+            img_latents = self._pack_latents(img_latents, batch_size, num_channels_latents, height, width)
+            ids = self._prepare_latent_image_ids(batch_size, height // 2, width // 2, device, dtype)
+            # Mark token type in the ids: first element -> type id.
+            ids[..., 0] = type_id
+            return img_latents, ids
+    
+        image_latents, image_ids = _process_image_input(image, type_id=0)
+        reference_latents, reference_ids = _process_image_input(reference_image, type_id=1)
+
+        add_img_latents_list = [reference_latents]
+        ids_list = [image_ids, reference_ids]
+
+        if geometry_image is not None:
+            geometry_latents, geometry_ids = _process_image_input(geometry_image, type_id=2)
+            add_img_latents_list.append(geometry_latents)
+            ids_list.append(geometry_ids)
+
+        noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
+        noise = self._pack_latents(noise, batch_size, num_channels_latents, height, width)
+        latents = self.scheduler.scale_noise(image_latents, timestep, noise)
+
+        return latents, add_img_latents_list, ids_list
+
+    @property
+    def guidance_scale(self):
+        return self._guidance_scale
+
+    @property
+    def joint_attention_kwargs(self):
+        return self._joint_attention_kwargs
+
+    @property
+    def num_timesteps(self):
+        return self._num_timesteps
+
+    @property
+    def interrupt(self):
+        return self._interrupt
+
+    @torch.no_grad()
+    def __call__(
+        self,
+        prompt: Union[str, List[str]] = None,
+        prompt_2: Optional[Union[str, List[str]]] = None,
+        image: Optional[torch.FloatTensor] = None,
+        mask_image: Optional[torch.FloatTensor] = None,
+        reference_image: Optional[torch.FloatTensor] = None,
+        geometry_image: Optional[torch.FloatTensor] = None,
+        masked_image_latents: Optional[torch.FloatTensor] = None,
+        height: Optional[int] = None,
+        width: Optional[int] = None,
+        strength: float = 1.0,
+        num_inference_steps: int = 50,
+        sigmas: Optional[List[float]] = None,
+        guidance_scale: float = 30.0,
+        num_images_per_prompt: Optional[int] = 1,
+        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
+        latents: Optional[torch.FloatTensor] = None,
+        prompt_embeds: Optional[torch.FloatTensor] = None,
+        pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
+        output_type: Optional[str] = "pil",
+        return_dict: bool = True,
+        joint_attention_kwargs: Optional[Dict[str, Any]] = None,
+        callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None,
+        callback_on_step_end_tensor_inputs: List[str] = ["latents"],
+        max_sequence_length: int = 512,
+        condition_embedder=None,
+        reference_guidance_scale=1.0,
+        **kwargs
+    ):
+        r"""
+        Function invoked when calling the pipeline for generation.
+
+        Args:
+            prompt (`str` or `List[str]`, *optional*):
+                The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
+                instead.
+            prompt_2 (`str` or `List[str]`, *optional*):
+                The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
+                will be used instead
+            image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`):
+                `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both
+                numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list
+                or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a
+                list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)`.
+            mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`):
+                `Image`, numpy array or tensor representing an image batch to mask `image`. White pixels in the mask
+                are repainted while black pixels are preserved. If `mask_image` is a PIL image, it is converted to a
+                single channel (luminance) before use. If it's a numpy array or pytorch tensor, it should contain one
+                color channel (L) instead of 3, so the expected shape for pytorch tensor would be `(B, 1, H, W)`, `(B,
+                H, W)`, `(1, H, W)`, `(H, W)`. And for numpy array would be for `(B, H, W, 1)`, `(B, H, W)`, `(H, W,
+                1)`, or `(H, W)`.
+            mask_image_latent (`torch.Tensor`, `List[torch.Tensor]`):
+                `Tensor` representing an image batch to mask `image` generated by VAE. If not provided, the mask
+                latents tensor will ge generated by `mask_image`.
+            height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
+                The height in pixels of the generated image. This is set to 1024 by default for the best results.
+            width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
+                The width in pixels of the generated image. This is set to 1024 by default for the best results.
+            strength (`float`, *optional*, defaults to 1.0):
+                Indicates extent to transform the reference `image`. Must be between 0 and 1. `image` is used as a
+                starting point and more noise is added the higher the `strength`. The number of denoising steps depends
+                on the amount of noise initially added. When `strength` is 1, added noise is maximum and the denoising
+                process runs for the full number of iterations specified in `num_inference_steps`. A value of 1
+                essentially ignores `image`.
+            num_inference_steps (`int`, *optional*, defaults to 50):
+                The number of denoising steps. More denoising steps usually lead to a higher quality image at the
+                expense of slower inference.
+            sigmas (`List[float]`, *optional*):
+                Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in
+                their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed
+                will be used.
+            guidance_scale (`float`, *optional*, defaults to 30.0):
+                Guidance scale as defined in [Classifier-Free Diffusion
+                Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2.
+                of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting
+                `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to
+                the text `prompt`, usually at the expense of lower image quality.
+            num_images_per_prompt (`int`, *optional*, defaults to 1):
+                The number of images to generate per prompt.
+            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):
+                One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
+                to make generation deterministic.
+            latents (`torch.FloatTensor`, *optional*):
+                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
+                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
+                tensor will ge generated by sampling using the supplied random `generator`.
+            prompt_embeds (`torch.FloatTensor`, *optional*):
+                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
+                provided, text embeddings will be generated from `prompt` input argument.
+            pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
+                Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
+                If not provided, pooled text embeddings will be generated from `prompt` input argument.
+            output_type (`str`, *optional*, defaults to `"pil"`):
+                The output format of the generate image. Choose between
+                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
+            return_dict (`bool`, *optional*, defaults to `True`):
+                Whether or not to return a [`~pipelines.flux.FluxPipelineOutput`] instead of a plain tuple.
+            joint_attention_kwargs (`dict`, *optional*):
+                A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
+                `self.processor` in
+                [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
+            callback_on_step_end (`Callable`, *optional*):
+                A function that calls at the end of each denoising steps during the inference. The function is called
+                with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,
+                callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by
+                `callback_on_step_end_tensor_inputs`.
+            callback_on_step_end_tensor_inputs (`List`, *optional*):
+                The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
+                will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
+                `._callback_tensor_inputs` attribute of your pipeline class.
+            max_sequence_length (`int` defaults to 512): Maximum sequence length to use with the `prompt`.
+
+        Examples:
+
+        Returns:
+            [`~pipelines.flux.FluxPipelineOutput`] or `tuple`: [`~pipelines.flux.FluxPipelineOutput`] if `return_dict`
+            is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the generated
+            images.
+        """
+
+        height = height or self.default_sample_size * self.vae_scale_factor
+        width = width or self.default_sample_size * self.vae_scale_factor
+
+        # 1. Check inputs. Raise error if not correct
+        self.check_inputs(
+            prompt,
+            prompt_2,
+            strength,
+            height,
+            width,
+            prompt_embeds=prompt_embeds,
+            pooled_prompt_embeds=pooled_prompt_embeds,
+            callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs,
+            max_sequence_length=max_sequence_length,
+            image=image,
+            mask_image=mask_image,
+            masked_image_latents=masked_image_latents,
+        )
+
+        self._guidance_scale = guidance_scale
+        self._joint_attention_kwargs = joint_attention_kwargs
+        self._interrupt = False
+        do_reference_guidance = reference_guidance_scale > 1
+
+        init_image = self.image_processor.preprocess(image, height=height, width=width)
+        reference_image = self.image_processor.preprocess(reference_image, height=height, width=width)
+        if geometry_image is not None:
+            geometry_image = self.image_processor.preprocess(geometry_image, height=height, width=width)
+        init_image = init_image.to(dtype=torch.float32)
+
+        # 2. Define call parameters
+        if prompt is not None and isinstance(prompt, str):
+            batch_size = 1
+        elif prompt is not None and isinstance(prompt, list):
+            batch_size = len(prompt)
+        else:
+            batch_size = prompt_embeds.shape[0]
+
+        device = self._execution_device
+
+        # 3. Prepare prompt embeddings
+        lora_scale = (
+            self.joint_attention_kwargs.get("scale", None) if self.joint_attention_kwargs is not None else None
+        )
+        (
+            prompt_embeds,
+            pooled_prompt_embeds,
+            text_ids,
+        ) = self.encode_prompt(
+            prompt=prompt,
+            prompt_2=prompt_2,
+            prompt_embeds=prompt_embeds,
+            pooled_prompt_embeds=pooled_prompt_embeds,
+            device=device,
+            num_images_per_prompt=num_images_per_prompt,
+            max_sequence_length=max_sequence_length,
+            lora_scale=lora_scale,
+        )
+
+        # 4. Prepare timesteps
+        sigmas = np.linspace(1.0, 1 / num_inference_steps, num_inference_steps) if sigmas is None else sigmas
+        image_seq_len = (int(height) // self.vae_scale_factor // 2) * (int(width) // self.vae_scale_factor // 2)
+        mu = calculate_shift(
+            image_seq_len,
+            self.scheduler.config.get("base_image_seq_len", 256),
+            self.scheduler.config.get("max_image_seq_len", 4096),
+            self.scheduler.config.get("base_shift", 0.5),
+            self.scheduler.config.get("max_shift", 1.15),
+        )
+        timesteps, num_inference_steps = retrieve_timesteps(
+            self.scheduler,
+            num_inference_steps,
+            device,
+            sigmas=sigmas,
+            mu=mu,
+        )
+        timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device)
+
+        if num_inference_steps < 1:
+            raise ValueError(
+                f"After adjusting the num_inference_steps by strength parameter: {strength}, the number of pipeline"
+                f"steps is {num_inference_steps} which is < 1 and not appropriate for this pipeline."
+            )
+        latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)
+
+        # 5. Prepare latent variables
+        num_channels_latents = self.vae.config.latent_channels
+        latents, cond_img_latents_list, img_ids_list = self.prepare_latents(
+            init_image,
+            reference_image,
+            geometry_image,
+            latent_timestep,
+            batch_size * num_images_per_prompt,
+            num_channels_latents,
+            height,
+            width,
+            prompt_embeds.dtype,
+            device,
+            generator,
+            latents,
+        )
+        latent_image_ids = torch.cat(img_ids_list, dim=0)  # dim 0 is sequence dimension
+
+        # 6. Prepare mask and masked image latents
+        if masked_image_latents is not None:
+            masked_image_latents = masked_image_latents.to(latents.device)
+        else:
+            mask_image = self.mask_processor.preprocess(mask_image, height=height, width=width)
+
+            masked_image = init_image
+            masked_image = masked_image.to(device=device, dtype=prompt_embeds.dtype)
+
+            height, width = init_image.shape[-2:]
+            mask, masked_image_latents = self.prepare_mask_latents(
+                mask_image,
+                masked_image,
+                batch_size,
+                num_channels_latents,
+                num_images_per_prompt,
+                height,
+                width,
+                prompt_embeds.dtype,
+                device,
+                generator,
+            )
+            masked_image_latents = torch.cat((masked_image_latents, mask), dim=-1)
+
+        num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
+        self._num_timesteps = len(timesteps)
+
+        # handle guidance
+        if self.transformer.config.guidance_embeds:
+            guidance = torch.full([1], guidance_scale, device=device, dtype=torch.float32)
+            guidance = guidance.expand(latents.shape[0])
+        else:
+            guidance = None
+
+        cond_latents = torch.cat(cond_img_latents_list, dim=1)
+        cond_hidden_states = condition_embedder(cond_latents)
+        if do_reference_guidance:
+            neg_reference_image = torch.zeros_like(reference_image)
+            neg_reference_latents = self._encode_vae_image(image=neg_reference_image, generator=generator)
+            neg_reference_latents = self._pack_latents(
+                neg_reference_latents,
+                batch_size,
+                num_channels_latents,
+                2 * (int(height) // (self.vae_scale_factor * 2)),
+                2 * (int(width) // (self.vae_scale_factor * 2)),
+            )
+            
+            neg_condition_latents = torch.cat([neg_reference_latents, *cond_img_latents_list[1:]], dim=1)
+            neg_hidden_state = condition_embedder(neg_condition_latents)
+            cond_hidden_states = torch.cat([neg_hidden_state, cond_hidden_states])
+            masked_image_latents = torch.cat([masked_image_latents]*2)
+            guidance = torch.cat([guidance]*2)
+            pooled_prompt_embeds = torch.cat([pooled_prompt_embeds]*2)
+            prompt_embeds = torch.cat([prompt_embeds]*2)
+
+        # 7. Denoising loop
+        with self.progress_bar(total=num_inference_steps) as progress_bar:
+            for i, t in enumerate(timesteps):
+                if self.interrupt:
+                    continue
+                latent_model_input = torch.cat([latents] * 2) if do_reference_guidance else latents
+                # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
+                timestep = t.expand(latent_model_input.shape[0]).to(latents.dtype)
+
+                transformer_output = self.transformer(
+                    hidden_states=torch.cat((latent_model_input, masked_image_latents), dim=2),
+                    timestep=timestep / 1000,
+                    guidance=guidance,
+                    pooled_projections=pooled_prompt_embeds,
+                    encoder_hidden_states=prompt_embeds,
+                    txt_ids=text_ids,
+                    img_ids=latent_image_ids,
+                    joint_attention_kwargs=self.joint_attention_kwargs,
+                    return_dict=False,
+                    cond_hidden_states=cond_hidden_states,
+                    **kwargs
+                )
+                noise_pred = transformer_output[0]
+
+                if do_reference_guidance:
+                    noise_pred_uncond, noise_pred_cond = noise_pred.chunk(2)
+                    noise_pred = noise_pred_uncond + reference_guidance_scale * (noise_pred_cond - noise_pred_uncond)
+
+                # compute the previous noisy sample x_t -> x_t-1
+                latents_dtype = latents.dtype
+                latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
+
+                if latents.dtype != latents_dtype:
+                    if torch.backends.mps.is_available():
+                        # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
+                        latents = latents.to(latents_dtype)
+
+                if callback_on_step_end is not None:
+                    callback_kwargs = {}
+                    for k in callback_on_step_end_tensor_inputs:
+                        callback_kwargs[k] = locals()[k]
+                    callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
+
+                    latents = callback_outputs.pop("latents", latents)
+                    prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
+
+                # call the callback, if provided
+                if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
+                    progress_bar.update()
+
+                if XLA_AVAILABLE:
+                    xm.mark_step()
+
+        # 8. Post-process the image
+        if output_type == "latent":
+            image = latents
+
+        else:
+            latents = self._unpack_latents(latents, height, width, self.vae_scale_factor)
+            latents = (latents / self.vae.config.scaling_factor) + self.vae.config.shift_factor
+            image = self.vae.decode(latents, return_dict=False)[0]
+            image = self.image_processor.postprocess(image, output_type=output_type)
+
+        # Offload all models
+        self.maybe_free_model_hooks()
+        if not return_dict:
+            return (image,)
+
+        return FluxPipelineOutput(images=image)
+
+
+class DirectPipeline:
+    def __init__(
+        self,
+        pipe: _DirectFluxFillPipeline,
+        condition_embedder,
+        image_encoder,
+        siglip_processor,
+        pooled_image_projector,
+        image_projector,
+        device,
+    ):
+        self.device = device
+        self.pipe = pipe.to(device)
+        self.condition_embedder = condition_embedder.to(device)
+        self.image_encoder = image_encoder.to(device)
+        self.siglip_processor = siglip_processor
+        self.pooled_image_projector = pooled_image_projector.to(device)
+        self.image_projector = image_projector.to(device)
+
+    @classmethod
+    def from_training_components(
+        cls,
+        flux_model_path: Union[str, os.PathLike],
+        transformer: FluxTransformer2DModelwithcond,
+        vae: AutoencoderKL,
+        condition_embedder: torch.nn.Module,
+        image_encoder: torch.nn.Module,
+        siglip_processor,
+        pooled_image_projector: torch.nn.Module,
+        image_projector: torch.nn.Module,
+        device: Optional[Union[str, torch.device]] = "cuda",
+        torch_dtype: Optional[Union[str, torch.dtype]] = None,
+        cache_dir: Optional[Union[str, os.PathLike]] = None,
+        local_files_only: bool = False,
+        token: Optional[Union[str, bool]] = None,
+        revision: Optional[str] = None,
+        variant: Optional[str] = None,
+    ) -> "DirectPipeline":
+        torch_dtype = cls._resolve_torch_dtype(torch_dtype)
+        hub_kwargs = {
+            "cache_dir": cache_dir,
+            "local_files_only": local_files_only,
+        }
+        if token is not None:
+            hub_kwargs["token"] = token
+        if revision is not None:
+            hub_kwargs["revision"] = revision
+        if variant is not None:
+            hub_kwargs["variant"] = variant
+
+        flux_pipeline = _DirectFluxFillPipeline.from_pretrained(
+            flux_model_path,
+            vae=vae,
+            text_encoder=None,
+            text_encoder_2=None,
+            transformer=transformer,
+            torch_dtype=torch_dtype,
+            **hub_kwargs,
+        )
+        return cls(
+            flux_pipeline,
+            condition_embedder.to(dtype=torch_dtype),
+            image_encoder,
+            siglip_processor,
+            pooled_image_projector.to(dtype=torch_dtype),
+            image_projector.to(dtype=torch_dtype),
+            device,
+        )
+
+    @staticmethod
+    def _load_config(
+        direct_model_path: Union[str, os.PathLike],
+        config_name: str,
+        cache_dir: Optional[Union[str, os.PathLike]],
+        local_files_only: bool,
+        token: Optional[Union[str, bool]],
+        revision: Optional[str],
+    ) -> Dict[str, Any]:
+        direct_model_path = str(direct_model_path)
+
+        if os.path.isdir(direct_model_path):
+            config_path = os.path.join(direct_model_path, config_name)
+            if not os.path.exists(config_path):
+                raise FileNotFoundError(f"Missing DIRECT config file: {config_path}")
+        else:
+            config_path = hf_hub_download(
+                direct_model_path,
+                filename=config_name,
+                cache_dir=cache_dir,
+                local_files_only=local_files_only,
+                token=token,
+                revision=revision,
+            )
+
+        with open(config_path, "r") as f:
+            return json.load(f)
+
+    @staticmethod
+    def _resolve_torch_dtype(torch_dtype: Optional[Union[str, torch.dtype]]) -> torch.dtype:
+        if isinstance(torch_dtype, torch.dtype):
+            return torch_dtype
+        if torch_dtype is None:
+            return torch.bfloat16
+        dtype_map = {
+            "float32": torch.float32,
+            "float": torch.float32,
+            "float16": torch.float16,
+            "fp16": torch.float16,
+            "bfloat16": torch.bfloat16,
+            "bf16": torch.bfloat16,
+        }
+        if torch_dtype not in dtype_map:
+            raise ValueError(f"Unsupported torch dtype: {torch_dtype}")
+        return dtype_map[torch_dtype]
+
+    @staticmethod
+    def _resolve_model_file(
+        model_path: Union[str, os.PathLike],
+        filename: str,
+        cache_dir: Optional[Union[str, os.PathLike]],
+        local_files_only: bool,
+        token: Optional[Union[str, bool]],
+        revision: Optional[str],
+    ) -> str:
+        model_path = str(model_path)
+        if os.path.isdir(model_path):
+            file_path = os.path.join(model_path, filename)
+            if not os.path.exists(file_path):
+                raise FileNotFoundError(f"Missing DIRECT weight file: {file_path}")
+            return file_path
+        return hf_hub_download(
+            model_path,
+            filename=filename,
+            cache_dir=cache_dir,
+            local_files_only=local_files_only,
+            token=token,
+            revision=revision,
+        )
+
+    @staticmethod
+    def _load_lora_processors(
+        transformer: FluxTransformer2DModelwithcond,
+        lora_path: str,
+        torch_dtype: torch.dtype,
+        lora_config: Dict[str, Any],
+    ) -> None:
+        ranks = lora_config["ranks"]
+        network_alphas = lora_config["alphas"]
+        lora_weights = lora_config.get("weights", [1] * len(ranks))
+        text_lora_config = lora_config["text"]
+        double_blocks_idx = set(range(lora_config["double_blocks"]))
+        single_blocks_idx = set(range(lora_config["single_blocks"]))
+
+        lora_attn_procs = {}
+        for name, attn_processor in transformer.attn_processors.items():
+            match = re.search(r"\.(\d+)\.", name)
+            layer_index = int(match.group(1)) if match else -1
+            if name.startswith("transformer_blocks") and layer_index in double_blocks_idx:
+                lora_attn_procs[name] = MultiDoubleStreamBlockLoraProcessor(
+                    dim=3072,
+                    ranks=ranks,
+                    network_alphas=network_alphas,
+                    lora_weights=lora_weights,
+                    device=transformer.device,
+                    dtype=torch_dtype,
+                    n_loras=lora_config["n_loras"],
+                    text_lora_config=text_lora_config,
+                )
+            elif name.startswith("single_transformer_blocks") and layer_index in single_blocks_idx:
+                lora_attn_procs[name] = MultiSingleStreamBlockLoraProcessor(
+                    dim=3072,
+                    ranks=ranks,
+                    network_alphas=network_alphas,
+                    lora_weights=lora_weights,
+                    device=transformer.device,
+                    dtype=torch_dtype,
+                    n_loras=lora_config["n_loras"],
+                    text_lora_config=text_lora_config,
+                )
+            else:
+                lora_attn_procs[name] = attn_processor
+
+        transformer.set_attn_processor(lora_attn_procs)
+        transformer.load_state_dict(load_file(lora_path), strict=False)
+
+    @staticmethod
+    def _load_state_dict(module: torch.nn.Module, state_path: str, strict: bool = True) -> torch.nn.Module:
+        module.load_state_dict(load_file(state_path), strict=strict)
+        return module
+
+    @classmethod
+    def _load_linear(
+        cls,
+        layer_config: Dict[str, int],
+        state_path: str,
+        torch_dtype: torch.dtype,
+    ) -> torch.nn.Linear:
+        layer = torch.nn.Linear(layer_config["input_dim"], layer_config["output_dim"]).to(dtype=torch_dtype)
+        return cls._load_state_dict(layer, state_path)
+
+    @classmethod
+    def _load_condition_embedder(
+        cls,
+        layer_config: Dict[str, int],
+        output_dim: int,
+        state_path: str,
+        torch_dtype: torch.dtype,
+    ) -> torch.nn.Linear:
+        layer = torch.nn.Linear(layer_config["input_dim"], output_dim).to(dtype=torch_dtype)
+        return cls._load_state_dict(layer, state_path)
+
+    @classmethod
+    def from_pretrained(
+        cls,
+        direct_model_path: Union[str, os.PathLike],
+        flux_model_path: Optional[Union[str, os.PathLike]] = None,
+        siglip_model_path: Optional[Union[str, os.PathLike]] = None,
+        device: Optional[Union[str, torch.device]] = "cuda",
+        torch_dtype: Optional[Union[str, torch.dtype]] = None,
+        config_name: str = "config.json",
+        cache_dir: Optional[Union[str, os.PathLike]] = None,
+        local_files_only: bool = False,
+        token: Optional[Union[str, bool]] = None,
+        revision: Optional[str] = None,
+    ) -> "DirectPipeline":
+        config = cls._load_config(direct_model_path, config_name, cache_dir, local_files_only, token, revision)
+        torch_dtype = cls._resolve_torch_dtype(torch_dtype or config.get("torch_dtype"))
+        device = torch.device(device)
+        flux_model_path = str(flux_model_path or config["flux_model"])
+        siglip_model_path = str(siglip_model_path or config["siglip_model"])
+
+        hub_kwargs = {
+            "cache_dir": cache_dir,
+            "local_files_only": local_files_only,
+        }
+        if token is not None:
+            hub_kwargs["token"] = token
+        if revision is not None:
+            hub_kwargs["revision"] = revision
+
+        weight_files = config["weight_files"]
+        get_weight = lambda key: cls._resolve_model_file(
+            direct_model_path, weight_files[key], cache_dir, local_files_only, token, revision
+        )
+
+        transformer = FluxTransformer2DModelwithcond.from_pretrained(
+            flux_model_path,
+            subfolder="transformer",
+            torch_dtype=torch_dtype,
+            **hub_kwargs,
+        )
+        vae = AutoencoderKL.from_pretrained(
+            flux_model_path,
+            subfolder="vae",
+            torch_dtype=torch_dtype,
+            **hub_kwargs,
+        )
+
+        cls._load_lora_processors(transformer, get_weight("lora"), torch_dtype, config["lora"])
+        cls._load_state_dict(transformer, get_weight("x_embedder"), strict=False)
+        cls._load_state_dict(transformer, get_weight("time_text_embed"), strict=False)
+        condition_embedder = cls._load_condition_embedder(
+            config["condition_embedder"],
+            transformer.inner_dim,
+            get_weight("condition_embedder"),
+            torch_dtype,
+        )
+        pooled_image_projector = cls._load_linear(
+            config["pooled_image_projector"],
+            get_weight("pooled_image_projector"),
+            torch_dtype,
+        )
+        image_projector = cls._load_linear(
+            config["image_projector"],
+            get_weight("image_projector"),
+            torch_dtype,
+        )
+
+        image_encoder = AutoModel.from_pretrained(siglip_model_path, **hub_kwargs).vision_model.eval().to(dtype=torch_dtype)
+        siglip_processor = AutoProcessor.from_pretrained(siglip_model_path, **hub_kwargs)
+
+        flux_pipeline = _DirectFluxFillPipeline.from_pretrained(
+            flux_model_path,
+            vae=vae,
+            text_encoder=None,
+            text_encoder_2=None,
+            transformer=transformer,
+            torch_dtype=torch_dtype,
+            **hub_kwargs,
+        )
+        return cls(
+            flux_pipeline,
+            condition_embedder.to(dtype=torch_dtype),
+            image_encoder,
+            siglip_processor,
+            pooled_image_projector,
+            image_projector,
+            device,
+        )
+
+    def prepare_images(
+        self,
+        composite_image,
+        inpaint_mask,
+        reference_image,
+        geometry_image,
+        context_image,
+        dtype,
+        device,
+    ):
+        composite_image = to_tensor_0_1(composite_image).to(dtype).to(device)
+        inpaint_mask = to_tensor_0_1(inpaint_mask).to(dtype).to(device)
+        reference_image = to_tensor_0_1(reference_image).to(dtype).to(device)
+        geometry_image = to_tensor_0_1(geometry_image).to(dtype).to(device)
+        return composite_image, inpaint_mask, reference_image, geometry_image, context_image
+
+    def encode_prompt(
+        self,
+        image,
+        dtype
+    ):
+        siglip_input = self.siglip_processor(images=image, return_tensors="pt").to(self.device)
+        siglip_output = self.image_encoder(**siglip_input)
+        pooled_prompt_embeds = self.pooled_image_projector(siglip_output.pooler_output).to(dtype)
+        prompt_embeds = self.image_projector(siglip_output.last_hidden_state).to(dtype)
+        return prompt_embeds, pooled_prompt_embeds
+
+    def __call__(
+        self,
+        composite_image,
+        inpaint_mask,
+        reference_image,
+        geometry_image,
+        context_image,
+        seed=None,
+        guidance_scale=7.5,
+        num_inference_steps=30,
+        height=512,
+        width=512,
+        use_autocast=True,
+        **kwargs,
+    ):
+        generator = torch.Generator(self.device).manual_seed(seed) if seed is not None else None
+        autocast_ctx = torch.autocast(self.device.type, dtype=torch.bfloat16) if use_autocast else nullcontext()
+        dtype = torch.bfloat16 if use_autocast else torch.float32
+        composite_image, inpaint_mask, reference_image, geometry_image, context_image = self.prepare_images(
+            composite_image,
+            inpaint_mask,
+            reference_image,
+            geometry_image,
+            context_image,
+            dtype,
+            self.device,
+        )
+
+        # Pre-calculate image prompt embeddings outside autocast.
+        with torch.no_grad():
+            prompt_embeds, pooled_prompt_embeds = self.encode_prompt(context_image, dtype)
+        with autocast_ctx:
+            images = self.pipe(
+                image=composite_image,
+                reference_image=reference_image,
+                geometry_image=geometry_image,
+                mask_image=inpaint_mask,
+                prompt_embeds=prompt_embeds,
+                pooled_prompt_embeds=pooled_prompt_embeds,
+                num_inference_steps=num_inference_steps,
+                guidance_scale=guidance_scale,
+                generator=generator,
+                height=height,
+                width=width,
+                condition_embedder=self.condition_embedder,
+                **kwargs
+            ).images
+        return images

direct/transformer_flux.py

@@ -0,0 +1,561 @@
+from typing import Any, Dict, Optional, Tuple, Union
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from diffusers.configuration_utils import ConfigMixin, register_to_config
+from diffusers.loaders import FluxTransformer2DLoadersMixin, FromOriginalModelMixin, PeftAdapterMixin
+from diffusers.models.attention import FeedForward
+from diffusers.models.attention_processor import (
+    Attention,
+    AttentionProcessor,
+    FluxAttnProcessor2_0,
+    FluxAttnProcessor2_0_NPU,
+    FusedFluxAttnProcessor2_0,
+)
+from diffusers.models.modeling_utils import ModelMixin
+from diffusers.models.normalization import AdaLayerNormContinuous, AdaLayerNormZero, AdaLayerNormZeroSingle
+from diffusers.utils import USE_PEFT_BACKEND, is_torch_version, logging, scale_lora_layers, unscale_lora_layers
+from diffusers.utils.import_utils import is_torch_npu_available
+from diffusers.utils.torch_utils import maybe_allow_in_graph
+from diffusers.models.embeddings import CombinedTimestepGuidanceTextProjEmbeddings, CombinedTimestepTextProjEmbeddings, FluxPosEmbed
+from diffusers.models.modeling_outputs import Transformer2DModelOutput
+from diffusers import CacheMixin
+
+logger = logging.get_logger(__name__)  # pylint: disable=invalid-name
+
+@maybe_allow_in_graph
+class FluxSingleTransformerBlock(nn.Module):
+
+    def __init__(self, dim, num_attention_heads, attention_head_dim, mlp_ratio=4.0):
+        super().__init__()
+        self.mlp_hidden_dim = int(dim * mlp_ratio)
+
+        self.norm = AdaLayerNormZeroSingle(dim)
+        self.proj_mlp = nn.Linear(dim, self.mlp_hidden_dim)
+        self.act_mlp = nn.GELU(approximate="tanh")
+        self.proj_out = nn.Linear(dim + self.mlp_hidden_dim, dim)
+
+        if is_torch_npu_available():
+            processor = FluxAttnProcessor2_0_NPU()
+        else:
+            processor = FluxAttnProcessor2_0()
+        self.attn = Attention(
+            query_dim=dim,
+            cross_attention_dim=None,
+            dim_head=attention_head_dim,
+            heads=num_attention_heads,
+            out_dim=dim,
+            bias=True,
+            processor=processor,
+            qk_norm="rms_norm",
+            eps=1e-6,
+            pre_only=True,
+        )
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        cond_hidden_states: torch.Tensor,
+        temb: torch.Tensor,
+        cond_temb: torch.Tensor,
+        image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+        joint_attention_kwargs: Optional[Dict[str, Any]] = None,
+    ) -> torch.Tensor:
+        use_cond = cond_hidden_states is not None
+        
+        residual = hidden_states
+        norm_hidden_states, gate = self.norm(hidden_states, emb=temb)
+        mlp_hidden_states = self.act_mlp(self.proj_mlp(norm_hidden_states))
+        
+        if use_cond:
+            residual_cond = cond_hidden_states
+            norm_cond_hidden_states, cond_gate = self.norm(cond_hidden_states, emb=cond_temb)
+            mlp_cond_hidden_states = self.act_mlp(self.proj_mlp(norm_cond_hidden_states))
+
+            norm_hidden_states_concat = torch.concat([norm_hidden_states, norm_cond_hidden_states], dim=-2)
+        else:
+            norm_hidden_states_concat = norm_hidden_states
+        joint_attention_kwargs = joint_attention_kwargs or {}
+        attn_output = self.attn(
+            hidden_states=norm_hidden_states_concat,
+            image_rotary_emb=image_rotary_emb,
+            use_cond=use_cond,
+            **joint_attention_kwargs,
+        )
+        if use_cond:
+            attn_output, cond_attn_output = attn_output
+
+        hidden_states = torch.cat([attn_output, mlp_hidden_states], dim=2)
+        gate = gate.unsqueeze(1)
+        hidden_states = gate * self.proj_out(hidden_states)
+        hidden_states = residual + hidden_states
+        
+        if use_cond:
+            condition_latents = torch.cat([cond_attn_output, mlp_cond_hidden_states], dim=2)
+            cond_gate = cond_gate.unsqueeze(1)
+            condition_latents = cond_gate * self.proj_out(condition_latents)
+            condition_latents = residual_cond + condition_latents
+        
+        if hidden_states.dtype == torch.float16:
+            hidden_states = hidden_states.clip(-65504, 65504)
+
+        return hidden_states, condition_latents if use_cond else None
+
+
+@maybe_allow_in_graph
+class FluxTransformerBlock(nn.Module):
+    def __init__(
+        self, dim: int, num_attention_heads: int, attention_head_dim: int, qk_norm: str = "rms_norm", eps: float = 1e-6
+    ):
+        super().__init__()
+
+        self.norm1 = AdaLayerNormZero(dim)
+
+        self.norm1_context = AdaLayerNormZero(dim)
+
+        if hasattr(F, "scaled_dot_product_attention"):
+            processor = FluxAttnProcessor2_0()
+        else:
+            raise ValueError(
+                "The current PyTorch version does not support the `scaled_dot_product_attention` function."
+            )
+        self.attn = Attention(
+            query_dim=dim,
+            cross_attention_dim=None,
+            added_kv_proj_dim=dim,
+            dim_head=attention_head_dim,
+            heads=num_attention_heads,
+            out_dim=dim,
+            context_pre_only=False,
+            bias=True,
+            processor=processor,
+            qk_norm=qk_norm,
+            eps=eps,
+        )
+
+        self.norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
+        self.ff = FeedForward(dim=dim, dim_out=dim, activation_fn="gelu-approximate")
+
+        self.norm2_context = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
+        self.ff_context = FeedForward(dim=dim, dim_out=dim, activation_fn="gelu-approximate")
+
+        # let chunk size default to None
+        self._chunk_size = None
+        self._chunk_dim = 0
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        cond_hidden_states: torch.Tensor,
+        encoder_hidden_states: torch.Tensor,
+        temb: torch.Tensor,
+        cond_temb: torch.Tensor,
+        image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+        joint_attention_kwargs: Optional[Dict[str, Any]] = None,
+    ) -> Tuple[torch.Tensor, torch.Tensor]:
+        use_cond = cond_hidden_states is not None
+        
+        norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1(hidden_states, emb=temb)
+        if use_cond:
+                (
+                    norm_cond_hidden_states,
+                    cond_gate_msa,
+                    cond_shift_mlp,
+                    cond_scale_mlp,
+                    cond_gate_mlp,
+                ) = self.norm1(cond_hidden_states, emb=cond_temb)
+
+                norm_hidden_states = torch.concat([norm_hidden_states, norm_cond_hidden_states], dim=-2)
+
+
+        norm_encoder_hidden_states, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = self.norm1_context(
+            encoder_hidden_states, emb=temb
+        )
+        
+        
+        joint_attention_kwargs = joint_attention_kwargs or {}
+        # Attention.
+        attention_outputs = self.attn(
+            hidden_states=norm_hidden_states,
+            encoder_hidden_states=norm_encoder_hidden_states,
+            image_rotary_emb=image_rotary_emb,
+            use_cond=use_cond,
+            **joint_attention_kwargs,
+        )
+
+        attn_output, context_attn_output = attention_outputs[:2]
+        cond_attn_output = attention_outputs[2] if use_cond else None
+
+        # Process attention outputs for the `hidden_states`.
+        attn_output = gate_msa.unsqueeze(1) * attn_output
+        hidden_states = hidden_states + attn_output
+
+        if use_cond:
+            cond_attn_output = cond_gate_msa.unsqueeze(1) * cond_attn_output
+            cond_hidden_states = cond_hidden_states + cond_attn_output
+
+        norm_hidden_states = self.norm2(hidden_states)
+        norm_hidden_states = norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]
+        
+        if use_cond:
+            norm_cond_hidden_states = self.norm2(cond_hidden_states)
+            norm_cond_hidden_states = (
+                norm_cond_hidden_states * (1 + cond_scale_mlp[:, None])
+                + cond_shift_mlp[:, None]
+            )
+
+        ff_output = self.ff(norm_hidden_states)
+        ff_output = gate_mlp.unsqueeze(1) * ff_output
+        hidden_states = hidden_states + ff_output
+        
+        if use_cond:
+            cond_ff_output = self.ff(norm_cond_hidden_states)
+            cond_ff_output = cond_gate_mlp.unsqueeze(1) * cond_ff_output
+            cond_hidden_states = cond_hidden_states + cond_ff_output
+
+        # Process attention outputs for the `encoder_hidden_states`.
+
+        context_attn_output = c_gate_msa.unsqueeze(1) * context_attn_output
+        encoder_hidden_states = encoder_hidden_states + context_attn_output
+
+        norm_encoder_hidden_states = self.norm2_context(encoder_hidden_states)
+        norm_encoder_hidden_states = norm_encoder_hidden_states * (1 + c_scale_mlp[:, None]) + c_shift_mlp[:, None]
+
+        context_ff_output = self.ff_context(norm_encoder_hidden_states)
+        encoder_hidden_states = encoder_hidden_states + c_gate_mlp.unsqueeze(1) * context_ff_output
+        if encoder_hidden_states.dtype == torch.float16:
+            encoder_hidden_states = encoder_hidden_states.clip(-65504, 65504)
+
+        return encoder_hidden_states, hidden_states, cond_hidden_states if use_cond else None
+
+
+class FluxTransformer2DModelwithcond(
+    ModelMixin, ConfigMixin, PeftAdapterMixin, FromOriginalModelMixin, FluxTransformer2DLoadersMixin, CacheMixin
+):
+    _supports_gradient_checkpointing = True
+    _no_split_modules = ["FluxTransformerBlock", "FluxSingleTransformerBlock"]
+
+    @register_to_config
+    def __init__(
+        self,
+        patch_size: int = 1,
+        in_channels: int = 64,
+        out_channels: Optional[int] = None,
+        num_layers: int = 19,
+        num_single_layers: int = 38,
+        attention_head_dim: int = 128,
+        num_attention_heads: int = 24,
+        joint_attention_dim: int = 4096,
+        pooled_projection_dim: int = 768,
+        guidance_embeds: bool = False,
+        axes_dims_rope: Tuple[int] = (16, 56, 56),
+    ):
+        super().__init__()
+        self.out_channels = out_channels or in_channels
+        self.inner_dim = num_attention_heads * attention_head_dim
+
+        self.pos_embed = FluxPosEmbed(theta=10000, axes_dim=axes_dims_rope)
+
+        text_time_guidance_cls = (
+            CombinedTimestepGuidanceTextProjEmbeddings if guidance_embeds else CombinedTimestepTextProjEmbeddings
+        )
+        self.time_text_embed = text_time_guidance_cls(
+            embedding_dim=self.inner_dim, pooled_projection_dim=pooled_projection_dim
+        )
+
+        self.context_embedder = nn.Linear(joint_attention_dim, self.inner_dim)
+        self.x_embedder = nn.Linear(in_channels, self.inner_dim)
+
+        self.transformer_blocks = nn.ModuleList(
+            [
+                FluxTransformerBlock(
+                    dim=self.inner_dim,
+                    num_attention_heads=num_attention_heads,
+                    attention_head_dim=attention_head_dim,
+                )
+                for _ in range(num_layers)
+            ]
+        )
+
+        self.single_transformer_blocks = nn.ModuleList(
+            [
+                FluxSingleTransformerBlock(
+                    dim=self.inner_dim,
+                    num_attention_heads=num_attention_heads,
+                    attention_head_dim=attention_head_dim,
+                )
+                for _ in range(num_single_layers)
+            ]
+        )
+
+        self.norm_out = AdaLayerNormContinuous(self.inner_dim, self.inner_dim, elementwise_affine=False, eps=1e-6)
+        self.proj_out = nn.Linear(self.inner_dim, patch_size * patch_size * self.out_channels, bias=True)
+
+        self.gradient_checkpointing = False
+
+    @property
+    # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors
+    def attn_processors(self) -> Dict[str, AttentionProcessor]:
+        r"""
+        Returns:
+            `dict` of attention processors: A dictionary containing all attention processors used in the model with
+            indexed by its weight name.
+        """
+        # set recursively
+        processors = {}
+
+        def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]):
+            if hasattr(module, "get_processor"):
+                processors[f"{name}.processor"] = module.get_processor()
+
+            for sub_name, child in module.named_children():
+                fn_recursive_add_processors(f"{name}.{sub_name}", child, processors)
+
+            return processors
+
+        for name, module in self.named_children():
+            fn_recursive_add_processors(name, module, processors)
+
+        return processors
+
+    # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor
+    def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]):
+        r"""
+        Sets the attention processor to use to compute attention.
+
+        Parameters:
+            processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`):
+                The instantiated processor class or a dictionary of processor classes that will be set as the processor
+                for **all** `Attention` layers.
+
+                If `processor` is a dict, the key needs to define the path to the corresponding cross attention
+                processor. This is strongly recommended when setting trainable attention processors.
+
+        """
+        count = len(self.attn_processors.keys())
+
+        if isinstance(processor, dict) and len(processor) != count:
+            raise ValueError(
+                f"A dict of processors was passed, but the number of processors {len(processor)} does not match the"
+                f" number of attention layers: {count}. Please make sure to pass {count} processor classes."
+            )
+
+        def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor):
+            if hasattr(module, "set_processor"):
+                if not isinstance(processor, dict):
+                    module.set_processor(processor)
+                else:
+                    module.set_processor(processor.pop(f"{name}.processor"))
+
+            for sub_name, child in module.named_children():
+                fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor)
+
+        for name, module in self.named_children():
+            fn_recursive_attn_processor(name, module, processor)
+
+    # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.fuse_qkv_projections with FusedAttnProcessor2_0->FusedFluxAttnProcessor2_0
+    def fuse_qkv_projections(self):
+        """
+        Enables fused QKV projections. For self-attention modules, all projection matrices (i.e., query, key, value)
+        are fused. For cross-attention modules, key and value projection matrices are fused.
+
+        <Tip warning={true}>
+
+        This API is 🧪 experimental.
+
+        </Tip>
+        """
+        self.original_attn_processors = None
+
+        for _, attn_processor in self.attn_processors.items():
+            if "Added" in str(attn_processor.__class__.__name__):
+                raise ValueError("`fuse_qkv_projections()` is not supported for models having added KV projections.")
+
+        self.original_attn_processors = self.attn_processors
+
+        for module in self.modules():
+            if isinstance(module, Attention):
+                module.fuse_projections(fuse=True)
+
+        self.set_attn_processor(FusedFluxAttnProcessor2_0())
+
+    # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.unfuse_qkv_projections
+    def unfuse_qkv_projections(self):
+        """Disables the fused QKV projection if enabled.
+
+        <Tip warning={true}>
+
+        This API is 🧪 experimental.
+
+        </Tip>
+
+        """
+        if self.original_attn_processors is not None:
+            self.set_attn_processor(self.original_attn_processors)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        cond_hidden_states: torch.Tensor = None,
+        encoder_hidden_states: torch.Tensor = None,
+        pooled_projections: torch.Tensor = None,
+        timestep: torch.LongTensor = None,
+        img_ids: torch.Tensor = None,
+        txt_ids: torch.Tensor = None,
+        guidance: torch.Tensor = None,
+        joint_attention_kwargs: Optional[Dict[str, Any]] = None,
+        controlnet_block_samples=None,
+        controlnet_single_block_samples=None,
+        return_dict: bool = True,
+        controlnet_blocks_repeat: bool = False,
+    ) -> Union[torch.Tensor, Transformer2DModelOutput]:
+        if cond_hidden_states is not None:
+            use_condition = True
+        else:
+            use_condition = False
+        
+        if joint_attention_kwargs is not None:
+            joint_attention_kwargs = joint_attention_kwargs.copy()
+            lora_scale = joint_attention_kwargs.pop("scale", 1.0)
+        else:
+            lora_scale = 1.0
+
+        if USE_PEFT_BACKEND:
+            # weight the lora layers by setting `lora_scale` for each PEFT layer
+            scale_lora_layers(self, lora_scale)
+        else:
+            if joint_attention_kwargs is not None and joint_attention_kwargs.get("scale", None) is not None:
+                logger.warning(
+                    "Passing `scale` via `joint_attention_kwargs` when not using the PEFT backend is ineffective."
+                )
+
+        hidden_states = self.x_embedder(hidden_states)
+
+        timestep = timestep.to(hidden_states.dtype) * 1000
+        if guidance is not None:
+            guidance = guidance.to(hidden_states.dtype) * 1000
+        else:
+            guidance = None
+
+        temb = (
+            self.time_text_embed(timestep, pooled_projections)
+            if guidance is None
+            else self.time_text_embed(timestep, guidance, pooled_projections)
+        )
+        
+        cond_temb = (
+            self.time_text_embed(torch.ones_like(timestep) * 0, pooled_projections)
+                if guidance is None
+                else self.time_text_embed(
+                    torch.ones_like(timestep) * 0, guidance, pooled_projections
+                )
+            )
+        
+        encoder_hidden_states = self.context_embedder(encoder_hidden_states)
+
+        if txt_ids.ndim == 3:
+            logger.warning(
+                "Passing `txt_ids` 3d torch.Tensor is deprecated."
+                "Please remove the batch dimension and pass it as a 2d torch Tensor"
+            )
+            txt_ids = txt_ids[0]
+        if img_ids.ndim == 3:
+            logger.warning(
+                "Passing `img_ids` 3d torch.Tensor is deprecated."
+                "Please remove the batch dimension and pass it as a 2d torch Tensor"
+            )
+            img_ids = img_ids[0]
+
+        ids = torch.cat((txt_ids, img_ids), dim=0)
+        image_rotary_emb = self.pos_embed(ids)
+
+        if joint_attention_kwargs is not None and "ip_adapter_image_embeds" in joint_attention_kwargs:
+            ip_adapter_image_embeds = joint_attention_kwargs.pop("ip_adapter_image_embeds")
+            ip_hidden_states = self.encoder_hid_proj(ip_adapter_image_embeds)
+            joint_attention_kwargs.update({"ip_hidden_states": ip_hidden_states})
+
+        if joint_attention_kwargs is None:
+            joint_attention_kwargs = {}  
+
+        for index_block, block in enumerate(self.transformer_blocks):
+            if torch.is_grad_enabled() and self.gradient_checkpointing:
+                encoder_hidden_states, hidden_states, cond_hidden_states = self._gradient_checkpointing_func(
+                    block,
+                    hidden_states,
+                    cond_hidden_states if use_condition else None,
+                    encoder_hidden_states,
+                    temb,
+                    cond_temb if use_condition else None,
+                    image_rotary_emb,
+                    joint_attention_kwargs,
+                ) 
+            else:
+                encoder_hidden_states, hidden_states, cond_hidden_states = block(
+                    hidden_states=hidden_states,
+                    encoder_hidden_states=encoder_hidden_states,
+                    cond_hidden_states=cond_hidden_states if use_condition else None,
+                    temb=temb,
+                    cond_temb=cond_temb if use_condition else None,
+                    image_rotary_emb=image_rotary_emb,
+                    joint_attention_kwargs=joint_attention_kwargs,
+                )
+
+            # controlnet residual
+            if controlnet_block_samples is not None:
+                interval_control = len(self.transformer_blocks) / len(controlnet_block_samples)
+                interval_control = int(np.ceil(interval_control))
+                # For Xlabs ControlNet.
+                if controlnet_blocks_repeat:
+                    hidden_states = (
+                        hidden_states + controlnet_block_samples[index_block % len(controlnet_block_samples)]
+                    )
+                else:
+                    hidden_states = hidden_states + controlnet_block_samples[index_block // interval_control]
+        hidden_states = torch.cat([encoder_hidden_states, hidden_states], dim=1)
+
+        for index_block, block in enumerate(self.single_transformer_blocks):
+            if torch.is_grad_enabled() and self.gradient_checkpointing:
+                hidden_states, cond_hidden_states = self._gradient_checkpointing_func(
+                    block,
+                    hidden_states,
+                    cond_hidden_states if use_condition else None,
+                    temb,
+                    cond_temb if use_condition else None,
+                    image_rotary_emb,
+                    joint_attention_kwargs,
+                )
+            else:
+                hidden_states, cond_hidden_states = block(
+                    hidden_states=hidden_states,
+                    cond_hidden_states=cond_hidden_states if use_condition else None,
+                    temb=temb,
+                    cond_temb=cond_temb if use_condition else None,
+                    image_rotary_emb=image_rotary_emb,
+                    joint_attention_kwargs=joint_attention_kwargs,
+                )
+
+            # controlnet residual
+            if controlnet_single_block_samples is not None:
+                interval_control = len(self.single_transformer_blocks) / len(controlnet_single_block_samples)
+                interval_control = int(np.ceil(interval_control))
+                hidden_states[:, encoder_hidden_states.shape[1] :, ...] = (
+                    hidden_states[:, encoder_hidden_states.shape[1] :, ...]
+                    + controlnet_single_block_samples[index_block // interval_control]
+                )
+
+        hidden_states = hidden_states[:, encoder_hidden_states.shape[1] :, ...]
+
+        hidden_states = self.norm_out(hidden_states, temb)
+        output = self.proj_out(hidden_states)
+
+        if USE_PEFT_BACKEND:
+            # remove `lora_scale` from each PEFT layer
+            unscale_lora_layers(self, lora_scale)
+
+        if not return_dict:
+            return (output,)
+
+        return Transformer2DModelOutput(sample=output)

requirements.txt

@@ -0,0 +1,17 @@
+diffusers==0.35.1
+transformers==4.53.1
+peft==0.17.1
+accelerate==1.10.1
+safetensors
+sentencepiece
+protobuf
+einops
+kornia
+timm
+numpy
+pillow
+opencv-python-headless
+scipy
+torchvision
+rembg==2.0.67
+onnxruntime==1.23.1