Initial commit with Xet-tracked images

.gitattributes

@@ -0,0 +1,41 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,22 @@
+lightning_logs/
+experiments/
+old/
+.DS_Store
+.venv/
+.idea/
+.vscode/
+.pytest_cache/
+downloads/
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+*.pyw
+images/sit.png
+.trash
+*.zip
+images/cup_torus/
+images/cup_torus_no_bg/
+*.pt
+models/
+.gradio/

README.md

@@ -0,0 +1,15 @@
+---
+title: VibeSpace
+emoji: 🚀
+colorFrom: purple
+colorTo: yellow
+sdk: gradio
+sdk_version: 5.24.0
+app_file: app.py
+pinned: false
+---
+
+
+step1: pip install -r ./requirements.txt
+
+step2: run `python app.py` or demo notebooks

app.py

@@ -0,0 +1,171 @@
+import logging
+import os
+from typing import List, Union
+
+import gradio as gr
+from PIL import Image
+import numpy as np
+
+from vibe_blending import run_vibe_blend_safe, run_vibe_blend_not_safe
+from ipadapter_model import create_image_grid
+
+USE_HUGGINGFACE_ZEROGPU = os.getenv("USE_HUGGINGFACE_ZEROGPU", "false").lower() == "false" #"true"
+DEFAULT_CONFIG_PATH = "./config.yaml"
+
+if USE_HUGGINGFACE_ZEROGPU:
+    try:
+        import spaces
+    except ImportError:
+        USE_HUGGINGFACE_ZEROGPU = False
+        logging.warning("HuggingFace Spaces not available, running without GPU acceleration")
+
+if USE_HUGGINGFACE_ZEROGPU:
+    run_vibe_blend_safe = spaces.GPU(duration=60)(run_vibe_blend_safe)
+    run_vibe_blend_not_safe = spaces.GPU(duration=60)(run_vibe_blend_not_safe)
+
+    try:
+        from download_models import download_ipadapter
+        download_ipadapter()
+    except ImportError:
+        logging.warning("Could not import download_models")
+
+
+def load_gradio_images_helper(pil_images: Union[List, Image.Image, str]) -> List[Image.Image]:
+    """
+    Convert various image input formats to a list of PIL Images.
+    """
+    if pil_images is None:
+        return []
+    
+    # Handle single image
+    if isinstance(pil_images, np.ndarray):
+        return Image.fromarray(pil_images).convert("RGB")
+    if isinstance(pil_images, Image.Image):
+        return pil_images.convert("RGB")
+    if isinstance(pil_images, str):
+        return Image.open(pil_images).convert("RGB")
+    
+    # Handle list of images
+    processed_images = []
+    for image in pil_images:
+        if isinstance(image, tuple):  # Gradio gallery format
+            image = image[0]
+        if isinstance(image, str):
+            image = Image.open(image)
+        elif isinstance(image, Image.Image):
+            pass  # Already PIL Image
+        else:
+            continue
+        processed_images.append(image.convert("RGB"))
+    
+    return processed_images
+
+
+def create_gradio_interface():
+    theme = gr.themes.Base(
+        spacing_size='md', 
+        text_size='lg', 
+        primary_hue='blue', 
+        neutral_hue='slate', 
+        secondary_hue='pink'
+    )
+    
+    demo = gr.Blocks(theme=theme)
+    with demo:
+        gr.Markdown("""
+        ## Vibe Blending Demo
+        
+        This is the demo for the paper "*Vibe Spaces for Creatively Connecting and Expressing Visual Concepts*".
+        
+        [Paper]() | [Code]() | [Website]()
+        
+        Given a pair of images, vibe blending will generate a set of images that creatively connect the input images.
+        
+        **[📝 Feedback Form](https://docs.google.com/forms/d/e/1FAIpQLSfS-2fdJ3eaG6JBUGNgHYD4zNRtoPUOc2OhF8J-uT-gyR3LyA/viewform?usp=dialog)** - Please submit your interesting images!
+        
+        """)
+        with gr.Row():
+            with gr.Column():
+                with gr.Group():
+                    with gr.Row():
+                        gr.Markdown("**Step 1:** Upload 2 images")
+                    with gr.Row():
+                        input1 = gr.Image(label="Input 1", show_label=True)
+                        input2 = gr.Image(label="Input 2", show_label=True)
+
+                with gr.Accordion("Options", open=False):
+                    with gr.Group():
+                        with gr.Row():
+                            alpha_start = gr.Slider(minimum=0, maximum=2, step=0.1, value=0.0, label="Start α", info="interpolation weight")
+                            alpha_end = gr.Slider(minimum=0, maximum=2, step=0.1, value=1.0, label="End α", info="use α>1 for extrapolation")
+                        # n_steps = gr.Slider(minimum=1, maximum=40, step=1, value=10, label="Number of Output Images")
+                        n_steps = gr.Number(value=12, label="Number of Output Images", interactive=True)
+                        with gr.Row():
+                            extra_images = gr.Gallery(label="Extra Images (optional)", show_label=True, columns=3, rows=2, height=150)
+                            negative_images = gr.Gallery(label="Negative Images (optional)", show_label=True, columns=3, rows=2, height=150)
+            with gr.Column():
+                with gr.Group():
+                    # blending_results = gr.Gallery(label="Vibe Blending Results", columns=5, rows=4, height=600)
+                    gr.Markdown("**Step 2:** Run Vibe Blending")
+                    blending_results = gr.Image(label="Vibe Blending Results", show_label=True, height=400)
+                    blend_button = gr.Button("🔴 Run Vibe Blending", variant="primary")
+        
+        # Training wrapper function
+        def blend_button_click(input1, input2, extra_images, negative_images, alpha_start, alpha_end, n_steps):
+            input1 = load_gradio_images_helper(input1)
+            input2 = load_gradio_images_helper(input2)
+            extra_images = load_gradio_images_helper(extra_images)
+            negative_images = load_gradio_images_helper(negative_images)
+
+            if extra_images is None:
+                extra_images = []
+            elif isinstance(extra_images, Image.Image):
+                extra_images = [extra_images]
+
+            if negative_images is None:
+                negative_images = []
+            elif isinstance(negative_images, Image.Image):
+                negative_images = [negative_images]
+
+            alpha_weights = np.linspace(alpha_start, alpha_end, n_steps+2)[1:-1].tolist()
+            blended_images = run_vibe_blend_not_safe(input1, input2, extra_images, negative_images, DEFAULT_CONFIG_PATH, alpha_weights)
+            blended_images = create_image_grid(blended_images, rows=np.ceil(len(blended_images)/4).astype(int), cols=4)
+            return blended_images
+        
+        blend_button.click(blend_button_click, inputs=[input1, input2, extra_images, negative_images, alpha_start, alpha_end, n_steps], outputs=[blending_results])
+        
+        example_cases = [
+            [Image.open("./images/playviolin_hr.png"), Image.open("./images/playguitar_hr.png")],
+            [Image.open("./images/input_cat.png"), Image.open("./images/input_bread.png")],
+            [Image.open("./images/02140_left.jpg"), Image.open("./images/02140_right.jpg")],
+            #[Image.open("./images/02718_l.jpg"), Image.open("./images/02718_r.jpg")],
+            [Image.open("./images/03969_l.jpg"), Image.open("./images/03969_r.jpg")],
+            [Image.open("./images/04963_l.jpg"), Image.open("./images/04963_r.jpg")],
+            #[Image.open("./images/05358_l.jpg"), Image.open("./images/05358_r.jpg")],
+            [Image.open("./images/00436_l.jpg"), Image.open("./images/00436_r.jpg")],
+            [Image.open("./images/archi/input_A.jpg"), Image.open("./images/archi/input_B.jpg")],
+        ]
+        gr.Examples(examples=example_cases, label="Example Cases", inputs=[input1, input2], outputs=[blending_results])
+        
+        extra_image_examples = [
+            [Image.open("./images/archi/input_A.jpg"), Image.open("./images/archi/input_B.jpg"), [Image.open("./images/archi/extra1.jpg"), Image.open("./images/archi/extra2.jpg"), Image.open("./images/archi/extra3.jpg")]],
+        ]
+        gr.Examples(examples=extra_image_examples, label="Extra Image Examples", inputs=[input1, input2, extra_images], outputs=[blending_results])
+        
+        negative_image_examples = [
+            [Image.open("./images/pink_bear1.jpg"), Image.open("./images/black_bear2.jpg"), [Image.open("./images/pink_bear1.jpg"), Image.open("./images/black_bear1.jpg")]],
+        ]
+        gr.Examples(examples=negative_image_examples, label="Negative Image Examples", inputs=[input1, input2, negative_images], outputs=[blending_results])
+        
+    return demo
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    
+    demo = create_gradio_interface()
+    demo.launch(
+        share=True,
+        server_name="0.0.0.0" if USE_HUGGINGFACE_ZEROGPU else None,
+        show_error=True
+    )

config.yaml

@@ -0,0 +1,26 @@
+in_dim: 768 #1024 #768 #384
+vibe_dim: -1
+out_dim: 1280
+latent_dim: 512
+n_layer: 2
+
+n_eig: 64
+flag_encoder_loss: 1.0
+flag_decoder_loss: 0.01
+recon_loss: 1.0
+
+negative_beta: 1.0
+do_decoder_negative_flag: false
+
+single_scale_flag: false
+
+log_dir: /tmp/logs/
+name: debug
+
+ipadapter_version: sd15  # sdxl, sd15
+
+steps: 1000
+batch_size: 8
+n_negative_sample: 100
+n_sample_eigsolve: 2000
+lr: 0.001

demo_vibe_blending.ipynb

@@ -0,0 +1,129 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a4447a99",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pathlib import Path\n",
+    "from typing import Iterable, List\n",
+    "\n",
+    "import numpy as np\n",
+    "from PIL import Image\n",
+    "from IPython.display import display\n",
+    "\n",
+    "from vibe_blending import run_vibe_blend_not_safe\n",
+    "from ipadapter_model import create_image_grid"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2f9e85da",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "CONFIG_PATH = Path('config.yaml')\n",
+    "IMAGES_DIR = Path('images')\n",
+    "\n",
+    "POSITIVE_IMAGE_PATHS = [\n",
+    "    IMAGES_DIR / 'playviolin_hr.png',\n",
+    "    IMAGES_DIR / 'playguitar_hr.png',\n",
+    "]\n",
+    "\n",
+    "EXTRA_IMAGE_PATHS = [\n",
+    "    # ...\n",
+    "]\n",
+    "\n",
+    "NEGATIVE_IMAGE_PATHS = [\n",
+    "    # ...\n",
+    "]\n",
+    "\n",
+    "def load_images(paths: Iterable[Path]) -> List[Image.Image]:\n",
+    "    return [Image.open(path).convert('RGB') for path in paths]\n",
+    "\n",
+    "positive_images = load_images(POSITIVE_IMAGE_PATHS)\n",
+    "extra_images = load_images(EXTRA_IMAGE_PATHS)\n",
+    "negative_images = load_images(NEGATIVE_IMAGE_PATHS)\n",
+    "\n",
+    "print(f'Loaded {len(positive_images)} positive, {len(extra_images)} extra, {len(negative_images)} negative images')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4b079a6a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "GRID_PREVIEW_SIZE = (256, 256)\n",
+    "GRID_BLEND_SIZE = (512, 512)\n",
+    "\n",
+    "def resize_for_grid(images: List[Image.Image], size: tuple[int, int]) -> List[Image.Image]:\n",
+    "    return [img.resize(size, Image.Resampling.LANCZOS) for img in images]\n",
+    "\n",
+    "def show_row(title: str, images: List[Image.Image]):\n",
+    "    if not images:\n",
+    "        return\n",
+    "    print(f\"{title} ({len(images)} images)\")\n",
+    "    thumbs = resize_for_grid(images, GRID_PREVIEW_SIZE)\n",
+    "    grid = create_image_grid(thumbs, rows=1, cols=len(thumbs))\n",
+    "    display(grid)\n",
+    "\n",
+    "show_row('Positives', positive_images)\n",
+    "show_row('Extra references', extra_images)\n",
+    "show_row('Negatives (attributes to suppress)', negative_images)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9d396972",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "alpha_weights = np.linspace(0, 1, 10).tolist()\n",
+    "print(f'α values: {alpha_weights}')\n",
+    "\n",
+    "blended_with_negatives = run_vibe_blend_not_safe(\n",
+    "    image1=positive_images[0],\n",
+    "    image2=positive_images[1],\n",
+    "    extra_images=extra_images,\n",
+    "    negative_images=negative_images,\n",
+    "    config_path=str(CONFIG_PATH),\n",
+    "    interpolation_weights=alpha_weights,\n",
+    "    n_clusters=20,\n",
+    ")\n",
+    "\n",
+    "sequence_with_neg = [positive_images[0], *blended_with_negatives, positive_images[1]]\n",
+    "rows = int(np.ceil(len(sequence_with_neg) / 4))\n",
+    "normalized_sequence = resize_for_grid(sequence_with_neg, GRID_BLEND_SIZE)\n",
+    "neg_grid = create_image_grid(normalized_sequence, rows=rows, cols=4)\n",
+    "display(neg_grid)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "mspace",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

dino_correspondence.py

@@ -0,0 +1,764 @@
+"""
+DINO Correspondence Analysis Module
+
+This module provides functions for analyzing visual correspondences between images
+using DINO features, normalized cuts (NCut), and clustering techniques.
+"""
+
+import numpy as np
+import torch
+from PIL import Image
+from scipy.optimize import linear_sum_assignment
+from einops import rearrange
+
+from extract_features import image_inverse_transform
+from ipadapter_model import image_grid
+from ncut_pytorch import ncut_fn, kway_ncut, convert_to_lab_color
+from ncut_pytorch.color import tsne_color
+from ncut_pytorch.utils.gamma import find_gamma_by_degree
+
+
+# ===== Core NCut and Clustering Functions =====
+
+def ncut_tsne_multiple_images(image_embeds, n_eig=50, gamma=None, degree=0.5):
+    """
+    Apply NCut and t-SNE coloring to multiple image embeddings.
+    
+    image_embeds is (batch, length, channels)
+    """
+    batch_size, length, channels = image_embeds.shape
+    flattened_input = image_embeds.flatten(end_dim=-2)
+    
+    if gamma is None:
+        gamma = find_gamma_by_degree(flattened_input, degree)
+    
+    eigenvectors, eigenvalues = ncut_fn(
+        flattened_input, n_eig=n_eig, gamma=gamma, device='cuda'
+    )
+    
+    rgb_colors = tsne_color(eigenvectors, n_dim=3, device='cuda', perplexity=50)
+    rgb_colors = convert_to_lab_color(rgb_colors)
+    
+    # Reshape back to original batch structure
+    rgb_colors = rearrange(rgb_colors, '(b l) c -> b l c', b=batch_size)
+    eigenvectors = rearrange(eigenvectors, '(b l) c -> b l c', b=batch_size)
+    
+    return eigenvectors, rgb_colors
+
+
+def _kway_cluster_single_image(image_embeds, n_clusters, gamma=None, degree=0.5):
+    length, channels = image_embeds.shape
+    flattened_input = image_embeds.flatten(end_dim=-2)
+    
+    if gamma is None:
+        gamma = find_gamma_by_degree(flattened_input, degree)
+    else:
+        gamma = gamma * image_embeds.var(0).sum().item()
+    
+    # Calculate number of eigenvectors needed
+    n_eig = min(n_clusters * 2 + 6, flattened_input.shape[0] // 2 - 1)
+    
+    eigenvectors, _ = ncut_fn(
+        flattened_input, n_eig=n_eig, gamma=gamma, device='cuda'
+    )
+    
+    continuous_clusters = kway_ncut(eigenvectors[:, :n_clusters])
+    return continuous_clusters
+
+
+def kway_cluster_per_image(image_embeds, n_clusters, gamma=None, degree=0.5):
+    """
+    Perform k-way clustering on each image separately.
+    
+    image_embeds is (batch, length, channels)
+    return (batch, length, clusters)
+    """
+    clustered_eigenvectors = []
+    
+    for i in range(image_embeds.shape[0]):
+        eigenvector = _kway_cluster_single_image(
+            image_embeds[i], n_clusters, gamma, degree
+        )
+        clustered_eigenvectors.append(eigenvector)
+    
+    return torch.stack(clustered_eigenvectors)
+
+
+def kway_cluster_multiple_images(image_embeds, n_clusters, gamma=None, degree=0.5):
+    """
+    Perform k-way clustering on multiple images jointly.
+    
+    image_embeds is (batch, length, channels)
+    return (batch, length, clusters)
+    """
+    batch_size, length, channels = image_embeds.shape
+    flattened_input = image_embeds.flatten(end_dim=-2)
+    
+    if gamma is None:
+        gamma = find_gamma_by_degree(flattened_input, degree)
+    
+    # Calculate number of eigenvectors needed
+    n_eig = min(n_clusters * 2 + 6, flattened_input.shape[0] // 2 - 1)
+    
+    eigenvectors, _ = ncut_fn(
+        flattened_input, n_eig=n_eig, gamma=gamma, device='cuda'
+    )
+    
+    continuous_clusters = kway_ncut(eigenvectors[:, :n_clusters])
+    continuous_clusters = rearrange(
+        continuous_clusters, '(b l) c -> b l c', b=batch_size
+    )
+    
+    return continuous_clusters
+
+
+# ===== Color and Visualization Functions =====
+
+def get_discrete_colors_from_clusters(joint_colors, cluster_eigenvectors):
+
+    n_clusters = cluster_eigenvectors.shape[-1]
+    discrete_colors = np.zeros_like(joint_colors)
+    
+    for img_idx in range(joint_colors.shape[0]):
+        colors = joint_colors[img_idx]
+        eigenvector = cluster_eigenvectors[img_idx].cpu().numpy()
+        cluster_labels = eigenvector.argmax(-1)
+        discrete_img_colors = np.zeros_like(colors)
+        
+        for cluster_idx in range(n_clusters):
+            cluster_mask = cluster_labels == cluster_idx
+            if cluster_mask.sum() > 0:
+                # Use mean color for each cluster
+                discrete_img_colors[cluster_mask] = colors[cluster_mask].mean(0)
+        
+        discrete_colors[img_idx] = discrete_img_colors
+    
+    # Convert to uint8 format
+    discrete_colors = (discrete_colors * 255).astype(np.uint8)
+    return discrete_colors
+
+
+# ===== Center Matching Functions =====
+
+def get_cluster_center_features(image_embeds, cluster_labels, n_clusters):
+
+    center_features = torch.zeros((n_clusters, image_embeds.shape[-1]))
+    
+    for cluster_idx in range(n_clusters):
+        cluster_mask = cluster_labels == cluster_idx
+        
+        if cluster_mask.sum() > 0:
+            center_features[cluster_idx] = image_embeds[cluster_mask].mean(0)
+        else:
+            # Use a unique identifier for empty clusters
+            center_features[cluster_idx] = torch.ones_like(image_embeds[0]) * 114514
+    
+    return center_features
+
+
+def cosine_similarity(matrix_a, matrix_b):
+    normalized_a = matrix_a / matrix_a.norm(dim=-1, keepdim=True)
+    normalized_b = matrix_b / matrix_b.norm(dim=-1, keepdim=True)
+    return normalized_a @ normalized_b.T
+
+
+def hungarian_match_centers(center_features1, center_features2):
+    distances = torch.cdist(center_features1, center_features2)
+    distances = distances.cpu().detach().numpy()
+    _, column_indices = linear_sum_assignment(distances)
+    return column_indices
+
+
+def argmin_matching(center_features1, center_features2):
+    distances = torch.cdist(center_features1, center_features2)
+    distances = distances.cpu().detach().numpy()
+    return np.argmin(distances, axis=-1)
+
+
+def match_cluster_centers(image_embed1, image_embed2, eigvec1, eigvec2, 
+                         match_method='hungarian'):
+    cluster_labels1 = eigvec1.argmax(-1).cpu().numpy()
+    cluster_labels2 = eigvec2.argmax(-1).cpu().numpy()
+    
+    center_features1 = get_cluster_center_features(
+        image_embed1, cluster_labels1, eigvec1.shape[-1]
+    )
+    center_features2 = get_cluster_center_features(
+        image_embed2, cluster_labels2, eigvec2.shape[-1]
+    )
+    
+    if match_method == 'hungarian':
+        mapping = hungarian_match_centers(center_features1, center_features2)
+    elif match_method == 'argmin':
+        mapping = argmin_matching(center_features1, center_features2)
+    else:
+        raise ValueError(f"Unknown match_method: {match_method}")
+    
+    return mapping
+
+
+def match_centers_three_images(image_embeds, eigenvectors, match_method='hungarian'):
+    """
+    Match cluster centers across three images (A2 -> A1 -> B1).
+    
+    Args:
+        image_embeds (torch.Tensor): Embeddings for 3 images [A2, A1, B1]
+        eigenvectors (torch.Tensor): Eigenvectors for 3 images
+        match_method (str): Matching method
+        
+    Returns:
+        tuple: (A2_to_A1_mapping, A1_to_B1_mapping)
+    """
+    a2_to_a1_mapping = match_cluster_centers(
+        image_embeds[0], image_embeds[1], 
+        eigenvectors[0], eigenvectors[1], 
+        match_method=match_method
+    )
+    
+    a1_to_b1_mapping = match_cluster_centers(
+        image_embeds[1], image_embeds[2], 
+        eigenvectors[1], eigenvectors[2], 
+        match_method=match_method
+    )
+    
+    return a2_to_a1_mapping, a1_to_b1_mapping
+
+
+def match_centers_two_images(image_embed1, image_embed2, eigvec1, eigvec2, 
+                            match_method='hungarian'):
+    return match_cluster_centers(
+        image_embed1, image_embed2, eigvec1, eigvec2, match_method=match_method
+    )
+
+
+# ===== Two-Step Clustering Functions =====
+
+def kway_cluster_per_image_two_step(
+    image_embeds,
+    n_superclusters,
+    n_subclusters_per_supercluster,
+    supercluster_gamma=None,
+    subcluster_gamma=None,
+    degree=0.5
+):
+    """
+    Perform 2-step hierarchical clustering on each image separately.
+    First finds superclusters, then subdivides each supercluster into subclusters.
+    
+    Args:
+        image_embeds: (batch, length, channels) - Image embeddings
+        n_superclusters: Number of coarse superclusters to find
+        n_subclusters_per_supercluster: Number of subclusters within each supercluster
+        supercluster_gamma: Gamma parameter for supercluster NCut (None = auto)
+        subcluster_gamma: Gamma parameter for subcluster NCut (None = auto)
+        degree: Degree parameter for gamma estimation
+        
+    Returns:
+        tuple: (supercluster_eigenvectors, subcluster_eigenvectors, subcluster_to_supercluster_mapping)
+            - supercluster_eigenvectors: (batch, length, n_superclusters)
+            - subcluster_eigenvectors: (batch, length, total_subclusters)
+            - subcluster_to_supercluster_mapping: (batch, total_subclusters) mapping each subcluster to its supercluster
+    """
+    batch_size = image_embeds.shape[0]
+    
+    # Step 1: Compute superclusters for each image
+    supercluster_eigenvectors = []
+    for i in range(batch_size):
+        eigenvector = _kway_cluster_single_image(
+            image_embeds[i], n_superclusters, supercluster_gamma, degree
+        )
+        supercluster_eigenvectors.append(eigenvector)
+    supercluster_eigenvectors = torch.stack(supercluster_eigenvectors)
+    
+    # Step 2: For each supercluster in each image, compute subclusters
+    subcluster_eigenvectors = []
+    subcluster_to_supercluster_mapping = []
+    
+    for img_idx in range(batch_size):
+        img_subclusters = []
+        img_mapping = []
+        
+        supercluster_labels = supercluster_eigenvectors[img_idx].argmax(-1)
+        
+        # For each supercluster, extract tokens and compute subclusters
+        for supercluster_idx in range(n_superclusters):
+            supercluster_mask = supercluster_labels == supercluster_idx
+            
+            if supercluster_mask.sum() == 0:
+                # Empty supercluster - create dummy subclusters
+                for sub_idx in range(n_subclusters_per_supercluster):
+                    img_mapping.append(supercluster_idx)
+                continue
+            
+            # Extract features belonging to this supercluster
+            supercluster_features = image_embeds[img_idx][supercluster_mask]
+            
+            # Perform clustering on this subset
+            if supercluster_features.shape[0] <= n_subclusters_per_supercluster:
+                # Too few tokens - each token becomes its own subcluster
+                n_actual_subclusters = supercluster_features.shape[0]
+                subcluster_labels = torch.arange(n_actual_subclusters).to(supercluster_features.device)
+                # Pad with dummy subclusters if needed
+                for sub_idx in range(n_subclusters_per_supercluster):
+                    img_mapping.append(supercluster_idx)
+            else:
+                # Perform subclustering
+                subcluster_eigvecs = _kway_cluster_single_image(
+                    supercluster_features, 
+                    n_subclusters_per_supercluster,
+                    subcluster_gamma,
+                    degree
+                )
+                subcluster_labels = subcluster_eigvecs.argmax(-1)
+                
+                # Track which supercluster these subclusters belong to
+                for sub_idx in range(n_subclusters_per_supercluster):
+                    img_mapping.append(supercluster_idx)
+            
+            # Store subcluster assignments for this supercluster
+            for sub_idx in range(n_subclusters_per_supercluster):
+                img_subclusters.append((supercluster_mask, subcluster_labels == sub_idx if supercluster_features.shape[0] > n_subclusters_per_supercluster else None))
+        
+        # Convert to full eigenvector representation
+        total_subclusters = n_superclusters * n_subclusters_per_supercluster
+        img_subcluster_eigvec = torch.zeros((image_embeds.shape[1], total_subclusters)).to(image_embeds.device)
+        
+        for subcluster_global_idx, (supercluster_mask, subcluster_mask) in enumerate(img_subclusters):
+            if subcluster_mask is not None:
+                # Combine masks: belongs to supercluster AND subcluster
+                final_mask = torch.zeros(image_embeds.shape[1], dtype=torch.bool).to(image_embeds.device)
+                supercluster_indices = torch.where(supercluster_mask)[0]
+                subcluster_within_super = torch.where(subcluster_mask)[0]
+                if len(subcluster_within_super) > 0:
+                    final_indices = supercluster_indices[subcluster_within_super]
+                    final_mask[final_indices] = True
+                    img_subcluster_eigvec[final_mask, subcluster_global_idx] = 1.0
+            # else: leave as zeros (empty subcluster)
+        
+        subcluster_eigenvectors.append(img_subcluster_eigvec)
+        subcluster_to_supercluster_mapping.append(torch.tensor(img_mapping))
+    
+    subcluster_eigenvectors = torch.stack(subcluster_eigenvectors)
+    subcluster_to_supercluster_mapping = torch.stack(subcluster_to_supercluster_mapping)
+    
+    return supercluster_eigenvectors, subcluster_eigenvectors, subcluster_to_supercluster_mapping
+
+
+def match_centers_two_step(
+    image_embed1,
+    image_embed2,
+    supercluster_eigvec1,
+    supercluster_eigvec2,
+    subcluster_eigvec1,
+    subcluster_eigvec2,
+    subcluster_to_supercluster_mapping1,
+    subcluster_to_supercluster_mapping2,
+    supercluster_match_method='hungarian',
+    subcluster_match_method='hungarian'
+):
+    """
+    Match clusters using 2-step hierarchical approach.
+    First matches superclusters, then matches subclusters only within matched superclusters.
+    
+    Args:
+        image_embed1, image_embed2: Image embeddings (length, channels)
+        supercluster_eigvec1, supercluster_eigvec2: Supercluster eigenvectors (length, n_superclusters)
+        subcluster_eigvec1, subcluster_eigvec2: Subcluster eigenvectors (length, total_subclusters)
+        subcluster_to_supercluster_mapping1, subcluster_to_supercluster_mapping2: (total_subclusters,)
+        supercluster_match_method: Matching method for superclusters
+        subcluster_match_method: Matching method for subclusters
+        
+    Returns:
+        np.ndarray: Mapping from image1 subclusters to image2 subclusters
+    """
+    n_superclusters = supercluster_eigvec1.shape[-1]
+    n_subclusters_total = subcluster_eigvec1.shape[-1]
+    
+    # Step 1: Match superclusters
+    supercluster_mapping = match_cluster_centers(
+        image_embed1, image_embed2,
+        supercluster_eigvec1, supercluster_eigvec2,
+        match_method=supercluster_match_method
+    )
+    
+    # Step 2: For each matched supercluster pair, match subclusters within them
+    subcluster_mapping = np.zeros(n_subclusters_total, dtype=np.int64)
+    
+    for supercluster1_idx in range(n_superclusters):
+        # Find which supercluster in image2 this maps to
+        supercluster2_idx = supercluster_mapping[supercluster1_idx]
+        
+        # Find all subclusters belonging to these superclusters
+        subclusters1_mask = (subcluster_to_supercluster_mapping1 == supercluster1_idx).cpu().numpy()
+        subclusters2_mask = (subcluster_to_supercluster_mapping2 == supercluster2_idx).cpu().numpy()
+        
+        subclusters1_indices = np.where(subclusters1_mask)[0]
+        subclusters2_indices = np.where(subclusters2_mask)[0]
+        
+        if len(subclusters1_indices) == 0 or len(subclusters2_indices) == 0:
+            # No subclusters in one or both superclusters - use identity mapping
+            for sub1_idx in subclusters1_indices:
+                if sub1_idx < len(subclusters2_indices):
+                    subcluster_mapping[sub1_idx] = subclusters2_indices[sub1_idx]
+                else:
+                    subcluster_mapping[sub1_idx] = subclusters2_indices[0] if len(subclusters2_indices) > 0 else 0
+            continue
+        
+        # Extract subcluster eigenvectors for matching
+        sub_eigvec1 = subcluster_eigvec1[:, subclusters1_indices]
+        sub_eigvec2 = subcluster_eigvec2[:, subclusters2_indices]
+        
+        # Compute cluster centers for these subclusters
+        cluster_labels1 = sub_eigvec1.argmax(-1).cpu()
+        cluster_labels2 = sub_eigvec2.argmax(-1).cpu()
+        
+        center_features1 = get_cluster_center_features(
+            image_embed1, cluster_labels1, len(subclusters1_indices)
+        )
+        center_features2 = get_cluster_center_features(
+            image_embed2, cluster_labels2, len(subclusters2_indices)
+        )
+        
+        # Match subclusters within this supercluster pair
+        if subcluster_match_method == 'hungarian':
+            local_mapping = hungarian_match_centers(center_features1, center_features2)
+        elif subcluster_match_method == 'argmin':
+            local_mapping = argmin_matching(center_features1, center_features2)
+        else:
+            raise ValueError(f"Unknown subcluster_match_method: {subcluster_match_method}")
+        
+        # Convert local mapping to global subcluster indices
+        for local_idx, global_idx1 in enumerate(subclusters1_indices):
+            global_idx2 = subclusters2_indices[local_mapping[local_idx]]
+            subcluster_mapping[global_idx1] = global_idx2
+    
+    return subcluster_mapping
+
+
+def kway_cluster_per_image_two_step_fgbg(
+    image_embeds,
+    n_foreground_subclusters,
+    n_background_subclusters,
+    supercluster_gamma=None,
+    subcluster_gamma=None,
+    degree=0.5
+):
+    """
+    Perform 2-step hierarchical clustering with automatic foreground/background separation.
+    First separates foreground (FG) and background (BG) using 2 clusters, identifying FG 
+    by the cluster with highest max eigenvector value. Then subdivides FG and BG separately.
+    
+    Args:
+        image_embeds: (batch, length, channels) - Image embeddings
+        n_foreground_subclusters: Number of subclusters within foreground
+        n_background_subclusters: Number of subclusters within background
+        supercluster_gamma: Gamma parameter for FG/BG clustering (None = auto)
+        subcluster_gamma: Gamma parameter for subcluster NCut (None = auto)
+        degree: Degree parameter for gamma estimation
+        
+    Returns:
+        tuple: (supercluster_eigenvectors, subcluster_eigenvectors, subcluster_to_supercluster_mapping, fg_indices)
+            - supercluster_eigenvectors: (batch, length, 2) - [BG, FG] clusters
+            - subcluster_eigenvectors: (batch, length, total_subclusters)
+            - subcluster_to_supercluster_mapping: (batch, total_subclusters) - 0=BG, 1=FG
+            - fg_indices: (batch,) - which supercluster index is foreground for each image
+    """
+    batch_size = image_embeds.shape[0]
+    n_superclusters = 2  # Always FG and BG
+    
+    # Step 1: Compute FG/BG separation for each image
+    supercluster_eigenvectors = []
+    fg_indices = []
+    
+    for i in range(batch_size):
+        eigenvector = _kway_cluster_single_image(
+            image_embeds[i], n_clusters=2, gamma=supercluster_gamma, degree=degree
+        )
+        supercluster_eigenvectors.append(eigenvector)
+        
+        # Identify foreground: cluster with highest max eigenvector value
+        fg_idx = eigenvector.max(0).values.argmax().item()
+        fg_indices.append(fg_idx)
+    
+    supercluster_eigenvectors = torch.stack(supercluster_eigenvectors)
+    fg_indices = torch.tensor(fg_indices)
+    
+    # Step 2: For each image, compute subclusters within FG and BG
+    subcluster_eigenvectors = []
+    subcluster_to_supercluster_mapping = []
+    
+    for img_idx in range(batch_size):
+        img_subclusters = []
+        img_mapping = []
+        
+        supercluster_labels = supercluster_eigenvectors[img_idx].argmax(-1)
+        fg_idx = fg_indices[img_idx].item()
+        bg_idx = 1 - fg_idx
+        
+        # Process BG and FG in order (BG first, then FG)
+        for is_foreground, n_subclusters in [(False, n_background_subclusters), (True, n_foreground_subclusters)]:
+            supercluster_idx = fg_idx if is_foreground else bg_idx
+            supercluster_mask = supercluster_labels == supercluster_idx
+            
+            # Mark which supercluster type (0=BG, 1=FG)
+            supercluster_type = 1 if is_foreground else 0
+            
+            if supercluster_mask.sum() == 0:
+                # Empty supercluster - create dummy subclusters
+                for sub_idx in range(n_subclusters):
+                    img_mapping.append(supercluster_type)
+                    img_subclusters.append((supercluster_mask, None))
+                continue
+            
+            # Extract features belonging to this supercluster
+            supercluster_features = image_embeds[img_idx][supercluster_mask]
+            
+            # Perform clustering on this subset
+            if supercluster_features.shape[0] <= n_subclusters:
+                # Too few tokens - each token becomes its own subcluster
+                n_actual_subclusters = supercluster_features.shape[0]
+                subcluster_labels = torch.arange(n_actual_subclusters).to(supercluster_features.device)
+                # Pad with dummy subclusters if needed
+                for sub_idx in range(n_subclusters):
+                    img_mapping.append(supercluster_type)
+                    if sub_idx < n_actual_subclusters:
+                        img_subclusters.append((supercluster_mask, subcluster_labels == sub_idx))
+                    else:
+                        img_subclusters.append((supercluster_mask, None))
+            else:
+                # Perform subclustering
+                subcluster_eigvecs = _kway_cluster_single_image(
+                    supercluster_features, 
+                    n_subclusters,
+                    subcluster_gamma,
+                    degree
+                )
+                subcluster_labels = subcluster_eigvecs.argmax(-1)
+                
+                # Store subcluster assignments
+                for sub_idx in range(n_subclusters):
+                    img_mapping.append(supercluster_type)
+                    img_subclusters.append((supercluster_mask, subcluster_labels == sub_idx))
+        
+        # Convert to full eigenvector representation
+        total_subclusters = n_background_subclusters + n_foreground_subclusters
+        img_subcluster_eigvec = torch.zeros((image_embeds.shape[1], total_subclusters)).to(image_embeds.device)
+        
+        for subcluster_global_idx, (supercluster_mask, subcluster_mask) in enumerate(img_subclusters):
+            if subcluster_mask is not None:
+                # Combine masks: belongs to supercluster AND subcluster
+                final_mask = torch.zeros(image_embeds.shape[1], dtype=torch.bool).to(image_embeds.device)
+                supercluster_indices = torch.where(supercluster_mask)[0]
+                subcluster_within_super = torch.where(subcluster_mask)[0]
+                if len(subcluster_within_super) > 0:
+                    final_indices = supercluster_indices[subcluster_within_super]
+                    final_mask[final_indices] = True
+                    img_subcluster_eigvec[final_mask, subcluster_global_idx] = 1.0
+            # else: leave as zeros (empty subcluster)
+        
+        subcluster_eigenvectors.append(img_subcluster_eigvec)
+        subcluster_to_supercluster_mapping.append(torch.tensor(img_mapping))
+    
+    subcluster_eigenvectors = torch.stack(subcluster_eigenvectors)
+    subcluster_to_supercluster_mapping = torch.stack(subcluster_to_supercluster_mapping)
+    
+    return supercluster_eigenvectors, subcluster_eigenvectors, subcluster_to_supercluster_mapping, fg_indices
+
+
+def match_centers_two_step_fgbg(
+    image_embed1,
+    image_embed2,
+    subcluster_eigvec1,
+    subcluster_eigvec2,
+    subcluster_to_supercluster_mapping1,
+    subcluster_to_supercluster_mapping2,
+    n_background_subclusters,
+    n_foreground_subclusters,
+    background_match_method='hungarian',
+    foreground_match_method='hungarian'
+):
+    """
+    Match clusters using 2-step FG/BG hierarchical approach.
+    FG and BG are automatically matched (no need for supercluster matching).
+    Subclusters are matched within their respective FG or BG groups.
+    
+    Args:
+        image_embed1, image_embed2: Image embeddings (length, channels)
+        subcluster_eigvec1, subcluster_eigvec2: Subcluster eigenvectors (length, total_subclusters)
+        subcluster_to_supercluster_mapping1, subcluster_to_supercluster_mapping2: (total_subclusters,) - 0=BG, 1=FG
+        n_background_subclusters: Number of background subclusters
+        n_foreground_subclusters: Number of foreground subclusters
+        background_match_method: Matching method for background subclusters
+        foreground_match_method: Matching method for foreground subclusters
+        
+    Returns:
+        np.ndarray: Mapping from image1 subclusters to image2 subclusters
+    """
+    total_subclusters = n_background_subclusters + n_foreground_subclusters
+    subcluster_mapping = np.zeros(total_subclusters, dtype=np.int64)
+    
+    # Process BG (supercluster_type=0) and FG (supercluster_type=1) separately
+    for supercluster_type in [0, 1]:  # 0=BG, 1=FG
+        # Find subclusters belonging to this supercluster type
+        subclusters1_mask = (subcluster_to_supercluster_mapping1 == supercluster_type).cpu().numpy()
+        subclusters2_mask = (subcluster_to_supercluster_mapping2 == supercluster_type).cpu().numpy()
+        
+        subclusters1_indices = np.where(subclusters1_mask)[0]
+        subclusters2_indices = np.where(subclusters2_mask)[0]
+        
+        if len(subclusters1_indices) == 0 or len(subclusters2_indices) == 0:
+            # No subclusters in one or both - use identity mapping
+            for sub1_idx in subclusters1_indices:
+                if sub1_idx < len(subclusters2_indices):
+                    subcluster_mapping[sub1_idx] = subclusters2_indices[sub1_idx]
+                else:
+                    subcluster_mapping[sub1_idx] = subclusters2_indices[0] if len(subclusters2_indices) > 0 else 0
+            continue
+        
+        # Extract subcluster eigenvectors for matching
+        sub_eigvec1 = subcluster_eigvec1[:, subclusters1_indices]
+        sub_eigvec2 = subcluster_eigvec2[:, subclusters2_indices]
+        
+        # Compute cluster centers for these subclusters
+        cluster_labels1 = sub_eigvec1.argmax(-1).cpu()
+        cluster_labels2 = sub_eigvec2.argmax(-1).cpu()
+        
+        center_features1 = get_cluster_center_features(
+            image_embed1, cluster_labels1, len(subclusters1_indices)
+        )
+        center_features2 = get_cluster_center_features(
+            image_embed2, cluster_labels2, len(subclusters2_indices)
+        )
+        
+        # Match subclusters within this FG/BG group
+        match_method = foreground_match_method if supercluster_type == 1 else background_match_method
+        
+        if match_method == 'hungarian':
+            local_mapping = hungarian_match_centers(center_features1, center_features2)
+        elif match_method == 'argmin':
+            local_mapping = argmin_matching(center_features1, center_features2)
+        else:
+            raise ValueError(f"Unknown match_method: {match_method}")
+        
+        # Convert local mapping to global subcluster indices
+        for local_idx, global_idx1 in enumerate(subclusters1_indices):
+            global_idx2 = subclusters2_indices[local_mapping[local_idx]]
+            subcluster_mapping[global_idx1] = global_idx2
+    
+    return subcluster_mapping
+
+
+# ===== Visualization Functions =====
+
+def plot_cluster_masks(image, eigenvector, cluster_order, hw=16):
+    """
+    blend the image with the cluster masks
+    # image is (c, h, w)
+    # eigenvector is (h*w, n_eig)
+    # cluster_order is (n_eig), the order of the clusters
+    """
+    cluster_images = []
+    base_img = image_inverse_transform(image).resize(
+        (128, 128), resample=Image.Resampling.NEAREST
+    )
+    
+    for cluster_idx in cluster_order:
+        # Create cluster mask
+        cluster_mask = eigenvector.argmax(-1) == cluster_idx
+        mask_array = cluster_mask.cpu().numpy()[1:].reshape(hw, hw)
+        mask_array = (mask_array * 255).astype(np.uint8)
+        
+        # Resize mask to match image
+        mask_img = Image.fromarray(mask_array).resize(
+            (128, 128), resample=Image.Resampling.NEAREST
+        )
+        
+        # Apply mask to image
+        mask_normalized = np.array(mask_img).astype(np.float32) / 255
+        img_array = np.array(base_img).astype(np.float32) / 255
+        
+        # Create 3-channel mask and apply
+        mask_3ch = np.stack([mask_normalized] * 3, axis=-1)
+        mask_3ch[mask_3ch == 0] = 0.1  # Dim non-masked areas
+        
+        masked_img = img_array * mask_3ch
+        masked_img = (masked_img * 255).astype(np.uint8)
+        
+        cluster_images.append(Image.fromarray(masked_img))
+    
+    return cluster_images
+
+
+def create_image_grid_row(image, eigenvector, cluster_order, discrete_colors, 
+                         hw=16, n_cols=10):
+
+    cluster_images = plot_cluster_masks(image, eigenvector, cluster_order, hw)
+    
+    # Prepare base images
+    base_img = image_inverse_transform(image).resize(
+        (128, 128), resample=Image.Resampling.NEAREST
+    )
+    
+    ncut_visualization = discrete_colors[1:].reshape(hw, hw, 3)
+    ncut_img = Image.fromarray(ncut_visualization).resize(
+        (128, 128), resample=Image.Resampling.NEAREST
+    )
+    
+    # Pad cluster images to fill grid
+    num_missing = n_cols - len(cluster_images) % n_cols
+    if num_missing != n_cols:
+        empty_img = Image.fromarray(np.zeros((128, 128, 3), dtype=np.uint8))
+        cluster_images.extend([empty_img] * num_missing)
+    
+    # Create grid rows
+    prepend_images = [base_img, ncut_img]
+    n_rows = len(cluster_images) // n_cols
+    grid_rows = []
+    
+    for row_idx in range(n_rows):
+        start_idx = row_idx * n_cols
+        end_idx = (row_idx + 1) * n_cols
+        row_images = prepend_images + cluster_images[start_idx:end_idx]
+        grid_rows.append(row_images)
+    
+    return grid_rows
+
+
+def create_multi_image_grid(images, eigenvectors, cluster_orders, discrete_colors, 
+                           hw=16, n_cols=10):
+    all_grid_rows = []
+    
+    for image, eigvec, cluster_order, discrete_rgb in zip(
+        images, eigenvectors, cluster_orders, discrete_colors
+    ):
+        grid_rows = create_image_grid_row(
+            image, eigvec, cluster_order, discrete_rgb, hw, n_cols
+        )
+        all_grid_rows.append(grid_rows)
+    
+    # Interleave rows from different images
+    interleaved_rows = []
+    for row_idx in range(len(all_grid_rows[0])):
+        for img_idx in range(len(all_grid_rows)):
+            interleaved_rows.append(all_grid_rows[img_idx][row_idx])
+    
+    return interleaved_rows
+
+
+def get_correspondence_plot(images, eigenvectors, cluster_orders, discrete_colors, 
+                           hw=16, n_cols=10):
+    n_clusters = eigenvectors.shape[-1]
+    n_cols = min(n_cols, n_clusters)
+    
+    interleaved_rows = create_multi_image_grid(
+        images, eigenvectors, cluster_orders, discrete_colors, hw, n_cols
+    )
+    
+    n_rows = len(interleaved_rows)
+    n_cols = len(interleaved_rows[0])
+    
+    # Flatten all images and create final grid
+    all_images = sum(interleaved_rows, [])
+    final_grid = image_grid(all_images, n_rows, n_cols)
+    
+    return final_grid

download_models.py

@@ -0,0 +1,10 @@
+def download_ipadapter():
+    from huggingface_hub import snapshot_download
+    # snapshot_download(repo_id="h94/IP-Adapter", ignore_patterns="sdxl_models/*", local_dir="./downloads/")
+    snapshot_download(repo_id="h94/IP-Adapter", local_dir="./downloads/")
+
+    from ipadapter_model import load_ipadapter
+    ip_model = load_ipadapter(device="cpu")
+    
+if __name__ == "__main__":
+    download_ipadapter()

extract_features.py

@@ -0,0 +1,135 @@
+"""
+Feature Extraction Module
+
+This module provides utilities for extracting features from images using various
+pre-trained models including DINO, DINOv3, and CLIP. It handles model loading,
+batch processing, and memory management for efficient feature extraction.
+"""
+
+import gc
+from typing import Tuple, Optional
+
+import torch
+import torch.nn as nn
+from einops import rearrange
+from torchvision import transforms
+
+from ipadapter_model import extract_clip_embedding_tensor
+from ipadapter_model import load_ipadapter
+
+
+# Default hyperparameters
+DEFAULT_BATCH_SIZE = 32
+
+
+# ===== Image Transforms =====
+
+# High-resolution transform for DINO models
+dino_image_transform = transforms.Compose([
+    transforms.Resize((256 * 2, 256 * 2)),  # High resolution for detailed features
+    transforms.ToTensor(),
+    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+])
+
+# Standard resolution transform for CLIP models  
+clip_image_transform = transforms.Compose([
+    transforms.Resize((224, 224)),  # Standard ImageNet resolution
+    transforms.ToTensor(),
+    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+])
+
+# Inverse transform to convert normalized tensors back to PIL images
+image_inverse_transform = transforms.Compose([
+    transforms.Normalize(mean=[0.0, 0.0, 0.0], std=[1/0.229, 1/0.224, 1/0.225]),
+    transforms.Normalize(mean=[-0.485, -0.456, -0.406], std=[1.0, 1.0, 1.0]),
+    transforms.ToPILImage(),
+])
+
+
+# ===== Memory Management =====
+
+def clear_gpu_memory():
+    """Clear GPU cache and run garbage collection to free memory."""
+    torch.cuda.empty_cache()
+    gc.collect()
+
+
+# ===== Feature Extraction Functions =====
+
+@torch.no_grad()
+def extract_dino_features(images: torch.Tensor, batch_size: int = DEFAULT_BATCH_SIZE) -> torch.Tensor:
+    """
+    Extract features using DINO ViT-S/16 model.
+    
+    Args:
+        images (torch.Tensor): Input images of shape (N, C, H, W)
+        batch_size (int): Batch size for processing
+        
+    Returns:
+        torch.Tensor: DINO features of shape (N, L, D)
+    """
+    # Load DINO model
+    #dino_model = torch.hub.load('facebookresearch/dino:main', 'dino_vits16')
+    dino_model = torch.hub.load('facebookresearch/dino:main', 'dino_vitb16')
+    dino_model = dino_model.eval().cuda()
+
+    # Process images in batches
+    num_batches = (images.shape[0] + batch_size - 1) // batch_size
+    feature_batches = []
+    
+    for batch_idx in range(num_batches):
+        start_idx = batch_idx * batch_size
+        end_idx = min((batch_idx + 1) * batch_size, images.shape[0])
+        
+        batch_images = images[start_idx:end_idx].cuda()
+        batch_features = dino_model.get_intermediate_layers(batch_images)[-1]
+        feature_batches.append(batch_features.cpu())
+    
+    # Concatenate all batches
+    all_features = torch.cat(feature_batches, dim=0)
+    
+    # Clean up memory
+    del dino_model
+    clear_gpu_memory()
+
+    return all_features
+
+
+@torch.no_grad()
+def extract_clip_features(images: torch.Tensor, batch_size: int = DEFAULT_BATCH_SIZE, ipadapter_version: str = "sd15") -> torch.Tensor:
+    """
+    Extract features using CLIP vision encoder.
+    
+    Args:
+        images (torch.Tensor): Input images of shape (N, C, H, W)
+        batch_size (int): Batch size for processing
+        
+    Returns:
+        torch.Tensor: CLIP features of shape (N, L, D)
+    """
+    # Load IP-Adapter model (contains CLIP encoder)
+    ip_adapter_model = load_ipadapter(version=ipadapter_version)
+    
+    # Process images in batches
+    num_batches = (images.shape[0] + batch_size - 1) // batch_size
+    feature_batches = []
+    
+    for batch_idx in range(num_batches):
+        start_idx = batch_idx * batch_size
+        end_idx = min((batch_idx + 1) * batch_size, images.shape[0])
+        
+        batch_images = images[start_idx:end_idx].cuda()
+        batch_features = extract_clip_embedding_tensor(
+            batch_images, ip_adapter_model, resize=False
+        )
+        feature_batches.append(batch_features.cpu())
+    
+    # Concatenate all batches
+    all_features = torch.cat(feature_batches, dim=0)
+    
+    # Clean up memory
+    del ip_adapter_model
+    clear_gpu_memory()
+
+    return all_features
+

intrinsic_dim.py

@@ -0,0 +1,149 @@
+"""
+Intrinsic Dimensionality Estimation Module
+
+This module provides utilities for estimating the intrinsic dimensionality of 
+high-dimensional feature representations using Maximum Likelihood Estimation (MLE).
+The intrinsic dimension represents the true underlying dimensionality of the data
+manifold, which is often much lower than the ambient feature space dimension.
+"""
+
+import logging
+from typing import Union, Optional
+
+import numpy as np
+import torch
+import skdim
+from ncut_pytorch.utils.sample import farthest_point_sampling
+
+
+# ===== Constants =====
+
+DEFAULT_MAX_SAMPLES = 2000
+MIN_SAMPLES_REQUIRED = 10
+
+
+# ===== Intrinsic Dimensionality Estimation =====
+
+def estimate_intrinsic_dimension(features: Union[torch.Tensor, np.ndarray], 
+                               max_samples: int = DEFAULT_MAX_SAMPLES,
+                               use_global_estimation: bool = True) -> float:
+    """
+    Estimate the intrinsic dimensionality of feature representations.
+    
+    This function uses Maximum Likelihood Estimation (MLE) to determine the intrinsic
+    dimensionality of high-dimensional features. If the dataset is large, it uses
+    farthest point sampling to select a representative subset for efficient computation.
+    
+    Args:
+        features (Union[torch.Tensor, np.ndarray]): Input features of any shape.
+                                                   Will be flattened to (N, D) format.
+        max_samples (int): Maximum number of samples to use for estimation.
+                          Larger values give more accurate estimates but are slower.
+        use_global_estimation (bool): Whether to prefer global over local estimation.
+        
+    Returns:
+        float: Estimated intrinsic dimensionality of the feature manifold.
+        
+    Raises:
+        ValueError: If input features are empty or have insufficient samples.
+        RuntimeError: If dimensionality estimation fails completely.
+        
+    Example:
+        >>> features = torch.randn(1000, 512)  # 1000 samples, 512-dim features
+        >>> intrinsic_dim = estimate_intrinsic_dimension(features)
+        >>> print(f"Intrinsic dimension: {intrinsic_dim:.2f}")
+    """
+    # Input validation
+    if features is None:
+        raise ValueError("Features cannot be None")
+    
+    # Convert to numpy if needed
+    if isinstance(features, torch.Tensor):
+        if features.numel() == 0:
+            raise ValueError("Input tensor is empty")
+        numpy_features = features.cpu().detach().numpy()
+    else:
+        numpy_features = np.asarray(features)
+        if numpy_features.size == 0:
+            raise ValueError("Input array is empty")
+    
+    # Reshape to 2D format (N_samples, N_features)
+    original_shape = numpy_features.shape
+    flattened_features = numpy_features.reshape(-1, numpy_features.shape[-1])
+    
+    n_samples, n_features = flattened_features.shape
+    
+    # Validate minimum requirements
+    if n_samples < MIN_SAMPLES_REQUIRED:
+        raise ValueError(
+            f"Insufficient samples for dimensionality estimation. "
+            f"Need at least {MIN_SAMPLES_REQUIRED}, got {n_samples}"
+        )
+    
+    if n_features < 2:
+        raise ValueError(
+            f"Feature dimension must be at least 2, got {n_features}"
+        )
+    
+    # Apply farthest point sampling if dataset is too large
+    if n_samples > max_samples:
+        logging.info(
+            f"Dataset has {n_samples} samples, downsampling to {max_samples} "
+            f"using farthest point sampling for efficiency"
+        )
+        
+        # Convert back to tensor for sampling
+        tensor_features = torch.tensor(flattened_features, dtype=torch.float32)
+        sample_indices = farthest_point_sampling(tensor_features, max_samples)
+        sampled_features = flattened_features[sample_indices]
+    else:
+        sampled_features = flattened_features
+    
+    # Validate sampled data quality
+    if np.any(np.isnan(sampled_features)) or np.any(np.isinf(sampled_features)):
+        logging.warning("Input features contain NaN or infinite values, which may affect estimation")
+    
+    # Estimate intrinsic dimensionality using MLE
+    try:
+        mle_estimator = skdim.id.MLE()
+        fitted_estimator = mle_estimator.fit(sampled_features)
+        estimated_dimension = fitted_estimator.dimension_
+        
+        # Handle failed global estimation
+        if estimated_dimension <= 0 or not np.isfinite(estimated_dimension):
+            if hasattr(fitted_estimator, 'dimension_pw_') and fitted_estimator.dimension_pw_ is not None:
+                # Fallback to local (pairwise) dimension estimates
+                local_dimensions = fitted_estimator.dimension_pw_
+                valid_local_dims = local_dimensions[np.isfinite(local_dimensions) & (local_dimensions > 0)]
+                
+                if len(valid_local_dims) > 0:
+                    estimated_dimension = float(np.mean(valid_local_dims))
+                    logging.warning(
+                        f"Global intrinsic dimension estimation failed (got {fitted_estimator.dimension_}). "
+                        f"Using mean of {len(valid_local_dims)} local estimates: {estimated_dimension:.2f}"
+                    )
+                else:
+                    raise RuntimeError("Both global and local dimensionality estimation failed")
+            else:
+                raise RuntimeError("Global dimensionality estimation failed and no local estimates available")
+        
+        # Sanity check: intrinsic dimension should not exceed ambient dimension
+        if estimated_dimension > n_features:
+            logging.warning(
+                f"Estimated intrinsic dimension ({estimated_dimension:.2f}) exceeds "
+                f"ambient dimension ({n_features}). Capping to ambient dimension."
+            )
+            estimated_dimension = float(n_features)
+        
+        # Log results
+        compression_ratio = n_features / estimated_dimension if estimated_dimension > 0 else np.inf
+        logging.info(
+            f"Intrinsic dimensionality estimation completed: "
+            f"{estimated_dimension:.2f} (compression ratio: {compression_ratio:.1f}x)"
+        )
+        
+        return float(estimated_dimension)
+        
+    except Exception as e:
+        raise RuntimeError(f"Intrinsic dimensionality estimation failed: {str(e)}") from e
+

ip_adapter/__init__.py

@@ -0,0 +1,9 @@
+from .ip_adapter import IPAdapter, IPAdapterPlus, IPAdapterPlusXL, IPAdapterXL, IPAdapterFull
+
+__all__ = [
+    "IPAdapter",
+    "IPAdapterPlus",
+    "IPAdapterPlusXL",
+    "IPAdapterXL",
+    "IPAdapterFull",
+]

ip_adapter/attention_processor.py

@@ -0,0 +1,568 @@
+# modified from https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class AttnProcessor(nn.Module):
+    r"""
+    Default processor for performing attention-related computations.
+    """
+
+    def __init__(
+        self,
+        hidden_size=None,
+        cross_attention_dim=None,
+    ):
+        super().__init__()
+
+    def __call__(
+        self,
+        attn,
+        hidden_states,
+        encoder_hidden_states=None,
+        attention_mask=None,
+        temb=None,
+        *args,
+        **kwargs,
+    ):
+        residual = hidden_states
+
+        if attn.spatial_norm is not None:
+            hidden_states = attn.spatial_norm(hidden_states, temb)
+
+        input_ndim = hidden_states.ndim
+
+        if input_ndim == 4:
+            batch_size, channel, height, width = hidden_states.shape
+            hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
+
+        batch_size, sequence_length, _ = (
+            hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape
+        )
+        attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
+
+        if attn.group_norm is not None:
+            hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)
+
+        query = attn.to_q(hidden_states)
+
+        if encoder_hidden_states is None:
+            encoder_hidden_states = hidden_states
+        elif attn.norm_cross:
+            encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)
+
+        key = attn.to_k(encoder_hidden_states)
+        value = attn.to_v(encoder_hidden_states)
+
+        query = attn.head_to_batch_dim(query)
+        key = attn.head_to_batch_dim(key)
+        value = attn.head_to_batch_dim(value)
+
+        attention_probs = attn.get_attention_scores(query, key, attention_mask)
+        hidden_states = torch.bmm(attention_probs, value)
+        hidden_states = attn.batch_to_head_dim(hidden_states)
+
+        # linear proj
+        hidden_states = attn.to_out[0](hidden_states)
+        # dropout
+        hidden_states = attn.to_out[1](hidden_states)
+
+        if input_ndim == 4:
+            hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
+
+        if attn.residual_connection:
+            hidden_states = hidden_states + residual
+
+        hidden_states = hidden_states / attn.rescale_output_factor
+
+        return hidden_states
+
+
+class IPAttnProcessor(nn.Module):
+    r"""
+    Attention processor for IP-Adapater.
+    Args:
+        hidden_size (`int`):
+            The hidden size of the attention layer.
+        cross_attention_dim (`int`):
+            The number of channels in the `encoder_hidden_states`.
+        scale (`float`, defaults to 1.0):
+            the weight scale of image prompt.
+        num_tokens (`int`, defaults to 4 when do ip_adapter_plus it should be 16):
+            The context length of the image features.
+    """
+
+    def __init__(self, hidden_size, cross_attention_dim=None, scale=1.0, num_tokens=4):
+        super().__init__()
+
+        self.hidden_size = hidden_size
+        self.cross_attention_dim = cross_attention_dim
+        self.scale = scale
+        self.num_tokens = num_tokens
+
+        self.to_k_ip = nn.Linear(cross_attention_dim or hidden_size, hidden_size, bias=False)
+        self.to_v_ip = nn.Linear(cross_attention_dim or hidden_size, hidden_size, bias=False)
+
+    def __call__(
+        self,
+        attn,
+        hidden_states,
+        encoder_hidden_states=None,
+        attention_mask=None,
+        temb=None,
+        *args,
+        **kwargs,
+    ):
+        residual = hidden_states
+
+        if attn.spatial_norm is not None:
+            hidden_states = attn.spatial_norm(hidden_states, temb)
+
+        input_ndim = hidden_states.ndim
+
+        if input_ndim == 4:
+            batch_size, channel, height, width = hidden_states.shape
+            hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
+
+        batch_size, sequence_length, _ = (
+            hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape
+        )
+        attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
+
+        if attn.group_norm is not None:
+            hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)
+
+        query = attn.to_q(hidden_states)
+
+        if encoder_hidden_states is None:
+            encoder_hidden_states = hidden_states
+        else:
+            # get encoder_hidden_states, ip_hidden_states
+            end_pos = encoder_hidden_states.shape[1] - self.num_tokens
+            encoder_hidden_states, ip_hidden_states = (
+                encoder_hidden_states[:, :end_pos, :],
+                encoder_hidden_states[:, end_pos:, :],
+            )
+            if attn.norm_cross:
+                encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)
+
+        key = attn.to_k(encoder_hidden_states)
+        value = attn.to_v(encoder_hidden_states)
+
+        query = attn.head_to_batch_dim(query)
+        key = attn.head_to_batch_dim(key)
+        value = attn.head_to_batch_dim(value)
+
+        attention_probs = attn.get_attention_scores(query, key, attention_mask)
+        hidden_states = torch.bmm(attention_probs, value)
+        hidden_states = attn.batch_to_head_dim(hidden_states)
+
+        # for ip-adapter
+        ip_key = self.to_k_ip(ip_hidden_states)
+        ip_value = self.to_v_ip(ip_hidden_states)
+
+        ip_key = attn.head_to_batch_dim(ip_key)
+        ip_value = attn.head_to_batch_dim(ip_value)
+
+        ip_attention_probs = attn.get_attention_scores(query, ip_key, None)
+        self.attn_map = ip_attention_probs
+        ip_hidden_states = torch.bmm(ip_attention_probs, ip_value)
+        ip_hidden_states = attn.batch_to_head_dim(ip_hidden_states)
+
+        hidden_states = hidden_states + self.scale * ip_hidden_states
+
+        # linear proj
+        hidden_states = attn.to_out[0](hidden_states)
+        # dropout
+        hidden_states = attn.to_out[1](hidden_states)
+
+        if input_ndim == 4:
+            hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
+
+        if attn.residual_connection:
+            hidden_states = hidden_states + residual
+
+        hidden_states = hidden_states / attn.rescale_output_factor
+
+        return hidden_states
+
+
+class AttnProcessor2_0(torch.nn.Module):
+    r"""
+    Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0).
+    """
+
+    def __init__(
+        self,
+        hidden_size=None,
+        cross_attention_dim=None,
+    ):
+        super().__init__()
+        if not hasattr(F, "scaled_dot_product_attention"):
+            raise ImportError("AttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.")
+
+    def __call__(
+        self,
+        attn,
+        hidden_states,
+        encoder_hidden_states=None,
+        attention_mask=None,
+        temb=None,
+        *args,
+        **kwargs,
+    ):
+        residual = hidden_states
+
+        if attn.spatial_norm is not None:
+            hidden_states = attn.spatial_norm(hidden_states, temb)
+
+        input_ndim = hidden_states.ndim
+
+        if input_ndim == 4:
+            batch_size, channel, height, width = hidden_states.shape
+            hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
+
+        batch_size, sequence_length, _ = (
+            hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape
+        )
+
+        if attention_mask is not None:
+            attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
+            # scaled_dot_product_attention expects attention_mask shape to be
+            # (batch, heads, source_length, target_length)
+            attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1])
+
+        if attn.group_norm is not None:
+            hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)
+
+        query = attn.to_q(hidden_states)
+
+        if encoder_hidden_states is None:
+            encoder_hidden_states = hidden_states
+        elif attn.norm_cross:
+            encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)
+
+        key = attn.to_k(encoder_hidden_states)
+        value = attn.to_v(encoder_hidden_states)
+
+        inner_dim = key.shape[-1]
+        head_dim = inner_dim // attn.heads
+
+        query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+
+        key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+        value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+
+        # the output of sdp = (batch, num_heads, seq_len, head_dim)
+        # TODO: add support for attn.scale when we move to Torch 2.1
+        hidden_states = F.scaled_dot_product_attention(
+            query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
+        )
+
+        hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
+        hidden_states = hidden_states.to(query.dtype)
+
+        # linear proj
+        hidden_states = attn.to_out[0](hidden_states)
+        # dropout
+        hidden_states = attn.to_out[1](hidden_states)
+
+        if input_ndim == 4:
+            hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
+
+        if attn.residual_connection:
+            hidden_states = hidden_states + residual
+
+        hidden_states = hidden_states / attn.rescale_output_factor
+
+        return hidden_states
+
+
+class IPAttnProcessor2_0(torch.nn.Module):
+    r"""
+    Attention processor for IP-Adapater for PyTorch 2.0.
+    Args:
+        hidden_size (`int`):
+            The hidden size of the attention layer.
+        cross_attention_dim (`int`):
+            The number of channels in the `encoder_hidden_states`.
+        scale (`float`, defaults to 1.0):
+            the weight scale of image prompt.
+        num_tokens (`int`, defaults to 4 when do ip_adapter_plus it should be 16):
+            The context length of the image features.
+    """
+
+    def __init__(self, hidden_size, cross_attention_dim=None, scale=1.0, num_tokens=4):
+        super().__init__()
+
+        if not hasattr(F, "scaled_dot_product_attention"):
+            raise ImportError("AttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.")
+
+        self.hidden_size = hidden_size
+        self.cross_attention_dim = cross_attention_dim
+        self.scale = scale
+        self.num_tokens = num_tokens
+
+        self.to_k_ip = nn.Linear(cross_attention_dim or hidden_size, hidden_size, bias=False)
+        self.to_v_ip = nn.Linear(cross_attention_dim or hidden_size, hidden_size, bias=False)
+
+    def __call__(
+        self,
+        attn,
+        hidden_states,
+        encoder_hidden_states=None,
+        attention_mask=None,
+        temb=None,
+        *args,
+        **kwargs,
+    ):
+        residual = hidden_states
+
+        if attn.spatial_norm is not None:
+            hidden_states = attn.spatial_norm(hidden_states, temb)
+
+        input_ndim = hidden_states.ndim
+
+        if input_ndim == 4:
+            batch_size, channel, height, width = hidden_states.shape
+            hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
+
+        batch_size, sequence_length, _ = (
+            hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape
+        )
+
+        if attention_mask is not None:
+            attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
+            # scaled_dot_product_attention expects attention_mask shape to be
+            # (batch, heads, source_length, target_length)
+            attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1])
+
+        if attn.group_norm is not None:
+            hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)
+
+        query = attn.to_q(hidden_states)
+
+        if encoder_hidden_states is None:
+            encoder_hidden_states = hidden_states
+        else:
+            # get encoder_hidden_states, ip_hidden_states
+            end_pos = encoder_hidden_states.shape[1] - self.num_tokens
+            encoder_hidden_states, ip_hidden_states = (
+                encoder_hidden_states[:, :end_pos, :],
+                encoder_hidden_states[:, end_pos:, :],
+            )
+            if attn.norm_cross:
+                encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)
+
+        key = attn.to_k(encoder_hidden_states)
+        value = attn.to_v(encoder_hidden_states)
+
+        inner_dim = key.shape[-1]
+        head_dim = inner_dim // attn.heads
+
+        query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+
+        key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+        value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+
+        # the output of sdp = (batch, num_heads, seq_len, head_dim)
+        # TODO: add support for attn.scale when we move to Torch 2.1
+        hidden_states = F.scaled_dot_product_attention(
+            query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
+        )
+
+        hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
+        hidden_states = hidden_states.to(query.dtype)
+
+        # for ip-adapter
+        ip_key = self.to_k_ip(ip_hidden_states)
+        ip_value = self.to_v_ip(ip_hidden_states)
+
+        ip_key = ip_key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+        ip_value = ip_value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+
+        # the output of sdp = (batch, num_heads, seq_len, head_dim)
+        # TODO: add support for attn.scale when we move to Torch 2.1
+        ip_hidden_states = F.scaled_dot_product_attention(
+            query, ip_key, ip_value, attn_mask=None, dropout_p=0.0, is_causal=False
+        )
+        with torch.no_grad():
+            self.attn_map = query @ ip_key.transpose(-2, -1).softmax(dim=-1)
+            #print(self.attn_map.shape)
+
+        ip_hidden_states = ip_hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
+        ip_hidden_states = ip_hidden_states.to(query.dtype)
+
+        hidden_states = hidden_states + self.scale * ip_hidden_states
+
+        # linear proj
+        hidden_states = attn.to_out[0](hidden_states)
+        # dropout
+        hidden_states = attn.to_out[1](hidden_states)
+
+        if input_ndim == 4:
+            hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
+
+        if attn.residual_connection:
+            hidden_states = hidden_states + residual
+
+        hidden_states = hidden_states / attn.rescale_output_factor
+
+        return hidden_states
+
+
+## for controlnet
+class CNAttnProcessor:
+    r"""
+    Default processor for performing attention-related computations.
+    """
+
+    def __init__(self, num_tokens=4):
+        self.num_tokens = num_tokens
+
+    def __call__(self, attn, hidden_states, encoder_hidden_states=None, attention_mask=None, temb=None, *args, **kwargs,):
+        residual = hidden_states
+
+        if attn.spatial_norm is not None:
+            hidden_states = attn.spatial_norm(hidden_states, temb)
+
+        input_ndim = hidden_states.ndim
+
+        if input_ndim == 4:
+            batch_size, channel, height, width = hidden_states.shape
+            hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
+
+        batch_size, sequence_length, _ = (
+            hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape
+        )
+        attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
+
+        if attn.group_norm is not None:
+            hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)
+
+        query = attn.to_q(hidden_states)
+
+        if encoder_hidden_states is None:
+            encoder_hidden_states = hidden_states
+        else:
+            end_pos = encoder_hidden_states.shape[1] - self.num_tokens
+            encoder_hidden_states = encoder_hidden_states[:, :end_pos]  # only use text
+            if attn.norm_cross:
+                encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)
+
+        key = attn.to_k(encoder_hidden_states)
+        value = attn.to_v(encoder_hidden_states)
+
+        query = attn.head_to_batch_dim(query)
+        key = attn.head_to_batch_dim(key)
+        value = attn.head_to_batch_dim(value)
+
+        attention_probs = attn.get_attention_scores(query, key, attention_mask)
+        hidden_states = torch.bmm(attention_probs, value)
+        hidden_states = attn.batch_to_head_dim(hidden_states)
+
+        # linear proj
+        hidden_states = attn.to_out[0](hidden_states)
+        # dropout
+        hidden_states = attn.to_out[1](hidden_states)
+
+        if input_ndim == 4:
+            hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
+
+        if attn.residual_connection:
+            hidden_states = hidden_states + residual
+
+        hidden_states = hidden_states / attn.rescale_output_factor
+
+        return hidden_states
+
+
+class CNAttnProcessor2_0:
+    r"""
+    Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0).
+    """
+
+    def __init__(self, num_tokens=4):
+        if not hasattr(F, "scaled_dot_product_attention"):
+            raise ImportError("AttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.")
+        self.num_tokens = num_tokens
+
+    def __call__(
+        self,
+        attn,
+        hidden_states,
+        encoder_hidden_states=None,
+        attention_mask=None,
+        temb=None,
+        *args,
+        **kwargs,
+    ):
+        residual = hidden_states
+
+        if attn.spatial_norm is not None:
+            hidden_states = attn.spatial_norm(hidden_states, temb)
+
+        input_ndim = hidden_states.ndim
+
+        if input_ndim == 4:
+            batch_size, channel, height, width = hidden_states.shape
+            hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
+
+        batch_size, sequence_length, _ = (
+            hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape
+        )
+
+        if attention_mask is not None:
+            attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
+            # scaled_dot_product_attention expects attention_mask shape to be
+            # (batch, heads, source_length, target_length)
+            attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1])
+
+        if attn.group_norm is not None:
+            hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)
+
+        query = attn.to_q(hidden_states)
+
+        if encoder_hidden_states is None:
+            encoder_hidden_states = hidden_states
+        else:
+            end_pos = encoder_hidden_states.shape[1] - self.num_tokens
+            encoder_hidden_states = encoder_hidden_states[:, :end_pos]  # only use text
+            if attn.norm_cross:
+                encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)
+
+        key = attn.to_k(encoder_hidden_states)
+        value = attn.to_v(encoder_hidden_states)
+
+        inner_dim = key.shape[-1]
+        head_dim = inner_dim // attn.heads
+
+        query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+
+        key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+        value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+
+        # the output of sdp = (batch, num_heads, seq_len, head_dim)
+        # TODO: add support for attn.scale when we move to Torch 2.1
+        hidden_states = F.scaled_dot_product_attention(
+            query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
+        )
+
+        hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
+        hidden_states = hidden_states.to(query.dtype)
+
+        # linear proj
+        hidden_states = attn.to_out[0](hidden_states)
+        # dropout
+        hidden_states = attn.to_out[1](hidden_states)
+
+        if input_ndim == 4:
+            hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
+
+        if attn.residual_connection:
+            hidden_states = hidden_states + residual
+
+        hidden_states = hidden_states / attn.rescale_output_factor
+
+        return hidden_states

ip_adapter/attention_processor_faceid.py

@@ -0,0 +1,433 @@
+# modified from https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from diffusers.models.lora import LoRALinearLayer
+
+
+class LoRAAttnProcessor(nn.Module):
+    r"""
+    Default processor for performing attention-related computations.
+    """
+
+    def __init__(
+        self,
+        hidden_size=None,
+        cross_attention_dim=None,
+        rank=4,
+        network_alpha=None,
+        lora_scale=1.0,
+    ):
+        super().__init__()
+
+        self.rank = rank
+        self.lora_scale = lora_scale
+        
+        self.to_q_lora = LoRALinearLayer(hidden_size, hidden_size, rank, network_alpha)
+        self.to_k_lora = LoRALinearLayer(cross_attention_dim or hidden_size, hidden_size, rank, network_alpha)
+        self.to_v_lora = LoRALinearLayer(cross_attention_dim or hidden_size, hidden_size, rank, network_alpha)
+        self.to_out_lora = LoRALinearLayer(hidden_size, hidden_size, rank, network_alpha)
+
+    def __call__(
+        self,
+        attn,
+        hidden_states,
+        encoder_hidden_states=None,
+        attention_mask=None,
+        temb=None,
+        *args,
+        **kwargs,
+    ):
+        residual = hidden_states
+
+        if attn.spatial_norm is not None:
+            hidden_states = attn.spatial_norm(hidden_states, temb)
+
+        input_ndim = hidden_states.ndim
+
+        if input_ndim == 4:
+            batch_size, channel, height, width = hidden_states.shape
+            hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
+
+        batch_size, sequence_length, _ = (
+            hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape
+        )
+        attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
+
+        if attn.group_norm is not None:
+            hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)
+
+        query = attn.to_q(hidden_states) + self.lora_scale * self.to_q_lora(hidden_states)
+
+        if encoder_hidden_states is None:
+            encoder_hidden_states = hidden_states
+        elif attn.norm_cross:
+            encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)
+
+        key = attn.to_k(encoder_hidden_states) + self.lora_scale * self.to_k_lora(encoder_hidden_states)
+        value = attn.to_v(encoder_hidden_states) + self.lora_scale * self.to_v_lora(encoder_hidden_states)
+
+        query = attn.head_to_batch_dim(query)
+        key = attn.head_to_batch_dim(key)
+        value = attn.head_to_batch_dim(value)
+
+        attention_probs = attn.get_attention_scores(query, key, attention_mask)
+        hidden_states = torch.bmm(attention_probs, value)
+        hidden_states = attn.batch_to_head_dim(hidden_states)
+
+        # linear proj
+        hidden_states = attn.to_out[0](hidden_states) + self.lora_scale * self.to_out_lora(hidden_states)
+        # dropout
+        hidden_states = attn.to_out[1](hidden_states)
+
+        if input_ndim == 4:
+            hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
+
+        if attn.residual_connection:
+            hidden_states = hidden_states + residual
+
+        hidden_states = hidden_states / attn.rescale_output_factor
+
+        return hidden_states
+
+
+class LoRAIPAttnProcessor(nn.Module):
+    r"""
+    Attention processor for IP-Adapater.
+    Args:
+        hidden_size (`int`):
+            The hidden size of the attention layer.
+        cross_attention_dim (`int`):
+            The number of channels in the `encoder_hidden_states`.
+        scale (`float`, defaults to 1.0):
+            the weight scale of image prompt.
+        num_tokens (`int`, defaults to 4 when do ip_adapter_plus it should be 16):
+            The context length of the image features.
+    """
+
+    def __init__(self, hidden_size, cross_attention_dim=None, rank=4, network_alpha=None, lora_scale=1.0, scale=1.0, num_tokens=4):
+        super().__init__()
+
+        self.rank = rank
+        self.lora_scale = lora_scale
+        
+        self.to_q_lora = LoRALinearLayer(hidden_size, hidden_size, rank, network_alpha)
+        self.to_k_lora = LoRALinearLayer(cross_attention_dim or hidden_size, hidden_size, rank, network_alpha)
+        self.to_v_lora = LoRALinearLayer(cross_attention_dim or hidden_size, hidden_size, rank, network_alpha)
+        self.to_out_lora = LoRALinearLayer(hidden_size, hidden_size, rank, network_alpha)
+
+        self.hidden_size = hidden_size
+        self.cross_attention_dim = cross_attention_dim
+        self.scale = scale
+        self.num_tokens = num_tokens
+
+        self.to_k_ip = nn.Linear(cross_attention_dim or hidden_size, hidden_size, bias=False)
+        self.to_v_ip = nn.Linear(cross_attention_dim or hidden_size, hidden_size, bias=False)
+
+    def __call__(
+        self,
+        attn,
+        hidden_states,
+        encoder_hidden_states=None,
+        attention_mask=None,
+        temb=None,
+        *args,
+        **kwargs,
+    ):
+        residual = hidden_states
+
+        if attn.spatial_norm is not None:
+            hidden_states = attn.spatial_norm(hidden_states, temb)
+
+        input_ndim = hidden_states.ndim
+
+        if input_ndim == 4:
+            batch_size, channel, height, width = hidden_states.shape
+            hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
+
+        batch_size, sequence_length, _ = (
+            hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape
+        )
+        attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
+
+        if attn.group_norm is not None:
+            hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)
+
+        query = attn.to_q(hidden_states) + self.lora_scale * self.to_q_lora(hidden_states)
+
+        if encoder_hidden_states is None:
+            encoder_hidden_states = hidden_states
+        else:
+            # get encoder_hidden_states, ip_hidden_states
+            end_pos = encoder_hidden_states.shape[1] - self.num_tokens
+            encoder_hidden_states, ip_hidden_states = (
+                encoder_hidden_states[:, :end_pos, :],
+                encoder_hidden_states[:, end_pos:, :],
+            )
+            if attn.norm_cross:
+                encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)
+
+        key = attn.to_k(encoder_hidden_states) + self.lora_scale * self.to_k_lora(encoder_hidden_states)
+        value = attn.to_v(encoder_hidden_states) + self.lora_scale * self.to_v_lora(encoder_hidden_states)
+
+        query = attn.head_to_batch_dim(query)
+        key = attn.head_to_batch_dim(key)
+        value = attn.head_to_batch_dim(value)
+
+        attention_probs = attn.get_attention_scores(query, key, attention_mask)
+        hidden_states = torch.bmm(attention_probs, value)
+        hidden_states = attn.batch_to_head_dim(hidden_states)
+
+        # for ip-adapter
+        ip_key = self.to_k_ip(ip_hidden_states)
+        ip_value = self.to_v_ip(ip_hidden_states)
+
+        ip_key = attn.head_to_batch_dim(ip_key)
+        ip_value = attn.head_to_batch_dim(ip_value)
+
+        ip_attention_probs = attn.get_attention_scores(query, ip_key, None)
+        self.attn_map = ip_attention_probs
+        ip_hidden_states = torch.bmm(ip_attention_probs, ip_value)
+        ip_hidden_states = attn.batch_to_head_dim(ip_hidden_states)
+
+        hidden_states = hidden_states + self.scale * ip_hidden_states
+
+        # linear proj
+        hidden_states = attn.to_out[0](hidden_states) + self.lora_scale * self.to_out_lora(hidden_states)
+        # dropout
+        hidden_states = attn.to_out[1](hidden_states)
+
+        if input_ndim == 4:
+            hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
+
+        if attn.residual_connection:
+            hidden_states = hidden_states + residual
+
+        hidden_states = hidden_states / attn.rescale_output_factor
+
+        return hidden_states
+
+
+class LoRAAttnProcessor2_0(nn.Module):
+    
+    r"""
+    Default processor for performing attention-related computations.
+    """
+    
+    def __init__(
+        self,
+        hidden_size=None,
+        cross_attention_dim=None,
+        rank=4,
+        network_alpha=None,
+        lora_scale=1.0,
+    ):
+        super().__init__()
+        
+        self.rank = rank
+        self.lora_scale = lora_scale
+        
+        self.to_q_lora = LoRALinearLayer(hidden_size, hidden_size, rank, network_alpha)
+        self.to_k_lora = LoRALinearLayer(cross_attention_dim or hidden_size, hidden_size, rank, network_alpha)
+        self.to_v_lora = LoRALinearLayer(cross_attention_dim or hidden_size, hidden_size, rank, network_alpha)
+        self.to_out_lora = LoRALinearLayer(hidden_size, hidden_size, rank, network_alpha)
+
+    def __call__(
+        self,
+        attn,
+        hidden_states,
+        encoder_hidden_states=None,
+        attention_mask=None,
+        temb=None,
+        *args,
+        **kwargs,
+    ):
+        residual = hidden_states
+
+        if attn.spatial_norm is not None:
+            hidden_states = attn.spatial_norm(hidden_states, temb)
+
+        input_ndim = hidden_states.ndim
+
+        if input_ndim == 4:
+            batch_size, channel, height, width = hidden_states.shape
+            hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
+
+        batch_size, sequence_length, _ = (
+            hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape
+        )
+        attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
+
+        if attn.group_norm is not None:
+            hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)
+
+        query = attn.to_q(hidden_states) + self.lora_scale * self.to_q_lora(hidden_states)
+
+        if encoder_hidden_states is None:
+            encoder_hidden_states = hidden_states
+        elif attn.norm_cross:
+            encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)
+
+        key = attn.to_k(encoder_hidden_states) + self.lora_scale * self.to_k_lora(encoder_hidden_states)
+        value = attn.to_v(encoder_hidden_states) + self.lora_scale * self.to_v_lora(encoder_hidden_states)
+
+        inner_dim = key.shape[-1]
+        head_dim = inner_dim // attn.heads
+
+        query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+
+        key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+        value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+
+        # the output of sdp = (batch, num_heads, seq_len, head_dim)
+        # TODO: add support for attn.scale when we move to Torch 2.1
+        hidden_states = F.scaled_dot_product_attention(
+            query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
+        )
+
+        hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
+        hidden_states = hidden_states.to(query.dtype)
+
+        # linear proj
+        hidden_states = attn.to_out[0](hidden_states) + self.lora_scale * self.to_out_lora(hidden_states)
+        # dropout
+        hidden_states = attn.to_out[1](hidden_states)
+
+        if input_ndim == 4:
+            hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
+
+        if attn.residual_connection:
+            hidden_states = hidden_states + residual
+
+        hidden_states = hidden_states / attn.rescale_output_factor
+
+        return hidden_states
+
+
+class LoRAIPAttnProcessor2_0(nn.Module):
+    r"""
+    Processor for implementing the LoRA attention mechanism.
+
+    Args:
+        hidden_size (`int`, *optional*):
+            The hidden size of the attention layer.
+        cross_attention_dim (`int`, *optional*):
+            The number of channels in the `encoder_hidden_states`.
+        rank (`int`, defaults to 4):
+            The dimension of the LoRA update matrices.
+        network_alpha (`int`, *optional*):
+            Equivalent to `alpha` but it's usage is specific to Kohya (A1111) style LoRAs.
+    """
+
+    def __init__(self, hidden_size, cross_attention_dim=None, rank=4, network_alpha=None, lora_scale=1.0, scale=1.0, num_tokens=4):
+        super().__init__()
+        
+        self.rank = rank
+        self.lora_scale = lora_scale
+        self.num_tokens = num_tokens
+        
+        self.to_q_lora = LoRALinearLayer(hidden_size, hidden_size, rank, network_alpha)
+        self.to_k_lora = LoRALinearLayer(cross_attention_dim or hidden_size, hidden_size, rank, network_alpha)
+        self.to_v_lora = LoRALinearLayer(cross_attention_dim or hidden_size, hidden_size, rank, network_alpha)
+        self.to_out_lora = LoRALinearLayer(hidden_size, hidden_size, rank, network_alpha)
+        
+        
+        self.hidden_size = hidden_size
+        self.cross_attention_dim = cross_attention_dim
+        self.scale = scale
+
+        self.to_k_ip = nn.Linear(cross_attention_dim or hidden_size, hidden_size, bias=False)
+        self.to_v_ip = nn.Linear(cross_attention_dim or hidden_size, hidden_size, bias=False)
+
+    def __call__(
+        self, attn, hidden_states, encoder_hidden_states=None, attention_mask=None, scale=1.0, temb=None, *args, **kwargs,
+    ):
+        residual = hidden_states
+
+        if attn.spatial_norm is not None:
+            hidden_states = attn.spatial_norm(hidden_states, temb)
+
+        input_ndim = hidden_states.ndim
+
+        if input_ndim == 4:
+            batch_size, channel, height, width = hidden_states.shape
+            hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
+
+        batch_size, sequence_length, _ = (
+            hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape
+        )
+        attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
+
+        if attn.group_norm is not None:
+            hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)
+
+        query = attn.to_q(hidden_states) + self.lora_scale * self.to_q_lora(hidden_states)
+        #query = attn.head_to_batch_dim(query)
+
+        if encoder_hidden_states is None:
+            encoder_hidden_states = hidden_states
+        else:
+            # get encoder_hidden_states, ip_hidden_states
+            end_pos = encoder_hidden_states.shape[1] - self.num_tokens
+            encoder_hidden_states, ip_hidden_states = (
+                encoder_hidden_states[:, :end_pos, :],
+                encoder_hidden_states[:, end_pos:, :],
+            )
+            if attn.norm_cross:
+                encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)
+
+        # for text
+        key = attn.to_k(encoder_hidden_states) + self.lora_scale * self.to_k_lora(encoder_hidden_states)
+        value = attn.to_v(encoder_hidden_states) + self.lora_scale * self.to_v_lora(encoder_hidden_states)
+
+        inner_dim = key.shape[-1]
+        head_dim = inner_dim // attn.heads
+
+        query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+
+        key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+        value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+
+        # the output of sdp = (batch, num_heads, seq_len, head_dim)
+        # TODO: add support for attn.scale when we move to Torch 2.1
+        hidden_states = F.scaled_dot_product_attention(
+            query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
+        )
+
+        hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
+        hidden_states = hidden_states.to(query.dtype)
+        
+        # for ip
+        ip_key = self.to_k_ip(ip_hidden_states)
+        ip_value = self.to_v_ip(ip_hidden_states)
+        
+        ip_key = ip_key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+        ip_value = ip_value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+
+        # the output of sdp = (batch, num_heads, seq_len, head_dim)
+        # TODO: add support for attn.scale when we move to Torch 2.1
+        ip_hidden_states = F.scaled_dot_product_attention(
+            query, ip_key, ip_value, attn_mask=None, dropout_p=0.0, is_causal=False
+        )
+        
+
+        ip_hidden_states = ip_hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
+        ip_hidden_states = ip_hidden_states.to(query.dtype)
+        
+        hidden_states = hidden_states + self.scale * ip_hidden_states
+
+        # linear proj
+        hidden_states = attn.to_out[0](hidden_states) + self.lora_scale * self.to_out_lora(hidden_states)
+        # dropout
+        hidden_states = attn.to_out[1](hidden_states)
+
+        if input_ndim == 4:
+            hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
+
+        if attn.residual_connection:
+            hidden_states = hidden_states + residual
+
+        hidden_states = hidden_states / attn.rescale_output_factor
+
+        return hidden_states

ip_adapter/custom_pipelines.py

@@ -0,0 +1,394 @@
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+
+import torch
+from diffusers import StableDiffusionXLPipeline
+from diffusers.pipelines.stable_diffusion_xl import StableDiffusionXLPipelineOutput
+from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl import rescale_noise_cfg
+
+from .utils import is_torch2_available
+
+if is_torch2_available():
+    from .attention_processor import IPAttnProcessor2_0 as IPAttnProcessor
+else:
+    from .attention_processor import IPAttnProcessor
+
+
+class StableDiffusionXLCustomPipeline(StableDiffusionXLPipeline):
+    def set_scale(self, scale):
+        for attn_processor in self.unet.attn_processors.values():
+            if isinstance(attn_processor, IPAttnProcessor):
+                attn_processor.scale = scale
+
+    @torch.no_grad()
+    def __call__(  # noqa: C901
+        self,
+        prompt: Optional[Union[str, List[str]]] = None,
+        prompt_2: Optional[Union[str, List[str]]] = None,
+        height: Optional[int] = None,
+        width: Optional[int] = None,
+        num_inference_steps: int = 50,
+        denoising_end: Optional[float] = None,
+        guidance_scale: float = 5.0,
+        negative_prompt: Optional[Union[str, List[str]]] = None,
+        negative_prompt_2: Optional[Union[str, List[str]]] = None,
+        num_images_per_prompt: Optional[int] = 1,
+        eta: float = 0.0,
+        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
+        latents: Optional[torch.FloatTensor] = None,
+        prompt_embeds: Optional[torch.FloatTensor] = None,
+        negative_prompt_embeds: Optional[torch.FloatTensor] = None,
+        pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
+        negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
+        output_type: Optional[str] = "pil",
+        return_dict: bool = True,
+        callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
+        callback_steps: int = 1,
+        cross_attention_kwargs: Optional[Dict[str, Any]] = None,
+        guidance_rescale: float = 0.0,
+        original_size: Optional[Tuple[int, int]] = None,
+        crops_coords_top_left: Tuple[int, int] = (0, 0),
+        target_size: Optional[Tuple[int, int]] = None,
+        negative_original_size: Optional[Tuple[int, int]] = None,
+        negative_crops_coords_top_left: Tuple[int, int] = (0, 0),
+        negative_target_size: Optional[Tuple[int, int]] = None,
+        control_guidance_start: float = 0.0,
+        control_guidance_end: float = 1.0,
+    ):
+        r"""
+        Function invoked when calling the pipeline for generation.
+
+        Args:
+            prompt (`str` or `List[str]`, *optional*):
+                The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
+                instead.
+            prompt_2 (`str` or `List[str]`, *optional*):
+                The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
+                used in both text-encoders
+            height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
+                The height in pixels of the generated image. This is set to 1024 by default for the best results.
+                Anything below 512 pixels won't work well for
+                [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)
+                and checkpoints that are not specifically fine-tuned on low resolutions.
+            width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
+                The width in pixels of the generated image. This is set to 1024 by default for the best results.
+                Anything below 512 pixels won't work well for
+                [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)
+                and checkpoints that are not specifically fine-tuned on low resolutions.
+            num_inference_steps (`int`, *optional*, defaults to 50):
+                The number of denoising steps. More denoising steps usually lead to a higher quality image at the
+                expense of slower inference.
+            denoising_end (`float`, *optional*):
+                When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be
+                completed before it is intentionally prematurely terminated. As a result, the returned sample will
+                still retain a substantial amount of noise as determined by the discrete timesteps selected by the
+                scheduler. The denoising_end parameter should ideally be utilized when this pipeline forms a part of a
+                "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image
+                Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output)
+            guidance_scale (`float`, *optional*, defaults to 5.0):
+                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
+                `guidance_scale` is defined as `w` of equation 2. of [Imagen
+                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
+                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
+                usually at the expense of lower image quality.
+            negative_prompt (`str` or `List[str]`, *optional*):
+                The prompt or prompts not to guide the image generation. If not defined, one has to pass
+                `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
+                less than `1`).
+            negative_prompt_2 (`str` or `List[str]`, *optional*):
+                The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
+                `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders
+            num_images_per_prompt (`int`, *optional*, defaults to 1):
+                The number of images to generate per prompt.
+            eta (`float`, *optional*, defaults to 0.0):
+                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
+                [`schedulers.DDIMScheduler`], will be ignored for others.
+            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):
+                One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
+                to make generation deterministic.
+            latents (`torch.FloatTensor`, *optional*):
+                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
+                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
+                tensor will ge generated by sampling using the supplied random `generator`.
+            prompt_embeds (`torch.FloatTensor`, *optional*):
+                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
+                provided, text embeddings will be generated from `prompt` input argument.
+            negative_prompt_embeds (`torch.FloatTensor`, *optional*):
+                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
+                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
+                argument.
+            pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
+                Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
+                If not provided, pooled text embeddings will be generated from `prompt` input argument.
+            negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
+                Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
+                weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
+                input argument.
+            output_type (`str`, *optional*, defaults to `"pil"`):
+                The output format of the generate image. Choose between
+                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
+            return_dict (`bool`, *optional*, defaults to `True`):
+                Whether or not to return a [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] instead
+                of a plain tuple.
+            callback (`Callable`, *optional*):
+                A function that will be called every `callback_steps` steps during inference. The function will be
+                called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
+            callback_steps (`int`, *optional*, defaults to 1):
+                The frequency at which the `callback` function will be called. If not specified, the callback will be
+                called at every step.
+            cross_attention_kwargs (`dict`, *optional*):
+                A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
+                `self.processor` in
+                [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
+            guidance_rescale (`float`, *optional*, defaults to 0.7):
+                Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are
+                Flawed](https://arxiv.org/pdf/2305.08891.pdf) `guidance_scale` is defined as `φ` in equation 16. of
+                [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf).
+                Guidance rescale factor should fix overexposure when using zero terminal SNR.
+            original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):
+                If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled.
+                `original_size` defaults to `(width, height)` if not specified. Part of SDXL's micro-conditioning as
+                explained in section 2.2 of
+                [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
+            crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)):
+                `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position
+                `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting
+                `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of
+                [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
+            target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):
+                For most cases, `target_size` should be set to the desired height and width of the generated image. If
+                not specified it will default to `(width, height)`. Part of SDXL's micro-conditioning as explained in
+                section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
+            negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):
+                To negatively condition the generation process based on a specific image resolution. Part of SDXL's
+                micro-conditioning as explained in section 2.2 of
+                [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
+                information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
+            negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)):
+                To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's
+                micro-conditioning as explained in section 2.2 of
+                [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
+                information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
+            negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):
+                To negatively condition the generation process based on a target image resolution. It should be as same
+                as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of
+                [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
+                information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
+            control_guidance_start (`float`, *optional*, defaults to 0.0):
+                The percentage of total steps at which the ControlNet starts applying.
+            control_guidance_end (`float`, *optional*, defaults to 1.0):
+                The percentage of total steps at which the ControlNet stops applying.
+
+        Examples:
+
+        Returns:
+            [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] or `tuple`:
+            [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] if `return_dict` is True, otherwise a
+            `tuple`. When returning a tuple, the first element is a list with the generated images.
+        """
+        # 0. Default height and width to unet
+        height = height or self.default_sample_size * self.vae_scale_factor
+        width = width or self.default_sample_size * self.vae_scale_factor
+
+        original_size = original_size or (height, width)
+        target_size = target_size or (height, width)
+
+        # 1. Check inputs. Raise error if not correct
+        self.check_inputs(
+            prompt,
+            prompt_2,
+            height,
+            width,
+            callback_steps,
+            negative_prompt,
+            negative_prompt_2,
+            prompt_embeds,
+            negative_prompt_embeds,
+            pooled_prompt_embeds,
+            negative_pooled_prompt_embeds,
+        )
+
+        # 2. Define call parameters
+        if prompt is not None and isinstance(prompt, str):
+            batch_size = 1
+        elif prompt is not None and isinstance(prompt, list):
+            batch_size = len(prompt)
+        else:
+            batch_size = prompt_embeds.shape[0]
+
+        device = self._execution_device
+
+        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
+        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
+        # corresponds to doing no classifier free guidance.
+        do_classifier_free_guidance = guidance_scale > 1.0
+
+        # 3. Encode input prompt
+        text_encoder_lora_scale = (
+            cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None
+        )
+        (
+            prompt_embeds,
+            negative_prompt_embeds,
+            pooled_prompt_embeds,
+            negative_pooled_prompt_embeds,
+        ) = self.encode_prompt(
+            prompt=prompt,
+            prompt_2=prompt_2,
+            device=device,
+            num_images_per_prompt=num_images_per_prompt,
+            do_classifier_free_guidance=do_classifier_free_guidance,
+            negative_prompt=negative_prompt,
+            negative_prompt_2=negative_prompt_2,
+            prompt_embeds=prompt_embeds,
+            negative_prompt_embeds=negative_prompt_embeds,
+            pooled_prompt_embeds=pooled_prompt_embeds,
+            negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
+            lora_scale=text_encoder_lora_scale,
+        )
+
+        # 4. Prepare timesteps
+        self.scheduler.set_timesteps(num_inference_steps, device=device)
+
+        timesteps = self.scheduler.timesteps
+
+        # 5. Prepare latent variables
+        num_channels_latents = self.unet.config.in_channels
+        latents = self.prepare_latents(
+            batch_size * num_images_per_prompt,
+            num_channels_latents,
+            height,
+            width,
+            prompt_embeds.dtype,
+            device,
+            generator,
+            latents,
+        )
+
+        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
+
+        # 7. Prepare added time ids & embeddings
+        add_text_embeds = pooled_prompt_embeds
+        if self.text_encoder_2 is None:
+            text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1])
+        else:
+            text_encoder_projection_dim = self.text_encoder_2.config.projection_dim
+
+        add_time_ids = self._get_add_time_ids(
+            original_size,
+            crops_coords_top_left,
+            target_size,
+            dtype=prompt_embeds.dtype,
+            text_encoder_projection_dim=text_encoder_projection_dim,
+        )
+        if negative_original_size is not None and negative_target_size is not None:
+            negative_add_time_ids = self._get_add_time_ids(
+                negative_original_size,
+                negative_crops_coords_top_left,
+                negative_target_size,
+                dtype=prompt_embeds.dtype,
+                text_encoder_projection_dim=text_encoder_projection_dim,
+            )
+        else:
+            negative_add_time_ids = add_time_ids
+
+        if do_classifier_free_guidance:
+            prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0)
+            add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0)
+            add_time_ids = torch.cat([negative_add_time_ids, add_time_ids], dim=0)
+
+        prompt_embeds = prompt_embeds.to(device)
+        add_text_embeds = add_text_embeds.to(device)
+        add_time_ids = add_time_ids.to(device).repeat(batch_size * num_images_per_prompt, 1)
+
+        # 8. Denoising loop
+        num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
+
+        # 7.1 Apply denoising_end
+        if denoising_end is not None and isinstance(denoising_end, float) and denoising_end > 0 and denoising_end < 1:
+            discrete_timestep_cutoff = int(
+                round(
+                    self.scheduler.config.num_train_timesteps
+                    - (denoising_end * self.scheduler.config.num_train_timesteps)
+                )
+            )
+            num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps)))
+            timesteps = timesteps[:num_inference_steps]
+
+        # get init conditioning scale
+        for attn_processor in self.unet.attn_processors.values():
+            if isinstance(attn_processor, IPAttnProcessor):
+                conditioning_scale = attn_processor.scale
+                break
+
+        with self.progress_bar(total=num_inference_steps) as progress_bar:
+            for i, t in enumerate(timesteps):
+                if (i / len(timesteps) < control_guidance_start) or ((i + 1) / len(timesteps) > control_guidance_end):
+                    self.set_scale(0.0)
+                else:
+                    self.set_scale(conditioning_scale)
+
+                # expand the latents if we are doing classifier free guidance
+                latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents
+
+                latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
+
+                # predict the noise residual
+                added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids}
+                noise_pred = self.unet(
+                    latent_model_input,
+                    t,
+                    encoder_hidden_states=prompt_embeds,
+                    cross_attention_kwargs=cross_attention_kwargs,
+                    added_cond_kwargs=added_cond_kwargs,
+                    return_dict=False,
+                )[0]
+
+                # perform guidance
+                if do_classifier_free_guidance:
+                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
+                    noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
+
+                if do_classifier_free_guidance and guidance_rescale > 0.0:
+                    # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf
+                    noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=guidance_rescale)
+
+                # compute the previous noisy sample x_t -> x_t-1
+                latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
+
+                # call the callback, if provided
+                if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
+                    progress_bar.update()
+                    if callback is not None and i % callback_steps == 0:
+                        callback(i, t, latents)
+
+        if not output_type == "latent":
+            # make sure the VAE is in float32 mode, as it overflows in float16
+            needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast
+
+            if needs_upcasting:
+                self.upcast_vae()
+                latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype)
+
+            image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0]
+
+            # cast back to fp16 if needed
+            if needs_upcasting:
+                self.vae.to(dtype=torch.float16)
+        else:
+            image = latents
+
+        if output_type != "latent":
+            # apply watermark if available
+            if self.watermark is not None:
+                image = self.watermark.apply_watermark(image)
+
+            image = self.image_processor.postprocess(image, output_type=output_type)
+
+        # Offload all models
+        self.maybe_free_model_hooks()
+
+        if not return_dict:
+            return (image,)
+
+        return StableDiffusionXLPipelineOutput(images=image)

ip_adapter/ip_adapter.py

@@ -0,0 +1,424 @@
+import os
+from typing import List
+
+import torch
+from diffusers import StableDiffusionPipeline
+from diffusers.pipelines.controlnet import MultiControlNetModel
+from PIL import Image
+from safetensors import safe_open
+from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection
+
+from .utils import is_torch2_available, get_generator
+
+if is_torch2_available():
+    from .attention_processor import (
+        AttnProcessor2_0 as AttnProcessor,
+    )
+    from .attention_processor import (
+        CNAttnProcessor2_0 as CNAttnProcessor,
+    )
+    from .attention_processor import (
+        IPAttnProcessor2_0 as IPAttnProcessor,
+    )
+else:
+    from .attention_processor import AttnProcessor, CNAttnProcessor, IPAttnProcessor
+from .resampler import Resampler
+
+
+class ImageProjModel(torch.nn.Module):
+    """Projection Model"""
+
+    def __init__(self, cross_attention_dim=1024, clip_embeddings_dim=1024, clip_extra_context_tokens=4):
+        super().__init__()
+
+        self.generator = None
+        self.cross_attention_dim = cross_attention_dim
+        self.clip_extra_context_tokens = clip_extra_context_tokens
+        self.proj = torch.nn.Linear(clip_embeddings_dim, self.clip_extra_context_tokens * cross_attention_dim)
+        self.norm = torch.nn.LayerNorm(cross_attention_dim)
+
+    def forward(self, image_embeds):
+        embeds = image_embeds
+        clip_extra_context_tokens = self.proj(embeds).reshape(
+            -1, self.clip_extra_context_tokens, self.cross_attention_dim
+        )
+        clip_extra_context_tokens = self.norm(clip_extra_context_tokens)
+        return clip_extra_context_tokens
+
+
+class MLPProjModel(torch.nn.Module):
+    """SD model with image prompt"""
+    def __init__(self, cross_attention_dim=1024, clip_embeddings_dim=1024):
+        super().__init__()
+        
+        self.proj = torch.nn.Sequential(
+            torch.nn.Linear(clip_embeddings_dim, clip_embeddings_dim),
+            torch.nn.GELU(),
+            torch.nn.Linear(clip_embeddings_dim, cross_attention_dim),
+            torch.nn.LayerNorm(cross_attention_dim)
+        )
+        
+    def forward(self, image_embeds):
+        clip_extra_context_tokens = self.proj(image_embeds)
+        return clip_extra_context_tokens
+
+
+class IPAdapter:
+    def __init__(self, sd_pipe, image_encoder_path, ip_ckpt, device, num_tokens=4):
+        self.device = device
+        self.image_encoder_path = image_encoder_path
+        self.ip_ckpt = ip_ckpt
+        self.num_tokens = num_tokens
+
+        self.pipe = sd_pipe.to(self.device)
+        self.set_ip_adapter()
+
+        # load image encoder
+        self.image_encoder = CLIPVisionModelWithProjection.from_pretrained(self.image_encoder_path).to(
+            self.device, dtype=torch.float16
+        )
+        self.clip_image_processor = CLIPImageProcessor()
+        # image proj model
+        self.image_proj_model = self.init_proj()
+
+        self.load_ip_adapter()
+
+    def init_proj(self):
+        image_proj_model = ImageProjModel(
+            cross_attention_dim=self.pipe.unet.config.cross_attention_dim,
+            clip_embeddings_dim=self.image_encoder.config.projection_dim,
+            clip_extra_context_tokens=self.num_tokens,
+        ).to(self.device, dtype=torch.float16)
+        return image_proj_model
+
+    def set_ip_adapter(self):
+        unet = self.pipe.unet
+        attn_procs = {}
+        for name in unet.attn_processors.keys():
+            cross_attention_dim = None if name.endswith("attn1.processor") else unet.config.cross_attention_dim
+            if name.startswith("mid_block"):
+                hidden_size = unet.config.block_out_channels[-1]
+            elif name.startswith("up_blocks"):
+                block_id = int(name[len("up_blocks.")])
+                hidden_size = list(reversed(unet.config.block_out_channels))[block_id]
+            elif name.startswith("down_blocks"):
+                block_id = int(name[len("down_blocks.")])
+                hidden_size = unet.config.block_out_channels[block_id]
+            if cross_attention_dim is None:
+                attn_procs[name] = AttnProcessor()
+            else:
+                attn_procs[name] = IPAttnProcessor(
+                    hidden_size=hidden_size,
+                    cross_attention_dim=cross_attention_dim,
+                    scale=1.0,
+                    num_tokens=self.num_tokens,
+                ).to(self.device, dtype=torch.float16)
+        unet.set_attn_processor(attn_procs)
+        if hasattr(self.pipe, "controlnet"):
+            if isinstance(self.pipe.controlnet, MultiControlNetModel):
+                for controlnet in self.pipe.controlnet.nets:
+                    controlnet.set_attn_processor(CNAttnProcessor(num_tokens=self.num_tokens))
+            else:
+                self.pipe.controlnet.set_attn_processor(CNAttnProcessor(num_tokens=self.num_tokens))
+
+    def load_ip_adapter(self):
+        if os.path.splitext(self.ip_ckpt)[-1] == ".safetensors":
+            state_dict = {"image_proj": {}, "ip_adapter": {}}
+            with safe_open(self.ip_ckpt, framework="pt", device="cpu") as f:
+                for key in f.keys():
+                    if key.startswith("image_proj."):
+                        state_dict["image_proj"][key.replace("image_proj.", "")] = f.get_tensor(key)
+                    elif key.startswith("ip_adapter."):
+                        state_dict["ip_adapter"][key.replace("ip_adapter.", "")] = f.get_tensor(key)
+        else:
+            state_dict = torch.load(self.ip_ckpt, map_location="cpu")
+        self.image_proj_model.load_state_dict(state_dict["image_proj"])
+        ip_layers = torch.nn.ModuleList(self.pipe.unet.attn_processors.values())
+        ip_layers.load_state_dict(state_dict["ip_adapter"])
+
+    @torch.inference_mode()
+    def get_image_embeds(self, pil_image=None, clip_image_embeds=None):
+        if pil_image is not None:
+            if isinstance(pil_image, Image.Image):
+                pil_image = [pil_image]
+            clip_image = self.clip_image_processor(images=pil_image, return_tensors="pt").pixel_values
+            clip_image_embeds = self.image_encoder(clip_image.to(self.device, dtype=torch.float16)).image_embeds
+        else:
+            clip_image_embeds = clip_image_embeds.to(self.device, dtype=torch.float16)
+        image_prompt_embeds = self.image_proj_model(clip_image_embeds)
+        uncond_image_prompt_embeds = self.image_proj_model(torch.zeros_like(clip_image_embeds))
+        return image_prompt_embeds, uncond_image_prompt_embeds
+
+    def set_scale(self, scale):
+        for attn_processor in self.pipe.unet.attn_processors.values():
+            if isinstance(attn_processor, IPAttnProcessor):
+                attn_processor.scale = scale
+
+    def generate(
+        self,
+        pil_image=None,
+        clip_image_embeds=None,
+        prompt=None,
+        negative_prompt=None,
+        scale=1.0,
+        num_samples=4,
+        seed=None,
+        guidance_scale=7.5,
+        num_inference_steps=30,
+        **kwargs,
+    ):
+        self.set_scale(scale)
+
+        if pil_image is not None:
+            num_prompts = 1 if isinstance(pil_image, Image.Image) else len(pil_image)
+        else:
+            num_prompts = clip_image_embeds.size(0)
+
+        if prompt is None:
+            prompt = "best quality, high quality"
+        if negative_prompt is None:
+            negative_prompt = "monochrome, lowres, bad anatomy, worst quality, low quality"
+
+        if not isinstance(prompt, List):
+            prompt = [prompt] * num_prompts
+        if not isinstance(negative_prompt, List):
+            negative_prompt = [negative_prompt] * num_prompts
+
+        image_prompt_embeds, uncond_image_prompt_embeds = self.get_image_embeds(
+            pil_image=pil_image, clip_image_embeds=clip_image_embeds
+        )
+        bs_embed, seq_len, _ = image_prompt_embeds.shape
+        image_prompt_embeds = image_prompt_embeds.repeat(1, num_samples, 1)
+        image_prompt_embeds = image_prompt_embeds.view(bs_embed * num_samples, seq_len, -1)
+        uncond_image_prompt_embeds = uncond_image_prompt_embeds.repeat(1, num_samples, 1)
+        uncond_image_prompt_embeds = uncond_image_prompt_embeds.view(bs_embed * num_samples, seq_len, -1)
+
+        with torch.inference_mode():
+            prompt_embeds_, negative_prompt_embeds_ = self.pipe.encode_prompt(
+                prompt,
+                device=self.device,
+                num_images_per_prompt=num_samples,
+                do_classifier_free_guidance=True,
+                negative_prompt=negative_prompt,
+            )
+            prompt_embeds = torch.cat([prompt_embeds_, image_prompt_embeds], dim=1)
+            negative_prompt_embeds = torch.cat([negative_prompt_embeds_, uncond_image_prompt_embeds], dim=1)
+
+        generator = get_generator(seed, self.device)
+
+        images = self.pipe(
+            prompt_embeds=prompt_embeds,
+            negative_prompt_embeds=negative_prompt_embeds,
+            guidance_scale=guidance_scale,
+            num_inference_steps=num_inference_steps,
+            generator=generator,
+            **kwargs,
+        ).images
+
+        return images
+
+
+class IPAdapterXL(IPAdapter):
+    """SDXL"""
+
+    def generate(
+        self,
+        pil_image,
+        prompt=None,
+        negative_prompt=None,
+        scale=1.0,
+        num_samples=4,
+        seed=None,
+        num_inference_steps=30,
+        **kwargs,
+    ):
+        self.set_scale(scale)
+
+        num_prompts = 1 if isinstance(pil_image, Image.Image) else len(pil_image)
+
+        if prompt is None:
+            prompt = "best quality, high quality"
+        if negative_prompt is None:
+            negative_prompt = "monochrome, lowres, bad anatomy, worst quality, low quality"
+
+        if not isinstance(prompt, List):
+            prompt = [prompt] * num_prompts
+        if not isinstance(negative_prompt, List):
+            negative_prompt = [negative_prompt] * num_prompts
+
+        image_prompt_embeds, uncond_image_prompt_embeds = self.get_image_embeds(pil_image)
+        bs_embed, seq_len, _ = image_prompt_embeds.shape
+        image_prompt_embeds = image_prompt_embeds.repeat(1, num_samples, 1)
+        image_prompt_embeds = image_prompt_embeds.view(bs_embed * num_samples, seq_len, -1)
+        uncond_image_prompt_embeds = uncond_image_prompt_embeds.repeat(1, num_samples, 1)
+        uncond_image_prompt_embeds = uncond_image_prompt_embeds.view(bs_embed * num_samples, seq_len, -1)
+
+        with torch.inference_mode():
+            (
+                prompt_embeds,
+                negative_prompt_embeds,
+                pooled_prompt_embeds,
+                negative_pooled_prompt_embeds,
+            ) = self.pipe.encode_prompt(
+                prompt,
+                num_images_per_prompt=num_samples,
+                do_classifier_free_guidance=True,
+                negative_prompt=negative_prompt,
+            )
+            prompt_embeds = torch.cat([prompt_embeds, image_prompt_embeds], dim=1)
+            negative_prompt_embeds = torch.cat([negative_prompt_embeds, uncond_image_prompt_embeds], dim=1)
+
+        self.generator = get_generator(seed, self.device)
+        
+        images = self.pipe(
+            prompt_embeds=prompt_embeds,
+            negative_prompt_embeds=negative_prompt_embeds,
+            pooled_prompt_embeds=pooled_prompt_embeds,
+            negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
+            num_inference_steps=num_inference_steps,
+            generator=self.generator,
+            **kwargs,
+        ).images
+
+        return images
+
+
+class IPAdapterPlus(IPAdapter):
+    """IP-Adapter with fine-grained features"""
+
+    def init_proj(self):
+        image_proj_model = Resampler(
+            dim=self.pipe.unet.config.cross_attention_dim,
+            depth=4,
+            dim_head=64,
+            heads=12,
+            num_queries=self.num_tokens,
+            embedding_dim=self.image_encoder.config.hidden_size,
+            output_dim=self.pipe.unet.config.cross_attention_dim,
+            ff_mult=4,
+        ).to(self.device, dtype=torch.float16)
+        return image_proj_model
+
+    @torch.inference_mode()
+    def get_image_embeds(self, pil_image=None, clip_image_embeds=None):
+        if isinstance(pil_image, Image.Image):
+            pil_image = [pil_image]
+        clip_image = self.clip_image_processor(images=pil_image, return_tensors="pt").pixel_values
+        clip_image = clip_image.to(self.device, dtype=torch.float16)
+        clip_image_embeds = self.image_encoder(clip_image, output_hidden_states=True).hidden_states[-2]
+        image_prompt_embeds = self.image_proj_model(clip_image_embeds)
+        uncond_clip_image_embeds = self.image_encoder(
+            torch.zeros_like(clip_image), output_hidden_states=True
+        ).hidden_states[-2]
+        uncond_image_prompt_embeds = self.image_proj_model(uncond_clip_image_embeds)
+        return image_prompt_embeds, uncond_image_prompt_embeds
+
+
+class IPAdapterFull(IPAdapterPlus):
+    """IP-Adapter with full features"""
+
+    def init_proj(self):
+        image_proj_model = MLPProjModel(
+            cross_attention_dim=self.pipe.unet.config.cross_attention_dim,
+            clip_embeddings_dim=self.image_encoder.config.hidden_size,
+        ).to(self.device, dtype=torch.float16)
+        return image_proj_model
+
+
+class IPAdapterPlusXL(IPAdapter):
+    """SDXL"""
+
+    def init_proj(self):
+        image_proj_model = Resampler(
+            dim=1280,
+            depth=4,
+            dim_head=64,
+            heads=20,
+            num_queries=self.num_tokens,
+            embedding_dim=self.image_encoder.config.hidden_size,
+            output_dim=self.pipe.unet.config.cross_attention_dim,
+            ff_mult=4,
+        ).to(self.device, dtype=torch.float16)
+        return image_proj_model
+
+    @torch.inference_mode()
+    def get_image_embeds(self, pil_image, clip_image_embeds=None):
+        if pil_image is not None:
+            if isinstance(pil_image, Image.Image):
+                pil_image = [pil_image]
+            clip_image = self.clip_image_processor(images=pil_image, return_tensors="pt").pixel_values
+            clip_image = clip_image.to(self.device, dtype=torch.float16)
+            clip_image_embeds = self.image_encoder(clip_image, output_hidden_states=True).hidden_states[-2]
+        else:
+            clip_image_embeds = clip_image_embeds.to(self.device, dtype=torch.float16)
+        image_prompt_embeds = self.image_proj_model(clip_image_embeds)
+        uncond_clip_image_embeds = self.image_encoder(
+            torch.zeros(clip_image_embeds.shape[0], 3, 224, 224).to(self.device, dtype=torch.float16), output_hidden_states=True
+        ).hidden_states[-2]
+        uncond_image_prompt_embeds = self.image_proj_model(uncond_clip_image_embeds)
+        return image_prompt_embeds, uncond_image_prompt_embeds
+
+    def generate(
+        self,
+        pil_image,
+        prompt=None,
+        clip_image_embeds=None,
+        negative_prompt=None,
+        scale=1.0,
+        num_samples=4,
+        seed=None,
+        num_inference_steps=30,
+        **kwargs,
+    ):
+        self.set_scale(scale)
+
+        if pil_image is not None:
+            num_prompts = 1 if isinstance(pil_image, Image.Image) else len(pil_image)
+        else:
+            num_prompts = clip_image_embeds.size(0)
+            
+        if prompt is None:
+            prompt = "best quality, high quality"
+        if negative_prompt is None:
+            negative_prompt = "monochrome, lowres, bad anatomy, worst quality, low quality"
+
+        if not isinstance(prompt, List):
+            prompt = [prompt] * num_prompts
+        if not isinstance(negative_prompt, List):
+            negative_prompt = [negative_prompt] * num_prompts
+
+        image_prompt_embeds, uncond_image_prompt_embeds = self.get_image_embeds(pil_image, clip_image_embeds)
+        bs_embed, seq_len, _ = image_prompt_embeds.shape
+        image_prompt_embeds = image_prompt_embeds.repeat(1, num_samples, 1)
+        image_prompt_embeds = image_prompt_embeds.view(bs_embed * num_samples, seq_len, -1)
+        uncond_image_prompt_embeds = uncond_image_prompt_embeds.repeat(1, num_samples, 1)
+        uncond_image_prompt_embeds = uncond_image_prompt_embeds.view(bs_embed * num_samples, seq_len, -1)
+
+        with torch.inference_mode():
+            (
+                prompt_embeds,
+                negative_prompt_embeds,
+                pooled_prompt_embeds,
+                negative_pooled_prompt_embeds,
+            ) = self.pipe.encode_prompt(
+                prompt,
+                num_images_per_prompt=num_samples,
+                do_classifier_free_guidance=True,
+                negative_prompt=negative_prompt,
+            )
+            prompt_embeds = torch.cat([prompt_embeds, image_prompt_embeds], dim=1)
+            negative_prompt_embeds = torch.cat([negative_prompt_embeds, uncond_image_prompt_embeds], dim=1)
+
+        generator = get_generator(seed, self.device)
+
+        images = self.pipe(
+            prompt_embeds=prompt_embeds,
+            negative_prompt_embeds=negative_prompt_embeds,
+            pooled_prompt_embeds=pooled_prompt_embeds,
+            negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
+            num_inference_steps=num_inference_steps,
+            generator=generator,
+            **kwargs,
+        ).images
+
+        return images

ip_adapter/ip_adapter_faceid.py

@@ -0,0 +1,542 @@
+import os
+from typing import List
+
+import torch
+from diffusers import StableDiffusionPipeline
+from diffusers.pipelines.controlnet import MultiControlNetModel
+from PIL import Image
+from safetensors import safe_open
+from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection
+
+from .attention_processor_faceid import LoRAAttnProcessor, LoRAIPAttnProcessor
+from .utils import is_torch2_available, get_generator
+
+USE_DAFAULT_ATTN = False # should be True for visualization_attnmap
+if is_torch2_available() and (not USE_DAFAULT_ATTN):
+    from .attention_processor_faceid import (
+        LoRAAttnProcessor2_0 as LoRAAttnProcessor,
+    )
+    from .attention_processor_faceid import (
+        LoRAIPAttnProcessor2_0 as LoRAIPAttnProcessor,
+    )
+else:
+    from .attention_processor_faceid import LoRAAttnProcessor, LoRAIPAttnProcessor
+from .resampler import PerceiverAttention, FeedForward
+
+
+class FacePerceiverResampler(torch.nn.Module):
+    def __init__(
+        self,
+        *,
+        dim=768,
+        depth=4,
+        dim_head=64,
+        heads=16,
+        embedding_dim=1280,
+        output_dim=768,
+        ff_mult=4,
+    ):
+        super().__init__()
+        
+        self.proj_in = torch.nn.Linear(embedding_dim, dim)
+        self.proj_out = torch.nn.Linear(dim, output_dim)
+        self.norm_out = torch.nn.LayerNorm(output_dim)
+        self.layers = torch.nn.ModuleList([])
+        for _ in range(depth):
+            self.layers.append(
+                torch.nn.ModuleList(
+                    [
+                        PerceiverAttention(dim=dim, dim_head=dim_head, heads=heads),
+                        FeedForward(dim=dim, mult=ff_mult),
+                    ]
+                )
+            )
+
+    def forward(self, latents, x):
+        x = self.proj_in(x)
+        for attn, ff in self.layers:
+            latents = attn(x, latents) + latents
+            latents = ff(latents) + latents
+        latents = self.proj_out(latents)
+        return self.norm_out(latents)
+
+
+class MLPProjModel(torch.nn.Module):
+    def __init__(self, cross_attention_dim=768, id_embeddings_dim=512, num_tokens=4):
+        super().__init__()
+        
+        self.cross_attention_dim = cross_attention_dim
+        self.num_tokens = num_tokens
+        
+        self.proj = torch.nn.Sequential(
+            torch.nn.Linear(id_embeddings_dim, id_embeddings_dim*2),
+            torch.nn.GELU(),
+            torch.nn.Linear(id_embeddings_dim*2, cross_attention_dim*num_tokens),
+        )
+        self.norm = torch.nn.LayerNorm(cross_attention_dim)
+        
+    def forward(self, id_embeds):
+        x = self.proj(id_embeds)
+        x = x.reshape(-1, self.num_tokens, self.cross_attention_dim)
+        x = self.norm(x)
+        return x
+
+
+class ProjPlusModel(torch.nn.Module):
+    def __init__(self, cross_attention_dim=768, id_embeddings_dim=512, clip_embeddings_dim=1280, num_tokens=4):
+        super().__init__()
+        
+        self.cross_attention_dim = cross_attention_dim
+        self.num_tokens = num_tokens
+        
+        self.proj = torch.nn.Sequential(
+            torch.nn.Linear(id_embeddings_dim, id_embeddings_dim*2),
+            torch.nn.GELU(),
+            torch.nn.Linear(id_embeddings_dim*2, cross_attention_dim*num_tokens),
+        )
+        self.norm = torch.nn.LayerNorm(cross_attention_dim)
+        
+        self.perceiver_resampler = FacePerceiverResampler(
+            dim=cross_attention_dim,
+            depth=4,
+            dim_head=64,
+            heads=cross_attention_dim // 64,
+            embedding_dim=clip_embeddings_dim,
+            output_dim=cross_attention_dim,
+            ff_mult=4,
+        )
+        
+    def forward(self, id_embeds, clip_embeds, shortcut=False, scale=1.0):
+        
+        x = self.proj(id_embeds)
+        x = x.reshape(-1, self.num_tokens, self.cross_attention_dim)
+        x = self.norm(x)
+        out = self.perceiver_resampler(x, clip_embeds)
+        if shortcut:
+            out = x + scale * out
+        return out
+
+
+class IPAdapterFaceID:
+    def __init__(self, sd_pipe, ip_ckpt, device, lora_rank=128, num_tokens=4, torch_dtype=torch.float16):
+        self.device = device
+        self.ip_ckpt = ip_ckpt
+        self.lora_rank = lora_rank
+        self.num_tokens = num_tokens
+        self.torch_dtype = torch_dtype
+
+        self.pipe = sd_pipe.to(self.device)
+        self.set_ip_adapter()
+
+        # image proj model
+        self.image_proj_model = self.init_proj()
+
+        self.load_ip_adapter()
+
+    def init_proj(self):
+        image_proj_model = MLPProjModel(
+            cross_attention_dim=self.pipe.unet.config.cross_attention_dim,
+            id_embeddings_dim=512,
+            num_tokens=self.num_tokens,
+        ).to(self.device, dtype=self.torch_dtype)
+        return image_proj_model
+
+    def set_ip_adapter(self):
+        unet = self.pipe.unet
+        attn_procs = {}
+        for name in unet.attn_processors.keys():
+            cross_attention_dim = None if name.endswith("attn1.processor") else unet.config.cross_attention_dim
+            if name.startswith("mid_block"):
+                hidden_size = unet.config.block_out_channels[-1]
+            elif name.startswith("up_blocks"):
+                block_id = int(name[len("up_blocks.")])
+                hidden_size = list(reversed(unet.config.block_out_channels))[block_id]
+            elif name.startswith("down_blocks"):
+                block_id = int(name[len("down_blocks.")])
+                hidden_size = unet.config.block_out_channels[block_id]
+            if cross_attention_dim is None:
+                attn_procs[name] = LoRAAttnProcessor(
+                    hidden_size=hidden_size, cross_attention_dim=cross_attention_dim, rank=self.lora_rank,
+                ).to(self.device, dtype=self.torch_dtype)
+            else:
+                attn_procs[name] = LoRAIPAttnProcessor(
+                    hidden_size=hidden_size, cross_attention_dim=cross_attention_dim, scale=1.0, rank=self.lora_rank, num_tokens=self.num_tokens,
+                ).to(self.device, dtype=self.torch_dtype)
+        unet.set_attn_processor(attn_procs)
+
+    def load_ip_adapter(self):
+        if os.path.splitext(self.ip_ckpt)[-1] == ".safetensors":
+            state_dict = {"image_proj": {}, "ip_adapter": {}}
+            with safe_open(self.ip_ckpt, framework="pt", device="cpu") as f:
+                for key in f.keys():
+                    if key.startswith("image_proj."):
+                        state_dict["image_proj"][key.replace("image_proj.", "")] = f.get_tensor(key)
+                    elif key.startswith("ip_adapter."):
+                        state_dict["ip_adapter"][key.replace("ip_adapter.", "")] = f.get_tensor(key)
+        else:
+            state_dict = torch.load(self.ip_ckpt, map_location="cpu")
+        self.image_proj_model.load_state_dict(state_dict["image_proj"])
+        ip_layers = torch.nn.ModuleList(self.pipe.unet.attn_processors.values())
+        ip_layers.load_state_dict(state_dict["ip_adapter"])
+
+    @torch.inference_mode()
+    def get_image_embeds(self, faceid_embeds):
+        
+        faceid_embeds = faceid_embeds.to(self.device, dtype=self.torch_dtype)
+        image_prompt_embeds = self.image_proj_model(faceid_embeds)
+        uncond_image_prompt_embeds = self.image_proj_model(torch.zeros_like(faceid_embeds))
+        return image_prompt_embeds, uncond_image_prompt_embeds
+
+    def set_scale(self, scale):
+        for attn_processor in self.pipe.unet.attn_processors.values():
+            if isinstance(attn_processor, LoRAIPAttnProcessor):
+                attn_processor.scale = scale
+
+    def generate(
+        self,
+        faceid_embeds=None,
+        prompt=None,
+        negative_prompt=None,
+        scale=1.0,
+        num_samples=4,
+        seed=None,
+        guidance_scale=7.5,
+        num_inference_steps=30,
+        **kwargs,
+    ):
+        self.set_scale(scale)
+
+       
+        num_prompts = faceid_embeds.size(0)
+
+        if prompt is None:
+            prompt = "best quality, high quality"
+        if negative_prompt is None:
+            negative_prompt = "monochrome, lowres, bad anatomy, worst quality, low quality"
+
+        if not isinstance(prompt, List):
+            prompt = [prompt] * num_prompts
+        if not isinstance(negative_prompt, List):
+            negative_prompt = [negative_prompt] * num_prompts
+
+        image_prompt_embeds, uncond_image_prompt_embeds = self.get_image_embeds(faceid_embeds)
+
+        bs_embed, seq_len, _ = image_prompt_embeds.shape
+        image_prompt_embeds = image_prompt_embeds.repeat(1, num_samples, 1)
+        image_prompt_embeds = image_prompt_embeds.view(bs_embed * num_samples, seq_len, -1)
+        uncond_image_prompt_embeds = uncond_image_prompt_embeds.repeat(1, num_samples, 1)
+        uncond_image_prompt_embeds = uncond_image_prompt_embeds.view(bs_embed * num_samples, seq_len, -1)
+
+        with torch.inference_mode():
+            prompt_embeds_, negative_prompt_embeds_ = self.pipe.encode_prompt(
+                prompt,
+                device=self.device,
+                num_images_per_prompt=num_samples,
+                do_classifier_free_guidance=True,
+                negative_prompt=negative_prompt,
+            )
+            prompt_embeds = torch.cat([prompt_embeds_, image_prompt_embeds], dim=1)
+            negative_prompt_embeds = torch.cat([negative_prompt_embeds_, uncond_image_prompt_embeds], dim=1)
+
+        generator = get_generator(seed, self.device)
+
+        images = self.pipe(
+            prompt_embeds=prompt_embeds,
+            negative_prompt_embeds=negative_prompt_embeds,
+            guidance_scale=guidance_scale,
+            num_inference_steps=num_inference_steps,
+            generator=generator,
+            **kwargs,
+        ).images
+
+        return images
+
+
+class IPAdapterFaceIDPlus:
+    def __init__(self, sd_pipe, image_encoder_path, ip_ckpt, device, lora_rank=128, num_tokens=4, torch_dtype=torch.float16):
+        self.device = device
+        self.image_encoder_path = image_encoder_path
+        self.ip_ckpt = ip_ckpt
+        self.lora_rank = lora_rank
+        self.num_tokens = num_tokens
+        self.torch_dtype = torch_dtype
+
+        self.pipe = sd_pipe.to(self.device)
+        self.set_ip_adapter()
+
+        # load image encoder
+        self.image_encoder = CLIPVisionModelWithProjection.from_pretrained(self.image_encoder_path).to(
+            self.device, dtype=self.torch_dtype
+        )
+        self.clip_image_processor = CLIPImageProcessor()
+        # image proj model
+        self.image_proj_model = self.init_proj()
+
+        self.load_ip_adapter()
+
+    def init_proj(self):
+        image_proj_model = ProjPlusModel(
+            cross_attention_dim=self.pipe.unet.config.cross_attention_dim,
+            id_embeddings_dim=512,
+            clip_embeddings_dim=self.image_encoder.config.hidden_size,
+            num_tokens=self.num_tokens,
+        ).to(self.device, dtype=self.torch_dtype)
+        return image_proj_model
+
+    def set_ip_adapter(self):
+        unet = self.pipe.unet
+        attn_procs = {}
+        for name in unet.attn_processors.keys():
+            cross_attention_dim = None if name.endswith("attn1.processor") else unet.config.cross_attention_dim
+            if name.startswith("mid_block"):
+                hidden_size = unet.config.block_out_channels[-1]
+            elif name.startswith("up_blocks"):
+                block_id = int(name[len("up_blocks.")])
+                hidden_size = list(reversed(unet.config.block_out_channels))[block_id]
+            elif name.startswith("down_blocks"):
+                block_id = int(name[len("down_blocks.")])
+                hidden_size = unet.config.block_out_channels[block_id]
+            if cross_attention_dim is None:
+                attn_procs[name] = LoRAAttnProcessor(
+                    hidden_size=hidden_size, cross_attention_dim=cross_attention_dim, rank=self.lora_rank,
+                ).to(self.device, dtype=self.torch_dtype)
+            else:
+                attn_procs[name] = LoRAIPAttnProcessor(
+                    hidden_size=hidden_size, cross_attention_dim=cross_attention_dim, scale=1.0, rank=self.lora_rank, num_tokens=self.num_tokens,
+                ).to(self.device, dtype=self.torch_dtype)
+        unet.set_attn_processor(attn_procs)
+
+    def load_ip_adapter(self):
+        if os.path.splitext(self.ip_ckpt)[-1] == ".safetensors":
+            state_dict = {"image_proj": {}, "ip_adapter": {}}
+            with safe_open(self.ip_ckpt, framework="pt", device="cpu") as f:
+                for key in f.keys():
+                    if key.startswith("image_proj."):
+                        state_dict["image_proj"][key.replace("image_proj.", "")] = f.get_tensor(key)
+                    elif key.startswith("ip_adapter."):
+                        state_dict["ip_adapter"][key.replace("ip_adapter.", "")] = f.get_tensor(key)
+        else:
+            state_dict = torch.load(self.ip_ckpt, map_location="cpu")
+        self.image_proj_model.load_state_dict(state_dict["image_proj"])
+        ip_layers = torch.nn.ModuleList(self.pipe.unet.attn_processors.values())
+        ip_layers.load_state_dict(state_dict["ip_adapter"])
+
+    @torch.inference_mode()
+    def get_image_embeds(self, faceid_embeds, face_image, s_scale, shortcut):
+        if isinstance(face_image, Image.Image):
+            pil_image = [face_image]
+        clip_image = self.clip_image_processor(images=face_image, return_tensors="pt").pixel_values
+        clip_image = clip_image.to(self.device, dtype=self.torch_dtype)
+        clip_image_embeds = self.image_encoder(clip_image, output_hidden_states=True).hidden_states[-2]
+        uncond_clip_image_embeds = self.image_encoder(
+            torch.zeros_like(clip_image), output_hidden_states=True
+        ).hidden_states[-2]
+        
+        faceid_embeds = faceid_embeds.to(self.device, dtype=self.torch_dtype)
+        image_prompt_embeds = self.image_proj_model(faceid_embeds, clip_image_embeds, shortcut=shortcut, scale=s_scale)
+        uncond_image_prompt_embeds = self.image_proj_model(torch.zeros_like(faceid_embeds), uncond_clip_image_embeds, shortcut=shortcut, scale=s_scale)
+        return image_prompt_embeds, uncond_image_prompt_embeds
+
+    def set_scale(self, scale):
+        for attn_processor in self.pipe.unet.attn_processors.values():
+            if isinstance(attn_processor, LoRAIPAttnProcessor):
+                attn_processor.scale = scale
+
+    def generate(
+        self,
+        face_image=None,
+        faceid_embeds=None,
+        prompt=None,
+        negative_prompt=None,
+        scale=1.0,
+        num_samples=4,
+        seed=None,
+        guidance_scale=7.5,
+        num_inference_steps=30,
+        s_scale=1.0,
+        shortcut=False,
+        **kwargs,
+    ):
+        self.set_scale(scale)
+
+       
+        num_prompts = faceid_embeds.size(0)
+
+        if prompt is None:
+            prompt = "best quality, high quality"
+        if negative_prompt is None:
+            negative_prompt = "monochrome, lowres, bad anatomy, worst quality, low quality"
+
+        if not isinstance(prompt, List):
+            prompt = [prompt] * num_prompts
+        if not isinstance(negative_prompt, List):
+            negative_prompt = [negative_prompt] * num_prompts
+
+        image_prompt_embeds, uncond_image_prompt_embeds = self.get_image_embeds(faceid_embeds, face_image, s_scale, shortcut)
+
+        bs_embed, seq_len, _ = image_prompt_embeds.shape
+        image_prompt_embeds = image_prompt_embeds.repeat(1, num_samples, 1)
+        image_prompt_embeds = image_prompt_embeds.view(bs_embed * num_samples, seq_len, -1)
+        uncond_image_prompt_embeds = uncond_image_prompt_embeds.repeat(1, num_samples, 1)
+        uncond_image_prompt_embeds = uncond_image_prompt_embeds.view(bs_embed * num_samples, seq_len, -1)
+
+        with torch.inference_mode():
+            prompt_embeds_, negative_prompt_embeds_ = self.pipe.encode_prompt(
+                prompt,
+                device=self.device,
+                num_images_per_prompt=num_samples,
+                do_classifier_free_guidance=True,
+                negative_prompt=negative_prompt,
+            )
+            prompt_embeds = torch.cat([prompt_embeds_, image_prompt_embeds], dim=1)
+            negative_prompt_embeds = torch.cat([negative_prompt_embeds_, uncond_image_prompt_embeds], dim=1)
+
+        generator = get_generator(seed, self.device)
+
+        images = self.pipe(
+            prompt_embeds=prompt_embeds,
+            negative_prompt_embeds=negative_prompt_embeds,
+            guidance_scale=guidance_scale,
+            num_inference_steps=num_inference_steps,
+            generator=generator,
+            **kwargs,
+        ).images
+
+        return images
+
+
+class IPAdapterFaceIDXL(IPAdapterFaceID):
+    """SDXL"""
+
+    def generate(
+        self,
+        faceid_embeds=None,
+        prompt=None,
+        negative_prompt=None,
+        scale=1.0,
+        num_samples=4,
+        seed=None,
+        num_inference_steps=30,
+        **kwargs,
+    ):
+        self.set_scale(scale)
+
+        num_prompts = faceid_embeds.size(0)
+
+        if prompt is None:
+            prompt = "best quality, high quality"
+        if negative_prompt is None:
+            negative_prompt = "monochrome, lowres, bad anatomy, worst quality, low quality"
+
+        if not isinstance(prompt, List):
+            prompt = [prompt] * num_prompts
+        if not isinstance(negative_prompt, List):
+            negative_prompt = [negative_prompt] * num_prompts
+
+        image_prompt_embeds, uncond_image_prompt_embeds = self.get_image_embeds(faceid_embeds)
+
+        bs_embed, seq_len, _ = image_prompt_embeds.shape
+        image_prompt_embeds = image_prompt_embeds.repeat(1, num_samples, 1)
+        image_prompt_embeds = image_prompt_embeds.view(bs_embed * num_samples, seq_len, -1)
+        uncond_image_prompt_embeds = uncond_image_prompt_embeds.repeat(1, num_samples, 1)
+        uncond_image_prompt_embeds = uncond_image_prompt_embeds.view(bs_embed * num_samples, seq_len, -1)
+
+        with torch.inference_mode():
+            (
+                prompt_embeds,
+                negative_prompt_embeds,
+                pooled_prompt_embeds,
+                negative_pooled_prompt_embeds,
+            ) = self.pipe.encode_prompt(
+                prompt,
+                num_images_per_prompt=num_samples,
+                do_classifier_free_guidance=True,
+                negative_prompt=negative_prompt,
+            )
+            prompt_embeds = torch.cat([prompt_embeds, image_prompt_embeds], dim=1)
+            negative_prompt_embeds = torch.cat([negative_prompt_embeds, uncond_image_prompt_embeds], dim=1)
+
+        generator = get_generator(seed, self.device)
+
+        images = self.pipe(
+            prompt_embeds=prompt_embeds,
+            negative_prompt_embeds=negative_prompt_embeds,
+            pooled_prompt_embeds=pooled_prompt_embeds,
+            negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
+            num_inference_steps=num_inference_steps,
+            generator=generator,
+            **kwargs,
+        ).images
+
+        return images
+
+
+class IPAdapterFaceIDPlusXL(IPAdapterFaceIDPlus):
+    """SDXL"""
+
+    def generate(
+        self,
+        face_image=None,
+        faceid_embeds=None,
+        prompt=None,
+        negative_prompt=None,
+        scale=1.0,
+        num_samples=4,
+        seed=None,
+        guidance_scale=7.5,
+        num_inference_steps=30,
+        s_scale=1.0,
+        shortcut=True,
+        **kwargs,
+    ):
+        self.set_scale(scale)
+
+        num_prompts = faceid_embeds.size(0)
+
+        if prompt is None:
+            prompt = "best quality, high quality"
+        if negative_prompt is None:
+            negative_prompt = "monochrome, lowres, bad anatomy, worst quality, low quality"
+
+        if not isinstance(prompt, List):
+            prompt = [prompt] * num_prompts
+        if not isinstance(negative_prompt, List):
+            negative_prompt = [negative_prompt] * num_prompts
+
+        image_prompt_embeds, uncond_image_prompt_embeds = self.get_image_embeds(faceid_embeds, face_image, s_scale, shortcut)
+
+        bs_embed, seq_len, _ = image_prompt_embeds.shape
+        image_prompt_embeds = image_prompt_embeds.repeat(1, num_samples, 1)
+        image_prompt_embeds = image_prompt_embeds.view(bs_embed * num_samples, seq_len, -1)
+        uncond_image_prompt_embeds = uncond_image_prompt_embeds.repeat(1, num_samples, 1)
+        uncond_image_prompt_embeds = uncond_image_prompt_embeds.view(bs_embed * num_samples, seq_len, -1)
+
+        with torch.inference_mode():
+            (
+                prompt_embeds,
+                negative_prompt_embeds,
+                pooled_prompt_embeds,
+                negative_pooled_prompt_embeds,
+            ) = self.pipe.encode_prompt(
+                prompt,
+                num_images_per_prompt=num_samples,
+                do_classifier_free_guidance=True,
+                negative_prompt=negative_prompt,
+            )
+            prompt_embeds = torch.cat([prompt_embeds, image_prompt_embeds], dim=1)
+            negative_prompt_embeds = torch.cat([negative_prompt_embeds, uncond_image_prompt_embeds], dim=1)
+
+        generator = get_generator(seed, self.device)
+
+        images = self.pipe(
+            prompt_embeds=prompt_embeds,
+            negative_prompt_embeds=negative_prompt_embeds,
+            pooled_prompt_embeds=pooled_prompt_embeds,
+            negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
+            num_inference_steps=num_inference_steps,
+            generator=generator,
+            guidance_scale=guidance_scale,
+            **kwargs,
+        ).images
+
+        return images

ip_adapter/ip_adapter_faceid_separate.py

@@ -0,0 +1,556 @@
+import os
+from typing import List
+
+import torch
+from diffusers import StableDiffusionPipeline
+from diffusers.pipelines.controlnet import MultiControlNetModel
+from PIL import Image
+from safetensors import safe_open
+from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection
+
+from .utils import is_torch2_available, get_generator
+
+USE_DAFAULT_ATTN = False # should be True for visualization_attnmap
+if is_torch2_available() and (not USE_DAFAULT_ATTN):
+    from .attention_processor import (
+        AttnProcessor2_0 as AttnProcessor,
+    )
+    from .attention_processor import (
+        IPAttnProcessor2_0 as IPAttnProcessor,
+    )
+else:
+    from .attention_processor import AttnProcessor, IPAttnProcessor
+from .resampler import PerceiverAttention, FeedForward
+
+
+class FacePerceiverResampler(torch.nn.Module):
+    def __init__(
+        self,
+        *,
+        dim=768,
+        depth=4,
+        dim_head=64,
+        heads=16,
+        embedding_dim=1280,
+        output_dim=768,
+        ff_mult=4,
+    ):
+        super().__init__()
+        
+        self.proj_in = torch.nn.Linear(embedding_dim, dim)
+        self.proj_out = torch.nn.Linear(dim, output_dim)
+        self.norm_out = torch.nn.LayerNorm(output_dim)
+        self.layers = torch.nn.ModuleList([])
+        for _ in range(depth):
+            self.layers.append(
+                torch.nn.ModuleList(
+                    [
+                        PerceiverAttention(dim=dim, dim_head=dim_head, heads=heads),
+                        FeedForward(dim=dim, mult=ff_mult),
+                    ]
+                )
+            )
+
+    def forward(self, latents, x):
+        x = self.proj_in(x)
+        for attn, ff in self.layers:
+            latents = attn(x, latents) + latents
+            latents = ff(latents) + latents
+        latents = self.proj_out(latents)
+        return self.norm_out(latents)
+
+
+class MLPProjModel(torch.nn.Module):
+    def __init__(self, cross_attention_dim=768, id_embeddings_dim=512, num_tokens=4):
+        super().__init__()
+        
+        self.cross_attention_dim = cross_attention_dim
+        self.num_tokens = num_tokens
+        
+        self.proj = torch.nn.Sequential(
+            torch.nn.Linear(id_embeddings_dim, id_embeddings_dim*2),
+            torch.nn.GELU(),
+            torch.nn.Linear(id_embeddings_dim*2, cross_attention_dim*num_tokens),
+        )
+        self.norm = torch.nn.LayerNorm(cross_attention_dim)
+        
+    def forward(self, id_embeds):
+        x = self.proj(id_embeds)
+        x = x.reshape(-1, self.num_tokens, self.cross_attention_dim)
+        x = self.norm(x)
+        return x
+
+
+class ProjPlusModel(torch.nn.Module):
+    def __init__(self, cross_attention_dim=768, id_embeddings_dim=512, clip_embeddings_dim=1280, num_tokens=4):
+        super().__init__()
+        
+        self.cross_attention_dim = cross_attention_dim
+        self.num_tokens = num_tokens
+        
+        self.proj = torch.nn.Sequential(
+            torch.nn.Linear(id_embeddings_dim, id_embeddings_dim*2),
+            torch.nn.GELU(),
+            torch.nn.Linear(id_embeddings_dim*2, cross_attention_dim*num_tokens),
+        )
+        self.norm = torch.nn.LayerNorm(cross_attention_dim)
+        
+        self.perceiver_resampler = FacePerceiverResampler(
+            dim=cross_attention_dim,
+            depth=4,
+            dim_head=64,
+            heads=cross_attention_dim // 64,
+            embedding_dim=clip_embeddings_dim,
+            output_dim=cross_attention_dim,
+            ff_mult=4,
+        )
+        
+    def forward(self, id_embeds, clip_embeds, shortcut=False, scale=1.0):
+        
+        x = self.proj(id_embeds)
+        x = x.reshape(-1, self.num_tokens, self.cross_attention_dim)
+        x = self.norm(x)
+        out = self.perceiver_resampler(x, clip_embeds)
+        if shortcut:
+            out = x + scale * out
+        return out
+
+
+class IPAdapterFaceID:
+    def __init__(self, sd_pipe, ip_ckpt, device, num_tokens=4, n_cond=1, torch_dtype=torch.float16):
+        self.device = device
+        self.ip_ckpt = ip_ckpt
+        self.num_tokens = num_tokens
+        self.n_cond = n_cond
+        self.torch_dtype = torch_dtype
+
+        self.pipe = sd_pipe.to(self.device)
+        self.set_ip_adapter()
+
+        # image proj model
+        self.image_proj_model = self.init_proj()
+
+        self.load_ip_adapter()
+
+    def init_proj(self):
+        image_proj_model = MLPProjModel(
+            cross_attention_dim=self.pipe.unet.config.cross_attention_dim,
+            id_embeddings_dim=512,
+            num_tokens=self.num_tokens,
+        ).to(self.device, dtype=self.torch_dtype)
+        return image_proj_model
+
+    def set_ip_adapter(self):
+        unet = self.pipe.unet
+        attn_procs = {}
+        for name in unet.attn_processors.keys():
+            cross_attention_dim = None if name.endswith("attn1.processor") else unet.config.cross_attention_dim
+            if name.startswith("mid_block"):
+                hidden_size = unet.config.block_out_channels[-1]
+            elif name.startswith("up_blocks"):
+                block_id = int(name[len("up_blocks.")])
+                hidden_size = list(reversed(unet.config.block_out_channels))[block_id]
+            elif name.startswith("down_blocks"):
+                block_id = int(name[len("down_blocks.")])
+                hidden_size = unet.config.block_out_channels[block_id]
+            if cross_attention_dim is None:
+                attn_procs[name] = AttnProcessor()
+            else:
+                attn_procs[name] = IPAttnProcessor(
+                    hidden_size=hidden_size, cross_attention_dim=cross_attention_dim, scale=1.0, num_tokens=self.num_tokens*self.n_cond,
+                ).to(self.device, dtype=self.torch_dtype)
+        unet.set_attn_processor(attn_procs)
+
+    def load_ip_adapter(self):
+        if os.path.splitext(self.ip_ckpt)[-1] == ".safetensors":
+            state_dict = {"image_proj": {}, "ip_adapter": {}}
+            with safe_open(self.ip_ckpt, framework="pt", device="cpu") as f:
+                for key in f.keys():
+                    if key.startswith("image_proj."):
+                        state_dict["image_proj"][key.replace("image_proj.", "")] = f.get_tensor(key)
+                    elif key.startswith("ip_adapter."):
+                        state_dict["ip_adapter"][key.replace("ip_adapter.", "")] = f.get_tensor(key)
+        else:
+            state_dict = torch.load(self.ip_ckpt, map_location="cpu")
+        self.image_proj_model.load_state_dict(state_dict["image_proj"])
+        ip_layers = torch.nn.ModuleList(self.pipe.unet.attn_processors.values())
+        ip_layers.load_state_dict(state_dict["ip_adapter"], strict=False)
+
+    @torch.inference_mode()
+    def get_image_embeds(self, faceid_embeds):
+
+        multi_face = False
+        if faceid_embeds.dim() == 3:
+            multi_face = True
+            b, n, c = faceid_embeds.shape
+            faceid_embeds = faceid_embeds.reshape(b*n, c)
+
+        faceid_embeds = faceid_embeds.to(self.device, dtype=self.torch_dtype)
+        image_prompt_embeds = self.image_proj_model(faceid_embeds)
+        uncond_image_prompt_embeds = self.image_proj_model(torch.zeros_like(faceid_embeds))
+        if multi_face:
+            c = image_prompt_embeds.size(-1)
+            image_prompt_embeds = image_prompt_embeds.reshape(b, -1, c)
+            uncond_image_prompt_embeds = uncond_image_prompt_embeds.reshape(b, -1, c)
+        
+        return image_prompt_embeds, uncond_image_prompt_embeds
+
+    def set_scale(self, scale):
+        for attn_processor in self.pipe.unet.attn_processors.values():
+            if isinstance(attn_processor, IPAttnProcessor):
+                attn_processor.scale = scale
+
+    def generate(
+        self,
+        faceid_embeds=None,
+        prompt=None,
+        negative_prompt=None,
+        scale=1.0,
+        num_samples=4,
+        seed=None,
+        guidance_scale=7.5,
+        num_inference_steps=30,
+        **kwargs,
+    ):
+        self.set_scale(scale)
+
+        num_prompts = faceid_embeds.size(0)
+
+        if prompt is None:
+            prompt = "best quality, high quality"
+        if negative_prompt is None:
+            negative_prompt = "monochrome, lowres, bad anatomy, worst quality, low quality"
+
+        if not isinstance(prompt, List):
+            prompt = [prompt] * num_prompts
+        else:
+            faceid_embeds = faceid_embeds.repeat(num_samples, 1, 1)
+            num_samples = 1
+
+        if not isinstance(negative_prompt, List):
+            negative_prompt = [negative_prompt] * num_prompts
+
+        image_prompt_embeds, uncond_image_prompt_embeds = self.get_image_embeds(faceid_embeds)
+
+        bs_embed, seq_len, _ = image_prompt_embeds.shape
+        image_prompt_embeds = image_prompt_embeds.repeat(1, num_samples, 1)
+        image_prompt_embeds = image_prompt_embeds.view(bs_embed * num_samples, seq_len, -1)
+        uncond_image_prompt_embeds = uncond_image_prompt_embeds.repeat(1, num_samples, 1)
+        uncond_image_prompt_embeds = uncond_image_prompt_embeds.view(bs_embed * num_samples, seq_len, -1)
+
+        with torch.inference_mode():
+            prompt_embeds_, negative_prompt_embeds_ = self.pipe.encode_prompt(
+                prompt,
+                device=self.device,
+                num_images_per_prompt=num_samples,
+                do_classifier_free_guidance=True,
+                negative_prompt=negative_prompt,
+            )
+            prompt_embeds = torch.cat([prompt_embeds_, image_prompt_embeds], dim=1)
+            negative_prompt_embeds = torch.cat([negative_prompt_embeds_, uncond_image_prompt_embeds], dim=1)
+
+        generator = get_generator(seed, self.device)
+
+        images = self.pipe(
+            prompt_embeds=prompt_embeds,
+            negative_prompt_embeds=negative_prompt_embeds,
+            guidance_scale=guidance_scale,
+            num_inference_steps=num_inference_steps,
+            generator=generator,
+            num_images_per_prompt=num_samples,
+            **kwargs,
+        ).images
+
+        return images
+
+
+class IPAdapterFaceIDPlus:
+    def __init__(self, sd_pipe, image_encoder_path, ip_ckpt, device, num_tokens=4, torch_dtype=torch.float16):
+        self.device = device
+        self.image_encoder_path = image_encoder_path
+        self.ip_ckpt = ip_ckpt
+        self.num_tokens = num_tokens
+        self.torch_dtype = torch_dtype
+
+        self.pipe = sd_pipe.to(self.device)
+        self.set_ip_adapter()
+
+        # load image encoder
+        self.image_encoder = CLIPVisionModelWithProjection.from_pretrained(self.image_encoder_path).to(
+            self.device, dtype=self.torch_dtype
+        )
+        self.clip_image_processor = CLIPImageProcessor()
+        # image proj model
+        self.image_proj_model = self.init_proj()
+
+        self.load_ip_adapter()
+
+    def init_proj(self):
+        image_proj_model = ProjPlusModel(
+            cross_attention_dim=self.pipe.unet.config.cross_attention_dim,
+            id_embeddings_dim=512,
+            clip_embeddings_dim=self.image_encoder.config.hidden_size,
+            num_tokens=self.num_tokens,
+        ).to(self.device, dtype=self.torch_dtype)
+        return image_proj_model
+
+    def set_ip_adapter(self):
+        unet = self.pipe.unet
+        attn_procs = {}
+        for name in unet.attn_processors.keys():
+            cross_attention_dim = None if name.endswith("attn1.processor") else unet.config.cross_attention_dim
+            if name.startswith("mid_block"):
+                hidden_size = unet.config.block_out_channels[-1]
+            elif name.startswith("up_blocks"):
+                block_id = int(name[len("up_blocks.")])
+                hidden_size = list(reversed(unet.config.block_out_channels))[block_id]
+            elif name.startswith("down_blocks"):
+                block_id = int(name[len("down_blocks.")])
+                hidden_size = unet.config.block_out_channels[block_id]
+            if cross_attention_dim is None:
+                attn_procs[name] = AttnProcessor()
+            else:
+                attn_procs[name] = IPAttnProcessor(
+                    hidden_size=hidden_size, cross_attention_dim=cross_attention_dim, scale=1.0, num_tokens=self.num_tokens,
+                ).to(self.device, dtype=self.torch_dtype)
+        unet.set_attn_processor(attn_procs)
+
+    def load_ip_adapter(self):
+        if os.path.splitext(self.ip_ckpt)[-1] == ".safetensors":
+            state_dict = {"image_proj": {}, "ip_adapter": {}}
+            with safe_open(self.ip_ckpt, framework="pt", device="cpu") as f:
+                for key in f.keys():
+                    if key.startswith("image_proj."):
+                        state_dict["image_proj"][key.replace("image_proj.", "")] = f.get_tensor(key)
+                    elif key.startswith("ip_adapter."):
+                        state_dict["ip_adapter"][key.replace("ip_adapter.", "")] = f.get_tensor(key)
+        else:
+            state_dict = torch.load(self.ip_ckpt, map_location="cpu")
+        self.image_proj_model.load_state_dict(state_dict["image_proj"])
+        ip_layers = torch.nn.ModuleList(self.pipe.unet.attn_processors.values())
+        ip_layers.load_state_dict(state_dict["ip_adapter"], strict=False)
+
+    @torch.inference_mode()
+    def get_image_embeds(self, faceid_embeds, face_image, s_scale, shortcut):
+        if isinstance(face_image, Image.Image):
+            pil_image = [face_image]
+        clip_image = self.clip_image_processor(images=face_image, return_tensors="pt").pixel_values
+        clip_image = clip_image.to(self.device, dtype=self.torch_dtype)
+        clip_image_embeds = self.image_encoder(clip_image, output_hidden_states=True).hidden_states[-2]
+        uncond_clip_image_embeds = self.image_encoder(
+            torch.zeros_like(clip_image), output_hidden_states=True
+        ).hidden_states[-2]
+        
+        faceid_embeds = faceid_embeds.to(self.device, dtype=self.torch_dtype)
+        image_prompt_embeds = self.image_proj_model(faceid_embeds, clip_image_embeds, shortcut=shortcut, scale=s_scale)
+        uncond_image_prompt_embeds = self.image_proj_model(torch.zeros_like(faceid_embeds), uncond_clip_image_embeds, shortcut=shortcut, scale=s_scale)
+        return image_prompt_embeds, uncond_image_prompt_embeds
+
+    def set_scale(self, scale):
+        for attn_processor in self.pipe.unet.attn_processors.values():
+            if isinstance(attn_processor, LoRAIPAttnProcessor):
+                attn_processor.scale = scale
+
+    def generate(
+        self,
+        face_image=None,
+        faceid_embeds=None,
+        prompt=None,
+        negative_prompt=None,
+        scale=1.0,
+        num_samples=4,
+        seed=None,
+        guidance_scale=7.5,
+        num_inference_steps=30,
+        s_scale=1.0,
+        shortcut=False,
+        **kwargs,
+    ):
+        self.set_scale(scale)
+
+       
+        num_prompts = faceid_embeds.size(0)
+
+        if prompt is None:
+            prompt = "best quality, high quality"
+        if negative_prompt is None:
+            negative_prompt = "monochrome, lowres, bad anatomy, worst quality, low quality"
+
+        if not isinstance(prompt, List):
+            prompt = [prompt] * num_prompts
+        if not isinstance(negative_prompt, List):
+            negative_prompt = [negative_prompt] * num_prompts
+
+        image_prompt_embeds, uncond_image_prompt_embeds = self.get_image_embeds(faceid_embeds, face_image, s_scale, shortcut)
+
+        bs_embed, seq_len, _ = image_prompt_embeds.shape
+        image_prompt_embeds = image_prompt_embeds.repeat(1, num_samples, 1)
+        image_prompt_embeds = image_prompt_embeds.view(bs_embed * num_samples, seq_len, -1)
+        uncond_image_prompt_embeds = uncond_image_prompt_embeds.repeat(1, num_samples, 1)
+        uncond_image_prompt_embeds = uncond_image_prompt_embeds.view(bs_embed * num_samples, seq_len, -1)
+
+        with torch.inference_mode():
+            prompt_embeds_, negative_prompt_embeds_ = self.pipe.encode_prompt(
+                prompt,
+                device=self.device,
+                num_images_per_prompt=num_samples,
+                do_classifier_free_guidance=True,
+                negative_prompt=negative_prompt,
+            )
+            prompt_embeds = torch.cat([prompt_embeds_, image_prompt_embeds], dim=1)
+            negative_prompt_embeds = torch.cat([negative_prompt_embeds_, uncond_image_prompt_embeds], dim=1)
+
+        generator = get_generator(seed, self.device)
+
+        images = self.pipe(
+            prompt_embeds=prompt_embeds,
+            negative_prompt_embeds=negative_prompt_embeds,
+            guidance_scale=guidance_scale,
+            num_inference_steps=num_inference_steps,
+            generator=generator,
+            **kwargs,
+        ).images
+
+        return images
+
+
+class IPAdapterFaceIDXL(IPAdapterFaceID):
+    """SDXL"""
+
+    def generate(
+        self,
+        faceid_embeds=None,
+        prompt=None,
+        negative_prompt=None,
+        scale=1.0,
+        num_samples=4,
+        seed=None,
+        num_inference_steps=30,
+        **kwargs,
+    ):
+        self.set_scale(scale)
+
+        num_prompts = faceid_embeds.size(0)
+
+        if prompt is None:
+            prompt = "best quality, high quality"
+        if negative_prompt is None:
+            negative_prompt = "monochrome, lowres, bad anatomy, worst quality, low quality"
+
+        if not isinstance(prompt, List):
+            prompt = [prompt] * num_prompts
+        else:
+            faceid_embeds = faceid_embeds.repeat(num_samples, 1, 1)
+            num_samples = 1
+
+        if not isinstance(negative_prompt, List):
+            negative_prompt = [negative_prompt] * num_prompts
+
+        image_prompt_embeds, uncond_image_prompt_embeds = self.get_image_embeds(faceid_embeds)
+
+        bs_embed, seq_len, _ = image_prompt_embeds.shape
+        image_prompt_embeds = image_prompt_embeds.repeat(1, num_samples, 1)
+        image_prompt_embeds = image_prompt_embeds.view(bs_embed * num_samples, seq_len, -1)
+        uncond_image_prompt_embeds = uncond_image_prompt_embeds.repeat(1, num_samples, 1)
+        uncond_image_prompt_embeds = uncond_image_prompt_embeds.view(bs_embed * num_samples, seq_len, -1)
+
+        with torch.inference_mode():
+            (
+                prompt_embeds,
+                negative_prompt_embeds,
+                pooled_prompt_embeds,
+                negative_pooled_prompt_embeds,
+            ) = self.pipe.encode_prompt(
+                prompt,
+                num_images_per_prompt=num_samples,
+                do_classifier_free_guidance=True,
+                negative_prompt=negative_prompt,
+            )
+            prompt_embeds = torch.cat([prompt_embeds, image_prompt_embeds], dim=1)
+            negative_prompt_embeds = torch.cat([negative_prompt_embeds, uncond_image_prompt_embeds], dim=1)
+
+        generator = get_generator(seed, self.device)
+
+        images = self.pipe(
+            prompt_embeds=prompt_embeds,
+            negative_prompt_embeds=negative_prompt_embeds,
+            pooled_prompt_embeds=pooled_prompt_embeds,
+            negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
+            num_inference_steps=num_inference_steps,
+            generator=generator,
+            num_images_per_prompt=num_samples,
+            **kwargs,
+        ).images
+
+        return images
+
+
+class IPAdapterFaceIDPlusXL(IPAdapterFaceIDPlus):
+    """SDXL"""
+
+    def generate(
+        self,
+        face_image=None,
+        faceid_embeds=None,
+        prompt=None,
+        negative_prompt=None,
+        scale=1.0,
+        num_samples=4,
+        seed=None,
+        guidance_scale=7.5,
+        num_inference_steps=30,
+        s_scale=1.0,
+        shortcut=True,
+        **kwargs,
+    ):
+        self.set_scale(scale)
+
+        num_prompts = faceid_embeds.size(0)
+
+        if prompt is None:
+            prompt = "best quality, high quality"
+        if negative_prompt is None:
+            negative_prompt = "monochrome, lowres, bad anatomy, worst quality, low quality"
+
+        if not isinstance(prompt, List):
+            prompt = [prompt] * num_prompts
+        if not isinstance(negative_prompt, List):
+            negative_prompt = [negative_prompt] * num_prompts
+
+        image_prompt_embeds, uncond_image_prompt_embeds = self.get_image_embeds(faceid_embeds, face_image, s_scale, shortcut)
+
+        bs_embed, seq_len, _ = image_prompt_embeds.shape
+        image_prompt_embeds = image_prompt_embeds.repeat(1, num_samples, 1)
+        image_prompt_embeds = image_prompt_embeds.view(bs_embed * num_samples, seq_len, -1)
+        uncond_image_prompt_embeds = uncond_image_prompt_embeds.repeat(1, num_samples, 1)
+        uncond_image_prompt_embeds = uncond_image_prompt_embeds.view(bs_embed * num_samples, seq_len, -1)
+
+        with torch.inference_mode():
+            (
+                prompt_embeds,
+                negative_prompt_embeds,
+                pooled_prompt_embeds,
+                negative_pooled_prompt_embeds,
+            ) = self.pipe.encode_prompt(
+                prompt,
+                num_images_per_prompt=num_samples,
+                do_classifier_free_guidance=True,
+                negative_prompt=negative_prompt,
+            )
+            prompt_embeds = torch.cat([prompt_embeds, image_prompt_embeds], dim=1)
+            negative_prompt_embeds = torch.cat([negative_prompt_embeds, uncond_image_prompt_embeds], dim=1)
+
+        generator = get_generator(seed, self.device)
+
+        images = self.pipe(
+            prompt_embeds=prompt_embeds,
+            negative_prompt_embeds=negative_prompt_embeds,
+            pooled_prompt_embeds=pooled_prompt_embeds,
+            negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
+            num_inference_steps=num_inference_steps,
+            generator=generator,
+            guidance_scale=guidance_scale,
+            **kwargs,
+        ).images
+
+        return images

ip_adapter/resampler.py

@@ -0,0 +1,158 @@
+# modified from https://github.com/mlfoundations/open_flamingo/blob/main/open_flamingo/src/helpers.py
+# and https://github.com/lucidrains/imagen-pytorch/blob/main/imagen_pytorch/imagen_pytorch.py
+
+import math
+
+import torch
+import torch.nn as nn
+from einops import rearrange
+from einops.layers.torch import Rearrange
+
+
+# FFN
+def FeedForward(dim, mult=4):
+    inner_dim = int(dim * mult)
+    return nn.Sequential(
+        nn.LayerNorm(dim),
+        nn.Linear(dim, inner_dim, bias=False),
+        nn.GELU(),
+        nn.Linear(inner_dim, dim, bias=False),
+    )
+
+
+def reshape_tensor(x, heads):
+    bs, length, width = x.shape
+    # (bs, length, width) --> (bs, length, n_heads, dim_per_head)
+    x = x.view(bs, length, heads, -1)
+    # (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head)
+    x = x.transpose(1, 2)
+    # (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head)
+    x = x.reshape(bs, heads, length, -1)
+    return x
+
+
+class PerceiverAttention(nn.Module):
+    def __init__(self, *, dim, dim_head=64, heads=8):
+        super().__init__()
+        self.scale = dim_head**-0.5
+        self.dim_head = dim_head
+        self.heads = heads
+        inner_dim = dim_head * heads
+
+        self.norm1 = nn.LayerNorm(dim)
+        self.norm2 = nn.LayerNorm(dim)
+
+        self.to_q = nn.Linear(dim, inner_dim, bias=False)
+        self.to_kv = nn.Linear(dim, inner_dim * 2, bias=False)
+        self.to_out = nn.Linear(inner_dim, dim, bias=False)
+
+    def forward(self, x, latents):
+        """
+        Args:
+            x (torch.Tensor): image features
+                shape (b, n1, D)
+            latent (torch.Tensor): latent features
+                shape (b, n2, D)
+        """
+        x = self.norm1(x)
+        latents = self.norm2(latents)
+
+        b, l, _ = latents.shape
+
+        q = self.to_q(latents)
+        kv_input = torch.cat((x, latents), dim=-2)
+        k, v = self.to_kv(kv_input).chunk(2, dim=-1)
+
+        q = reshape_tensor(q, self.heads)
+        k = reshape_tensor(k, self.heads)
+        v = reshape_tensor(v, self.heads)
+
+        # attention
+        scale = 1 / math.sqrt(math.sqrt(self.dim_head))
+        weight = (q * scale) @ (k * scale).transpose(-2, -1)  # More stable with f16 than dividing afterwards
+        weight = torch.softmax(weight.float(), dim=-1).type(weight.dtype)
+        out = weight @ v
+
+        out = out.permute(0, 2, 1, 3).reshape(b, l, -1)
+
+        return self.to_out(out)
+
+
+class Resampler(nn.Module):
+    def __init__(
+        self,
+        dim=1024,
+        depth=8,
+        dim_head=64,
+        heads=16,
+        num_queries=8,
+        embedding_dim=768,
+        output_dim=1024,
+        ff_mult=4,
+        max_seq_len: int = 257,  # CLIP tokens + CLS token
+        apply_pos_emb: bool = False,
+        num_latents_mean_pooled: int = 0,  # number of latents derived from mean pooled representation of the sequence
+    ):
+        super().__init__()
+        self.pos_emb = nn.Embedding(max_seq_len, embedding_dim) if apply_pos_emb else None
+
+        self.latents = nn.Parameter(torch.randn(1, num_queries, dim) / dim**0.5)
+
+        self.proj_in = nn.Linear(embedding_dim, dim)
+
+        self.proj_out = nn.Linear(dim, output_dim)
+        self.norm_out = nn.LayerNorm(output_dim)
+
+        self.to_latents_from_mean_pooled_seq = (
+            nn.Sequential(
+                nn.LayerNorm(dim),
+                nn.Linear(dim, dim * num_latents_mean_pooled),
+                Rearrange("b (n d) -> b n d", n=num_latents_mean_pooled),
+            )
+            if num_latents_mean_pooled > 0
+            else None
+        )
+
+        self.layers = nn.ModuleList([])
+        for _ in range(depth):
+            self.layers.append(
+                nn.ModuleList(
+                    [
+                        PerceiverAttention(dim=dim, dim_head=dim_head, heads=heads),
+                        FeedForward(dim=dim, mult=ff_mult),
+                    ]
+                )
+            )
+
+    def forward(self, x):
+        if self.pos_emb is not None:
+            n, device = x.shape[1], x.device
+            pos_emb = self.pos_emb(torch.arange(n, device=device))
+            x = x + pos_emb
+
+        latents = self.latents.repeat(x.size(0), 1, 1)
+
+        x = self.proj_in(x)
+
+        if self.to_latents_from_mean_pooled_seq:
+            meanpooled_seq = masked_mean(x, dim=1, mask=torch.ones(x.shape[:2], device=x.device, dtype=torch.bool))
+            meanpooled_latents = self.to_latents_from_mean_pooled_seq(meanpooled_seq)
+            latents = torch.cat((meanpooled_latents, latents), dim=-2)
+
+        for attn, ff in self.layers:
+            latents = attn(x, latents) + latents
+            latents = ff(latents) + latents
+
+        latents = self.proj_out(latents)
+        return self.norm_out(latents)
+
+
+def masked_mean(t, *, dim, mask=None):
+    if mask is None:
+        return t.mean(dim=dim)
+
+    denom = mask.sum(dim=dim, keepdim=True)
+    mask = rearrange(mask, "b n -> b n 1")
+    masked_t = t.masked_fill(~mask, 0.0)
+
+    return masked_t.sum(dim=dim) / denom.clamp(min=1e-5)

ip_adapter/sd3_attention_processor.py

@@ -0,0 +1,179 @@
+from typing import Callable, List, Optional, Union
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+from diffusers.models.attention_processor import Attention
+
+
+class JointAttnProcessor2_0:
+    """Attention processor used typically in processing the SD3-like self-attention projections."""
+
+    def __init__(self):
+        if not hasattr(F, "scaled_dot_product_attention"):
+            raise ImportError("AttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.")
+
+    def __call__(
+        self,
+        attn: Attention,
+        hidden_states: torch.FloatTensor,
+        encoder_hidden_states: torch.FloatTensor = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        *args,
+        **kwargs,
+    ) -> torch.FloatTensor:
+        residual = hidden_states
+
+        input_ndim = hidden_states.ndim
+        if input_ndim == 4:
+            batch_size, channel, height, width = hidden_states.shape
+            hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
+        context_input_ndim = encoder_hidden_states.ndim
+        if context_input_ndim == 4:
+            batch_size, channel, height, width = encoder_hidden_states.shape
+            encoder_hidden_states = encoder_hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
+
+        batch_size = encoder_hidden_states.shape[0]
+
+        # `sample` projections.
+        query = attn.to_q(hidden_states)
+        key = attn.to_k(hidden_states)
+        value = attn.to_v(hidden_states)
+
+        # `context` projections.
+        encoder_hidden_states_query_proj = attn.add_q_proj(encoder_hidden_states)
+        encoder_hidden_states_key_proj = attn.add_k_proj(encoder_hidden_states)
+        encoder_hidden_states_value_proj = attn.add_v_proj(encoder_hidden_states)
+
+        # attention
+        query = torch.cat([query, encoder_hidden_states_query_proj], dim=1)
+        key = torch.cat([key, encoder_hidden_states_key_proj], dim=1)
+        value = torch.cat([value, encoder_hidden_states_value_proj], dim=1)
+
+        inner_dim = key.shape[-1]
+        head_dim = inner_dim // attn.heads
+        query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+        key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+        value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+
+        hidden_states = F.scaled_dot_product_attention(query, key, value, dropout_p=0.0, is_causal=False)
+        hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
+        hidden_states = hidden_states.to(query.dtype)
+
+        # Split the attention outputs.
+        hidden_states, encoder_hidden_states = (
+            hidden_states[:, : residual.shape[1]],
+            hidden_states[:, residual.shape[1] :],
+        )
+
+        # linear proj
+        hidden_states = attn.to_out[0](hidden_states)
+        # dropout
+        hidden_states = attn.to_out[1](hidden_states)
+        if not attn.context_pre_only:
+            encoder_hidden_states = attn.to_add_out(encoder_hidden_states)
+
+        if input_ndim == 4:
+            hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
+        if context_input_ndim == 4:
+            encoder_hidden_states = encoder_hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
+
+        return hidden_states, encoder_hidden_states
+
+
+class IPJointAttnProcessor2_0(torch.nn.Module):
+    """Attention processor used typically in processing the SD3-like self-attention projections."""
+
+    def __init__(self, context_dim, hidden_dim, scale=1.0):
+        if not hasattr(F, "scaled_dot_product_attention"):
+            raise ImportError("AttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.")
+        super().__init__()
+        self.scale = scale
+
+        self.add_k_proj_ip = nn.Linear(context_dim, hidden_dim)
+        self.add_v_proj_ip = nn.Linear(context_dim, hidden_dim)
+
+
+    def __call__(
+        self,
+        attn: Attention,
+        hidden_states: torch.FloatTensor,
+        encoder_hidden_states: torch.FloatTensor = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        ip_hidden_states: torch.FloatTensor = None,
+        *args,
+        **kwargs,
+    ) -> torch.FloatTensor:
+        residual = hidden_states
+
+        input_ndim = hidden_states.ndim
+        if input_ndim == 4:
+            batch_size, channel, height, width = hidden_states.shape
+            hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
+        context_input_ndim = encoder_hidden_states.ndim
+        if context_input_ndim == 4:
+            batch_size, channel, height, width = encoder_hidden_states.shape
+            encoder_hidden_states = encoder_hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
+
+        batch_size = encoder_hidden_states.shape[0]
+
+        # `sample` projections.
+        query = attn.to_q(hidden_states)
+        key = attn.to_k(hidden_states)
+        value = attn.to_v(hidden_states)
+
+        sample_query = query # latent query
+
+        # `context` projections.
+        encoder_hidden_states_query_proj = attn.add_q_proj(encoder_hidden_states)
+        encoder_hidden_states_key_proj = attn.add_k_proj(encoder_hidden_states)
+        encoder_hidden_states_value_proj = attn.add_v_proj(encoder_hidden_states)
+
+        # attention
+        query = torch.cat([query, encoder_hidden_states_query_proj], dim=1)
+        key = torch.cat([key, encoder_hidden_states_key_proj], dim=1)
+        value = torch.cat([value, encoder_hidden_states_value_proj], dim=1)
+
+        inner_dim = key.shape[-1]
+        head_dim = inner_dim // attn.heads
+        query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+        key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+        value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+
+        hidden_states = F.scaled_dot_product_attention(query, key, value, dropout_p=0.0, is_causal=False)
+        hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
+        hidden_states = hidden_states.to(query.dtype)
+
+        # Split the attention outputs.
+        hidden_states, encoder_hidden_states = (
+            hidden_states[:, : residual.shape[1]],
+            hidden_states[:, residual.shape[1] :],
+        )
+
+        # for ip-adapter
+        ip_key = self.add_k_proj_ip(ip_hidden_states)
+        ip_value = self.add_v_proj_ip(ip_hidden_states)
+        ip_query = sample_query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+        ip_key = ip_key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+        ip_value = ip_value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) 
+        
+        ip_hidden_states = F.scaled_dot_product_attention(ip_query, ip_key, ip_value, dropout_p=0.0, is_causal=False)
+        ip_hidden_states = ip_hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
+        ip_hidden_states = ip_hidden_states.to(ip_query.dtype)
+
+        hidden_states = hidden_states + self.scale * ip_hidden_states
+
+        # linear proj
+        hidden_states = attn.to_out[0](hidden_states)
+        # dropout
+        hidden_states = attn.to_out[1](hidden_states)
+        if not attn.context_pre_only:
+            encoder_hidden_states = attn.to_add_out(encoder_hidden_states)
+
+        if input_ndim == 4:
+            hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
+        if context_input_ndim == 4:
+            encoder_hidden_states = encoder_hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
+
+        return hidden_states, encoder_hidden_states
+

ip_adapter/test_resampler.py

@@ -0,0 +1,44 @@
+import torch
+from resampler import Resampler
+from transformers import CLIPVisionModel
+
+BATCH_SIZE = 2
+OUTPUT_DIM = 1280
+NUM_QUERIES = 8
+NUM_LATENTS_MEAN_POOLED = 4  # 0 for no mean pooling (previous behavior)
+APPLY_POS_EMB = True  # False for no positional embeddings (previous behavior)
+IMAGE_ENCODER_NAME_OR_PATH = "laion/CLIP-ViT-H-14-laion2B-s32B-b79K"
+
+
+def main():
+    image_encoder = CLIPVisionModel.from_pretrained(IMAGE_ENCODER_NAME_OR_PATH)
+    embedding_dim = image_encoder.config.hidden_size
+    print(f"image_encoder hidden size: ", embedding_dim)
+
+    image_proj_model = Resampler(
+        dim=1024,
+        depth=2,
+        dim_head=64,
+        heads=16,
+        num_queries=NUM_QUERIES,
+        embedding_dim=embedding_dim,
+        output_dim=OUTPUT_DIM,
+        ff_mult=2,
+        max_seq_len=257,
+        apply_pos_emb=APPLY_POS_EMB,
+        num_latents_mean_pooled=NUM_LATENTS_MEAN_POOLED,
+    )
+
+    dummy_images = torch.randn(BATCH_SIZE, 3, 224, 224)
+    with torch.no_grad():
+        image_embeds = image_encoder(dummy_images, output_hidden_states=True).hidden_states[-2]
+    print("image_embds shape: ", image_embeds.shape)
+
+    with torch.no_grad():
+        ip_tokens = image_proj_model(image_embeds)
+    print("ip_tokens shape:", ip_tokens.shape)
+    assert ip_tokens.shape == (BATCH_SIZE, NUM_QUERIES + NUM_LATENTS_MEAN_POOLED, OUTPUT_DIM)
+
+
+if __name__ == "__main__":
+    main()

ip_adapter/utils.py

@@ -0,0 +1,93 @@
+import torch
+import torch.nn.functional as F
+import numpy as np
+from PIL import Image
+
+attn_maps = {}
+def hook_fn(name):
+    def forward_hook(module, input, output):
+        if hasattr(module.processor, "attn_map"):
+            attn_maps[name] = module.processor.attn_map
+            del module.processor.attn_map
+
+    return forward_hook
+
+def register_cross_attention_hook(unet):
+    for name, module in unet.named_modules():
+        if name.split('.')[-1].startswith('attn2'):
+            module.register_forward_hook(hook_fn(name))
+
+    return unet
+
+def upscale(attn_map, target_size):
+    attn_map = torch.mean(attn_map, dim=0)
+    attn_map = attn_map.permute(1,0)
+    temp_size = None
+
+    for i in range(0,5):
+        scale = 2 ** i
+        if ( target_size[0] // scale ) * ( target_size[1] // scale) == attn_map.shape[1]*64:
+            temp_size = (target_size[0]//(scale*8), target_size[1]//(scale*8))
+            break
+
+    assert temp_size is not None, "temp_size cannot is None"
+
+    attn_map = attn_map.view(attn_map.shape[0], *temp_size)
+
+    attn_map = F.interpolate(
+        attn_map.unsqueeze(0).to(dtype=torch.float32),
+        size=target_size,
+        mode='bilinear',
+        align_corners=False
+    )[0]
+
+    attn_map = torch.softmax(attn_map, dim=0)
+    return attn_map
+def get_net_attn_map(image_size, batch_size=2, instance_or_negative=False, detach=True):
+
+    idx = 0 if instance_or_negative else 1
+    net_attn_maps = []
+
+    for name, attn_map in attn_maps.items():
+        attn_map = attn_map.cpu() if detach else attn_map
+        attn_map = torch.chunk(attn_map, batch_size)[idx].squeeze()
+        attn_map = upscale(attn_map, image_size) 
+        net_attn_maps.append(attn_map) 
+
+    net_attn_maps = torch.mean(torch.stack(net_attn_maps,dim=0),dim=0)
+
+    return net_attn_maps
+
+def attnmaps2images(net_attn_maps):
+
+    #total_attn_scores = 0
+    images = []
+
+    for attn_map in net_attn_maps:
+        attn_map = attn_map.cpu().numpy()
+        #total_attn_scores += attn_map.mean().item()
+
+        normalized_attn_map = (attn_map - np.min(attn_map)) / (np.max(attn_map) - np.min(attn_map)) * 255
+        normalized_attn_map = normalized_attn_map.astype(np.uint8)
+        #print("norm: ", normalized_attn_map.shape)
+        image = Image.fromarray(normalized_attn_map)
+
+        #image = fix_save_attn_map(attn_map)
+        images.append(image)
+
+    #print(total_attn_scores)
+    return images
+def is_torch2_available():
+    return hasattr(F, "scaled_dot_product_attention")
+
+def get_generator(seed, device):
+
+    if seed is not None:
+        if isinstance(seed, list):
+            generator = [torch.Generator(device).manual_seed(seed_item) for seed_item in seed]
+        else:
+            generator = torch.Generator(device).manual_seed(seed)
+    else:
+        generator = None
+
+    return generator

ipadapter_model.py

@@ -0,0 +1,314 @@
+"""
+IP-Adapter Model Interface
+
+This module provides utilities for working with IP-Adapter models, including:
+- Loading Stable Diffusion pipelines with IP-Adapter
+- Extracting CLIP embeddings from images
+- Generating images from CLIP embeddings
+- Utility functions for image processing
+"""
+
+from typing import List, Optional, Union, Tuple
+
+import numpy as np
+import torch
+from PIL import Image
+from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline, DDIMScheduler, AutoencoderKL
+
+# Fix for torch 2.5.0 compatibility
+torch.backends.cuda.enable_cudnn_sdp(False)
+
+from ip_adapter import IPAdapterPlus, IPAdapterPlusXL
+
+
+# ===== Image Utility Functions =====
+
+def create_image_grid(images: List[Image.Image], rows: int, cols: int) -> Image.Image:
+    # Get dimensions from first image (assumes all images are same size)
+    width, height = images[0].size
+    
+    # Create empty grid canvas
+    grid = Image.new('RGB', size=(cols * width, rows * height))
+    
+    # Paste each image into the grid
+    for i, img in enumerate(images):
+        x_pos = (i % cols) * width
+        y_pos = (i // cols) * height
+        grid.paste(img, box=(x_pos, y_pos))
+    
+    return grid
+
+
+# ===== CLIP Embedding Extraction Functions =====
+
+@torch.inference_mode()
+def extract_clip_embeddings_from_pil(pil_image: Union[Image.Image, List[Image.Image]], 
+                                    ip_model) -> torch.Tensor:
+    """
+    Returns:
+        torch.Tensor: CLIP embeddings of shape (batch_size, seq_len, embed_dim)
+    """
+    if isinstance(pil_image, Image.Image):
+        pil_image = [pil_image]
+    
+    # Process images through CLIP processor
+    processed_images = ip_model.clip_image_processor(
+        images=pil_image, return_tensors="pt"
+    ).pixel_values
+    
+    # Move to model device with appropriate dtype
+    processed_images = processed_images.to(ip_model.device, dtype=torch.float16)
+    
+    # Extract embeddings from penultimate layer (better for downstream tasks)
+    clip_embeddings = ip_model.image_encoder(
+        processed_images, output_hidden_states=True
+    ).hidden_states[-2]
+    
+    # Convert to float32 for better numerical stability
+    return clip_embeddings.float()
+
+
+@torch.inference_mode()
+def extract_clip_embeddings_from_pil_batch(pil_images: List[Image.Image], 
+                                          ip_model) -> torch.Tensor:
+    """
+    Returns:
+        torch.Tensor: Concatenated CLIP embeddings of shape (batch, seq_len, embed_dim)
+    """
+    embeddings_batch = []
+    
+    for image in pil_images:
+        embeddings = extract_clip_embeddings_from_pil(image, ip_model)
+        embeddings_batch.append(embeddings)
+    
+    return torch.cat(embeddings_batch, dim=0)
+
+
+@torch.inference_mode()
+def extract_clip_embeddings_from_tensor(tensor_image: torch.Tensor, 
+                                       ip_model, 
+                                       resize: bool = True) -> torch.Tensor:
+    """
+    Returns:
+        torch.Tensor: CLIP embeddings of shape (batch_size, seq_len, embed_dim)
+    """
+    # Move tensor to model device with appropriate dtype
+    tensor_image = tensor_image.to(ip_model.device, dtype=torch.float16)
+    
+    # Resize to CLIP input resolution if requested
+    if resize:
+        tensor_image = torch.nn.functional.interpolate(
+            tensor_image, 
+            size=(224, 224), 
+            mode="bilinear", 
+            align_corners=False
+        )
+    
+    # Extract embeddings with positional encoding interpolation
+    clip_embeddings = ip_model.image_encoder(
+        tensor_image, 
+        output_hidden_states=True, 
+        interpolate_pos_encoding=True
+    ).hidden_states[-2]
+    
+    # Convert to float32 for numerical stability
+    return clip_embeddings.float()
+
+
+# ===== IP-Adapter Helper Functions =====
+
+@torch.inference_mode()
+def _enhanced_get_image_embeds(self, pil_image=None, clip_image_embeds=None):
+    """
+    Enhanced version of IP-Adapter's get_image_embeds method.
+    
+    This method processes either PIL images or pre-computed CLIP embeddings
+    and returns both conditional and unconditional embeddings for generation.
+    
+    Args:
+        pil_image: PIL Image(s) to process (optional)
+        clip_image_embeds: Pre-computed CLIP embeddings (optional)
+        
+    Returns:
+        Tuple of (conditional_embeds, unconditional_embeds)
+    """
+    # Process PIL images if provided
+    if pil_image is not None:
+        if isinstance(pil_image, Image.Image):
+            pil_image = [pil_image]
+        
+        # Convert PIL to tensor and extract CLIP embeddings
+        processed_images = self.clip_image_processor(
+            images=pil_image, return_tensors="pt"
+        ).pixel_values
+        processed_images = processed_images.to(self.device, dtype=torch.float16)
+        
+        clip_image_embeds = self.image_encoder(
+            processed_images, output_hidden_states=True
+        ).hidden_states[-2]
+    
+    # Project CLIP embeddings to IP-Adapter space
+    conditional_embeds = self.image_proj_model(clip_image_embeds)
+    
+    # Generate unconditional embeddings (for classifier-free guidance)
+    zero_tensor = torch.zeros(1, 3, 224, 224).to(self.device, dtype=torch.float16)
+    uncond_clip_embeds = self.image_encoder(
+        zero_tensor, output_hidden_states=True
+    ).hidden_states[-2]
+    unconditional_embeds = self.image_proj_model(uncond_clip_embeds)
+    
+    return conditional_embeds, unconditional_embeds
+
+
+# ===== Model Loading Functions =====
+
+@torch.inference_mode()
+def load_stable_diffusion_pipeline(device: str = "cuda") -> StableDiffusionPipeline:
+    # Model paths
+    base_model_path = "SG161222/Realistic_Vision_V4.0_noVAE"
+    vae_model_path = "stabilityai/sd-vae-ft-mse"
+
+    # Configure DDIM scheduler for high-quality sampling
+    noise_scheduler = DDIMScheduler(
+        num_train_timesteps=1000,
+        beta_start=0.00085,
+        beta_end=0.012,
+        beta_schedule="scaled_linear",
+        clip_sample=False,
+        set_alpha_to_one=False,
+        steps_offset=1,
+    )
+    
+    # Load VAE separately for better quality
+    vae = AutoencoderKL.from_pretrained(vae_model_path).to(dtype=torch.float16)
+    
+    # Create Stable Diffusion pipeline
+    pipeline = StableDiffusionPipeline.from_pretrained(
+        base_model_path,
+        torch_dtype=torch.float16,
+        scheduler=noise_scheduler,
+        vae=vae,
+        feature_extractor=None,  # Disable safety checker for faster inference
+        safety_checker=None,
+    )
+    
+    return pipeline
+
+
+@torch.inference_mode()
+def load_ip_adapter_model(device: str = "cuda", sd_only: bool = False) -> IPAdapterPlus:
+    # Model and checkpoint paths
+    base_model_path = "SG161222/Realistic_Vision_V4.0_noVAE"
+    vae_model_path = "stabilityai/sd-vae-ft-mse"
+    image_encoder_path = "./downloads/models/image_encoder"
+    ip_checkpoint_path = "./downloads/models/ip-adapter-plus_sd15.bin"
+
+    # Configure DDIM scheduler
+    noise_scheduler = DDIMScheduler(
+        num_train_timesteps=1000,
+        beta_start=0.00085,
+        beta_end=0.012,
+        beta_schedule="scaled_linear",
+        clip_sample=False,
+        set_alpha_to_one=False,
+        steps_offset=1,
+    )
+    
+    # Load high-quality VAE
+    vae = AutoencoderKL.from_pretrained(vae_model_path).to(dtype=torch.float16)
+    
+    # Create base Stable Diffusion pipeline
+    pipeline = StableDiffusionPipeline.from_pretrained(
+        base_model_path,
+        torch_dtype=torch.float16,
+        scheduler=noise_scheduler,
+        vae=vae,
+        feature_extractor=None,
+        safety_checker=None,
+    )
+    
+    if sd_only:
+        return pipeline
+    
+    # Initialize IP-Adapter with 16 tokens for better image conditioning
+    ip_model = IPAdapterPlus(
+        pipeline, 
+        image_encoder_path, 
+        ip_checkpoint_path, 
+        device, 
+        num_tokens=16
+    )
+
+    # Enhance the model with our improved get_image_embeds method
+    setattr(ip_model.__class__, "get_image_embeds", _enhanced_get_image_embeds)
+    
+    return ip_model
+
+
+def load_ip_adapter_xl_model(device: str = "cuda") -> IPAdapterPlusXL:
+    base_model_path = "SG161222/RealVisXL_V1.0"
+    image_encoder_path = "./downloads/models/image_encoder"
+    ip_ckpt = "./downloads/sdxl_models/ip-adapter-plus_sdxl_vit-h.bin"
+
+    pipe = StableDiffusionXLPipeline.from_pretrained(
+        base_model_path,
+        torch_dtype=torch.float16,
+        add_watermarker=False,
+    )
+    ip_model = IPAdapterPlusXL(pipe, image_encoder_path, ip_ckpt, device, num_tokens=16)
+
+    return ip_model
+
+def load_ipadapter(version: str = "sd15", device: str = "cuda") -> IPAdapterPlus | IPAdapterPlusXL:
+    if version == "sd15":
+        return load_ip_adapter_model(device)
+    elif version == "sdxl":
+        return load_ip_adapter_xl_model(device)
+    else:
+        raise ValueError(f"Invalid version: {version}")
+
+
+# ===== Image Generation Functions =====
+
+@torch.inference_mode()
+def generate_images_from_clip_embeddings(ip_model : IPAdapterPlus,
+                                       clip_embeddings: torch.Tensor,
+                                       num_samples: int = 4, 
+                                       num_inference_steps: int = 50, 
+                                       seed: Optional[int] = 42) -> List[Image.Image]:
+    """Generate images from CLIP embeddings using IP-Adapter.
+    clip_embeddings is (batch, seq_len, embed_dim)
+    """
+    # Ensure embeddings have correct shape and dtype
+    if clip_embeddings.ndim == 2:
+        clip_embeddings = clip_embeddings.unsqueeze(0)
+    
+    if clip_embeddings.ndim != 3:
+        raise ValueError(f"Expected 3D embeddings (batch, seq, dim), got {clip_embeddings.shape}")
+    
+    # Move to appropriate device and dtype
+    clip_embeddings = clip_embeddings.half().to(ip_model.device)
+    
+    # Generate images using IP-Adapter
+    negative_prompt = "nsfw, lowres, (bad), text, error, fewer, extra, missing, worst quality, jpeg artifacts, low quality, watermark, unfinished, displeasing, oldest, early, chromatic aberration, signature, extra digits, artistic error, username, scan, [abstract]"
+    generated_images = ip_model.generate(
+        clip_image_embeds=clip_embeddings,
+        negative_prompt=negative_prompt,
+        pil_image=None,
+        num_samples=num_samples,
+        num_inference_steps=num_inference_steps,
+        seed=seed
+    )
+    
+    return generated_images
+
+
+# ===== Legacy Function Aliases =====
+
+# Maintain backward compatibility with existing code
+image_grid = create_image_grid
+extract_clip_embedding_pil = extract_clip_embeddings_from_pil
+extract_clip_embedding_pil_batch = extract_clip_embeddings_from_pil_batch
+extract_clip_embedding_tensor = extract_clip_embeddings_from_tensor
+load_sdxl = load_stable_diffusion_pipeline
+generate = generate_images_from_clip_embeddings

requirements.txt

@@ -0,0 +1,14 @@
+torch
+torchvision
+einops
+matplotlib
+opencv-python
+pillow
+scikit-image
+omegaconf
+scikit-dimension
+pytorch-lightning==1.9.4
+diffusers==0.33.1
+transformers==4.47.0
+triton==3.0.0
+ncut_pytorch==2.3.0

vibe_blending.py

@@ -0,0 +1,230 @@
+import numpy as np
+import torch
+from PIL import Image
+from typing import List, Optional, Tuple
+import torch.nn.functional as F
+from omegaconf import OmegaConf
+from ipadapter_model import generate_images_from_clip_embeddings
+from ipadapter_model import load_ipadapter
+from intrinsic_dim import estimate_intrinsic_dimension
+from vibespace_model import VibeSpaceModel, train_vibe_space, clear_gpu_memory
+from dino_correspondence import kway_cluster_per_image, match_centers_two_images, get_cluster_center_features
+
+from extract_features import extract_dino_features, extract_clip_features, dino_image_transform, clip_image_transform
+import logging
+import gradio as gr
+
+
+
+DEFAULT_CONFIG_PATH = "./config.yaml"
+def load_config(config_path: str):
+    cfg_base = OmegaConf.load(DEFAULT_CONFIG_PATH)
+    cfg = OmegaConf.load(config_path)
+    cfg_base.update(cfg)
+    return cfg_base
+
+
+def run_vibe_blend_safe(image1, image2, extra_images, negative_images, config_path, interpolation_weights: List[float], n_clusters: int = 25):
+    success = False
+    while not success:
+        try:
+            model, trainer = run_vibe_space_training(
+                positive_images=[image1, image2, *extra_images],
+                negative_images=negative_images,
+                config_path=config_path,
+            )
+            success = True
+        except Exception as e:
+            logging.error(f"Error training model: {e}")
+            torch.cuda.empty_cache()
+            continue
+        
+    success = False
+    while not success:
+        try:
+            blended_images = generate_blend_images(
+                image1, 
+                image2, 
+                model,
+                interpolation_weights,
+                n_clusters=n_clusters, 
+            )
+            success = True
+        except Exception as e:
+            logging.error(f"Error generating images: {e}")
+            torch.cuda.empty_cache()
+            continue
+
+    return blended_images
+
+
+def run_vibe_blend_not_safe(image1, image2, extra_images, negative_images, config_path, interpolation_weights: List[float], n_clusters: int = 20):
+    
+    model, trainer = run_vibe_space_training(
+        positive_images=[image1, image2, *extra_images],
+        negative_images=negative_images,
+        config_path=config_path,
+    )
+    blended_images = generate_blend_images(
+        image1, 
+        image2, 
+        model,
+        interpolation_weights,
+        n_clusters=n_clusters, 
+    )
+    return blended_images
+
+
+def run_vibe_space_training(positive_images: List[Image.Image], 
+                            negative_images: List[Image.Image],
+                            config_path: str = DEFAULT_CONFIG_PATH) -> Tuple[VibeSpaceModel, object]:
+    """
+    Train a Mood Space compression model from input images.
+    
+    This function extracts DINO and CLIP features from the input images,
+    estimates the intrinsic dimensionality if not provided, and trains
+    a neural compression model to learn a meaningful embedding space.
+    
+    Args:
+        pil_images: List of PIL Images for training
+    """
+    # Load and configure training parameters
+    config = load_config(config_path)
+    positive_images = [img for img in positive_images if img is not None]
+    negative_images = [img for img in negative_images or [] if img is not None]
+    if len(positive_images) == 0:
+        raise ValueError("No valid positive images provided for Vibe Space training")
+    has_negative_images = len(negative_images) > 0
+    
+    # Transform images for feature extraction
+    dino_input_images = torch.stack([dino_image_transform(image) for image in positive_images])
+    clip_input_images = torch.stack([clip_image_transform(image) for image in positive_images])
+    if has_negative_images:
+        negative_dino_input_images = torch.stack([dino_image_transform(image) for image in negative_images])
+    else:
+        negative_dino_input_images = None
+    
+    # Extract features using pre-trained models
+    dino_image_embeds = extract_dino_features(dino_input_images)
+    clip_image_embeds = extract_clip_features(clip_input_images)
+    if has_negative_images:
+        negative_dino_embeds = extract_dino_features(negative_dino_input_images)
+    else:
+        negative_dino_embeds = None
+    
+    # Determine intrinsic dimensionality
+    flattened_features = dino_image_embeds.flatten(end_dim=-2)
+    estimated_dim = estimate_intrinsic_dimension(flattened_features)
+    hidden_dim = int(estimated_dim)
+    config.vibe_dim = hidden_dim
+    
+    if len(positive_images) > 2:
+        # increase training steps for extra images
+        config.steps = config.steps * 2
+    
+    # Create and train model
+    model = VibeSpaceModel(config, enable_gradio_progress=True)
+    trainer = train_vibe_space(
+        model,
+        config,
+        dino_image_embeds,
+        clip_image_embeds,
+        negative_dino_embeds,
+    )
+    
+    return model, trainer
+
+
+def _compute_direction_from_two_images(image_embeds: torch.Tensor, 
+                                    eigenvectors: torch.Tensor | List[torch.Tensor],
+                                    a_to_b_mapping: np.ndarray, 
+                                    use_unit_norm: bool = False) -> torch.Tensor:
+    
+    # Compute cluster centers
+    a_center_features = get_cluster_center_features(
+        image_embeds[0], eigenvectors[0].argmax(-1).cpu(), eigenvectors[0].shape[-1])
+    b_center_features = get_cluster_center_features(
+        image_embeds[1], eigenvectors[1].argmax(-1).cpu(), eigenvectors[1].shape[-1])
+    
+    # Compute direction vectors
+    direction_vectors = []
+    for i_a, i_b in enumerate(a_to_b_mapping):
+        direction = b_center_features[i_b] - a_center_features[i_a]
+        if use_unit_norm:
+            direction = F.normalize(direction, dim=-1)
+        direction_vectors.append(direction)
+    direction_vectors = torch.stack(direction_vectors)
+    
+    
+    # Apply direction based on cluster assignments
+    cluster_labels = eigenvectors[0].argmax(-1).cpu()
+    direction_field = torch.zeros_like(image_embeds[0])
+    
+    for i_cluster in range(eigenvectors[0].shape[-1]):
+        cluster_mask = cluster_labels == i_cluster
+        if cluster_mask.sum() > 0:
+            direction_field[cluster_mask] = direction_vectors[i_cluster]
+    
+    return direction_field
+
+
+def generate_blend_images(image1: Image.Image, 
+                        image2: Image.Image,
+                        model: VibeSpaceModel, 
+                        interpolation_weights: List[float],
+                        n_clusters: int = 20, 
+                        seed: Optional[int] = None,
+                        ) -> List[Image.Image]:
+    """
+    Interpolate between two images using the trained compression model.
+    
+    Args:
+        image1, image2: Input PIL Images
+        model: Trained compression model
+        interpolation_weights: Weights for interpolation
+        n_clusters: Number of clusters for correspondence matching
+        seed: Random seed for generation
+        
+    Returns:
+        List[Image.Image]: Generated interpolated images
+    """
+    clear_gpu_memory()
+    
+    # Prepare images and extract features
+    images = torch.stack([dino_image_transform(img) for img in [image1, image2]])
+    dino_image_embeds = extract_dino_features(images)
+    compressed_image_embeds = model.encoder(dino_image_embeds)
+ 
+    cluster_eigenvectors = kway_cluster_per_image(dino_image_embeds, n_clusters=n_clusters, gamma=None)
+    a_to_b_mapping = match_centers_two_images(
+        dino_image_embeds[0], dino_image_embeds[1],
+        cluster_eigenvectors[0], cluster_eigenvectors[1], 
+        match_method='hungarian'
+    )
+    direction_field = _compute_direction_from_two_images(
+        compressed_image_embeds, cluster_eigenvectors, a_to_b_mapping, use_unit_norm=False
+    )
+    
+    # Generate interpolated images
+    ip_model = load_ipadapter()
+    
+    progress_tracker = gr.Progress()
+    generated_images = []
+    for i, weight in enumerate(interpolation_weights):
+        progress_tracker(i / len(interpolation_weights), desc=f"Generating images, α = {weight:.2f}")
+        interpolated_embedding = compressed_image_embeds[0] + direction_field * weight
+        decompressed_embedding = model.decoder(interpolated_embedding)
+        
+        batch_images = generate_images_from_clip_embeddings(
+            ip_model, decompressed_embedding, num_samples=1, seed=seed
+        )
+        if np.all(np.array(batch_images[0]) == 0):
+            raise ValueError("Generated image is all black")
+        generated_images.extend(batch_images)
+    
+    # Clean up
+    del ip_model
+    clear_gpu_memory()
+    
+    return generated_images
+

vibespace_model.py

@@ -0,0 +1,504 @@
+"""
+Neural Compression Model for Feature Space Learning
+
+This module implements a compression model that learns to compress and decompress
+image features while preserving their geometric and semantic properties using
+normalized cuts (NCut).
+"""
+
+import gc
+from collections import defaultdict
+from typing import List, Optional, Tuple
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import pytorch_lightning as pl
+from einops import rearrange
+from omegaconf import DictConfig
+import gradio as gr
+
+from ncut_pytorch.ncuts.ncut_nystrom import _plain_ncut
+from ncut_pytorch.utils.math import rbf_affinity
+
+
+def compute_ncut_eigenvectors(features: torch.Tensor, n_eig: int) -> Tuple[torch.Tensor, torch.Tensor]:
+    gamma = features.var(0).sum().item()
+    affinity_matrix = rbf_affinity(features, gamma=gamma)
+    eigenvectors, eigenvalues = _plain_ncut(affinity_matrix, n_eig)
+    return eigenvectors, eigenvalues
+
+
+# ===== Neural Network Components =====
+
+class MultiLayerPerceptron(nn.Module):
+    
+    def __init__(self, input_dim: int, output_dim: int, num_layers: int = 4, hidden_dim: int = 4096):
+        super().__init__()
+        
+        layers = [nn.Linear(input_dim, hidden_dim), nn.GELU()]
+        
+        # Add hidden layers
+        for _ in range(num_layers):
+            layers.extend([nn.Linear(hidden_dim, hidden_dim), nn.GELU()])
+        
+        # Output layer
+        layers.append(nn.Linear(hidden_dim, output_dim))
+        
+        self.mlp = nn.Sequential(*layers)
+    
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return self.mlp(x)
+
+class SpatialPoolingAvgPool(nn.Module):
+    """
+    AvgPool layer for spatial pooling of feature maps with support for sequence inputs.
+    
+    Handles inputs with CLS tokens and reshapes appropriately for 2D convolution.
+    """
+    def __init__(self, downsample_factor: int = 2):
+        super().__init__()
+        self.downsample_factor = downsample_factor
+        self.avg_pool = nn.AvgPool2d(downsample_factor)
+    
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        """
+        Forward pass supporting both (batch, seq_len, channels) and (seq_len, channels) inputs.
+        """
+        # Handle input shape variations
+        added_batch_dim = False
+        if x.dim() == 2:
+            x = x.unsqueeze(0)
+            added_batch_dim = True
+        elif x.dim() != 3:
+            raise ValueError(f"Expected input shape (B, L, C) or (L, C), got {x.shape}")
+
+        batch_size, seq_len, channels = x.shape
+        
+        if seq_len < 2:
+            raise ValueError("Sequence length must be at least 2 (1 CLS token + 1 patch)")
+
+        # Validate that seq_len-1 is a perfect square (for spatial arrangement)
+        spatial_size = int(round((seq_len - 1) ** 0.5))
+        if spatial_size * spatial_size != (seq_len - 1):
+            raise ValueError(f"seq_len-1 must be perfect square. Got {seq_len-1}")
+
+        # Separate CLS token and spatial features
+        cls_tokens = x[:, :1, :]  # (B, 1, C)
+        spatial_features = x[:, 1:, :]  # (B, H*W, C)
+        
+        # Reshape to 2D for convolution
+        spatial_2d = rearrange(
+            spatial_features, 'b (h w) c -> b c h w', 
+            h=spatial_size, w=spatial_size
+        )
+        
+        # Apply pooling
+        pooled_features = self.avg_pool(spatial_2d)
+        
+        # Reshape back to sequence format
+        pooled_sequence = rearrange(pooled_features, 'b c h w -> b (h w) c')
+        
+        # Concatenate CLS token back
+        output = torch.cat([cls_tokens, pooled_sequence], dim=1)
+
+        # Remove batch dimension if it was added
+        if added_batch_dim:
+            output = output.squeeze(0)
+        
+        return output
+
+class MLPWithSpatialPooling(nn.Module):
+    def __init__(self, input_dim: int, output_dim: int, num_layers: int = 4, 
+                 hidden_dim: int = 4096, downsample_factor: int = 2):
+        super().__init__()
+        
+        self.pooling = SpatialPoolingAvgPool(downsample_factor)
+        
+        layers = [nn.Linear(input_dim, hidden_dim), nn.GELU()]
+        
+        # Add hidden layers
+        for _ in range(num_layers):
+            layers.extend([nn.Linear(hidden_dim, hidden_dim), nn.GELU()])
+        
+        # Output layer
+        layers.append(nn.Linear(hidden_dim, output_dim))
+        
+        self.network = nn.Sequential(*layers)
+    
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = self.pooling(x)
+        return self.network(x)
+
+
+# ===== Main Compression Model =====
+
+class VibeSpaceModel(pl.LightningModule):
+    """
+    Neural compression model for learning compressed feature representations.
+    
+    This model compresses input features to a lower-dimensional "vibe space" and
+    then decompresses them back, while preserving geometric and semantic properties
+    through various loss functions including NCut-based losses.
+    """
+    
+    def __init__(self, config: DictConfig, enable_gradio_progress: bool = False, downsample_factor: int = 2):
+        super().__init__()
+        
+        self.config = config
+        self.downsample_factor = downsample_factor
+        
+        self.encoder = MultiLayerPerceptron(
+            config.in_dim, config.vibe_dim, config.n_layer, config.latent_dim
+        )
+        
+        self.decoder = MLPWithSpatialPooling(
+            config.vibe_dim, config.out_dim, config.n_layer, 
+            config.latent_dim, self.downsample_factor
+        )
+        
+        self.loss_history = defaultdict(list)
+        self.enable_gradio_progress = enable_gradio_progress
+        if enable_gradio_progress:
+            self.progress_tracker = gr.Progress()
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        compressed = self.encoder(x)
+        reconstructed = self.decoder(compressed)
+        return reconstructed
+
+    def training_step(self, batch, batch_idx):
+        # Update progress bar if using Gradio
+        if (self.enable_gradio_progress and 
+            self.trainer.global_step % 10 == 0 and 
+            self.trainer.global_step > 0 and
+            self.loss_history['recon']):
+            
+            progress = self.trainer.global_step / self.config.steps
+            recent_loss = self.loss_history['recon'][-1]
+            self.progress_tracker(progress, desc=f"Training Vibe Space, loss = {recent_loss:.4f}")
+
+        positive_features, negative_features, target_features, negative_mask = batch
+        negative_mask = negative_mask.bool()
+        has_negatives = bool(negative_mask.any().item())
+
+        if has_negatives:
+            if bool(negative_mask.all().item()):
+                batch_negative_features = negative_features
+            else:
+                batch_negative_features = negative_features[negative_mask]
+        else:
+            batch_negative_features = None
+        
+        compressed_features = self.encoder(positive_features)
+        reconstructed_features = self.decoder(compressed_features)
+        
+        
+        total_loss = self._compute_total_loss(
+            positive_features,
+            batch_negative_features,
+            target_features,
+            compressed_features,
+            reconstructed_features,
+        )
+        
+        self.log("loss/total", total_loss, prog_bar=True)
+        return total_loss
+    
+    def _compute_ncut_eigenvectors(self, features: torch.Tensor) -> torch.Tensor:
+        """Compute NCut eigenvectors for features."""
+        # Accept inputs shaped either (batch, length, channels) or (length, channels)
+        flattened_features = features
+        if flattened_features.dim() >= 3:
+            flattened_features = flattened_features.flatten(0, 1)
+        elif flattened_features.dim() == 1:
+            # rbf_affinity expects at least 2D; treat single vector as one sample with channels
+            flattened_features = flattened_features.unsqueeze(0)
+
+        if flattened_features.numel() > 0 and flattened_features.dim() == 2:
+            eigenvectors, _ = compute_ncut_eigenvectors(flattened_features, self.config.n_eig)
+            return eigenvectors
+        else:
+            # Return zero tensor if no features
+            device = features.device if isinstance(features, torch.Tensor) else 'cpu'
+            return torch.zeros((1, self.config.n_eig), device=device)
+    
+    def _compute_multiscale_similarity(self, eigenvectors: torch.Tensor, 
+                                      start_n_eig: int = 4, step_mult: int = 2) -> torch.Tensor:
+        """Compute multi-scale similarity matrix from eigenvectors.
+        eigenvectors is (batch*length, n_eig)
+        """
+        total_similarity = 0.0
+        num_scales = 0
+        max_available = eigenvectors.shape[1]
+        current_n_eig = min(start_n_eig, max_available)
+        
+        if self.config.single_scale_flag:
+            current_n_eig = max_available
+        
+        while current_n_eig <= max_available:
+            eigvec_subset = eigenvectors[:, :current_n_eig]
+            eigvec_normalized = F.normalize(eigvec_subset, dim=-1)
+            
+            total_similarity += eigvec_normalized @ eigvec_normalized.T
+            
+            num_scales += 1
+            current_n_eig *= step_mult
+        
+        return total_similarity / num_scales if num_scales > 0 else total_similarity
+    
+    def _compute_flag_decoder_loss(
+        self,
+        compressed_features: torch.Tensor,
+        reconstructed_features: torch.Tensor,
+        negative_input_features: Optional[torch.Tensor] = None,
+    ) -> torch.Tensor:
+        """
+        compressed_features is (batch, length, channels)
+        reconstructed_features is (batch, length, channels)
+        """
+        pooled_compressed = self.decoder.pooling(compressed_features)
+        pooled_compressed = pooled_compressed.flatten(0, 1)
+        reconstructed_features = reconstructed_features.flatten(0, 1)
+
+        has_negative = (
+            negative_input_features is not None and negative_input_features.numel() > 0
+        )
+
+        # sample points from the compressed feature space (only when no negatives available)
+        dim_mins = pooled_compressed.min(0).values
+        dim_maxs = pooled_compressed.max(0).values
+        dim_mins -= 0.25 * (dim_maxs - dim_mins) * torch.rand_like(dim_mins)
+        dim_maxs += 0.25 * (dim_maxs - dim_mins) * torch.rand_like(dim_maxs)
+        
+        num_samples = 0 if has_negative else self.config.n_negative_sample
+        sample_points = torch.rand(num_samples, pooled_compressed.shape[1], device=pooled_compressed.device)
+        sample_points = sample_points * (dim_maxs - dim_mins) + dim_mins
+        
+        # reconstruct the sample points
+        sample_reconstructed = self.decoder.network(sample_points)
+        
+        all_compressed = torch.cat([pooled_compressed, sample_points], dim=0)
+        all_reconstructed = torch.cat([reconstructed_features, sample_reconstructed], dim=0)
+        
+        # flag loss on the sample points 
+        similarity = all_compressed @ all_compressed.T
+        eigenvectors_pos, _ = compute_ncut_eigenvectors(all_reconstructed, self.config.n_eig)
+
+        if has_negative and self.config.get('do_decoder_negative_flag', False):
+            negative_compressed = self.encoder(negative_input_features)
+            negative_reconstructed = self.decoder(negative_compressed)
+            negative_reconstructed = negative_reconstructed.flatten(0, 1)
+
+            neg_eigenvectors, _ = compute_ncut_eigenvectors(negative_reconstructed, self.config.n_eig)
+
+            max_available = min(eigenvectors_pos.shape[1], neg_eigenvectors.shape[1])
+            if max_available == 0:
+                eig_similarity = self._compute_multiscale_similarity(eigenvectors_pos)
+            else:
+                if self.config.single_scale_flag:
+                    current_n_eig = max_available
+                else:
+                    current_n_eig = min(self.config.get('start_n_eig', 4), max_available)
+                    current_n_eig = max(current_n_eig, 1)
+
+                total_filtered_similarity = similarity.new_zeros(similarity.shape)
+                num_scales = 0
+                beta = self.config.get('decoder_negative_beta', self.config.get('negative_beta', 1.0))
+                step_mult = self.config.get('step_mult', 2)
+
+                while current_n_eig <= max_available:
+                    P = eigenvectors_pos[:, :current_n_eig]
+                    N = neg_eigenvectors[:, :current_n_eig]
+
+                    N_norm = F.normalize(N, dim=0)
+                    projection = torch.matmul(N_norm.T, P)
+                    P_filtered = P - beta * torch.matmul(N_norm, projection)
+
+                    P_filtered_norm = F.normalize(P_filtered, dim=-1)
+                    total_filtered_similarity += P_filtered_norm @ P_filtered_norm.T
+
+                    num_scales += 1
+                    current_n_eig *= step_mult
+
+                if num_scales > 0:
+                    eig_similarity = total_filtered_similarity / num_scales
+                else:
+                    eig_similarity = self._compute_multiscale_similarity(eigenvectors_pos)
+        else:
+            eig_similarity = self._compute_multiscale_similarity(eigenvectors_pos)
+
+        loss = F.smooth_l1_loss(eig_similarity, similarity)
+        return loss
+    
+    def _compute_flag_encoder_loss(self, input_features: torch.Tensor, compressed_features: torch.Tensor) -> torch.Tensor:
+        """
+        input_features is (batch, length, channels)
+        compressed_features is (batch, length, channels)
+        """
+        sample_indices = torch.randperm(input_features.shape[0])[:self.config.n_sample_eigsolve]
+        gt_eigenvectors = self._compute_ncut_eigenvectors(input_features.flatten(0, 1)[sample_indices])
+        gt_similarity = self._compute_multiscale_similarity(gt_eigenvectors)
+        flattened_compressed = compressed_features.flatten(0, 1)[sample_indices]
+        pred_similarity = flattened_compressed @ flattened_compressed.T
+        loss = F.smooth_l1_loss(gt_similarity, pred_similarity)
+        return loss
+    
+    def _compute_total_loss(
+        self,
+        positive_features: torch.Tensor,
+        negative_features: Optional[torch.Tensor],
+        target_features: torch.Tensor,
+        compressed_features: torch.Tensor,
+        reconstructed_features: torch.Tensor,
+    ) -> torch.Tensor:
+        """
+        positive_features is (batch, length, channels)
+        target_features is (batch, length, channels)
+        compressed_features is (batch, length, channels)
+        reconstructed_features is (batch, length, channels)
+        """
+        total_loss = positive_features.new_tensor(0.0)
+        has_negative_features = (
+            negative_features is not None and negative_features.numel() > 0
+        )
+        beta = self.config.get('negative_beta', 1.0)
+        
+        # Flag encoder loss - guide the structure from encoder to compressed features
+        if self.config.flag_encoder_loss > 0 and has_negative_features:
+            gt_eigenvectors_pos = self._compute_ncut_eigenvectors(positive_features)
+            gt_eigenvectors_neg = self._compute_ncut_eigenvectors(negative_features)
+
+            total_filtered_similarity = 0.0
+            num_scales = 0
+            max_available = min(gt_eigenvectors_pos.shape[1], gt_eigenvectors_neg.shape[1])
+
+            if max_available == 0:
+                gt_similarity = self._compute_multiscale_similarity(gt_eigenvectors_pos)
+            else:
+                if self.config.single_scale_flag:
+                    current_n_eig = max_available
+                else:
+                    current_n_eig = min(self.config.get('start_n_eig', 4), max_available)
+                    current_n_eig = max(current_n_eig, 1)
+
+                step_mult = self.config.get('step_mult', 2)
+                while current_n_eig <= max_available and current_n_eig > 0:
+                    P = gt_eigenvectors_pos[:, :current_n_eig]
+                    N = gt_eigenvectors_neg[:, :current_n_eig]
+
+                    N_norm = F.normalize(N, dim=0)
+                    projection = torch.matmul(N_norm.T, P)
+                    P_filtered = P - beta * torch.matmul(N_norm, projection)
+
+                    P_filtered_norm = F.normalize(P_filtered, dim=-1)
+                    total_filtered_similarity += P_filtered_norm @ P_filtered_norm.T
+
+                    num_scales += 1
+                    current_n_eig *= step_mult
+
+                if num_scales > 0:
+                    gt_similarity = total_filtered_similarity / num_scales
+                else:
+                    gt_similarity = self._compute_multiscale_similarity(gt_eigenvectors_pos)
+            flattened_compressed = compressed_features.flatten(0, 1)
+            pred_similarity = flattened_compressed @ flattened_compressed.T
+
+            flag_encoder_loss = F.smooth_l1_loss(gt_similarity, pred_similarity)
+            self.log("loss/flag_encoder", flag_encoder_loss, prog_bar=True)
+            total_loss += flag_encoder_loss * self.config.flag_encoder_loss
+            self.loss_history['flag_encoder'].append(flag_encoder_loss.item())
+        elif self.config.flag_encoder_loss > 0:
+            flag_encoder_loss = self._compute_flag_encoder_loss(positive_features, compressed_features)
+            self.log("loss/flag_encoder", flag_encoder_loss, prog_bar=True)
+            total_loss += flag_encoder_loss * self.config.flag_encoder_loss
+            self.loss_history['flag_encoder'].append(flag_encoder_loss.item())
+        
+        # Flag decoder loss - guide the structure from compressed to decoded features
+        if self.config.flag_decoder_loss > 0:
+            if self.trainer.global_step >= 500:  # warmup period
+                flag_decoder_loss = self._compute_flag_decoder_loss(
+                    compressed_features,
+                    reconstructed_features,
+                    negative_features,
+                )
+                self.log("loss/flag_decoder", flag_decoder_loss, prog_bar=True)
+                total_loss += flag_decoder_loss * self.config.flag_decoder_loss
+                self.loss_history['flag_decoder'].append(flag_decoder_loss.item())
+
+        # Reconstruction loss
+        if self.config.recon_loss > 0:
+            recon_loss = F.smooth_l1_loss(target_features, reconstructed_features)
+            self.log("loss/recon", recon_loss, prog_bar=True)
+            total_loss += recon_loss * self.config.recon_loss
+            self.loss_history['recon'].append(recon_loss.item())
+
+        return total_loss
+    
+    def configure_optimizers(self):
+        return torch.optim.NAdam(self.parameters(), lr=self.config.lr)
+
+
+# ===== Dataset and Training Utilities =====
+
+class FeatureDataset(torch.utils.data.Dataset):
+    
+    def __init__(
+        self,
+        positive_features: torch.Tensor,
+        target_features: torch.Tensor,
+        negative_features: Optional[torch.Tensor] = None,
+    ):
+        self.positive_features = positive_features
+        self.target_features = target_features
+        if negative_features is not None and negative_features.numel() > 0:
+            self.negative_features = negative_features
+        else:
+            self.negative_features = None
+    
+    def __len__(self) -> int:
+        return len(self.positive_features)
+    
+    def __getitem__(self, idx: int) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
+        positive = self.positive_features[idx]
+        target = self.target_features[idx]
+
+        if self.negative_features is None:
+            negative = torch.zeros_like(positive)
+            has_negative = torch.tensor(False, dtype=torch.bool)
+        else:
+            neg_idx = torch.randint(0, self.negative_features.shape[0], (1,)).item()
+            negative = self.negative_features[neg_idx]
+            has_negative = torch.tensor(True, dtype=torch.bool)
+
+        return positive, negative, target, has_negative
+
+
+def clear_gpu_memory():
+    torch.cuda.empty_cache()
+    torch.cuda.ipc_collect()
+    gc.collect()
+
+
+def train_vibe_space(model: VibeSpaceModel, 
+                          config: DictConfig,
+                          input_features: torch.Tensor,
+                          target_features: torch.Tensor,
+                          negative_features: Optional[torch.Tensor] = None,
+                          devices: List[int] = [0]) -> pl.Trainer:
+    clear_gpu_memory()
+    dataset = FeatureDataset(input_features, target_features, negative_features)
+    dataloader = torch.utils.data.DataLoader(dataset, batch_size=8, shuffle=True, num_workers=0)
+    trainer = pl.Trainer(
+        max_steps=config.steps,
+        gradient_clip_val=1.0,
+        accelerator="gpu", 
+        devices=devices,
+        enable_checkpointing=False,
+        enable_progress_bar=True,
+        logger=False  # Disable default logger
+    )
+    
+    trainer.fit(model, dataloader)
+    
+    return trainer