diffusers/pr_14313/en/_app/immutable/nodes/332.yRGhOW0Z.js

import"../chunks/DsnmJJEf.js";import{i as U,h as T,C as w,H as m,a as l,E as Z,s as f}from"../chunks/CmJXCtRL.js";import{p as b,o as j,s as a,f as B,a as y,b as W,d as h,n as G}from"../chunks/DK803DsY.js";const I='{"title":"T2I-Adapter","local":"t2i-adapter","sections":[{"title":"MultiAdapter","local":"multiadapter","sections":[],"depth":2}],"depth":1}';var C=h('<meta name="hf:doc:metadata"/>'),k=h('<p></p> <!> <!> <p><a href="https://huggingface.co/papers/2302.08453" rel="nofollow">T2I-Adapter</a> is an adapter that enables controllable generation like <a href="./controlnet">ControlNet</a>. A T2I-Adapter works by learning a <em>mapping</em> between a control signal (for example, a depth map) and a pretrained model’s internal knowledge. The adapter is plugged in to the base model to provide extra guidance based on the control signal during generation.</p> <p>Load a T2I-Adapter conditioned on a specific control, such as canny edge, and pass it to the pipeline in <a href="/docs/diffusers/pr_14313/en/api/pipelines/overview#diffusers.DiffusionPipeline.from_pretrained">from_pretrained()</a>.</p> <!> <p>Generate a canny image with <a href="https://github.com/opencv/opencv-python" rel="nofollow">opencv-python</a>.</p> <!> <p>Pass the canny image to the pipeline to generate an image.</p> <!> <div style="display: flex; gap: 10px; justify-content: space-around; align-items: flex-end;"><figure><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/non-enhanced-prompt.png" width="300" alt="Generated image (prompt only)"/> <figcaption style="text-align: center;">original image</figcaption></figure> <figure><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/canny-cat.png" width="300" alt="Control image (Canny edges)"/> <figcaption style="text-align: center;">canny image</figcaption></figure> <figure><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/t2i-canny-cat-generated.png" width="300" alt="Generated image (ControlNet + prompt)"/> <figcaption style="text-align: center;">generated image</figcaption></figure></div> <!> <p>You can compose multiple controls, such as canny image and a depth map, with the <code>MultiAdapter</code> class.</p> <p>The example below composes a canny image and depth map.</p> <p>Load the control images and T2I-Adapters as a list.</p> <!> <p>Pass the adapters, prompt, and control images to <a href="/docs/diffusers/pr_14313/en/api/pipelines/stable_diffusion/adapter#diffusers.StableDiffusionXLAdapterPipeline">StableDiffusionXLAdapterPipeline</a>. Use the <code>adapter_conditioning_scale</code> parameter to determine how much weight to assign to each control.</p> <!> <div style="display: flex; gap: 10px; justify-content: space-around; align-items: flex-end;"><figure><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/canny-cat.png" width="300" alt="Generated image (prompt only)"/> <figcaption style="text-align: center;">canny image</figcaption></figure> <figure><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/sdxl_depth_image.png" width="300" alt="Control image (Canny edges)"/> <figcaption style="text-align: center;">depth map</figcaption></figure> <figure><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/t2i-multi-rabbit.png" width="300" alt="Generated image (ControlNet + prompt)"/> <figcaption style="text-align: center;">generated image</figcaption></figure></div> <!> <p></p>',1);function v(g,J){b(J,!1),j(()=>{new URLSearchParams(window.location.search).get("fw")}),U();var e=k();T("1iuclj9",M=>{var c=C();f(c,"content",I),y(M,c)});var t=a(B(e),2);w(t,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var n=a(t,2);m(n,{title:"T2I-Adapter",local:"t2i-adapter",headingTag:"h1"});var s=a(n,6);l(s,{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwVDJJQWRhcHRlciUyQyUyMFN0YWJsZURpZmZ1c2lvblhMQWRhcHRlclBpcGVsaW5lJTJDJTIwQXV0b2VuY29kZXJLTCUwQSUwQXQyaV9hZGFwdGVyJTIwJTNEJTIwVDJJQWRhcHRlci5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyVGVuY2VudEFSQyUyRnQyaS1hZGFwdGVyLWNhbm55LXNkeGwtMS4wJTIyJTJDJTBBJTIwJTIwJTIwJTIwZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> T2IAdapter, StableDiffusionXLAdapterPipeline, AutoencoderKL

t2i_adapter = T2IAdapter.from_pretrained(
    <span class="hljs-string">&quot;TencentARC/t2i-adapter-canny-sdxl-1.0&quot;</span>,
    dtype=torch.float16,
)`,lang:"py",wrap:!1});var i=a(s,4);l(i,{code:"aW1wb3J0JTIwY3YyJTBBaW1wb3J0JTIwbnVtcHklMjBhcyUyMG5wJTBBZnJvbSUyMFBJTCUyMGltcG9ydCUyMEltYWdlJTBBZnJvbSUyMGRpZmZ1c2Vycy51dGlscyUyMGltcG9ydCUyMGxvYWRfaW1hZ2UlMEElMEFvcmlnaW5hbF9pbWFnZSUyMCUzRCUyMGxvYWRfaW1hZ2UoJTBBJTIwJTIwJTIwJTIwJTIyaHR0cHMlM0ElMkYlMkZodWdnaW5nZmFjZS5jbyUyRmRhdGFzZXRzJTJGaHVnZ2luZ2ZhY2UlMkZkb2N1bWVudGF0aW9uLWltYWdlcyUyRnJlc29sdmUlMkZtYWluJTJGZGlmZnVzZXJzJTJGbm9uLWVuaGFuY2VkLXByb21wdC5wbmclMjIlMEEpJTBBJTBBaW1hZ2UlMjAlM0QlMjBucC5hcnJheShvcmlnaW5hbF9pbWFnZSklMEElMEFsb3dfdGhyZXNob2xkJTIwJTNEJTIwMTAwJTBBaGlnaF90aHJlc2hvbGQlMjAlM0QlMjAyMDAlMEElMEFpbWFnZSUyMCUzRCUyMGN2Mi5DYW5ueShpbWFnZSUyQyUyMGxvd190aHJlc2hvbGQlMkMlMjBoaWdoX3RocmVzaG9sZCklMEFpbWFnZSUyMCUzRCUyMGltYWdlJTVCJTNBJTJDJTIwJTNBJTJDJTIwTm9uZSU1RCUwQWltYWdlJTIwJTNEJTIwbnAuY29uY2F0ZW5hdGUoJTVCaW1hZ2UlMkMlMjBpbWFnZSUyQyUyMGltYWdlJTVEJTJDJTIwYXhpcyUzRDIpJTBBY2FubnlfaW1hZ2UlMjAlM0QlMjBJbWFnZS5mcm9tYXJyYXkoaW1hZ2Up",highlighted:`<span class="hljs-keyword">import</span> cv2
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image

original_image = load_image(
    <span class="hljs-string">&quot;https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/non-enhanced-prompt.png&quot;</span>
)

image = np.array(original_image)

low_threshold = <span class="hljs-number">100</span>
high_threshold = <span class="hljs-number">200</span>

image = cv2.Canny(image, low_threshold, high_threshold)
image = image[:, :, <span class="hljs-literal">None</span>]
image = np.concatenate([image, image, image], axis=<span class="hljs-number">2</span>)
canny_image = Image.fromarray(image)`,lang:"py",wrap:!1});var p=a(i,4);l(p,{code:"dmFlJTIwJTNEJTIwQXV0b2VuY29kZXJLTC5mcm9tX3ByZXRyYWluZWQoJTIybWFkZWJ5b2xsaW4lMkZzZHhsLXZhZS1mcDE2LWZpeCUyMiUyQyUyMGR0eXBlJTNEdG9yY2guZmxvYXQxNiklMEFwaXBlbGluZSUyMCUzRCUyMFN0YWJsZURpZmZ1c2lvblhMQWRhcHRlclBpcGVsaW5lLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJzdGFiaWxpdHlhaSUyRnN0YWJsZS1kaWZmdXNpb24teGwtYmFzZS0xLjAlMjIlMkMlMEElMjAlMjAlMjAlMjBhZGFwdGVyJTNEdDJpX2FkYXB0ZXIlMkMlMEElMjAlMjAlMjAlMjB2YWUlM0R2YWUlMkMlMEElMjAlMjAlMjAlMjBkdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMEEpLnRvKCUyMmN1ZGElMjIpJTBBJTBBcHJvbXB0JTIwJTNEJTIwJTIyJTIyJTIyJTBBQSUyMHBob3RvcmVhbGlzdGljJTIwb3ZlcmhlYWQlMjBpbWFnZSUyMG9mJTIwYSUyMGNhdCUyMHJlY2xpbmluZyUyMHNpZGV3YXlzJTIwaW4lMjBhJTIwZmxhbWluZ28lMjBwb29sJTIwZmxvYXRpZSUyMGhvbGRpbmclMjBhJTIwbWFyZ2FyaXRhLiUyMCUwQVRoZSUyMGNhdCUyMGlzJTIwZmxvYXRpbmclMjBsZWlzdXJlbHklMjBpbiUyMHRoZSUyMHBvb2wlMjBhbmQlMjBjb21wbGV0ZWx5JTIwcmVsYXhlZCUyMGFuZCUyMGhhcHB5LiUwQSUyMiUyMiUyMiUwQSUwQXBpcGVsaW5lKCUwQSUyMCUyMCUyMCUyMHByb21wdCUyQyUyMCUwQSUyMCUyMCUyMCUyMGltYWdlJTNEY2FubnlfaW1hZ2UlMkMlMEElMjAlMjAlMjAlMjBudW1faW5mZXJlbmNlX3N0ZXBzJTNEMTAwJTJDJTIwJTBBJTIwJTIwJTIwJTIwZ3VpZGFuY2Vfc2NhbGUlM0QxMCUyQyUwQSkuaW1hZ2VzJTVCMCU1RA==",highlighted:`vae = AutoencoderKL.from_pretrained(<span class="hljs-string">&quot;madebyollin/sdxl-vae-fp16-fix&quot;</span>, dtype=torch.float16)
pipeline = StableDiffusionXLAdapterPipeline.from_pretrained(
    <span class="hljs-string">&quot;stabilityai/stable-diffusion-xl-base-1.0&quot;</span>,
    adapter=t2i_adapter,
    vae=vae,
    dtype=torch.float16,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)

prompt = <span class="hljs-string">&quot;&quot;&quot;
A photorealistic overhead image of a cat reclining sideways in a flamingo pool floatie holding a margarita. 
The cat is floating leisurely in the pool and completely relaxed and happy.
&quot;&quot;&quot;</span>

pipeline(
    prompt, 
    image=canny_image,
    num_inference_steps=<span class="hljs-number">100</span>, 
    guidance_scale=<span class="hljs-number">10</span>,
).images[<span class="hljs-number">0</span>]`,lang:"py",wrap:!1});var o=a(p,4);m(o,{title:"MultiAdapter",local:"multiadapter",headingTag:"h2"});var d=a(o,8);l(d,{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzLnV0aWxzJTIwaW1wb3J0JTIwbG9hZF9pbWFnZSUwQWZyb20lMjBkaWZmdXNlcnMlMjBpbXBvcnQlMjBTdGFibGVEaWZmdXNpb25YTEFkYXB0ZXJQaXBlbGluZSUyQyUyMEF1dG9lbmNvZGVyS0wlMkMlMjBNdWx0aUFkYXB0ZXIlMkMlMjBUMklBZGFwdGVyJTBBJTBBY2FubnlfaW1hZ2UlMjAlM0QlMjBsb2FkX2ltYWdlKCUwQSUyMCUyMCUyMCUyMCUyMmh0dHBzJTNBJTJGJTJGaHVnZ2luZ2ZhY2UuY28lMkZkYXRhc2V0cyUyRmh1Z2dpbmdmYWNlJTJGZG9jdW1lbnRhdGlvbi1pbWFnZXMlMkZyZXNvbHZlJTJGbWFpbiUyRmRpZmZ1c2VycyUyRmNhbm55LWNhdC5wbmclMjIlMEEpJTBBZGVwdGhfaW1hZ2UlMjAlM0QlMjBsb2FkX2ltYWdlKCUwQSUyMCUyMCUyMCUyMCUyMmh0dHBzJTNBJTJGJTJGaHVnZ2luZ2ZhY2UuY28lMkZkYXRhc2V0cyUyRmh1Z2dpbmdmYWNlJTJGZG9jdW1lbnRhdGlvbi1pbWFnZXMlMkZyZXNvbHZlJTJGbWFpbiUyRmRpZmZ1c2VycyUyRnNkeGxfZGVwdGhfaW1hZ2UucG5nJTIyJTBBKSUwQWNvbnRyb2xzJTIwJTNEJTIwJTVCY2FubnlfaW1hZ2UlMkMlMjBkZXB0aF9pbWFnZSU1RCUwQXByb21wdCUyMCUzRCUyMCU1QiUyMiUyMiUyMiUwQWElMjByZWxheGVkJTIwcmFiYml0JTIwc2l0dGluZyUyMG9uJTIwYSUyMHN0cmlwZWQlMjB0b3dlbCUyMG5leHQlMjB0byUyMGElMjBwb29sJTIwd2l0aCUyMGElMjB0cm9waWNhbCUyMGRyaW5rJTIwbmVhcmJ5JTJDJTIwJTBBYnJpZ2h0JTIwc3VubnklMjBkYXklMkMlMjB2YWNhdGlvbiUyMHNjZW5lJTJDJTIwMzVtbSUyMHBob3RvZ3JhcGglMkMlMjBmaWxtJTJDJTIwcHJvZmVzc2lvbmFsJTJDJTIwNGslMkMlMjBoaWdobHklMjBkZXRhaWxlZCUwQSUyMiUyMiUyMiU1RCUwQSUwQWFkYXB0ZXJzJTIwJTNEJTIwTXVsdGlBZGFwdGVyKCUwQSUyMCUyMCUyMCUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMFQySUFkYXB0ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMlRlbmNlbnRBUkMlMkZ0MmktYWRhcHRlci1jYW5ueS1zZHhsLTEuMCUyMiUyQyUyMGR0eXBlJTNEdG9yY2guZmxvYXQxNiklMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBUMklBZGFwdGVyLmZyb21fcHJldHJhaW5lZCglMjJUZW5jZW50QVJDJTJGdDJpLWFkYXB0ZXItZGVwdGgtbWlkYXMtc2R4bC0xLjAlMjIlMkMlMjBkdHlwZSUzRHRvcmNoLmZsb2F0MTYpJTJDJTBBJTIwJTIwJTIwJTIwJTVEJTBBKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionXLAdapterPipeline, AutoencoderKL, MultiAdapter, T2IAdapter

canny_image = load_image(
    <span class="hljs-string">&quot;https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/canny-cat.png&quot;</span>
)
depth_image = load_image(
    <span class="hljs-string">&quot;https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/sdxl_depth_image.png&quot;</span>
)
controls = [canny_image, depth_image]
prompt = [<span class="hljs-string">&quot;&quot;&quot;
a relaxed rabbit sitting on a striped towel next to a pool with a tropical drink nearby, 
bright sunny day, vacation scene, 35mm photograph, film, professional, 4k, highly detailed
&quot;&quot;&quot;</span>]

adapters = MultiAdapter(
    [
        T2IAdapter.from_pretrained(<span class="hljs-string">&quot;TencentARC/t2i-adapter-canny-sdxl-1.0&quot;</span>, dtype=torch.float16),
        T2IAdapter.from_pretrained(<span class="hljs-string">&quot;TencentARC/t2i-adapter-depth-midas-sdxl-1.0&quot;</span>, dtype=torch.float16),
    ]
)`,lang:"py",wrap:!1});var r=a(d,4);l(r,{code:"dmFlJTIwJTNEJTIwQXV0b2VuY29kZXJLTC5mcm9tX3ByZXRyYWluZWQoJTIybWFkZWJ5b2xsaW4lMkZzZHhsLXZhZS1mcDE2LWZpeCUyMiUyQyUyMGR0eXBlJTNEdG9yY2guZmxvYXQxNiklMEFwaXBlbGluZSUyMCUzRCUyMFN0YWJsZURpZmZ1c2lvblhMQWRhcHRlclBpcGVsaW5lLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJzdGFiaWxpdHlhaSUyRnN0YWJsZS1kaWZmdXNpb24teGwtYmFzZS0xLjAlMjIlMkMlMEElMjAlMjAlMjAlMjBkdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMEElMjAlMjAlMjAlMjB2YWUlM0R2YWUlMkMlMEElMjAlMjAlMjAlMjBhZGFwdGVyJTNEYWRhcHRlcnMlMkMlMEEpLnRvKCUyMmN1ZGElMjIpJTBBJTBBcGlwZWxpbmUoJTBBJTIwJTIwJTIwJTIwcHJvbXB0JTJDJTBBJTIwJTIwJTIwJTIwaW1hZ2UlM0Rjb250cm9scyUyQyUwQSUyMCUyMCUyMCUyMGhlaWdodCUzRDEwMjQlMkMlMEElMjAlMjAlMjAlMjB3aWR0aCUzRDEwMjQlMkMlMEElMjAlMjAlMjAlMjBhZGFwdGVyX2NvbmRpdGlvbmluZ19zY2FsZSUzRCU1QjAuNyUyQyUyMDAuNyU1RCUwQSkuaW1hZ2VzJTVCMCU1RA==",highlighted:`vae = AutoencoderKL.from_pretrained(<span class="hljs-string">&quot;madebyollin/sdxl-vae-fp16-fix&quot;</span>, dtype=torch.float16)
pipeline = StableDiffusionXLAdapterPipeline.from_pretrained(
    <span class="hljs-string">&quot;stabilityai/stable-diffusion-xl-base-1.0&quot;</span>,
    dtype=torch.float16,
    vae=vae,
    adapter=adapters,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)

pipeline(
    prompt,
    image=controls,
    height=<span class="hljs-number">1024</span>,
    width=<span class="hljs-number">1024</span>,
    adapter_conditioning_scale=[<span class="hljs-number">0.7</span>, <span class="hljs-number">0.7</span>]
).images[<span class="hljs-number">0</span>]`,lang:"py",wrap:!1});var u=a(r,4);Z(u,{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/using-diffusers/t2i_adapter.md"}),G(2),y(g,e),W()}export{v as component};