diffusers/pr_12652/en/hybrid_inference/overview.html

<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Remote inference&quot;,&quot;local&quot;:&quot;remote-inference&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Encoding&quot;,&quot;local&quot;:&quot;encoding&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Decoding&quot;,&quot;local&quot;:&quot;decoding&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Queuing&quot;,&quot;local&quot;:&quot;queuing&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Benchmarks&quot;,&quot;local&quot;:&quot;benchmarks&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Resources&quot;,&quot;local&quot;:&quot;resources&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}">
		<link href="/docs/diffusers/pr_12652/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
		<link rel="modulepreload" href="/docs/diffusers/pr_12652/en/_app/immutable/entry/start.78b62fee.js">
		<link rel="modulepreload" href="/docs/diffusers/pr_12652/en/_app/immutable/chunks/scheduler.53228c21.js">
		<link rel="modulepreload" href="/docs/diffusers/pr_12652/en/_app/immutable/chunks/singletons.89d0b97a.js">
		<link rel="modulepreload" href="/docs/diffusers/pr_12652/en/_app/immutable/chunks/index.e93d0901.js">
		<link rel="modulepreload" href="/docs/diffusers/pr_12652/en/_app/immutable/chunks/paths.67f826e3.js">
		<link rel="modulepreload" href="/docs/diffusers/pr_12652/en/_app/immutable/entry/app.062e1615.js">
		<link rel="modulepreload" href="/docs/diffusers/pr_12652/en/_app/immutable/chunks/preload-helper.222e0275.js">
		<link rel="modulepreload" href="/docs/diffusers/pr_12652/en/_app/immutable/chunks/index.100fac89.js">
		<link rel="modulepreload" href="/docs/diffusers/pr_12652/en/_app/immutable/nodes/0.fe8af227.js">
		<link rel="modulepreload" href="/docs/diffusers/pr_12652/en/_app/immutable/chunks/each.e59479a4.js">
		<link rel="modulepreload" href="/docs/diffusers/pr_12652/en/_app/immutable/nodes/266.11ecf0ae.js">
		<link rel="modulepreload" href="/docs/diffusers/pr_12652/en/_app/immutable/chunks/CopyLLMTxtMenu.50ab6782.js">
		<link rel="modulepreload" href="/docs/diffusers/pr_12652/en/_app/immutable/chunks/globals.7f7f1b26.js">
		<link rel="modulepreload" href="/docs/diffusers/pr_12652/en/_app/immutable/chunks/IconCopy.38cf8f56.js">
		<link rel="modulepreload" href="/docs/diffusers/pr_12652/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.720a8c3c.js">
		<link rel="modulepreload" href="/docs/diffusers/pr_12652/en/_app/immutable/chunks/CodeBlock.d30a6509.js">
		<link rel="modulepreload" href="/docs/diffusers/pr_12652/en/_app/immutable/chunks/HfOption.fad27e59.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Remote inference&quot;,&quot;local&quot;:&quot;remote-inference&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Encoding&quot;,&quot;local&quot;:&quot;encoding&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Decoding&quot;,&quot;local&quot;:&quot;decoding&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Queuing&quot;,&quot;local&quot;:&quot;queuing&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Benchmarks&quot;,&quot;local&quot;:&quot;benchmarks&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Resources&quot;,&quot;local&quot;:&quot;resources&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"><!-- HEAD_svelte-u9bgzb_END -->      <p></p>   <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div>  <h1 class="relative group"><a id="remote-inference" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#remote-inference"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Remote inference</span></h1> <blockquote class="tip" data-svelte-h="svelte-3czh2q"><p>This is currently an experimental feature, and if you have any feedback, please feel free to leave it <a href="https://github.com/huggingface/diffusers/issues/new?template=remote-vae-pilot-feedback.yml" rel="nofollow">here</a>.</p></blockquote> <p data-svelte-h="svelte-zcw5r">Remote inference offloads the decoding and encoding process to a remote endpoint to relax the memory requirements for local inference with large models. This feature is powered by <a href="https://huggingface.co/docs/inference-endpoints/index" rel="nofollow">Inference Endpoints</a>. Refer to the table below for the supported models and endpoint.</p> <table data-svelte-h="svelte-1on2smg"><thead><tr><th>Model</th> <th>Endpoint</th> <th>Checkpoint</th> <th>Support</th></tr></thead> <tbody><tr><td>Stable Diffusion v1</td> <td><a href="https://q1bj3bpq6kzilnsu.us-east-1.aws.endpoints.huggingface.cloud" rel="nofollow">https://q1bj3bpq6kzilnsu.us-east-1.aws.endpoints.huggingface.cloud</a></td> <td><a href="https://huggingface.co/stabilityai/sd-vae-ft-mse" rel="nofollow">stabilityai/sd-vae-ft-mse</a></td> <td>encode/decode</td></tr> <tr><td>Stable Diffusion XL</td> <td><a href="https://x2dmsqunjd6k9prw.us-east-1.aws.endpoints.huggingface.cloud" rel="nofollow">https://x2dmsqunjd6k9prw.us-east-1.aws.endpoints.huggingface.cloud</a></td> <td><a href="https://huggingface.co/madebyollin/sdxl-vae-fp16-fix" rel="nofollow">madebyollin/sdxl-vae-fp16-fix</a></td> <td>encode/decode</td></tr> <tr><td>Flux</td> <td><a href="https://whhx50ex1aryqvw6.us-east-1.aws.endpoints.huggingface.cloud" rel="nofollow">https://whhx50ex1aryqvw6.us-east-1.aws.endpoints.huggingface.cloud</a></td> <td><a href="https://huggingface.co/black-forest-labs/FLUX.1-schnell" rel="nofollow">black-forest-labs/FLUX.1-schnell</a></td> <td>encode/decode</td></tr> <tr><td>HunyuanVideo</td> <td><a href="https://o7ywnmrahorts457.us-east-1.aws.endpoints.huggingface.cloud" rel="nofollow">https://o7ywnmrahorts457.us-east-1.aws.endpoints.huggingface.cloud</a></td> <td><a href="https://huggingface.co/hunyuanvideo-community/HunyuanVideo" rel="nofollow">hunyuanvideo-community/HunyuanVideo</a></td> <td>decode</td></tr></tbody></table> <p data-svelte-h="svelte-d8nohm">This guide will show you how to encode and decode latents with remote inference.</p>  <h2 class="relative group"><a id="encoding" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#encoding"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Encoding</span></h2> <p data-svelte-h="svelte-1jus9h0">Encoding converts images and videos into latent representations. Refer to the table below for the supported VAEs.</p> <p data-svelte-h="svelte-1iu76qj">Pass an image to <code>~utils.remote_encode</code> to encode it. The specific <code>scaling_factor</code> and <code>shift_factor</code> values for each model can be found in the <a href="../hybrid_inference/api_reference">Remote inference</a> API reference.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxPipeline
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image
<span class="hljs-keyword">from</span> diffusers.utils.remote_utils <span class="hljs-keyword">import</span> remote_encode

pipeline = FluxPipeline.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-schnell&quot;</span>,
    torch_dtype=torch.float16,
    vae=<span class="hljs-literal">None</span>,
    device_map=<span class="hljs-string">&quot;cuda&quot;</span>
)

init_image = load_image(
    <span class="hljs-string">&quot;https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/astronaut.jpg&quot;</span>
)
init_image = init_image.resize((<span class="hljs-number">768</span>, <span class="hljs-number">512</span>))

init_latent = remote_encode(
    endpoint=<span class="hljs-string">&quot;https://whhx50ex1aryqvw6.us-east-1.aws.endpoints.huggingface.cloud&quot;</span>,
    image=init_image,
    scaling_factor=<span class="hljs-number">0.3611</span>,
    shift_factor=<span class="hljs-number">0.1159</span>
)<!-- HTML_TAG_END --></pre></div>  <h2 class="relative group"><a id="decoding" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#decoding"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Decoding</span></h2> <p data-svelte-h="svelte-pi515h">Decoding converts latent representations back into images or videos. Refer to the table below for the available and supported VAEs.</p> <p data-svelte-h="svelte-lei4uj">Set the output type to <code>&quot;latent&quot;</code> in the pipeline and set the <code>vae</code> to <code>None</code>. Pass the latents to the <a href="/docs/diffusers/pr_12652/en/hybrid_inference/api_reference#diffusers.utils.remote_decode">remote_decode()</a> function. For Flux, the latents are packed so the <code>height</code> and <code>width</code> also need to be passed. The specific <code>scaling_factor</code> and <code>shift_factor</code> values for each model can be found in the <a href="../hybrid_inference/api_reference">Remote inference</a> API reference.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">Flux </div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">HunyuanVideo </div></div> <div class="language-select"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxPipeline

pipeline = FluxPipeline.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-schnell&quot;</span>,
    torch_dtype=torch.bfloat16,
    vae=<span class="hljs-literal">None</span>,
    device_map=<span class="hljs-string">&quot;cuda&quot;</span>
)

prompt = <span class="hljs-string">&quot;&quot;&quot;
A photorealistic Apollo-era photograph of a cat in a small astronaut suit with a bubble helmet, standing on the Moon and holding a flagpole planted in the dusty lunar soil. The flag shows a colorful paw-print emblem. Earth glows in the black sky above the stark gray surface, with sharp shadows and high-contrast lighting like vintage NASA photos.
&quot;&quot;&quot;</span>

latent = pipeline(
    prompt=prompt,
    guidance_scale=<span class="hljs-number">0.0</span>,
    num_inference_steps=<span class="hljs-number">4</span>,
    output_type=<span class="hljs-string">&quot;latent&quot;</span>,
).images
image = remote_decode(
    endpoint=<span class="hljs-string">&quot;https://whhx50ex1aryqvw6.us-east-1.aws.endpoints.huggingface.cloud/&quot;</span>,
    tensor=latent,
    height=<span class="hljs-number">1024</span>,
    width=<span class="hljs-number">1024</span>,
    scaling_factor=<span class="hljs-number">0.3611</span>,
    shift_factor=<span class="hljs-number">0.1159</span>,
)
image.save(<span class="hljs-string">&quot;image.jpg&quot;</span>)<!-- HTML_TAG_END --></pre></div> </div>  <h2 class="relative group"><a id="queuing" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#queuing"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Queuing</span></h2> <p data-svelte-h="svelte-3feanv">Remote inference supports queuing to process multiple generation requests. While the current latent is being decoded, you can queue the next prompt.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> queue
<span class="hljs-keyword">import</span> threading
<span class="hljs-keyword">from</span> IPython.display <span class="hljs-keyword">import</span> display
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionXLPipeline

<span class="hljs-keyword">def</span> <span class="hljs-title function_">decode_worker</span>(<span class="hljs-params">q: queue.Queue</span>):
    <span class="hljs-keyword">while</span> <span class="hljs-literal">True</span>:
        item = q.get()
        <span class="hljs-keyword">if</span> item <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
            <span class="hljs-keyword">break</span>
        image = remote_decode(
            endpoint=<span class="hljs-string">&quot;https://q1bj3bpq6kzilnsu.us-east-1.aws.endpoints.huggingface.cloud/&quot;</span>,
            tensor=item,
            scaling_factor=<span class="hljs-number">0.13025</span>,
        )
        display(image)
        q.task_done()

q = queue.Queue()
thread = threading.Thread(target=decode_worker, args=(q,), daemon=<span class="hljs-literal">True</span>)
thread.start()

<span class="hljs-keyword">def</span> <span class="hljs-title function_">decode</span>(<span class="hljs-params">latent: torch.Tensor</span>):
    q.put(latent)

prompts = [
    <span class="hljs-string">&quot;A grainy Apollo-era style photograph of a cat in a snug astronaut suit with a bubble helmet, standing on the lunar surface and gripping a flag with a paw-print emblem. The gray Moon landscape stretches behind it, Earth glowing vividly in the black sky, shadows crisp and high-contrast.&quot;</span>,
    <span class="hljs-string">&quot;A vintage 1960s sci-fi pulp magazine cover illustration of a heroic cat astronaut planting a flag on the Moon. Bold, saturated colors, exaggerated space gear, playful typography floating in the background, Earth painted in bright blues and greens.&quot;</span>,
    <span class="hljs-string">&quot;A hyper-detailed cinematic shot of a cat astronaut on the Moon holding a fluttering flag, fur visible through the helmet glass, lunar dust scattering under its feet. The vastness of space and Earth in the distance create an epic, awe-inspiring tone.&quot;</span>,
    <span class="hljs-string">&quot;A colorful cartoon drawing of a happy cat wearing a chunky, oversized spacesuit, proudly holding a flag with a big paw print on it. The Moon’s surface is simplified with craters drawn like doodles, and Earth in the sky has a smiling face.&quot;</span>,
    <span class="hljs-string">&quot;A monochrome 1969-style press photo of a “first cat on the Moon” moment. The cat, in a tiny astronaut suit, stands by a planted flag, with grainy textures, scratches, and a blurred Earth in the background, mimicking old archival space photos.&quot;</span>
]


pipeline = StableDiffusionXLPipeline.from_pretrained(
    <span class="hljs-string">&quot;https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0&quot;</span>,
    torch_dtype=torch.float16,
    vae=<span class="hljs-literal">None</span>,
    device_map=<span class="hljs-string">&quot;cuda&quot;</span>
)

pipeline.unet = pipeline.unet.to(memory_format=torch.channels_last)
pipeline.unet = torch.<span class="hljs-built_in">compile</span>(pipe.unet, mode=<span class="hljs-string">&quot;reduce-overhead&quot;</span>, fullgraph=<span class="hljs-literal">True</span>)

_ = pipeline(
    prompt=prompts[<span class="hljs-number">0</span>],
    output_type=<span class="hljs-string">&quot;latent&quot;</span>,
)

<span class="hljs-keyword">for</span> prompt <span class="hljs-keyword">in</span> prompts:
    latent = pipeline(
        prompt=prompt,
        output_type=<span class="hljs-string">&quot;latent&quot;</span>,
    ).images
    decode(latent)

q.put(<span class="hljs-literal">None</span>)
thread.join()<!-- HTML_TAG_END --></pre></div>  <h2 class="relative group"><a id="benchmarks" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#benchmarks"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Benchmarks</span></h2> <p data-svelte-h="svelte-7qt7c5">The tables demonstrate the memory requirements for encoding and decoding with Stable Diffusion v1.5 and SDXL on different GPUs.</p> <p data-svelte-h="svelte-1e09lvj">For the majority of these GPUs, the memory usage dictates whether other models (text encoders, UNet/transformer) need to be offloaded or required tiled encoding. The latter two techniques increases inference time and impacts quality.</p> <details data-svelte-h="svelte-1tipbpf"><summary>Encoding - Stable Diffusion v1.5</summary> <table><thead><tr><th align="left">GPU</th> <th align="left">Resolution</th> <th align="right">Time (seconds)</th> <th align="right">Memory (%)</th> <th align="right">Tiled Time (secs)</th> <th align="right">Tiled Memory (%)</th></tr></thead> <tbody><tr><td align="left">NVIDIA GeForce RTX 4090</td> <td align="left">512x512</td> <td align="right">0.015</td> <td align="right">3.51901</td> <td align="right">0.015</td> <td align="right">3.51901</td></tr> <tr><td align="left">NVIDIA GeForce RTX 4090</td> <td align="left">256x256</td> <td align="right">0.004</td> <td align="right">1.3154</td> <td align="right">0.005</td> <td align="right">1.3154</td></tr> <tr><td align="left">NVIDIA GeForce RTX 4090</td> <td align="left">2048x2048</td> <td align="right">0.402</td> <td align="right">47.1852</td> <td align="right">0.496</td> <td align="right">3.51901</td></tr> <tr><td align="left">NVIDIA GeForce RTX 4090</td> <td align="left">1024x1024</td> <td align="right">0.078</td> <td align="right">12.2658</td> <td align="right">0.094</td> <td align="right">3.51901</td></tr> <tr><td align="left">NVIDIA GeForce RTX 4080 SUPER</td> <td align="left">512x512</td> <td align="right">0.023</td> <td align="right">5.30105</td> <td align="right">0.023</td> <td align="right">5.30105</td></tr> <tr><td align="left">NVIDIA GeForce RTX 4080 SUPER</td> <td align="left">256x256</td> <td align="right">0.006</td> <td align="right">1.98152</td> <td align="right">0.006</td> <td align="right">1.98152</td></tr> <tr><td align="left">NVIDIA GeForce RTX 4080 SUPER</td> <td align="left">2048x2048</td> <td align="right">0.574</td> <td align="right">71.08</td> <td align="right">0.656</td> <td align="right">5.30105</td></tr> <tr><td align="left">NVIDIA GeForce RTX 4080 SUPER</td> <td align="left">1024x1024</td> <td align="right">0.111</td> <td align="right">18.4772</td> <td align="right">0.14</td> <td align="right">5.30105</td></tr> <tr><td align="left">NVIDIA GeForce RTX 3090</td> <td align="left">512x512</td> <td align="right">0.032</td> <td align="right">3.52782</td> <td align="right">0.032</td> <td align="right">3.52782</td></tr> <tr><td align="left">NVIDIA GeForce RTX 3090</td> <td align="left">256x256</td> <td align="right">0.01</td> <td align="right">1.31869</td> <td align="right">0.009</td> <td align="right">1.31869</td></tr> <tr><td align="left">NVIDIA GeForce RTX 3090</td> <td align="left">2048x2048</td> <td align="right">0.742</td> <td align="right">47.3033</td> <td align="right">0.954</td> <td align="right">3.52782</td></tr> <tr><td align="left">NVIDIA GeForce RTX 3090</td> <td align="left">1024x1024</td> <td align="right">0.136</td> <td align="right">12.2965</td> <td align="right">0.207</td> <td align="right">3.52782</td></tr> <tr><td align="left">NVIDIA GeForce RTX 3080</td> <td align="left">512x512</td> <td align="right">0.036</td> <td align="right">8.51761</td> <td align="right">0.036</td> <td align="right">8.51761</td></tr> <tr><td align="left">NVIDIA GeForce RTX 3080</td> <td align="left">256x256</td> <td align="right">0.01</td> <td align="right">3.18387</td> <td align="right">0.01</td> <td align="right">3.18387</td></tr> <tr><td align="left">NVIDIA GeForce RTX 3080</td> <td align="left">2048x2048</td> <td align="right">0.863</td> <td align="right">86.7424</td> <td align="right">1.191</td> <td align="right">8.51761</td></tr> <tr><td align="left">NVIDIA GeForce RTX 3080</td> <td align="left">1024x1024</td> <td align="right">0.157</td> <td align="right">29.6888</td> <td align="right">0.227</td> <td align="right">8.51761</td></tr> <tr><td align="left">NVIDIA GeForce RTX 3070</td> <td align="left">512x512</td> <td align="right">0.051</td> <td align="right">10.6941</td> <td align="right">0.051</td> <td align="right">10.6941</td></tr> <tr><td align="left">NVIDIA GeForce RTX 3070</td> <td align="left">256x256</td> <td align="right">0.015</td> <td align="right">3.99743</td> <td align="right">0.015</td> <td align="right">3.99743</td></tr> <tr><td align="left">NVIDIA GeForce RTX 3070</td> <td align="left">2048x2048</td> <td align="right">1.217</td> <td align="right">96.054</td> <td align="right">1.482</td> <td align="right">10.6941</td></tr> <tr><td align="left">NVIDIA GeForce RTX 3070</td> <td align="left">1024x1024</td> <td align="right">0.223</td> <td align="right">37.2751</td> <td align="right">0.327</td> <td align="right">10.6941</td></tr></tbody></table></details> <details data-svelte-h="svelte-4bsp3g"><summary>Encoding SDXL</summary> <table><thead><tr><th align="left">GPU</th> <th align="left">Resolution</th> <th align="right">Time (seconds)</th> <th align="right">Memory Consumed (%)</th> <th align="right">Tiled Time (seconds)</th> <th align="right">Tiled Memory (%)</th></tr></thead> <tbody><tr><td align="left">NVIDIA GeForce RTX 4090</td> <td align="left">512x512</td> <td align="right">0.029</td> <td align="right">4.95707</td> <td align="right">0.029</td> <td align="right">4.95707</td></tr> <tr><td align="left">NVIDIA GeForce RTX 4090</td> <td align="left">256x256</td> <td align="right">0.007</td> <td align="right">2.29666</td> <td align="right">0.007</td> <td align="right">2.29666</td></tr> <tr><td align="left">NVIDIA GeForce RTX 4090</td> <td align="left">2048x2048</td> <td align="right">0.873</td> <td align="right">66.3452</td> <td align="right">0.863</td> <td align="right">15.5649</td></tr> <tr><td align="left">NVIDIA GeForce RTX 4090</td> <td align="left">1024x1024</td> <td align="right">0.142</td> <td align="right">15.5479</td> <td align="right">0.143</td> <td align="right">15.5479</td></tr> <tr><td align="left">NVIDIA GeForce RTX 4080 SUPER</td> <td align="left">512x512</td> <td align="right">0.044</td> <td align="right">7.46735</td> <td align="right">0.044</td> <td align="right">7.46735</td></tr> <tr><td align="left">NVIDIA GeForce RTX 4080 SUPER</td> <td align="left">256x256</td> <td align="right">0.01</td> <td align="right">3.4597</td> <td align="right">0.01</td> <td align="right">3.4597</td></tr> <tr><td align="left">NVIDIA GeForce RTX 4080 SUPER</td> <td align="left">2048x2048</td> <td align="right">1.317</td> <td align="right">87.1615</td> <td align="right">1.291</td> <td align="right">23.447</td></tr> <tr><td align="left">NVIDIA GeForce RTX 4080 SUPER</td> <td align="left">1024x1024</td> <td align="right">0.213</td> <td align="right">23.4215</td> <td align="right">0.214</td> <td align="right">23.4215</td></tr> <tr><td align="left">NVIDIA GeForce RTX 3090</td> <td align="left">512x512</td> <td align="right">0.058</td> <td align="right">5.65638</td> <td align="right">0.058</td> <td align="right">5.65638</td></tr> <tr><td align="left">NVIDIA GeForce RTX 3090</td> <td align="left">256x256</td> <td align="right">0.016</td> <td align="right">2.45081</td> <td align="right">0.016</td> <td align="right">2.45081</td></tr> <tr><td align="left">NVIDIA GeForce RTX 3090</td> <td align="left">2048x2048</td> <td align="right">1.755</td> <td align="right">77.8239</td> <td align="right">1.614</td> <td align="right">18.4193</td></tr> <tr><td align="left">NVIDIA GeForce RTX 3090</td> <td align="left">1024x1024</td> <td align="right">0.265</td> <td align="right">18.4023</td> <td align="right">0.265</td> <td align="right">18.4023</td></tr> <tr><td align="left">NVIDIA GeForce RTX 3080</td> <td align="left">512x512</td> <td align="right">0.064</td> <td align="right">13.6568</td> <td align="right">0.064</td> <td align="right">13.6568</td></tr> <tr><td align="left">NVIDIA GeForce RTX 3080</td> <td align="left">256x256</td> <td align="right">0.018</td> <td align="right">5.91728</td> <td align="right">0.018</td> <td align="right">5.91728</td></tr> <tr><td align="left">NVIDIA GeForce RTX 3080</td> <td align="left">2048x2048</td> <td align="right">OOM</td> <td align="right">OOM</td> <td align="right">1.866</td> <td align="right">44.4717</td></tr> <tr><td align="left">NVIDIA GeForce RTX 3080</td> <td align="left">1024x1024</td> <td align="right">0.302</td> <td align="right">44.4308</td> <td align="right">0.302</td> <td align="right">44.4308</td></tr> <tr><td align="left">NVIDIA GeForce RTX 3070</td> <td align="left">512x512</td> <td align="right">0.093</td> <td align="right">17.1465</td> <td align="right">0.093</td> <td align="right">17.1465</td></tr> <tr><td align="left">NVIDIA GeForce RTX 3070</td> <td align="left">256x256</td> <td align="right">0.025</td> <td align="right">7.42931</td> <td align="right">0.026</td> <td align="right">7.42931</td></tr> <tr><td align="left">NVIDIA GeForce RTX 3070</td> <td align="left">2048x2048</td> <td align="right">OOM</td> <td align="right">OOM</td> <td align="right">2.674</td> <td align="right">55.8355</td></tr> <tr><td align="left">NVIDIA GeForce RTX 3070</td> <td align="left">1024x1024</td> <td align="right">0.443</td> <td align="right">55.7841</td> <td align="right">0.443</td> <td align="right">55.7841</td></tr></tbody></table></details> <details data-svelte-h="svelte-fyn0h1"><summary>Decoding - Stable Diffusion v1.5</summary> <table><thead><tr><th>GPU</th> <th>Resolution</th> <th>Time (seconds)</th> <th>Memory (%)</th> <th>Tiled Time (secs)</th> <th>Tiled Memory (%)</th></tr></thead> <tbody><tr><td>NVIDIA GeForce RTX 4090</td> <td>512x512</td> <td>0.031</td> <td>5.60%</td> <td>0.031 (0%)</td> <td>5.60%</td></tr> <tr><td>NVIDIA GeForce RTX 4090</td> <td>1024x1024</td> <td>0.148</td> <td>20.00%</td> <td>0.301 (+103%)</td> <td>5.60%</td></tr> <tr><td>NVIDIA GeForce RTX 4080</td> <td>512x512</td> <td>0.05</td> <td>8.40%</td> <td>0.050 (0%)</td> <td>8.40%</td></tr> <tr><td>NVIDIA GeForce RTX 4080</td> <td>1024x1024</td> <td>0.224</td> <td>30.00%</td> <td>0.356 (+59%)</td> <td>8.40%</td></tr> <tr><td>NVIDIA GeForce RTX 4070 Ti</td> <td>512x512</td> <td>0.066</td> <td>11.30%</td> <td>0.066 (0%)</td> <td>11.30%</td></tr> <tr><td>NVIDIA GeForce RTX 4070 Ti</td> <td>1024x1024</td> <td>0.284</td> <td>40.50%</td> <td>0.454 (+60%)</td> <td>11.40%</td></tr> <tr><td>NVIDIA GeForce RTX 3090</td> <td>512x512</td> <td>0.062</td> <td>5.20%</td> <td>0.062 (0%)</td> <td>5.20%</td></tr> <tr><td>NVIDIA GeForce RTX 3090</td> <td>1024x1024</td> <td>0.253</td> <td>18.50%</td> <td>0.464 (+83%)</td> <td>5.20%</td></tr> <tr><td>NVIDIA GeForce RTX 3080</td> <td>512x512</td> <td>0.07</td> <td>12.80%</td> <td>0.070 (0%)</td> <td>12.80%</td></tr> <tr><td>NVIDIA GeForce RTX 3080</td> <td>1024x1024</td> <td>0.286</td> <td>45.30%</td> <td>0.466 (+63%)</td> <td>12.90%</td></tr> <tr><td>NVIDIA GeForce RTX 3070</td> <td>512x512</td> <td>0.102</td> <td>15.90%</td> <td>0.102 (0%)</td> <td>15.90%</td></tr> <tr><td>NVIDIA GeForce RTX 3070</td> <td>1024x1024</td> <td>0.421</td> <td>56.30%</td> <td>0.746 (+77%)</td> <td>16.00%</td></tr></tbody></table></details> <details data-svelte-h="svelte-bqeu9g"><summary>Decoding SDXL</summary> <table><thead><tr><th>GPU</th> <th>Resolution</th> <th>Time (seconds)</th> <th>Memory Consumed (%)</th> <th>Tiled Time (seconds)</th> <th>Tiled Memory (%)</th></tr></thead> <tbody><tr><td>NVIDIA GeForce RTX 4090</td> <td>512x512</td> <td>0.057</td> <td>10.00%</td> <td>0.057 (0%)</td> <td>10.00%</td></tr> <tr><td>NVIDIA GeForce RTX 4090</td> <td>1024x1024</td> <td>0.256</td> <td>35.50%</td> <td>0.257 (+0.4%)</td> <td>35.50%</td></tr> <tr><td>NVIDIA GeForce RTX 4080</td> <td>512x512</td> <td>0.092</td> <td>15.00%</td> <td>0.092 (0%)</td> <td>15.00%</td></tr> <tr><td>NVIDIA GeForce RTX 4080</td> <td>1024x1024</td> <td>0.406</td> <td>53.30%</td> <td>0.406 (0%)</td> <td>53.30%</td></tr> <tr><td>NVIDIA GeForce RTX 4070 Ti</td> <td>512x512</td> <td>0.121</td> <td>20.20%</td> <td>0.120 (-0.8%)</td> <td>20.20%</td></tr> <tr><td>NVIDIA GeForce RTX 4070 Ti</td> <td>1024x1024</td> <td>0.519</td> <td>72.00%</td> <td>0.519 (0%)</td> <td>72.00%</td></tr> <tr><td>NVIDIA GeForce RTX 3090</td> <td>512x512</td> <td>0.107</td> <td>10.50%</td> <td>0.107 (0%)</td> <td>10.50%</td></tr> <tr><td>NVIDIA GeForce RTX 3090</td> <td>1024x1024</td> <td>0.459</td> <td>38.00%</td> <td>0.460 (+0.2%)</td> <td>38.00%</td></tr> <tr><td>NVIDIA GeForce RTX 3080</td> <td>512x512</td> <td>0.121</td> <td>25.60%</td> <td>0.121 (0%)</td> <td>25.60%</td></tr> <tr><td>NVIDIA GeForce RTX 3080</td> <td>1024x1024</td> <td>0.524</td> <td>93.00%</td> <td>0.524 (0%)</td> <td>93.00%</td></tr> <tr><td>NVIDIA GeForce RTX 3070</td> <td>512x512</td> <td>0.183</td> <td>31.80%</td> <td>0.183 (0%)</td> <td>31.80%</td></tr> <tr><td>NVIDIA GeForce RTX 3070</td> <td>1024x1024</td> <td>0.794</td> <td>96.40%</td> <td>0.794 (0%)</td> <td>96.40%</td></tr></tbody></table></details>  <h2 class="relative group"><a id="resources" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#resources"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Resources</span></h2> <ul data-svelte-h="svelte-fhy9uy"><li>Remote inference is also supported in <a href="https://github.com/vladmandic/sdnext" rel="nofollow">SD.Next</a> and <a href="https://github.com/kijai/ComfyUI-HFRemoteVae" rel="nofollow">ComfyUI-HFRemoteVae</a>.</li> <li>Refer to the <a href="https://huggingface.co/blog/remote_vae" rel="nofollow">Remote VAEs for decoding with Inference Endpoints</a> blog post to learn more.</li></ul> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/hybrid_inference/overview.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a>  <p></p> 
			
			<script>
				{
					__sveltekit_1te7xiv = {
						assets: "/docs/diffusers/pr_12652/en",
						base: "/docs/diffusers/pr_12652/en",
						env: {}
					};

					const element = document.currentScript.parentElement;

					const data = [null,null];

					Promise.all([
						import("/docs/diffusers/pr_12652/en/_app/immutable/entry/start.78b62fee.js"),
						import("/docs/diffusers/pr_12652/en/_app/immutable/entry/app.062e1615.js")
					]).then(([kit, app]) => {
						kit.start(app, element, {
							node_ids: [0, 266],
							data,
							form: null,
							error: null
						});
					});
				}
			</script>