diffusers/pr_14229/en/_app/immutable/nodes/31.DeQL8uVC.js

import"../chunks/DsnmJJEf.js";import{i as N,h as B,C as G,H as o,a as l,D as n,E as C,s as X}from"../chunks/BtE7mKSK.js";import{p as L,o as V,s as e,f as k,a as R,b as Q,c as t,d as U,r as a,n as j}from"../chunks/jDjavuwI.js";const S='{"title":"AutoencoderRAE","local":"autoencoderrae","sections":[{"title":"Loading a pretrained model","local":"loading-a-pretrained-model","sections":[],"depth":2},{"title":"Encoding and decoding a real image","local":"encoding-and-decoding-a-real-image","sections":[],"depth":2},{"title":"Latent normalization","local":"latent-normalization","sections":[],"depth":2},{"title":"AutoencoderRAE","local":"diffusers.AutoencoderRAE","sections":[],"depth":2},{"title":"DecoderOutput","local":"diffusers.models.autoencoders.vae.DecoderOutput","sections":[],"depth":2}],"depth":1}';var D=U('<meta name="hf:doc:metadata"/>'),F=U(`<p></p> <!> <!> <p>The Representation Autoencoder (RAE) model introduced in <a href="https://huggingface.co/papers/2510.11690" rel="nofollow">Diffusion Transformers with Representation Autoencoders</a> by Boyang Zheng, Nanye Ma, Shengbang Tong, Saining Xie from NYU VISIONx.</p> <p>RAE combines a frozen pretrained vision encoder (DINOv2, SigLIP2, or MAE) with a trainable ViT-MAE-style decoder. In the two-stage RAE training recipe, the autoencoder is trained in stage 1 (reconstruction), and then a diffusion model is trained on the resulting latent space in stage 2 (generation).</p> <p>The following RAE models are released and supported in Diffusers:</p> <table><thead><tr><th align="left">Model</th><th align="left">Encoder</th><th align="left">Latent shape (224px input)</th></tr></thead><tbody><tr><td align="left"><a href="https://huggingface.co/nyu-visionx/RAE-dinov2-wReg-base-ViTXL-n08" rel="nofollow"><code>nyu-visionx/RAE-dinov2-wReg-base-ViTXL-n08</code></a></td><td align="left">DINOv2-base</td><td align="left">768 x 16 x 16</td></tr><tr><td align="left"><a href="https://huggingface.co/nyu-visionx/RAE-dinov2-wReg-base-ViTXL-n08-i512" rel="nofollow"><code>nyu-visionx/RAE-dinov2-wReg-base-ViTXL-n08-i512</code></a></td><td align="left">DINOv2-base (512px)</td><td align="left">768 x 32 x 32</td></tr><tr><td align="left"><a href="https://huggingface.co/nyu-visionx/RAE-dinov2-wReg-small-ViTXL-n08" rel="nofollow"><code>nyu-visionx/RAE-dinov2-wReg-small-ViTXL-n08</code></a></td><td align="left">DINOv2-small</td><td align="left">384 x 16 x 16</td></tr><tr><td align="left"><a href="https://huggingface.co/nyu-visionx/RAE-dinov2-wReg-large-ViTXL-n08" rel="nofollow"><code>nyu-visionx/RAE-dinov2-wReg-large-ViTXL-n08</code></a></td><td align="left">DINOv2-large</td><td align="left">1024 x 16 x 16</td></tr><tr><td align="left"><a href="https://huggingface.co/nyu-visionx/RAE-siglip2-base-p16-i256-ViTXL-n08" rel="nofollow"><code>nyu-visionx/RAE-siglip2-base-p16-i256-ViTXL-n08</code></a></td><td align="left">SigLIP2-base</td><td align="left">768 x 16 x 16</td></tr><tr><td align="left"><a href="https://huggingface.co/nyu-visionx/RAE-mae-base-p16-ViTXL-n08" rel="nofollow"><code>nyu-visionx/RAE-mae-base-p16-ViTXL-n08</code></a></td><td align="left">MAE-base</td><td align="left">768 x 16 x 16</td></tr></tbody></table> <!> <!> <!> <!> <!> <p>Some pretrained checkpoints include per-channel <code>latents_mean</code> and <code>latents_std</code> statistics for normalizing the latent space. When present, <code>encode</code> and <code>decode</code> automatically apply the normalization and denormalization, respectively.</p> <!> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Representation Autoencoder (RAE) model for encoding images to latents and decoding latents to images.</p> <p>This model uses a frozen pretrained encoder (DINOv2, SigLIP2, or MAE) with a trainable ViT decoder to reconstruct
images from learned representations.</p> <p>This model inherits from <a href="/docs/diffusers/pr_14229/en/api/models/overview#diffusers.ModelMixin">ModelMixin</a>. Check the superclass documentation for its generic methods implemented for
all models (such as downloading or saving).</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Output of decoding method.</p></div> <!> <p></p>`,1);function H(J,E){L(E,!1),V(()=>{new URLSearchParams(window.location.search).get("fw")}),N();var c=F();B("1la7ssz",w=>{var A=D();X(A,"content",S),R(w,A)});var p=e(k(c),2);G(p,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var m=e(p,2);o(m,{title:"AutoencoderRAE",local:"autoencoderrae",headingTag:"h1"});var u=e(m,10);o(u,{title:"Loading a pretrained model",local:"loading-a-pretrained-model",headingTag:"h2"});var h=e(u,2);l(h,{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEF1dG9lbmNvZGVyUkFFJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvZW5jb2RlclJBRS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIybnl1LXZpc2lvbnglMkZSQUUtZGlub3YyLXdSZWctYmFzZS1WaVRYTC1uMDglMjIlMEEpLnRvKCUyMmN1ZGElMjIpLmV2YWwoKQ==",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoencoderRAE

model = AutoencoderRAE.from_pretrained(
    <span class="hljs-string">&quot;nyu-visionx/RAE-dinov2-wReg-base-ViTXL-n08&quot;</span>
).to(<span class="hljs-string">&quot;cuda&quot;</span>).<span class="hljs-built_in">eval</span>()`,lang:"python",wrap:!1});var g=e(h,2);o(g,{title:"Encoding and decoding a real image",local:"encoding-and-decoding-a-real-image",headingTag:"h2"});var f=e(g,2);l(f,{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQXV0b2VuY29kZXJSQUUlMEFmcm9tJTIwZGlmZnVzZXJzLnV0aWxzJTIwaW1wb3J0JTIwbG9hZF9pbWFnZSUwQWZyb20lMjB0b3JjaHZpc2lvbi50cmFuc2Zvcm1zLmZ1bmN0aW9uYWwlMjBpbXBvcnQlMjB0b190ZW5zb3IlMkMlMjB0b19waWxfaW1hZ2UlMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9lbmNvZGVyUkFFLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJueXUtdmlzaW9ueCUyRlJBRS1kaW5vdjItd1JlZy1iYXNlLVZpVFhMLW4wOCUyMiUwQSkudG8oJTIyY3VkYSUyMikuZXZhbCgpJTBBJTBBaW1hZ2UlMjAlM0QlMjBsb2FkX2ltYWdlKCUyMmh0dHBzJTNBJTJGJTJGaHVnZ2luZ2ZhY2UuY28lMkZkYXRhc2V0cyUyRmh1Z2dpbmdmYWNlJTJGZG9jdW1lbnRhdGlvbi1pbWFnZXMlMkZyZXNvbHZlJTJGbWFpbiUyRmRpZmZ1c2VycyUyRmNhdC5wbmclMjIpJTBBaW1hZ2UlMjAlM0QlMjBpbWFnZS5jb252ZXJ0KCUyMlJHQiUyMikucmVzaXplKCgyMjQlMkMlMjAyMjQpKSUwQXglMjAlM0QlMjB0b190ZW5zb3IoaW1hZ2UpLnVuc3F1ZWV6ZSgwKS50byglMjJjdWRhJTIyKSUyMCUyMCUyMyUyMCgxJTJDJTIwMyUyQyUyMDIyNCUyQyUyMDIyNCklMkMlMjB2YWx1ZXMlMjBpbiUyMCU1QjAlMkMlMjAxJTVEJTBBJTBBd2l0aCUyMHRvcmNoLm5vX2dyYWQoKSUzQSUwQSUyMCUyMCUyMCUyMGxhdGVudHMlMjAlM0QlMjBtb2RlbC5lbmNvZGUoeCkubGF0ZW50JTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwKDElMkMlMjA3NjglMkMlMjAxNiUyQyUyMDE2KSUwQSUyMCUyMCUyMCUyMHJlY29uJTIwJTNEJTIwbW9kZWwuZGVjb2RlKGxhdGVudHMpLnNhbXBsZSUyMCUyMCUyMCUyMCUyMCUyMyUyMCgxJTJDJTIwMyUyQyUyMDI1NiUyQyUyMDI1NiklMEElMEFyZWNvbl9pbWFnZSUyMCUzRCUyMHRvX3BpbF9pbWFnZShyZWNvbiU1QjAlNUQuY2xhbXAoMCUyQyUyMDEpLmNwdSgpKSUwQXJlY29uX2ltYWdlLnNhdmUoJTIycmVjb24ucG5nJTIyKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoencoderRAE
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image
<span class="hljs-keyword">from</span> torchvision.transforms.functional <span class="hljs-keyword">import</span> to_tensor, to_pil_image

model = AutoencoderRAE.from_pretrained(
    <span class="hljs-string">&quot;nyu-visionx/RAE-dinov2-wReg-base-ViTXL-n08&quot;</span>
).to(<span class="hljs-string">&quot;cuda&quot;</span>).<span class="hljs-built_in">eval</span>()

image = load_image(<span class="hljs-string">&quot;https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png&quot;</span>)
image = image.convert(<span class="hljs-string">&quot;RGB&quot;</span>).resize((<span class="hljs-number">224</span>, <span class="hljs-number">224</span>))
x = to_tensor(image).unsqueeze(<span class="hljs-number">0</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)  <span class="hljs-comment"># (1, 3, 224, 224), values in [0, 1]</span>

<span class="hljs-keyword">with</span> torch.no_grad():
    latents = model.encode(x).latent        <span class="hljs-comment"># (1, 768, 16, 16)</span>
    recon = model.decode(latents).sample     <span class="hljs-comment"># (1, 3, 256, 256)</span>

recon_image = to_pil_image(recon[<span class="hljs-number">0</span>].clamp(<span class="hljs-number">0</span>, <span class="hljs-number">1</span>).cpu())
recon_image.save(<span class="hljs-string">&quot;recon.png&quot;</span>)`,lang:"python",wrap:!1});var y=e(f,2);o(y,{title:"Latent normalization",local:"latent-normalization",headingTag:"h2"});var _=e(y,4);l(_,{code:"bW9kZWwlMjAlM0QlMjBBdXRvZW5jb2RlclJBRS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIybnl1LXZpc2lvbnglMkZSQUUtZGlub3YyLXdSZWctYmFzZS1WaVRYTC1uMDglMjIlMEEpLnRvKCUyMmN1ZGElMjIpLmV2YWwoKSUwQSUwQSUyMyUyMExhdGVudCUyMG5vcm1hbGl6YXRpb24lMjBpcyUyMGhhbmRsZWQlMjBhdXRvbWF0aWNhbGx5JTIwaW5zaWRlJTIwZW5jb2RlJTJGZGVjb2RlJTBBJTIzJTIwd2hlbiUyMHRoZSUyMGNoZWNrcG9pbnQlMjBjb25maWclMjBpbmNsdWRlcyUyMGxhdGVudHNfbWVhbiUyRmxhdGVudHNfc3RkLiUwQXdpdGglMjB0b3JjaC5ub19ncmFkKCklM0ElMEElMjAlMjAlMjAlMjBsYXRlbnRzJTIwJTNEJTIwbW9kZWwuZW5jb2RlKHgpLmxhdGVudCUyMCUyMCUyMCUyMyUyMG5vcm1hbGl6ZWQlMjBsYXRlbnRzJTBBJTIwJTIwJTIwJTIwcmVjb24lMjAlM0QlMjBtb2RlbC5kZWNvZGUobGF0ZW50cykuc2FtcGxl",highlighted:`model = AutoencoderRAE.from_pretrained(
    <span class="hljs-string">&quot;nyu-visionx/RAE-dinov2-wReg-base-ViTXL-n08&quot;</span>
).to(<span class="hljs-string">&quot;cuda&quot;</span>).<span class="hljs-built_in">eval</span>()

<span class="hljs-comment"># Latent normalization is handled automatically inside encode/decode</span>
<span class="hljs-comment"># when the checkpoint config includes latents_mean/latents_std.</span>
<span class="hljs-keyword">with</span> torch.no_grad():
    latents = model.encode(x).latent   <span class="hljs-comment"># normalized latents</span>
    recon = model.decode(latents).sample`,lang:"python",wrap:!1});var b=e(_,2);o(b,{title:"AutoencoderRAE",local:"diffusers.AutoencoderRAE",headingTag:"h2"});var s=e(b,2),v=t(s);n(v,{name:"class diffusers.AutoencoderRAE",anchor:"diffusers.AutoencoderRAE",source:"https://github.com/huggingface/diffusers/blob/vr_14229/src/diffusers/models/autoencoders/autoencoder_rae.py#L393",parameters:[{name:"encoder_type",val:": str = 'dinov2'"},{name:"encoder_hidden_size",val:": int = 768"},{name:"encoder_patch_size",val:": int = 14"},{name:"encoder_num_hidden_layers",val:": int = 12"},{name:"decoder_hidden_size",val:": int = 512"},{name:"decoder_num_hidden_layers",val:": int = 8"},{name:"decoder_num_attention_heads",val:": int = 16"},{name:"decoder_intermediate_size",val:": int = 2048"},{name:"patch_size",val:": int = 16"},{name:"encoder_input_size",val:": int = 224"},{name:"image_size",val:": int | None = None"},{name:"num_channels",val:": int = 3"},{name:"encoder_norm_mean",val:": list | None = None"},{name:"encoder_norm_std",val:": list | None = None"},{name:"latents_mean",val:": typing.Union[list, tuple, torch.Tensor, NoneType] = None"},{name:"latents_std",val:": typing.Union[list, tuple, torch.Tensor, NoneType] = None"},{name:"noise_tau",val:": float = 0.0"},{name:"reshape_to_2d",val:": bool = True"},{name:"use_encoder_loss",val:": bool = False"},{name:"scaling_factor",val:": float = 1.0"}],parametersDescription:[{anchor:"diffusers.AutoencoderRAE.encoder_type",description:`<strong>encoder_type</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;dinov2&quot;</code>) &#x2014;
Type of frozen encoder to use. One of <code>&quot;dinov2&quot;</code>, <code>&quot;siglip2&quot;</code>, or <code>&quot;mae&quot;</code>.`,name:"encoder_type"},{anchor:"diffusers.AutoencoderRAE.encoder_hidden_size",description:`<strong>encoder_hidden_size</strong> (<code>int</code>, <em>optional</em>, defaults to <code>768</code>) &#x2014;
Hidden size of the encoder model.`,name:"encoder_hidden_size"},{anchor:"diffusers.AutoencoderRAE.encoder_patch_size",description:`<strong>encoder_patch_size</strong> (<code>int</code>, <em>optional</em>, defaults to <code>14</code>) &#x2014;
Patch size of the encoder model.`,name:"encoder_patch_size"},{anchor:"diffusers.AutoencoderRAE.encoder_num_hidden_layers",description:`<strong>encoder_num_hidden_layers</strong> (<code>int</code>, <em>optional</em>, defaults to <code>12</code>) &#x2014;
Number of hidden layers in the encoder model.`,name:"encoder_num_hidden_layers"},{anchor:"diffusers.AutoencoderRAE.patch_size",description:`<strong>patch_size</strong> (<code>int</code>, <em>optional</em>, defaults to <code>16</code>) &#x2014;
Decoder patch size (used for unpatchify and decoder head).`,name:"patch_size"},{anchor:"diffusers.AutoencoderRAE.encoder_input_size",description:`<strong>encoder_input_size</strong> (<code>int</code>, <em>optional</em>, defaults to <code>224</code>) &#x2014;
Input size expected by the encoder.`,name:"encoder_input_size"},{anchor:"diffusers.AutoencoderRAE.image_size",description:`<strong>image_size</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Decoder output image size. If <code>None</code>, it is derived from encoder token count and <code>patch_size</code> like
RAE-main: <code>image_size = patch_size * sqrt(num_patches)</code>, where <code>num_patches = (encoder_input_size // encoder_patch_size) ** 2</code>.`,name:"image_size"},{anchor:"diffusers.AutoencoderRAE.num_channels",description:`<strong>num_channels</strong> (<code>int</code>, <em>optional</em>, defaults to <code>3</code>) &#x2014;
Number of input/output channels.`,name:"num_channels"},{anchor:"diffusers.AutoencoderRAE.encoder_norm_mean",description:`<strong>encoder_norm_mean</strong> (<code>list</code>, <em>optional</em>, defaults to <code>[0.485, 0.456, 0.406]</code>) &#x2014;
Channel-wise mean for encoder input normalization (ImageNet defaults).`,name:"encoder_norm_mean"},{anchor:"diffusers.AutoencoderRAE.encoder_norm_std",description:`<strong>encoder_norm_std</strong> (<code>list</code>, <em>optional</em>, defaults to <code>[0.229, 0.224, 0.225]</code>) &#x2014;
Channel-wise std for encoder input normalization (ImageNet defaults).`,name:"encoder_norm_std"},{anchor:"diffusers.AutoencoderRAE.latents_mean",description:`<strong>latents_mean</strong> (<code>list</code> or <code>tuple</code>, <em>optional</em>) &#x2014;
Optional mean for latent normalization. Tensor inputs are accepted and converted to config-serializable
lists.`,name:"latents_mean"},{anchor:"diffusers.AutoencoderRAE.latents_std",description:`<strong>latents_std</strong> (<code>list</code> or <code>tuple</code>, <em>optional</em>) &#x2014;
Optional standard deviation for latent normalization. Tensor inputs are accepted and converted to
config-serializable lists.`,name:"latents_std"},{anchor:"diffusers.AutoencoderRAE.noise_tau",description:`<strong>noise_tau</strong> (<code>float</code>, <em>optional</em>, defaults to <code>0.0</code>) &#x2014;
Noise level for training (adds noise to latents during training).`,name:"noise_tau"},{anchor:"diffusers.AutoencoderRAE.reshape_to_2d",description:`<strong>reshape_to_2d</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to reshape latents to 2D (B, C, H, W) format.`,name:"reshape_to_2d"},{anchor:"diffusers.AutoencoderRAE.use_encoder_loss",description:`<strong>use_encoder_loss</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to use encoder hidden states in the loss (for advanced training).`,name:"use_encoder_loss"}]});var d=e(v,8),Z=t(d);n(Z,{name:"encode",anchor:"diffusers.AutoencoderRAE.encode",source:"https://github.com/huggingface/diffusers/blob/vr_14229/src/diffusers/models/autoencoders/autoencoder_rae.py#L640",parameters:[{name:"x",val:": Tensor"},{name:"return_dict",val:": bool = True"},{name:"generator",val:": typing.Optional[torch.Generator] = None"}]}),a(d);var r=e(d,2),x=t(r);n(x,{name:"decode",anchor:"diffusers.AutoencoderRAE.decode",source:"https://github.com/huggingface/diffusers/blob/vr_14229/src/diffusers/models/autoencoders/autoencoder_rae.py#L671",parameters:[{name:"z",val:": Tensor"},{name:"return_dict",val:": bool = True"}]}),a(r);var M=e(r,2),z=t(M);n(z,{name:"forward",anchor:"diffusers.AutoencoderRAE.forward",source:"https://github.com/huggingface/diffusers/blob/vr_14229/src/diffusers/models/autoencoders/autoencoder_rae.py#L682",parameters:[{name:"sample",val:": Tensor"},{name:"return_dict",val:": bool = True"},{name:"generator",val:": typing.Optional[torch.Generator] = None"}],parametersDescription:[{anchor:"diffusers.AutoencoderRAE.forward.sample",description:"<strong>sample</strong> (<code>torch.Tensor</code>) &#x2014; Input sample.",name:"sample"},{anchor:"diffusers.AutoencoderRAE.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not to return a <code>DecoderOutput</code> instead of a plain tuple.`,name:"return_dict"},{anchor:"diffusers.AutoencoderRAE.forward.generator",description:`<strong>generator</strong> (<code>torch.Generator</code>, <em>optional</em>) &#x2014;
A <a href="https://pytorch.org/docs/stable/generated/torch.Generator.html" rel="nofollow"><code>torch.Generator</code></a> to make sampling
deterministic.`,name:"generator"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>If <code>return_dict</code> is True, a <code>DecoderOutput</code> is returned, otherwise a plain <code>tuple</code> is returned.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>DecoderOutput</code> or <code>tuple</code></p>
`}),a(M),a(s);var T=e(s,2);o(T,{title:"DecoderOutput",local:"diffusers.models.autoencoders.vae.DecoderOutput",headingTag:"h2"});var i=e(T,2),I=t(i);n(I,{name:"class diffusers.models.autoencoders.vae.DecoderOutput",anchor:"diffusers.models.autoencoders.vae.DecoderOutput",source:"https://github.com/huggingface/diffusers/blob/vr_14229/src/diffusers/models/autoencoders/vae.py#L46",parameters:[{name:"sample",val:": Tensor"},{name:"commit_loss",val:": typing.Optional[torch.FloatTensor] = None"}],parametersDescription:[{anchor:"diffusers.models.autoencoders.vae.DecoderOutput.sample",description:`<strong>sample</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, num_channels, height, width)</code>) &#x2014;
The decoded output sample from the last layer of the model.`,name:"sample"}]}),j(2),a(i);var W=e(i,2);C(W,{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/api/models/autoencoder_rae.md"}),j(2),R(J,c),Q()}export{H as component};