diffusers/pr_13881/en/_app/immutable/nodes/90.BsD1eL92.js

import"../chunks/DsnmJJEf.js";import{i as W,h as w,C as N,H as d,a as A,D as r,E as C,s as z}from"../chunks/BtE7mKSK.js";import{p as U,o as B,s as e,f as q,a as T,b as J,c as n,d as b,n as o,r as a}from"../chunks/jDjavuwI.js";const L='{"title":"SanaWMTransformer3DModel","local":"sanawmtransformer3dmodel","sections":[{"title":"SanaWMTransformer3DModel","local":"diffusers.SanaWMTransformer3DModel","sections":[],"depth":2},{"title":"Transformer2DModelOutput","local":"diffusers.models.modeling_outputs.Transformer2DModelOutput","sections":[],"depth":2}],"depth":1}';var j=b('<meta name="hf:doc:metadata"/>'),E=b(`<p></p> <!> <!> <p>A 3D Diffusion Transformer (1.6B parameters) for camera-controlled image-to-video generation, used as the stage-1
sampler of <a href="/docs/diffusers/pr_13881/en/api/pipelines/sana_wm#diffusers.SanaWMPipeline">SanaWMPipeline</a>. The transformer combines:</p> <ul><li>a bidirectional GDN-Triton linear-attention main branch (depth 20, hidden 2240, 20 heads),</li> <li>a UCPE (Unified Camera Pose Embedding) camera-control branch that consumes a raymap + Plücker representation of
the requested trajectory, and</li> <li>a Wan-style 3D rotary position embedding plus periodic softmax-attention blocks injected every <code>softmax_every_n</code> layers.</li></ul> <p>The state-dict layout matches the public SANA-WM release one-to-one — the diffusers wrapper places the inner DiT
under a <code>_inner.</code> prefix. See <a href="/docs/diffusers/pr_13881/en/api/models/sana_wm_transformer3d#diffusers.SanaWMTransformer3DModel.add_inner_prefix">SanaWMTransformer3DModel.add_inner_prefix()</a> for the helper used by the conversion
script.</p> <p>The model can be loaded with:</p> <!> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>SANA-WM 1600M bidirectional camera-controlled DiT.</p> <p>Wraps <code>SanaMSVideoCamCtrl</code> (depth=20, hidden_size=2240, patch_size=(1,1,1), num_heads=20 — i.e. the public <code>Efficient-Large-Model/SANA-WM_bidirectional</code> release). <code>save_pretrained</code> / <code>from_pretrained</code> work out of the
box via <a href="/docs/diffusers/pr_13881/en/api/configuration#diffusers.ConfigMixin">ConfigMixin</a>.</p> <p>The state-dict is identical to the public sana checkpoint apart from the fixed <code>_inner.</code> prefix the wrapper adds
(see <code>add_inner_prefix</code>).</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Re-key a public SANA-WM state-dict for loading into this wrapper.</p> <p>The public release ships keys like <code>blocks.0.attn.qkv.weight</code>; the diffusers wrapper holds those parameters
under the <code>_inner.</code> prefix. Use this helper before <code>load_state_dict</code>:</p> <p>state = load_file(release_safetensors) state.pop(“pos_embed”, None)
model.load_state_dict(model.add_inner_prefix(state), strict=False)</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Run the SANA-WM DiT.</p></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The output of <a href="/docs/diffusers/pr_13881/en/api/models/transformer2d#diffusers.Transformer2DModel">Transformer2DModel</a>.</p></div> <!> <p></p>`,1);function I(v,k){U(k,!1),B(()=>{new URLSearchParams(window.location.search).get("fw")}),W();var l=E();w("x0f9w2",g=>{var M=j();z(M,"content",L),T(g,M)});var m=e(q(l),2);N(m,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var c=e(m,2);d(c,{title:"SanaWMTransformer3DModel",local:"sanawmtransformer3dmodel",headingTag:"h1"});var f=e(c,10);A(f,{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwU2FuYVdNVHJhbnNmb3JtZXIzRE1vZGVsJTBBJTBBdHJhbnNmb3JtZXIlMjAlM0QlMjBTYW5hV01UcmFuc2Zvcm1lcjNETW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMkVmZmljaWVudC1MYXJnZS1Nb2RlbCUyRlNBTkEtV01fYmlkaXJlY3Rpb25hbC1kaWZmdXNlcnMlMjIlMkMlMEElMjAlMjAlMjAlMjBzdWJmb2xkZXIlM0QlMjJ0cmFuc2Zvcm1lciUyMiUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYlMkMlMEEp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> SanaWMTransformer3DModel

transformer = SanaWMTransformer3DModel.from_pretrained(
    <span class="hljs-string">&quot;Efficient-Large-Model/SANA-WM_bidirectional-diffusers&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    torch_dtype=torch.bfloat16,
)`,lang:"python",wrap:!1});var p=e(f,2);d(p,{title:"SanaWMTransformer3DModel",local:"diffusers.SanaWMTransformer3DModel",headingTag:"h2"});var s=e(p,2),_=n(s);r(_,{name:"class diffusers.SanaWMTransformer3DModel",anchor:"diffusers.SanaWMTransformer3DModel",source:"https://github.com/huggingface/diffusers/blob/vr_13881/src/diffusers/models/transformers/transformer_sana_wm.py#L7375",parameters:[{name:"in_channels",val:": int = 128"},{name:"attn_type",val:": str = 'BidirectionalGDNTriton'"},{name:"camctrl_type",val:": str = 'BidirectionalGDNUCPESinglePathLiteLABothTriton'"},{name:"softmax_every_n",val:": int = 4"},{name:"linear_head_dim",val:": int = 112"},{name:"ffn_type",val:": str = 'GLUMBConvTemp'"},{name:"t_kernel_size",val:": int = 3"},{name:"conv_kernel_size",val:": int = 4"},{name:"k_conv_only",val:": bool = True"},{name:"pos_embed_type",val:": str = 'wan_rope'"},{name:"qk_norm",val:": bool = True"},{name:"cross_norm",val:": bool = True"},{name:"y_norm",val:": bool = True"},{name:"y_norm_scale_factor",val:": float = 0.01"},{name:"cam_attn_compress",val:": int = 1"},{name:"init_cam_from_base",val:": bool = True"},{name:"chunk_split_strategy",val:": str = 'first_chunk_plus_one'"},{name:"use_chunk_plucker_post_attn",val:": bool = True"},{name:"chunk_plucker_channels",val:": int = 48"},{name:"chunk_plucker_post_attn_blocks",val:": int = 20"},{name:"fp32_attention",val:": bool = True"},{name:"image_size",val:": int = 720"},{name:"caption_channels",val:": int = 2304"},{name:"model_max_length",val:": int = 300"},{name:"mlp_ratio",val:": float = 3.0"},{name:"mlp_acts",val:": tuple = ('silu', 'silu', None)"},{name:"use_pe",val:": bool = True"},{name:"learn_sigma",val:": bool = False"},{name:"pred_sigma",val:": bool = False"},{name:"mixed_precision",val:": str = 'bf16'"}],parametersDescription:[{anchor:"diffusers.SanaWMTransformer3DModel.in_channels",description:"<strong>in_channels</strong> (<em>int</em>, defaults to 128) &#x2014; VAE latent channels (LTX-2).",name:"in_channels"},{anchor:"diffusers.SanaWMTransformer3DModel.attn_type",description:"<strong>attn_type</strong> (<em>str</em>) &#x2014; Main-branch attention, e.g. <code>&quot;BidirectionalGDNTriton&quot;</code>.",name:"attn_type"},{anchor:"diffusers.SanaWMTransformer3DModel.camctrl_type",description:`<strong>camctrl_type</strong> (<em>str</em>) &#x2014; Camera-branch attention, e.g.
<code>&quot;BidirectionalGDNUCPESinglePathLiteLABothTriton&quot;</code>.`,name:"camctrl_type"},{anchor:"diffusers.SanaWMTransformer3DModel.softmax_every_n",description:"<strong>softmax_every_n</strong> (<em>int</em>, defaults to 4) &#x2014; Inject a softmax block every N blocks.",name:"softmax_every_n"},{anchor:"diffusers.SanaWMTransformer3DModel.linear_head_dim",description:"<strong>linear_head_dim</strong> (<em>int</em>, defaults to 112) &#x2014; GDN head dimension.",name:"linear_head_dim"},{anchor:"diffusers.SanaWMTransformer3DModel.ffn_type",description:"<strong>ffn_type</strong> (<em>str</em>, defaults to <code>&quot;GLUMBConvTemp&quot;</code>) &#x2014; FFN.",name:"ffn_type"},{anchor:"diffusers.SanaWMTransformer3DModel.t_kernel_size",description:"<strong>t_kernel_size</strong> (<em>int</em>, defaults to 3) &#x2014; Temporal conv kernel.",name:"t_kernel_size"},{anchor:"diffusers.SanaWMTransformer3DModel.conv_kernel_size",description:"<strong>conv_kernel_size</strong> (<em>int</em>, defaults to 4) &#x2014; Spatial conv kernel inside attention.",name:"conv_kernel_size"},{anchor:"diffusers.SanaWMTransformer3DModel.k_conv_only",description:"<strong>k_conv_only</strong> (<em>bool</em>, defaults to True) &#x2014; Apply conv only on K.",name:"k_conv_only"},{anchor:"diffusers.SanaWMTransformer3DModel.pos_embed_type",description:"<strong>pos_embed_type</strong> (<em>str</em>, defaults to <code>&quot;wan_rope&quot;</code>) &#x2014; Position embedding.",name:"pos_embed_type"},{anchor:"diffusers.SanaWMTransformer3DModel.qk_norm",description:"<strong>qk_norm</strong> (<em>bool</em>, defaults to True) &#x2014; RMSNorm on Q/K.",name:"qk_norm"},{anchor:"diffusers.SanaWMTransformer3DModel.cross_norm",description:"<strong>cross_norm</strong> (<em>bool</em>, defaults to True) &#x2014; RMSNorm on cross-attention K.",name:"cross_norm"},{anchor:"diffusers.SanaWMTransformer3DModel.y_norm",description:"<strong>y_norm</strong> (<em>bool</em>, defaults to True) &#x2014; Apply <code>attention_y_norm</code> to text embeddings.",name:"y_norm"},{anchor:"diffusers.SanaWMTransformer3DModel.y_norm_scale_factor",description:"<strong>y_norm_scale_factor</strong> (<em>float</em>, defaults to 0.01) &#x2014; Scale factor for <code>attention_y_norm</code>.",name:"y_norm_scale_factor"},{anchor:"diffusers.SanaWMTransformer3DModel.init_cam_from_base",description:"<strong>init_cam_from_base</strong> (<em>bool</em>, defaults to True) &#x2014; Initialize camera branch QKV from main.",name:"init_cam_from_base"},{anchor:"diffusers.SanaWMTransformer3DModel.chunk_split_strategy",description:"<strong>chunk_split_strategy</strong> (<em>str</em>, defaults to <code>&quot;first_chunk_plus_one&quot;</code>). &#x2014;",name:"chunk_split_strategy"},{anchor:"diffusers.SanaWMTransformer3DModel.use_chunk_plucker_post_attn",description:"<strong>use_chunk_plucker_post_attn</strong> (<em>bool</em>, defaults to True). &#x2014;",name:"use_chunk_plucker_post_attn"},{anchor:"diffusers.SanaWMTransformer3DModel.chunk_plucker_channels",description:"<strong>chunk_plucker_channels</strong> (<em>int</em>, defaults to 48) &#x2014; <code>6 dims * temporal_stride 8</code>.",name:"chunk_plucker_channels"},{anchor:"diffusers.SanaWMTransformer3DModel.chunk_plucker_post_attn_blocks",description:"<strong>chunk_plucker_post_attn_blocks</strong> (<em>int</em>, defaults to 20) &#x2014; All blocks.",name:"chunk_plucker_post_attn_blocks"},{anchor:"diffusers.SanaWMTransformer3DModel.fp32_attention",description:"<strong>fp32_attention</strong> (<em>bool</em>, defaults to True) &#x2014; Run attention in fp32.",name:"fp32_attention"},{anchor:"diffusers.SanaWMTransformer3DModel.image_size",description:"<strong>image_size</strong> (<em>int</em>, defaults to 720) &#x2014; Nominal image size.",name:"image_size"},{anchor:"diffusers.SanaWMTransformer3DModel.caption_channels",description:"<strong>caption_channels</strong> (<em>int</em>, defaults to 2304) &#x2014; Gemma-2 hidden size.",name:"caption_channels"},{anchor:"diffusers.SanaWMTransformer3DModel.model_max_length",description:"<strong>model_max_length</strong> (<em>int</em>, defaults to 300) &#x2014; Max prompt tokens.",name:"model_max_length"}]});var t=e(_,8),y=n(t);r(y,{name:"add_inner_prefix",anchor:"diffusers.SanaWMTransformer3DModel.add_inner_prefix",source:"https://github.com/huggingface/diffusers/blob/vr_13881/src/diffusers/models/transformers/transformer_sana_wm.py#L7493",parameters:[{name:"state_dict",val:": dict"}]}),o(6),a(t);var u=e(t,2),x=n(u);r(x,{name:"forward",anchor:"diffusers.SanaWMTransformer3DModel.forward",source:"https://github.com/huggingface/diffusers/blob/vr_13881/src/diffusers/models/transformers/transformer_sana_wm.py#L7505",parameters:[{name:"hidden_states",val:": torch.Tensor"},{name:"timestep",val:": torch.Tensor"},{name:"encoder_hidden_states",val:": torch.Tensor"},{name:"encoder_attention_mask",val:": torch.Tensor | None = None"},{name:"mask",val:": torch.Tensor | None = None"},{name:"return_dict",val:": bool = True"},{name:"**kwargs",val:": Any"}],parametersDescription:[{anchor:"diffusers.SanaWMTransformer3DModel.forward.hidden_states",description:"<strong>hidden_states</strong> &#x2014; <code>(B, C, T, H, W)</code> latents.",name:"hidden_states"},{anchor:"diffusers.SanaWMTransformer3DModel.forward.timestep",description:"<strong>timestep</strong> &#x2014; <code>(B, 1, T)</code> per-frame diffusion timesteps (LTX style).",name:"timestep"},{anchor:"diffusers.SanaWMTransformer3DModel.forward.encoder_hidden_states",description:"<strong>encoder_hidden_states</strong> &#x2014; <code>(B, 1, L, D_caption)</code> text embeddings.",name:"encoder_hidden_states"},{anchor:"diffusers.SanaWMTransformer3DModel.forward.encoder_attention_mask",description:"<strong>encoder_attention_mask</strong> &#x2014; <code>(B, L)</code> text attention mask (diffusers convention).",name:"encoder_attention_mask"},{anchor:"diffusers.SanaWMTransformer3DModel.forward.mask",description:`<strong>mask</strong> &#x2014; Alias for <code>encoder_attention_mask</code> matching the inner Sana DiT&#x2019;s
kwarg name. If both are passed, <code>mask</code> takes precedence.`,name:"mask"},{anchor:"diffusers.SanaWMTransformer3DModel.forward.return_dict",description:`<strong>return_dict</strong> &#x2014; If <code>True</code> (default), returns a <code>Transformer2DModelOutput</code>;
otherwise returns a one-tuple <code>(sample,)</code>.`,name:"return_dict"},{anchor:"diffusers.SanaWMTransformer3DModel.forward.*kwargs",description:`*<strong>*kwargs</strong> &#x2014; SANA-WM-specific conditioning &#x2014; at minimum
<code>data_info</code>, <code>camera_conditions</code>, <code>chunk_plucker</code>.`,name:"*kwargs"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>Transformer2DModelOutput</code> with <code>sample</code> of shape <code>(B, C, T, H, W)</code>.</p>
`}),o(2),a(u),a(s);var h=e(s,2);d(h,{title:"Transformer2DModelOutput",local:"diffusers.models.modeling_outputs.Transformer2DModelOutput",headingTag:"h2"});var i=e(h,2),D=n(i);r(D,{name:"class diffusers.models.modeling_outputs.Transformer2DModelOutput",anchor:"diffusers.models.modeling_outputs.Transformer2DModelOutput",source:"https://github.com/huggingface/diffusers/blob/vr_13881/src/diffusers/models/modeling_outputs.py#L21",parameters:[{name:"sample",val:": torch.Tensor"}],parametersDescription:[{anchor:"diffusers.models.modeling_outputs.Transformer2DModelOutput.sample",description:`<strong>sample</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, num_channels, height, width)</code> or <code>(batch size, num_vector_embeds - 1, num_latent_pixels)</code> if <a href="/docs/diffusers/pr_13881/en/api/models/transformer2d#diffusers.Transformer2DModel">Transformer2DModel</a> is discrete) &#x2014;
The hidden states output conditioned on the <code>encoder_hidden_states</code> input. If discrete, returns probability
distributions for the unnoised latent pixels.`,name:"sample"}]}),o(2),a(i);var S=e(i,2);C(S,{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/api/models/sana_wm_transformer3d.md"}),o(2),T(v,l),J()}export{I as component};