diffusers/pr_13813/en/_app/immutable/nodes/60.d4b0e9eb.js

import{s as ie,n as ae,o as me}from"../chunks/scheduler.53228c21.js";import{S as de,i as le,e as i,s as t,c as b,h as fe,a,d as n,b as r,f as U,g as $,j as N,k as F,l as M,m as o,n as y,t as C,o as k,p as O}from"../chunks/index.cac5d66a.js";import{C as ue}from"../chunks/CopyLLMTxtMenu.5ac9ab94.js";import{D as te}from"../chunks/Docstring.8a316450.js";import{C as pe}from"../chunks/CodeBlock.606cbaf4.js";import{H as re,E as _e}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.92b2cd9d.js";function ce(A){let m,j,q,P,f,z,u,E,p,K="A Mixture-of-Transformer (MoT) joint vision-language transformer introduced as part of NVIDIA’s Cosmos3 world foundation model family. The model runs two parallel computation pathways over a packed joint sequence:",Z,_,ee="<li>a <strong>causal “understanding” pathway</strong> that self-attends over text tokens with causal masking, and</li> <li>a <strong>bi-directional “generation” pathway</strong> that cross-attends from generation tokens (vision + optional sound latents) over the full understanding-plus-generation key/value set.</li>",L,c,se="The two pathways share the same hidden size and number of layers but maintain <strong>separate Q/K/V/O projections, MLPs, and RMSNorm parameters</strong>, which is what makes the architecture a Mixture-of-Transformer rather than a standard Mixture-of-Experts. Position information is supplied through a 3D multimodal RoPE (mRoPE) that interleaves temporal / height / width frequencies for video latents and reuses the temporal axis for text and audio.",D,h,ne="The model can be loaded as follows.",W,v,H,g,R,d,x,X,l,T,Y,J,oe="Run a full denoising-step forward pass.",Q,w,V,I,B;return f=new ue({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),u=new re({props:{title:"Cosmos3OmniTransformer",local:"cosmos3omnitransformer",headingTag:"h1"}}),v=new pe({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQ29zbW9zM09tbmlUcmFuc2Zvcm1lciUwQSUwQXRyYW5zZm9ybWVyJTIwJTNEJTIwQ29zbW9zM09tbmlUcmFuc2Zvcm1lci5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIybnZpZGlhJTJGQ29zbW9zMy1OYW5vJTIyJTJDJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTBBKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniTransformer

transformer = Cosmos3OmniTransformer.from_pretrained(
    <span class="hljs-string">&quot;nvidia/Cosmos3-Nano&quot;</span>, subfolder=<span class="hljs-string">&quot;transformer&quot;</span>, torch_dtype=torch.bfloat16
)`,lang:"python",wrap:!1}}),g=new re({props:{title:"Cosmos3OmniTransformer",local:"diffusers.Cosmos3OmniTransformer",headingTag:"h2"}}),x=new te({props:{name:"class diffusers.Cosmos3OmniTransformer",anchor:"diffusers.Cosmos3OmniTransformer",parameters:[{name:"attention_bias",val:": bool = False"},{name:"attention_dropout",val:": float = 0.0"},{name:"dtype",val:": str = 'bfloat16'"},{name:"head_dim",val:": int = 128"},{name:"hidden_size",val:": int = 4096"},{name:"intermediate_size",val:": int = 12288"},{name:"base_fps",val:": int = 24"},{name:"enable_fps_modulation",val:": bool = True"},{name:"latent_channel",val:": int = 48"},{name:"unified_3d_mrope_reset_spatial_ids",val:": bool = True"},{name:"unified_3d_mrope_temporal_modality_margin",val:": int = 15000"},{name:"latent_patch_size",val:": int = 2"},{name:"num_attention_heads",val:": int = 32"},{name:"num_hidden_layers",val:": int = 36"},{name:"num_key_value_heads",val:": int = 8"},{name:"patch_latent_dim",val:": int = 192"},{name:"rms_norm_eps",val:": float = 1e-06"},{name:"rope_scaling",val:": dict | None = None"},{name:"rope_theta",val:": float = 5000000.0"},{name:"sound_dim",val:": int | None = None"},{name:"sound_gen",val:": bool = False"},{name:"sound_latent_fps",val:": float = 25.0"},{name:"timestep_scale",val:": float = 0.001"},{name:"vocab_size",val:": int = 151936"}],source:"https://github.com/huggingface/diffusers/blob/vr_13813/src/diffusers/models/transformers/transformer_cosmos3.py#L261"}}),T=new te({props:{name:"forward",anchor:"diffusers.Cosmos3OmniTransformer.forward",parameters:[{name:"input_ids",val:": Tensor"},{name:"text_indexes",val:": Tensor"},{name:"position_ids",val:": Tensor"},{name:"und_len",val:": int"},{name:"sequence_length",val:": int"},{name:"vision_tokens",val:": list"},{name:"vision_token_shapes",val:": list"},{name:"vision_sequence_indexes",val:": Tensor"},{name:"vision_mse_loss_indexes",val:": Tensor"},{name:"vision_timesteps",val:": Tensor"},{name:"vision_noisy_frame_indexes",val:": list"},{name:"sound_tokens",val:": list[torch.Tensor] | None = None"},{name:"sound_token_shapes",val:": list[tuple[int, int, int]] | None = None"},{name:"sound_sequence_indexes",val:": torch.Tensor | None = None"},{name:"sound_mse_loss_indexes",val:": torch.Tensor | None = None"},{name:"sound_timesteps",val:": torch.Tensor | None = None"},{name:"sound_noisy_frame_indexes",val:": list[torch.Tensor] | None = None"}],parametersDescription:[{anchor:"diffusers.Cosmos3OmniTransformer.forward.input_ids",description:"<strong>input_ids</strong> &#x2014; Text token IDs placed at <code>text_indexes</code> in the joint sequence.",name:"input_ids"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.text_indexes",description:"<strong>text_indexes</strong> &#x2014; Indices of text tokens in the joint sequence.",name:"text_indexes"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.position_ids",description:"<strong>position_ids</strong> &#x2014; <code>[3, sequence_length]</code> mRoPE position IDs for the full joint sequence.",name:"position_ids"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.und_len",description:"<strong>und_len</strong> &#x2014; Length of the causal text (understanding) prefix; generation tokens follow.",name:"und_len"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.sequence_length",description:"<strong>sequence_length</strong> &#x2014; Total length of the joint packed sequence.",name:"sequence_length"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.vision_tokens",description:"<strong>vision_tokens</strong> &#x2014; Per-item vision latent tensors before patchify.",name:"vision_tokens"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.vision_token_shapes",description:"<strong>vision_token_shapes</strong> &#x2014; Patch grid shapes <code>(T, H, W)</code> per vision item.",name:"vision_token_shapes"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.vision_sequence_indexes",description:"<strong>vision_sequence_indexes</strong> &#x2014; Indices of vision tokens in the joint sequence.",name:"vision_sequence_indexes"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.vision_mse_loss_indexes",description:"<strong>vision_mse_loss_indexes</strong> &#x2014; Indices used to read vision predictions after the backbone.",name:"vision_mse_loss_indexes"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.vision_timesteps",description:"<strong>vision_timesteps</strong> &#x2014; Per-patch diffusion timesteps for vision tokens.",name:"vision_timesteps"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.vision_noisy_frame_indexes",description:"<strong>vision_noisy_frame_indexes</strong> &#x2014; Noisy frame indices per vision item.",name:"vision_noisy_frame_indexes"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.sound_tokens",description:"<strong>sound_tokens</strong> &#x2014; Optional sound latent tensors before packing.",name:"sound_tokens"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.sound_token_shapes",description:"<strong>sound_token_shapes</strong> &#x2014; Optional patch grid shapes for sound items.",name:"sound_token_shapes"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.sound_sequence_indexes",description:"<strong>sound_sequence_indexes</strong> &#x2014; Optional indices of sound tokens in the joint sequence.",name:"sound_sequence_indexes"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.sound_mse_loss_indexes",description:"<strong>sound_mse_loss_indexes</strong> &#x2014; Optional indices used to read sound predictions.",name:"sound_mse_loss_indexes"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.sound_timesteps",description:"<strong>sound_timesteps</strong> &#x2014; Optional per-token diffusion timesteps for sound.",name:"sound_timesteps"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.sound_noisy_frame_indexes",description:"<strong>sound_noisy_frame_indexes</strong> &#x2014; Optional noisy frame indices per sound item.",name:"sound_noisy_frame_indexes"}],source:"https://github.com/huggingface/diffusers/blob/vr_13813/src/diffusers/models/transformers/transformer_cosmos3.py#L472",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>(preds_vision, preds_sound)</code> — list of per-modality latents (<code>preds_sound</code> is <code>None</code> when the model
has no sound branch or sound inputs are omitted).</p>
`}}),w=new _e({props:{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/api/models/cosmos3_omni_transformer.md"}}),{c(){m=i("meta"),j=t(),q=i("p"),P=t(),b(f.$$.fragment),z=t(),b(u.$$.fragment),E=t(),p=i("p"),p.textContent=K,Z=t(),_=i("ul"),_.innerHTML=ee,L=t(),c=i("p"),c.innerHTML=se,D=t(),h=i("p"),h.textContent=ne,W=t(),b(v.$$.fragment),H=t(),b(g.$$.fragment),R=t(),d=i("div"),b(x.$$.fragment),X=t(),l=i("div"),b(T.$$.fragment),Y=t(),J=i("p"),J.textContent=oe,Q=t(),b(w.$$.fragment),V=t(),I=i("p"),this.h()},l(e){const s=fe("svelte-u9bgzb",document.head);m=a(s,"META",{name:!0,content:!0}),s.forEach(n),j=r(e),q=a(e,"P",{}),U(q).forEach(n),P=r(e),$(f.$$.fragment,e),z=r(e),$(u.$$.fragment,e),E=r(e),p=a(e,"P",{"data-svelte-h":!0}),N(p)!=="svelte-1sw275u"&&(p.textContent=K),Z=r(e),_=a(e,"UL",{"data-svelte-h":!0}),N(_)!=="svelte-1c79fxj"&&(_.innerHTML=ee),L=r(e),c=a(e,"P",{"data-svelte-h":!0}),N(c)!=="svelte-1tbij9r"&&(c.innerHTML=se),D=r(e),h=a(e,"P",{"data-svelte-h":!0}),N(h)!=="svelte-jb636u"&&(h.textContent=ne),W=r(e),$(v.$$.fragment,e),H=r(e),$(g.$$.fragment,e),R=r(e),d=a(e,"DIV",{class:!0});var G=U(d);$(x.$$.fragment,G),X=r(G),l=a(G,"DIV",{class:!0});var S=U(l);$(T.$$.fragment,S),Y=r(S),J=a(S,"P",{"data-svelte-h":!0}),N(J)!=="svelte-s5ds3c"&&(J.textContent=oe),S.forEach(n),G.forEach(n),Q=r(e),$(w.$$.fragment,e),V=r(e),I=a(e,"P",{}),U(I).forEach(n),this.h()},h(){F(m,"name","hf:doc:metadata"),F(m,"content",he),F(l,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),F(d,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,s){M(document.head,m),o(e,j,s),o(e,q,s),o(e,P,s),y(f,e,s),o(e,z,s),y(u,e,s),o(e,E,s),o(e,p,s),o(e,Z,s),o(e,_,s),o(e,L,s),o(e,c,s),o(e,D,s),o(e,h,s),o(e,W,s),y(v,e,s),o(e,H,s),y(g,e,s),o(e,R,s),o(e,d,s),y(x,d,null),M(d,X),M(d,l),y(T,l,null),M(l,Y),M(l,J),o(e,Q,s),y(w,e,s),o(e,V,s),o(e,I,s),B=!0},p:ae,i(e){B||(C(f.$$.fragment,e),C(u.$$.fragment,e),C(v.$$.fragment,e),C(g.$$.fragment,e),C(x.$$.fragment,e),C(T.$$.fragment,e),C(w.$$.fragment,e),B=!0)},o(e){k(f.$$.fragment,e),k(u.$$.fragment,e),k(v.$$.fragment,e),k(g.$$.fragment,e),k(x.$$.fragment,e),k(T.$$.fragment,e),k(w.$$.fragment,e),B=!1},d(e){e&&(n(j),n(q),n(P),n(z),n(E),n(p),n(Z),n(_),n(L),n(c),n(D),n(h),n(W),n(H),n(R),n(d),n(Q),n(V),n(I)),n(m),O(f,e),O(u,e),O(v,e),O(g,e),O(x),O(T),O(w,e)}}}const he='{"title":"Cosmos3OmniTransformer","local":"cosmos3omnitransformer","sections":[{"title":"Cosmos3OmniTransformer","local":"diffusers.Cosmos3OmniTransformer","sections":[],"depth":2}],"depth":1}';function ve(A){return me(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ye extends de{constructor(m){super(),le(this,m,ve,ce,ie,{})}}export{ye as component};