diffusers/pr_13921/en/_app/immutable/nodes/60.b93a1cd7.js

import{s as re,n as ae,o as me}from"../chunks/scheduler.53228c21.js";import{S as de,i as le,e as r,s as t,c as b,h as fe,a,d as s,b as i,f as S,g as $,j as I,k as U,l as j,m as o,n as y,t as k,o as C,p as O}from"../chunks/index.cac5d66a.js";import{C as ce}from"../chunks/CopyLLMTxtMenu.0ef49226.js";import{D as te}from"../chunks/Docstring.9de32ff4.js";import{C as _e}from"../chunks/CodeBlock.606cbaf4.js";import{H as ie,E as pe}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.48d5cb47.js";function ue(A){let m,M,q,P,f,z,c,E,_,K="A Mixture-of-Transformer (MoT) joint vision-language transformer introduced as part of NVIDIA’s Cosmos3 world foundation model family. The model runs two parallel computation pathways over a packed joint sequence:",Z,p,ee="<li>a <strong>causal “understanding” pathway</strong> that self-attends over text tokens with causal masking, and</li> <li>a <strong>bi-directional “generation” pathway</strong> that cross-attends from generation tokens (vision + optional sound latents) over the full understanding-plus-generation key/value set.</li>",D,u,ne="The two pathways share the same hidden size and number of layers but maintain <strong>separate Q/K/V/O projections, MLPs, and RMSNorm parameters</strong>, which is what makes the architecture a Mixture-of-Transformer rather than a standard Mixture-of-Experts. Position information is supplied through a 3D multimodal RoPE (mRoPE) that interleaves temporal / height / width frequencies for video latents and reuses the temporal axis for text and audio.",L,h,se="The model can be loaded as follows.",W,g,H,v,R,d,x,X,l,T,Y,N,oe="Run a full denoising-step forward pass.",Q,w,V,J,B;return f=new ce({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),c=new ie({props:{title:"Cosmos3OmniTransformer",local:"cosmos3omnitransformer",headingTag:"h1"}}),g=new _e({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQ29zbW9zM09tbmlUcmFuc2Zvcm1lciUwQSUwQXRyYW5zZm9ybWVyJTIwJTNEJTIwQ29zbW9zM09tbmlUcmFuc2Zvcm1lci5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIybnZpZGlhJTJGQ29zbW9zMy1OYW5vJTIyJTJDJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTBBKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniTransformer

transformer = Cosmos3OmniTransformer.from_pretrained(
    <span class="hljs-string">&quot;nvidia/Cosmos3-Nano&quot;</span>, subfolder=<span class="hljs-string">&quot;transformer&quot;</span>, torch_dtype=torch.bfloat16
)`,lang:"python",wrap:!1}}),v=new ie({props:{title:"Cosmos3OmniTransformer",local:"diffusers.Cosmos3OmniTransformer",headingTag:"h2"}}),x=new te({props:{name:"class diffusers.Cosmos3OmniTransformer",anchor:"diffusers.Cosmos3OmniTransformer",parameters:[{name:"attention_bias",val:": bool = False"},{name:"attention_dropout",val:": float = 0.0"},{name:"dtype",val:": str = 'bfloat16'"},{name:"head_dim",val:": int = 128"},{name:"hidden_size",val:": int = 4096"},{name:"intermediate_size",val:": int = 12288"},{name:"base_fps",val:": int = 24"},{name:"enable_fps_modulation",val:": bool = True"},{name:"latent_channel",val:": int = 48"},{name:"unified_3d_mrope_reset_spatial_ids",val:": bool = True"},{name:"unified_3d_mrope_temporal_modality_margin",val:": int = 15000"},{name:"latent_patch_size",val:": int = 2"},{name:"num_attention_heads",val:": int = 32"},{name:"num_hidden_layers",val:": int = 36"},{name:"num_key_value_heads",val:": int = 8"},{name:"patch_latent_dim",val:": int = 192"},{name:"rms_norm_eps",val:": float = 1e-06"},{name:"rope_scaling",val:": dict | None = None"},{name:"rope_theta",val:": float = 5000000.0"},{name:"action_dim",val:": int | None = None"},{name:"action_gen",val:": bool = False"},{name:"num_embodiment_domains",val:": int = 32"},{name:"sound_dim",val:": int | None = None"},{name:"sound_gen",val:": bool = False"},{name:"sound_latent_fps",val:": float = 25.0"},{name:"timestep_scale",val:": float = 0.001"},{name:"vocab_size",val:": int = 151936"}],source:"https://github.com/huggingface/diffusers/blob/vr_13921/src/diffusers/models/transformers/transformer_cosmos3.py#L292"}}),T=new te({props:{name:"forward",anchor:"diffusers.Cosmos3OmniTransformer.forward",parameters:[{name:"input_ids",val:": Tensor"},{name:"text_indexes",val:": Tensor"},{name:"position_ids",val:": Tensor"},{name:"und_len",val:": int"},{name:"sequence_length",val:": int"},{name:"vision_tokens",val:": list"},{name:"vision_token_shapes",val:": list"},{name:"vision_sequence_indexes",val:": Tensor"},{name:"vision_mse_loss_indexes",val:": Tensor"},{name:"vision_timesteps",val:": Tensor"},{name:"vision_noisy_frame_indexes",val:": list"},{name:"sound_tokens",val:": list[torch.Tensor] | None = None"},{name:"sound_token_shapes",val:": list[tuple[int, int, int]] | None = None"},{name:"sound_sequence_indexes",val:": torch.Tensor | None = None"},{name:"sound_mse_loss_indexes",val:": torch.Tensor | None = None"},{name:"sound_timesteps",val:": torch.Tensor | None = None"},{name:"sound_noisy_frame_indexes",val:": list[torch.Tensor] | None = None"},{name:"action_tokens",val:": list[torch.Tensor] | None = None"},{name:"action_token_shapes",val:": list[tuple[int, int, int]] | None = None"},{name:"action_sequence_indexes",val:": torch.Tensor | None = None"},{name:"action_mse_loss_indexes",val:": torch.Tensor | None = None"},{name:"action_timesteps",val:": torch.Tensor | None = None"},{name:"action_noisy_frame_indexes",val:": list[torch.Tensor] | None = None"},{name:"action_domain_ids",val:": list[torch.Tensor] | None = None"}],parametersDescription:[{anchor:"diffusers.Cosmos3OmniTransformer.forward.input_ids",description:"<strong>input_ids</strong> &#x2014; Text token IDs placed at <code>text_indexes</code> in the joint sequence.",name:"input_ids"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.text_indexes",description:"<strong>text_indexes</strong> &#x2014; Indices of text tokens in the joint sequence.",name:"text_indexes"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.position_ids",description:"<strong>position_ids</strong> &#x2014; <code>[3, sequence_length]</code> mRoPE position IDs for the full joint sequence.",name:"position_ids"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.und_len",description:"<strong>und_len</strong> &#x2014; Length of the causal text (understanding) prefix; generation tokens follow.",name:"und_len"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.sequence_length",description:"<strong>sequence_length</strong> &#x2014; Total length of the joint packed sequence.",name:"sequence_length"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.vision_tokens",description:"<strong>vision_tokens</strong> &#x2014; Per-item vision latent tensors before patchify.",name:"vision_tokens"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.vision_token_shapes",description:"<strong>vision_token_shapes</strong> &#x2014; Patch grid shapes <code>(T, H, W)</code> per vision item.",name:"vision_token_shapes"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.vision_sequence_indexes",description:"<strong>vision_sequence_indexes</strong> &#x2014; Indices of vision tokens in the joint sequence.",name:"vision_sequence_indexes"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.vision_mse_loss_indexes",description:"<strong>vision_mse_loss_indexes</strong> &#x2014; Indices used to read vision predictions after the backbone.",name:"vision_mse_loss_indexes"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.vision_timesteps",description:"<strong>vision_timesteps</strong> &#x2014; Per-patch diffusion timesteps for vision tokens.",name:"vision_timesteps"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.vision_noisy_frame_indexes",description:"<strong>vision_noisy_frame_indexes</strong> &#x2014; Noisy frame indices per vision item.",name:"vision_noisy_frame_indexes"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.sound_tokens",description:"<strong>sound_tokens</strong> &#x2014; Optional sound latent tensors before packing.",name:"sound_tokens"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.sound_token_shapes",description:"<strong>sound_token_shapes</strong> &#x2014; Optional patch grid shapes for sound items.",name:"sound_token_shapes"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.sound_sequence_indexes",description:"<strong>sound_sequence_indexes</strong> &#x2014; Optional indices of sound tokens in the joint sequence.",name:"sound_sequence_indexes"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.sound_mse_loss_indexes",description:"<strong>sound_mse_loss_indexes</strong> &#x2014; Optional indices used to read sound predictions.",name:"sound_mse_loss_indexes"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.sound_timesteps",description:"<strong>sound_timesteps</strong> &#x2014; Optional per-token diffusion timesteps for sound.",name:"sound_timesteps"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.sound_noisy_frame_indexes",description:"<strong>sound_noisy_frame_indexes</strong> &#x2014; Optional noisy frame indices per sound item.",name:"sound_noisy_frame_indexes"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.action_tokens",description:"<strong>action_tokens</strong> &#x2014; Optional action latent tensors before packing.",name:"action_tokens"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.action_token_shapes",description:"<strong>action_token_shapes</strong> &#x2014; Optional patch grid shapes <code>(T, H, W)</code> per action item.",name:"action_token_shapes"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.action_sequence_indexes",description:"<strong>action_sequence_indexes</strong> &#x2014; Optional indices of action tokens in the joint sequence.",name:"action_sequence_indexes"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.action_mse_loss_indexes",description:"<strong>action_mse_loss_indexes</strong> &#x2014; Optional indices used to read action predictions after the backbone.",name:"action_mse_loss_indexes"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.action_timesteps",description:"<strong>action_timesteps</strong> &#x2014; Optional per-token diffusion timesteps for action tokens.",name:"action_timesteps"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.action_noisy_frame_indexes",description:"<strong>action_noisy_frame_indexes</strong> &#x2014; Optional noisy frame indices per action item.",name:"action_noisy_frame_indexes"},{anchor:"diffusers.Cosmos3OmniTransformer.forward.action_domain_ids",description:"<strong>action_domain_ids</strong> &#x2014; Optional per-item domain IDs selecting the action head weights.",name:"action_domain_ids"}],source:"https://github.com/huggingface/diffusers/blob/vr_13921/src/diffusers/models/transformers/transformer_cosmos3.py#L549",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>(preds_vision, preds_sound, preds_action)</code> — lists of per-modality predictions. Optional modalities
return <code>None</code> when their inputs are omitted.</p>
`}}),w=new pe({props:{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/api/models/cosmos3_omni_transformer.md"}}),{c(){m=r("meta"),M=t(),q=r("p"),P=t(),b(f.$$.fragment),z=t(),b(c.$$.fragment),E=t(),_=r("p"),_.textContent=K,Z=t(),p=r("ul"),p.innerHTML=ee,D=t(),u=r("p"),u.innerHTML=ne,L=t(),h=r("p"),h.textContent=se,W=t(),b(g.$$.fragment),H=t(),b(v.$$.fragment),R=t(),d=r("div"),b(x.$$.fragment),X=t(),l=r("div"),b(T.$$.fragment),Y=t(),N=r("p"),N.textContent=oe,Q=t(),b(w.$$.fragment),V=t(),J=r("p"),this.h()},l(e){const n=fe("svelte-u9bgzb",document.head);m=a(n,"META",{name:!0,content:!0}),n.forEach(s),M=i(e),q=a(e,"P",{}),S(q).forEach(s),P=i(e),$(f.$$.fragment,e),z=i(e),$(c.$$.fragment,e),E=i(e),_=a(e,"P",{"data-svelte-h":!0}),I(_)!=="svelte-1sw275u"&&(_.textContent=K),Z=i(e),p=a(e,"UL",{"data-svelte-h":!0}),I(p)!=="svelte-1c79fxj"&&(p.innerHTML=ee),D=i(e),u=a(e,"P",{"data-svelte-h":!0}),I(u)!=="svelte-1tbij9r"&&(u.innerHTML=ne),L=i(e),h=a(e,"P",{"data-svelte-h":!0}),I(h)!=="svelte-jb636u"&&(h.textContent=se),W=i(e),$(g.$$.fragment,e),H=i(e),$(v.$$.fragment,e),R=i(e),d=a(e,"DIV",{class:!0});var F=S(d);$(x.$$.fragment,F),X=i(F),l=a(F,"DIV",{class:!0});var G=S(l);$(T.$$.fragment,G),Y=i(G),N=a(G,"P",{"data-svelte-h":!0}),I(N)!=="svelte-s5ds3c"&&(N.textContent=oe),G.forEach(s),F.forEach(s),Q=i(e),$(w.$$.fragment,e),V=i(e),J=a(e,"P",{}),S(J).forEach(s),this.h()},h(){U(m,"name","hf:doc:metadata"),U(m,"content",he),U(l,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),U(d,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,n){j(document.head,m),o(e,M,n),o(e,q,n),o(e,P,n),y(f,e,n),o(e,z,n),y(c,e,n),o(e,E,n),o(e,_,n),o(e,Z,n),o(e,p,n),o(e,D,n),o(e,u,n),o(e,L,n),o(e,h,n),o(e,W,n),y(g,e,n),o(e,H,n),y(v,e,n),o(e,R,n),o(e,d,n),y(x,d,null),j(d,X),j(d,l),y(T,l,null),j(l,Y),j(l,N),o(e,Q,n),y(w,e,n),o(e,V,n),o(e,J,n),B=!0},p:ae,i(e){B||(k(f.$$.fragment,e),k(c.$$.fragment,e),k(g.$$.fragment,e),k(v.$$.fragment,e),k(x.$$.fragment,e),k(T.$$.fragment,e),k(w.$$.fragment,e),B=!0)},o(e){C(f.$$.fragment,e),C(c.$$.fragment,e),C(g.$$.fragment,e),C(v.$$.fragment,e),C(x.$$.fragment,e),C(T.$$.fragment,e),C(w.$$.fragment,e),B=!1},d(e){e&&(s(M),s(q),s(P),s(z),s(E),s(_),s(Z),s(p),s(D),s(u),s(L),s(h),s(W),s(H),s(R),s(d),s(Q),s(V),s(J)),s(m),O(f,e),O(c,e),O(g,e),O(v,e),O(x),O(T),O(w,e)}}}const he='{"title":"Cosmos3OmniTransformer","local":"cosmos3omnitransformer","sections":[{"title":"Cosmos3OmniTransformer","local":"diffusers.Cosmos3OmniTransformer","sections":[],"depth":2}],"depth":1}';function ge(A){return me(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ye extends de{constructor(m){super(),le(this,m,ge,ue,re,{})}}export{ye as component};