diffusers/pr_10312/en/_app/immutable/nodes/33.32f85044.js

import{s as Pe,n as ye,o as Ce}from"../chunks/scheduler.8c3d61f6.js";import{S as be,i as ve,g as i,s as n,r as m,A as Te,h as a,f as o,c as r,j as I,u as _,x as p,k as X,y as c,a as s,v as h,d as g,t as A,w as P}from"../chunks/index.da70eac4.js";import{D as de}from"../chunks/Docstring.6b390b9a.js";import{C as De}from"../chunks/CodeBlock.00a903b3.js";import{H as le,E as xe}from"../chunks/EditOnGithub.1e64e623.js";function Me(ce){let l,z,V,U,y,E,C,fe='HunyuanDiT2DControlNetModel is an implementation of ControlNet for <a href="https://arxiv.org/abs/2405.08748" rel="nofollow">Hunyuan-DiT</a>.',J,b,ue='ControlNet was introduced in <a href="https://huggingface.co/papers/2302.05543" rel="nofollow">Adding Conditional Control to Text-to-Image Diffusion Models</a> by Lvmin Zhang, Anyi Rao, and Maneesh Agrawala.',j,v,me="With a ControlNet model, you can provide an additional control image to condition and control Hunyuan-DiT generation. For example, if you provide a depth map, the ControlNet model generates an image that’ll preserve the spatial information from the depth map. It is a more flexible and accurate way to control the image generation process.",Z,T,_e="The abstract from the paper is:",W,D,pe="<em>We present ControlNet, a neural network architecture to add spatial conditioning controls to large, pretrained text-to-image diffusion models. ControlNet locks the production-ready large diffusion models, and reuses their deep and robust encoding layers pretrained with billions of images as a strong backbone to learn a diverse set of conditional controls. The neural architecture is connected with “zero convolutions” (zero-initialized convolution layers) that progressively grow the parameters from zero and ensure that no harmful noise could affect the finetuning. We test various conditioning controls, eg, edges, depth, segmentation, human pose, etc, with Stable Diffusion, using single or multiple conditions, with or without prompts. We show that the training of ControlNets is robust with small (&lt;50k) and large (&gt;1m) datasets. Extensive results show that ControlNet may facilitate wider applications to control image diffusion models.</em>",K,x,he='This code is implemented by Tencent Hunyuan Team. You can find pre-trained checkpoints for Hunyuan-DiT ControlNets on <a href="https://huggingface.co/Tencent-Hunyuan" rel="nofollow">Tencent Hunyuan</a>.',B,M,Y,$,q,H,O,d,N,ne,f,w,re,G,ge='The <a href="/docs/diffusers/pr_10312/en/api/models/controlnet_hunyuandit#diffusers.HunyuanDiT2DControlNetModel">HunyuanDiT2DControlNetModel</a> forward method.',ie,u,F,ae,S,Ae="Sets the attention processor to use to compute attention.",Q,L,ee,R,te;return y=new le({props:{title:"HunyuanDiT2DControlNetModel",local:"hunyuandit2dcontrolnetmodel",headingTag:"h1"}}),M=new le({props:{title:"Example For Loading HunyuanDiT2DControlNetModel",local:"example-for-loading-hunyuandit2dcontrolnetmodel",headingTag:"h2"}}),$=new De({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEh1bnl1YW5EaVQyRENvbnRyb2xOZXRNb2RlbCUwQWltcG9ydCUyMHRvcmNoJTBBY29udHJvbG5ldCUyMCUzRCUyMEh1bnl1YW5EaVQyRENvbnRyb2xOZXRNb2RlbC5mcm9tX3ByZXRyYWluZWQoJTIyVGVuY2VudC1IdW55dWFuJTJGSHVueXVhbkRpVC12MS4xLUNvbnRyb2xOZXQtRGlmZnVzZXJzLVBvc2UlMjIlMkMlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYp",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> HunyuanDiT2DControlNetModel
<span class="hljs-keyword">import</span> torch
controlnet = HunyuanDiT2DControlNetModel.from_pretrained(<span class="hljs-string">&quot;Tencent-Hunyuan/HunyuanDiT-v1.1-ControlNet-Diffusers-Pose&quot;</span>, torch_dtype=torch.float16)`,wrap:!1}}),H=new le({props:{title:"HunyuanDiT2DControlNetModel",local:"diffusers.HunyuanDiT2DControlNetModel",headingTag:"h2"}}),N=new de({props:{name:"class diffusers.HunyuanDiT2DControlNetModel",anchor:"diffusers.HunyuanDiT2DControlNetModel",parameters:[{name:"conditioning_channels",val:": int = 3"},{name:"num_attention_heads",val:": int = 16"},{name:"attention_head_dim",val:": int = 88"},{name:"in_channels",val:": typing.Optional[int] = None"},{name:"patch_size",val:": typing.Optional[int] = None"},{name:"activation_fn",val:": str = 'gelu-approximate'"},{name:"sample_size",val:" = 32"},{name:"hidden_size",val:" = 1152"},{name:"transformer_num_layers",val:": int = 40"},{name:"mlp_ratio",val:": float = 4.0"},{name:"cross_attention_dim",val:": int = 1024"},{name:"cross_attention_dim_t5",val:": int = 2048"},{name:"pooled_projection_dim",val:": int = 1024"},{name:"text_len",val:": int = 77"},{name:"text_len_t5",val:": int = 256"},{name:"use_style_cond_and_image_meta_size",val:": bool = True"}],source:"https://github.com/huggingface/diffusers/blob/vr_10312/src/diffusers/models/controlnets/controlnet_hunyuan.py#L41"}}),w=new de({props:{name:"forward",anchor:"diffusers.HunyuanDiT2DControlNetModel.forward",parameters:[{name:"hidden_states",val:""},{name:"timestep",val:""},{name:"controlnet_cond",val:": Tensor"},{name:"conditioning_scale",val:": float = 1.0"},{name:"encoder_hidden_states",val:" = None"},{name:"text_embedding_mask",val:" = None"},{name:"encoder_hidden_states_t5",val:" = None"},{name:"text_embedding_mask_t5",val:" = None"},{name:"image_meta_size",val:" = None"},{name:"style",val:" = None"},{name:"image_rotary_emb",val:" = None"},{name:"return_dict",val:" = True"}],parametersDescription:[{anchor:"diffusers.HunyuanDiT2DControlNetModel.forward.hidden_states",description:`<strong>hidden_states</strong> (<code>torch.Tensor</code> of shape <code>(batch size, dim, height, width)</code>) &#x2014;
The input tensor.`,name:"hidden_states"},{anchor:"diffusers.HunyuanDiT2DControlNetModel.forward.timestep",description:`<strong>timestep</strong> ( <code>torch.LongTensor</code>, <em>optional</em>) &#x2014;
Used to indicate denoising step.`,name:"timestep"},{anchor:"diffusers.HunyuanDiT2DControlNetModel.forward.controlnet_cond",description:`<strong>controlnet_cond</strong> ( <code>torch.Tensor</code> ) &#x2014;
The conditioning input to ControlNet.`,name:"controlnet_cond"},{anchor:"diffusers.HunyuanDiT2DControlNetModel.forward.conditioning_scale",description:`<strong>conditioning_scale</strong> ( <code>float</code> ) &#x2014;
Indicate the conditioning scale.`,name:"conditioning_scale"},{anchor:"diffusers.HunyuanDiT2DControlNetModel.forward.encoder_hidden_states",description:`<strong>encoder_hidden_states</strong> ( <code>torch.Tensor</code> of shape <code>(batch size, sequence len, embed dims)</code>, <em>optional</em>) &#x2014;
Conditional embeddings for cross attention layer. This is the output of <code>BertModel</code>.`,name:"encoder_hidden_states"},{anchor:"diffusers.HunyuanDiT2DControlNetModel.forward.text_embedding_mask",description:`<strong>text_embedding_mask</strong> &#x2014; torch.Tensor
An attention mask of shape <code>(batch, key_tokens)</code> is applied to <code>encoder_hidden_states</code>. This is the output
of <code>BertModel</code>.`,name:"text_embedding_mask"},{anchor:"diffusers.HunyuanDiT2DControlNetModel.forward.encoder_hidden_states_t5",description:`<strong>encoder_hidden_states_t5</strong> ( <code>torch.Tensor</code> of shape <code>(batch size, sequence len, embed dims)</code>, <em>optional</em>) &#x2014;
Conditional embeddings for cross attention layer. This is the output of T5 Text Encoder.`,name:"encoder_hidden_states_t5"},{anchor:"diffusers.HunyuanDiT2DControlNetModel.forward.text_embedding_mask_t5",description:`<strong>text_embedding_mask_t5</strong> &#x2014; torch.Tensor
An attention mask of shape <code>(batch, key_tokens)</code> is applied to <code>encoder_hidden_states</code>. This is the output
of T5 Text Encoder.`,name:"text_embedding_mask_t5"},{anchor:"diffusers.HunyuanDiT2DControlNetModel.forward.image_meta_size",description:`<strong>image_meta_size</strong> (torch.Tensor) &#x2014;
Conditional embedding indicate the image sizes`,name:"image_meta_size"},{anchor:"diffusers.HunyuanDiT2DControlNetModel.forward.style",description:`<strong>style</strong> &#x2014; torch.Tensor:
Conditional embedding indicate the style`,name:"style"},{anchor:"diffusers.HunyuanDiT2DControlNetModel.forward.image_rotary_emb",description:`<strong>image_rotary_emb</strong> (<code>torch.Tensor</code>) &#x2014;
The image rotary embeddings to apply on query and key tensors during attention calculation.`,name:"image_rotary_emb"},{anchor:"diffusers.HunyuanDiT2DControlNetModel.forward.return_dict",description:`<strong>return_dict</strong> &#x2014; bool
Whether to return a dictionary.`,name:"return_dict"}],source:"https://github.com/huggingface/diffusers/blob/vr_10312/src/diffusers/models/controlnets/controlnet_hunyuan.py#L216"}}),F=new de({props:{name:"set_attn_processor",anchor:"diffusers.HunyuanDiT2DControlNetModel.set_attn_processor",parameters:[{name:"processor",val:": typing.Union[diffusers.models.attention_processor.AttnProcessor, diffusers.models.attention_processor.CustomDiffusionAttnProcessor, diffusers.models.attention_processor.AttnAddedKVProcessor, diffusers.models.attention_processor.AttnAddedKVProcessor2_0, diffusers.models.attention_processor.JointAttnProcessor2_0, diffusers.models.attention_processor.PAGJointAttnProcessor2_0, diffusers.models.attention_processor.PAGCFGJointAttnProcessor2_0, diffusers.models.attention_processor.FusedJointAttnProcessor2_0, diffusers.models.attention_processor.AllegroAttnProcessor2_0, diffusers.models.attention_processor.AuraFlowAttnProcessor2_0, diffusers.models.attention_processor.FusedAuraFlowAttnProcessor2_0, diffusers.models.attention_processor.FluxAttnProcessor2_0, diffusers.models.attention_processor.FluxAttnProcessor2_0_NPU, diffusers.models.attention_processor.FusedFluxAttnProcessor2_0, diffusers.models.attention_processor.FusedFluxAttnProcessor2_0_NPU, diffusers.models.attention_processor.CogVideoXAttnProcessor2_0, diffusers.models.attention_processor.FusedCogVideoXAttnProcessor2_0, diffusers.models.attention_processor.XFormersAttnAddedKVProcessor, diffusers.models.attention_processor.XFormersAttnProcessor, diffusers.models.attention_processor.XLAFlashAttnProcessor2_0, diffusers.models.attention_processor.AttnProcessorNPU, diffusers.models.attention_processor.AttnProcessor2_0, diffusers.models.attention_processor.MochiVaeAttnProcessor2_0, diffusers.models.attention_processor.MochiAttnProcessor2_0, diffusers.models.attention_processor.StableAudioAttnProcessor2_0, diffusers.models.attention_processor.HunyuanAttnProcessor2_0, diffusers.models.attention_processor.FusedHunyuanAttnProcessor2_0, diffusers.models.attention_processor.PAGHunyuanAttnProcessor2_0, diffusers.models.attention_processor.PAGCFGHunyuanAttnProcessor2_0, diffusers.models.attention_processor.LuminaAttnProcessor2_0, diffusers.models.attention_processor.FusedAttnProcessor2_0, diffusers.models.attention_processor.CustomDiffusionXFormersAttnProcessor, diffusers.models.attention_processor.CustomDiffusionAttnProcessor2_0, diffusers.models.attention_processor.SlicedAttnProcessor, diffusers.models.attention_processor.SlicedAttnAddedKVProcessor, diffusers.models.attention_processor.SanaLinearAttnProcessor2_0, diffusers.models.attention_processor.PAGCFGSanaLinearAttnProcessor2_0, diffusers.models.attention_processor.PAGIdentitySanaLinearAttnProcessor2_0, diffusers.models.attention_processor.SanaMultiscaleLinearAttention, diffusers.models.attention_processor.SanaMultiscaleAttnProcessor2_0, diffusers.models.attention_processor.SanaMultiscaleAttentionProjection, diffusers.models.attention_processor.IPAdapterAttnProcessor, diffusers.models.attention_processor.IPAdapterAttnProcessor2_0, diffusers.models.attention_processor.IPAdapterXFormersAttnProcessor, diffusers.models.attention_processor.SD3IPAdapterJointAttnProcessor2_0, diffusers.models.attention_processor.PAGIdentitySelfAttnProcessor2_0, diffusers.models.attention_processor.PAGCFGIdentitySelfAttnProcessor2_0, diffusers.models.attention_processor.LoRAAttnProcessor, diffusers.models.attention_processor.LoRAAttnProcessor2_0, diffusers.models.attention_processor.LoRAXFormersAttnProcessor, diffusers.models.attention_processor.LoRAAttnAddedKVProcessor, typing.Dict[str, typing.Union[diffusers.models.attention_processor.AttnProcessor, diffusers.models.attention_processor.CustomDiffusionAttnProcessor, diffusers.models.attention_processor.AttnAddedKVProcessor, diffusers.models.attention_processor.AttnAddedKVProcessor2_0, diffusers.models.attention_processor.JointAttnProcessor2_0, diffusers.models.attention_processor.PAGJointAttnProcessor2_0, diffusers.models.attention_processor.PAGCFGJointAttnProcessor2_0, diffusers.models.attention_processor.FusedJointAttnProcessor2_0, diffusers.models.attention_processor.AllegroAttnProcessor2_0, diffusers.models.attention_processor.AuraFlowAttnProcessor2_0, diffusers.models.attention_processor.FusedAuraFlowAttnProcessor2_0, diffusers.models.attention_processor.FluxAttnProcessor2_0, diffusers.models.attention_processor.FluxAttnProcessor2_0_NPU, diffusers.models.attention_processor.FusedFluxAttnProcessor2_0, diffusers.models.attention_processor.FusedFluxAttnProcessor2_0_NPU, diffusers.models.attention_processor.CogVideoXAttnProcessor2_0, diffusers.models.attention_processor.FusedCogVideoXAttnProcessor2_0, diffusers.models.attention_processor.XFormersAttnAddedKVProcessor, diffusers.models.attention_processor.XFormersAttnProcessor, diffusers.models.attention_processor.XLAFlashAttnProcessor2_0, diffusers.models.attention_processor.AttnProcessorNPU, diffusers.models.attention_processor.AttnProcessor2_0, diffusers.models.attention_processor.MochiVaeAttnProcessor2_0, diffusers.models.attention_processor.MochiAttnProcessor2_0, diffusers.models.attention_processor.StableAudioAttnProcessor2_0, diffusers.models.attention_processor.HunyuanAttnProcessor2_0, diffusers.models.attention_processor.FusedHunyuanAttnProcessor2_0, diffusers.models.attention_processor.PAGHunyuanAttnProcessor2_0, diffusers.models.attention_processor.PAGCFGHunyuanAttnProcessor2_0, diffusers.models.attention_processor.LuminaAttnProcessor2_0, diffusers.models.attention_processor.FusedAttnProcessor2_0, diffusers.models.attention_processor.CustomDiffusionXFormersAttnProcessor, diffusers.models.attention_processor.CustomDiffusionAttnProcessor2_0, diffusers.models.attention_processor.SlicedAttnProcessor, diffusers.models.attention_processor.SlicedAttnAddedKVProcessor, diffusers.models.attention_processor.SanaLinearAttnProcessor2_0, diffusers.models.attention_processor.PAGCFGSanaLinearAttnProcessor2_0, diffusers.models.attention_processor.PAGIdentitySanaLinearAttnProcessor2_0, diffusers.models.attention_processor.SanaMultiscaleLinearAttention, diffusers.models.attention_processor.SanaMultiscaleAttnProcessor2_0, diffusers.models.attention_processor.SanaMultiscaleAttentionProjection, diffusers.models.attention_processor.IPAdapterAttnProcessor, diffusers.models.attention_processor.IPAdapterAttnProcessor2_0, diffusers.models.attention_processor.IPAdapterXFormersAttnProcessor, diffusers.models.attention_processor.SD3IPAdapterJointAttnProcessor2_0, diffusers.models.attention_processor.PAGIdentitySelfAttnProcessor2_0, diffusers.models.attention_processor.PAGCFGIdentitySelfAttnProcessor2_0, diffusers.models.attention_processor.LoRAAttnProcessor, diffusers.models.attention_processor.LoRAAttnProcessor2_0, diffusers.models.attention_processor.LoRAXFormersAttnProcessor, diffusers.models.attention_processor.LoRAAttnAddedKVProcessor]]]"}],parametersDescription:[{anchor:"diffusers.HunyuanDiT2DControlNetModel.set_attn_processor.processor",description:`<strong>processor</strong> (<code>dict</code> of <code>AttentionProcessor</code> or only <code>AttentionProcessor</code>) &#x2014;
The instantiated processor class or a dictionary of processor classes that will be set as the processor
for <strong>all</strong> <code>Attention</code> layers. If <code>processor</code> is a dict, the key needs to define the path to the
corresponding cross attention processor. This is strongly recommended when setting trainable attention
processors.`,name:"processor"}],source:"https://github.com/huggingface/diffusers/blob/vr_10312/src/diffusers/models/controlnets/controlnet_hunyuan.py#L142"}}),L=new xe({props:{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/api/models/controlnet_hunyuandit.md"}}),{c(){l=i("meta"),z=n(),V=i("p"),U=n(),m(y.$$.fragment),E=n(),C=i("p"),C.innerHTML=fe,J=n(),b=i("p"),b.innerHTML=ue,j=n(),v=i("p"),v.textContent=me,Z=n(),T=i("p"),T.textContent=_e,W=n(),D=i("p"),D.innerHTML=pe,K=n(),x=i("p"),x.innerHTML=he,B=n(),m(M.$$.fragment),Y=n(),m($.$$.fragment),q=n(),m(H.$$.fragment),O=n(),d=i("div"),m(N.$$.fragment),ne=n(),f=i("div"),m(w.$$.fragment),re=n(),G=i("p"),G.innerHTML=ge,ie=n(),u=i("div"),m(F.$$.fragment),ae=n(),S=i("p"),S.textContent=Ae,Q=n(),m(L.$$.fragment),ee=n(),R=i("p"),this.h()},l(e){const t=Te("svelte-u9bgzb",document.head);l=a(t,"META",{name:!0,content:!0}),t.forEach(o),z=r(e),V=a(e,"P",{}),I(V).forEach(o),U=r(e),_(y.$$.fragment,e),E=r(e),C=a(e,"P",{"data-svelte-h":!0}),p(C)!=="svelte-r9b48f"&&(C.innerHTML=fe),J=r(e),b=a(e,"P",{"data-svelte-h":!0}),p(b)!=="svelte-1v2xz23"&&(b.innerHTML=ue),j=r(e),v=a(e,"P",{"data-svelte-h":!0}),p(v)!=="svelte-w3tnjc"&&(v.textContent=me),Z=r(e),T=a(e,"P",{"data-svelte-h":!0}),p(T)!=="svelte-1cwsb16"&&(T.textContent=_e),W=r(e),D=a(e,"P",{"data-svelte-h":!0}),p(D)!=="svelte-fbiw6t"&&(D.innerHTML=pe),K=r(e),x=a(e,"P",{"data-svelte-h":!0}),p(x)!=="svelte-v5nvtj"&&(x.innerHTML=he),B=r(e),_(M.$$.fragment,e),Y=r(e),_($.$$.fragment,e),q=r(e),_(H.$$.fragment,e),O=r(e),d=a(e,"DIV",{class:!0});var k=I(d);_(N.$$.fragment,k),ne=r(k),f=a(k,"DIV",{class:!0});var oe=I(f);_(w.$$.fragment,oe),re=r(oe),G=a(oe,"P",{"data-svelte-h":!0}),p(G)!=="svelte-ops3iu"&&(G.innerHTML=ge),oe.forEach(o),ie=r(k),u=a(k,"DIV",{class:!0});var se=I(u);_(F.$$.fragment,se),ae=r(se),S=a(se,"P",{"data-svelte-h":!0}),p(S)!=="svelte-1o77hl2"&&(S.textContent=Ae),se.forEach(o),k.forEach(o),Q=r(e),_(L.$$.fragment,e),ee=r(e),R=a(e,"P",{}),I(R).forEach(o),this.h()},h(){X(l,"name","hf:doc:metadata"),X(l,"content",$e),X(f,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),X(u,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),X(d,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){c(document.head,l),s(e,z,t),s(e,V,t),s(e,U,t),h(y,e,t),s(e,E,t),s(e,C,t),s(e,J,t),s(e,b,t),s(e,j,t),s(e,v,t),s(e,Z,t),s(e,T,t),s(e,W,t),s(e,D,t),s(e,K,t),s(e,x,t),s(e,B,t),h(M,e,t),s(e,Y,t),h($,e,t),s(e,q,t),h(H,e,t),s(e,O,t),s(e,d,t),h(N,d,null),c(d,ne),c(d,f),h(w,f,null),c(f,re),c(f,G),c(d,ie),c(d,u),h(F,u,null),c(u,ae),c(u,S),s(e,Q,t),h(L,e,t),s(e,ee,t),s(e,R,t),te=!0},p:ye,i(e){te||(g(y.$$.fragment,e),g(M.$$.fragment,e),g($.$$.fragment,e),g(H.$$.fragment,e),g(N.$$.fragment,e),g(w.$$.fragment,e),g(F.$$.fragment,e),g(L.$$.fragment,e),te=!0)},o(e){A(y.$$.fragment,e),A(M.$$.fragment,e),A($.$$.fragment,e),A(H.$$.fragment,e),A(N.$$.fragment,e),A(w.$$.fragment,e),A(F.$$.fragment,e),A(L.$$.fragment,e),te=!1},d(e){e&&(o(z),o(V),o(U),o(E),o(C),o(J),o(b),o(j),o(v),o(Z),o(T),o(W),o(D),o(K),o(x),o(B),o(Y),o(q),o(O),o(d),o(Q),o(ee),o(R)),o(l),P(y,e),P(M,e),P($,e),P(H,e),P(N),P(w),P(F),P(L,e)}}}const $e='{"title":"HunyuanDiT2DControlNetModel","local":"hunyuandit2dcontrolnetmodel","sections":[{"title":"Example For Loading HunyuanDiT2DControlNetModel","local":"example-for-loading-hunyuandit2dcontrolnetmodel","sections":[],"depth":2},{"title":"HunyuanDiT2DControlNetModel","local":"diffusers.HunyuanDiT2DControlNetModel","sections":[],"depth":2}],"depth":1}';function He(ce){return Ce(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Se extends be{constructor(l){super(),ve(this,l,He,Me,Pe,{})}}export{Se as component};