diffusers/pr_13881/zh/_app/immutable/nodes/41.CYp97ox1.js

import"../chunks/DsnmJJEf.js";import{i as D,h as q,C as P,H as M,a,b as y,E as $,s as O}from"../chunks/DdZvggmf.js";import{p as K,o as ll,s as l,f as i,a as o,b as al,c,n as Q}from"../chunks/BbekZcyp.js";import{H as b}from"../chunks/BcnRgdDK.js";const el='{"title":"ControlNet","local":"controlnet","sections":[{"title":"脚本参数","local":"脚本参数","sections":[{"title":"Min-SNR加权策略","local":"min-snr加权策略","sections":[],"depth":3}],"depth":2},{"title":"训练脚本","local":"训练脚本","sections":[],"depth":2},{"title":"启动训练","local":"启动训练","sections":[],"depth":2},{"title":"Stable Diffusion XL","local":"stable-diffusion-xl","sections":[],"depth":2},{"title":"后续步骤","local":"后续步骤","sections":[],"depth":2}],"depth":1}';var tl=c('<meta name="hf:doc:metadata"/>'),ol=c('<p>若可访问TPU设备，Flax训练脚本将运行得更快！以下是在 <a href="https://cloud.google.com/tpu/docs/run-calculation-jax" rel="nofollow">Google Cloud TPU VM</a> 上的配置流程。创建单个TPU v4-8虚拟机并连接：</p> <!> <p>安装JAX 0.4.5：</p> <!> <p>然后安装Flax脚本的依赖：</p> <!>',1),H=c("<!> <!>",1),nl=c("<p>16GB显卡可使用bitsandbytes 8-bit优化器和梯度检查点：</p> <!> <p>训练命令添加以下参数：</p> <!>",1),sl=c("<p>12GB显卡需组合使用bitsandbytes 8-bit优化器、梯度检查点、xFormers，并将梯度置为None而非0：</p> <!>",1),il=c('<p>8GB显卡需使用 <a href="https://www.deepspeed.ai/" rel="nofollow">DeepSpeed</a> 将张量卸载到CPU或NVME：</p> <p>运行以下命令配置环境：</p> <!> <p>选择DeepSpeed stage 2，结合fp16混合精度和参数卸载到CPU的方案。注意这会增加约25GB内存占用。配置示例如下：</p> <!> <p>建议将优化器替换为DeepSpeed特化版 <a href="https://deepspeed.readthedocs.io/en/latest/optimizers.html#adam-cpu" rel="nofollow"><code>deepspeed.ops.adam.DeepSpeedCPUAdam</code></a>，注意CUDA工具链版本需与PyTorch匹配。</p> <p>当前bitsandbytes与DeepSpeed存在兼容性问题。</p> <p>无需额外添加训练参数。</p>',1),cl=c("<!> <!> <!>",1),rl=c('<p>Flax版本支持通过 <code>--profile_steps==5</code> 参数进行性能分析：</p> <!> <p>在 <a href="http://localhost:6006/#profile" rel="nofollow">http://localhost:6006/#profile</a> 查看分析结果。</p> <blockquote class="warning"><p>若遇到插件版本冲突，建议重新安装TensorFlow和Tensorboard。注意性能分析插件仍处实验阶段，部分视图可能不完整。<code>trace_viewer</code> 会截断超过1M的事件记录，在编译步骤分析时可能导致设备轨迹丢失。</p></blockquote> <!>',1),pl=c('<p></p> <!> <!> <p><a href="https://hf.co/papers/2302.05543" rel="nofollow">ControlNet</a> 是一种基于预训练模型的适配器架构。它通过额外输入的条件图像（如边缘检测图、深度图、人体姿态图等），实现对生成图像的精细化控制。</p> <p>在显存有限的GPU上训练时，建议启用训练命令中的 <code>gradient_checkpointing</code>（梯度检查点）、<code>gradient_accumulation_steps</code>（梯度累积步数）和 <code>mixed_precision</code>（混合精度）参数。还可使用 <a href="../optimization/xformers">xFormers</a> 的内存高效注意力机制进一步降低显存占用。虽然JAX/Flax训练支持在TPU和GPU上高效运行，但不支持梯度检查点和xFormers。若需通过Flax加速训练，建议使用显存大于30GB的GPU。</p> <p>本指南将解析 <a href="https://github.com/huggingface/diffusers/blob/main/examples/controlnet/train_controlnet.py" rel="nofollow">train_controlnet.py</a> 训练脚本，帮助您理解其逻辑并适配自定义需求。</p> <p>运行脚本前，请确保从源码安装库：</p> <!> <p>然后进入包含训练脚本的示例目录，安装所需依赖：</p> <!> <blockquote class="tip"><p>🤗 Accelerate 是一个支持多GPU/TPU训练和混合精度的库，它能根据硬件环境自动配置训练方案。参阅 🤗 Accelerate <a href="https://huggingface.co/docs/accelerate/quicktour" rel="nofollow">快速入门</a> 了解更多。</p></blockquote> <p>初始化🤗 Accelerate环境：</p> <!> <p>若要创建默认配置（不进行交互式选择）：</p> <!> <p>若环境不支持交互式shell（如notebook），可使用：</p> <!> <p>最后，如需训练自定义数据集，请参阅 <a href="create_dataset">创建训练数据集</a> 指南了解数据准备方法。</p> <blockquote class="tip"><p>下文重点解析脚本中的关键模块，但不会覆盖所有实现细节。如需深入了解，建议直接阅读 <a href="https://github.com/huggingface/diffusers/blob/main/examples/controlnet/train_controlnet.py" rel="nofollow">脚本源码</a>，如有疑问欢迎反馈。</p></blockquote> <!> <p>训练脚本提供了丰富的可配置参数，所有参数及其说明详见 <a href="https://github.com/huggingface/diffusers/blob/64603389da01082055a901f2883c4810d1144edb/examples/controlnet/train_controlnet.py#L231" rel="nofollow"><code>parse_args()</code></a> 函数。虽然该函数已为每个参数提供默认值（如训练批大小、学习率等），但您可以通过命令行参数覆盖这些默认值。</p> <p>例如，使用fp16混合精度加速训练, 可使用<code>--mixed_precision</code>参数</p> <!> <p>基础参数说明可参考 <a href="text2image#script-parameters">文生图</a> 训练指南，此处重点介绍ControlNet相关参数：</p> <ul><li><code>--max_train_samples</code>: 训练样本数量，减少该值可加快训练，但对超大数据集需配合 <code>--streaming</code> 参数使用</li> <li><code>--gradient_accumulation_steps</code>: 梯度累积步数，通过分步计算实现显存受限情况下的更大批次训练</li></ul> <!> <p><a href="https://huggingface.co/papers/2303.09556" rel="nofollow">Min-SNR</a> 加权策略通过重新平衡损失函数加速模型收敛。虽然训练脚本支持预测 <code>epsilon</code>（噪声）或 <code>v_prediction</code>，但Min-SNR对两种预测类型均兼容。该策略仅适用于PyTorch版本，Flax训练脚本暂不支持。</p> <p>推荐值设为5.0：</p> <!> <!> <p>与参数说明类似，训练流程的通用解析可参考 <a href="text2image#training-script">文生图</a> 指南。此处重点分析ControlNet特有的实现。</p> <p>脚本中的 <a href="https://github.com/huggingface/diffusers/blob/64603389da01082055a901f2883c4810d1144edb/examples/controlnet/train_controlnet.py#L582" rel="nofollow"><code>make_train_dataset</code></a> 函数负责数据预处理，除常规的文本标注分词和图像变换外，还包含条件图像的特效处理：</p> <blockquote class="tip"><p>在TPU上流式加载数据集时，🤗 Datasets库可能成为性能瓶颈（因其未针对图像数据优化）。建议考虑 <a href="https://webdataset.github.io/webdataset/" rel="nofollow">WebDataset</a>、<a href="https://github.com/pytorch/data" rel="nofollow">TorchData</a> 或 <a href="https://www.tensorflow.org/datasets/tfless_tfds" rel="nofollow">TensorFlow Datasets</a> 等高效数据格式。</p></blockquote> <!> <p>在 <a href="https://github.com/huggingface/diffusers/blob/64603389da01082055a901f2883c4810d1144edb/examples/controlnet/train_controlnet.py#L713" rel="nofollow"><code>main()</code></a> 函数中，代码会加载分词器、文本编码器、调度器和模型。此处也是ControlNet模型的加载点（支持从现有权重加载或从UNet随机初始化）：</p> <!> <p><a href="https://github.com/huggingface/diffusers/blob/64603389da01082055a901f2883c4810d1144edb/examples/controlnet/train_controlnet.py#L871" rel="nofollow">优化器</a> 专门针对ControlNet参数进行更新：</p> <!> <p>在 <a href="https://github.com/huggingface/diffusers/blob/64603389da01082055a901f2883c4810d1144edb/examples/controlnet/train_controlnet.py#L943" rel="nofollow">训练循环</a> 中，条件文本嵌入和图像被输入到ControlNet的下采样和中层模块：</p> <!> <p>若想深入理解训练循环机制，可参阅 <a href="../using-diffusers/write_own_pipeline">理解管道、模型与调度器</a> 教程，该教程详细解析了去噪过程的基本原理。</p> <!> <p>现在可以启动训练脚本了！🚀</p> <p>本指南使用 <a href="https://huggingface.co/datasets/fusing/fill50k" rel="nofollow">fusing/fill50k</a> 数据集，当然您也可以按照 <a href="create_dataset">创建训练数据集</a> 指南准备自定义数据。</p> <p>设置环境变量 <code>MODEL_NAME</code> 为Hub模型ID或本地路径，<code>OUTPUT_DIR</code> 为模型保存路径。</p> <p>下载训练用的条件图像：</p> <!> <p>根据GPU型号，可能需要启用特定优化。默认配置需要约38GB显存。若使用多GPU训练，请在 <code>accelerate launch</code> 命令中添加 <code>--multi_gpu</code> 参数。</p> <!> <!> <p>训练完成后即可进行推理：</p> <!> <!> <p>Stable Diffusion XL (SDXL) 是新一代文生图模型，通过添加第二文本编码器支持生成更高分辨率图像。使用 <a href="https://github.com/huggingface/diffusers/blob/main/examples/controlnet/train_controlnet_sdxl.py" rel="nofollow"><code>train_controlnet_sdxl.py</code></a> 脚本可为SDXL训练ControlNet适配器。</p> <p>SDXL训练脚本的详细解析请参阅 <a href="sdxl">SDXL训练</a> 指南。</p> <!> <p>恭喜完成ControlNet训练！如需进一步了解模型应用，以下指南可能有所帮助：</p> <ul><li>学习如何 <a href="../using-diffusers/controlnet">使用ControlNet</a> 进行多样化任务的推理</li></ul> <!> <p></p>',1);function Jl(A,z){K(z,!1),ll(()=>{new URLSearchParams(window.location.search).get("fw")}),D();var g=pl();q("1n8wm8g",r=>{var m=tl();O(m,"content",el),o(r,m)});var T=l(i(g),2);P(T,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var f=l(T,2);M(f,{title:"ControlNet",local:"controlnet",headingTag:"h1"});var w=l(f,10);a(w,{code:"Z2l0JTIwY2xvbmUlMjBodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZodWdnaW5nZmFjZSUyRmRpZmZ1c2VycyUwQWNkJTIwZGlmZnVzZXJzJTBBcGlwJTIwaW5zdGFsbCUyMC4=",highlighted:`git <span class="hljs-built_in">clone</span> https://github.com/huggingface/diffusers
<span class="hljs-built_in">cd</span> diffusers
pip install .`,lang:"bash",wrap:!1});var U=l(w,4);y(U,{id:"installation",options:["PyTorch","Flax"],children:(r,m)=>{var p=H(),d=i(p);b(d,{id:"installation",option:"PyTorch",children:(n,s)=>{a(n,{code:"Y2QlMjBleGFtcGxlcyUyRmNvbnRyb2xuZXQlMEFwaXAlMjBpbnN0YWxsJTIwLXIlMjByZXF1aXJlbWVudHMudHh0",highlighted:`<span class="hljs-built_in">cd</span> examples/controlnet
pip install -r requirements.txt`,lang:"bash",wrap:!1})},$$slots:{default:!0}});var J=l(d,2);b(J,{id:"installation",option:"Flax",children:(n,s)=>{var h=ol(),e=l(i(h),2);a(e,{code:"Wk9ORSUzRHVzLWNlbnRyYWwyLWIlMEFUUFVfVFlQRSUzRHY0LTglMEFWTV9OQU1FJTNEaGdfZmxheCUwQSUwQWdjbG91ZCUyMGFscGhhJTIwY29tcHV0ZSUyMHRwdXMlMjB0cHUtdm0lMjBjcmVhdGUlMjAlMjRWTV9OQU1FJTIwJTVDJTBBJTIwLS16b25lJTIwJTI0Wk9ORSUyMCU1QyUwQSUyMC0tYWNjZWxlcmF0b3ItdHlwZSUyMCUyNFRQVV9UWVBFJTIwJTVDJTBBJTIwLS12ZXJzaW9uJTIwJTIwdHB1LXZtLXY0LWJhc2UlMEElMEFnY2xvdWQlMjBhbHBoYSUyMGNvbXB1dGUlMjB0cHVzJTIwdHB1LXZtJTIwc3NoJTIwJTI0Vk1fTkFNRSUyMC0tem9uZSUyMCUyNFpPTkUlMjAtLSUyMCU1Qw==",highlighted:`ZONE=us-central2-b
TPU_TYPE=v4-8
VM_NAME=hg_flax

gcloud alpha compute tpus tpu-vm create <span class="hljs-variable">$VM_NAME</span> \\
 --zone <span class="hljs-variable">$ZONE</span> \\
 --accelerator-type <span class="hljs-variable">$TPU_TYPE</span> \\
 --version  tpu-vm-v4-base

gcloud alpha compute tpus tpu-vm ssh <span class="hljs-variable">$VM_NAME</span> --zone <span class="hljs-variable">$ZONE</span> -- \\`,lang:"bash",wrap:!1});var t=l(e,4);a(t,{code:"cGlwJTIwaW5zdGFsbCUyMCUyMmpheCU1QnRwdSU1RCUzRCUzRDAuNC41JTIyJTIwLWYlMjBodHRwcyUzQSUyRiUyRnN0b3JhZ2UuZ29vZ2xlYXBpcy5jb20lMkZqYXgtcmVsZWFzZXMlMkZsaWJ0cHVfcmVsZWFzZXMuaHRtbA==",highlighted:'pip install <span class="hljs-string">&quot;jax[tpu]==0.4.5&quot;</span> -f https://storage.googleapis.com/jax-releases/libtpu_releases.html',lang:"bash",wrap:!1});var u=l(t,4);a(u,{code:"Y2QlMjBleGFtcGxlcyUyRmNvbnRyb2xuZXQlMEFwaXAlMjBpbnN0YWxsJTIwLXIlMjByZXF1aXJlbWVudHNfZmxheC50eHQ=",highlighted:`<span class="hljs-built_in">cd</span> examples/controlnet
pip install -r requirements_flax.txt`,lang:"bash",wrap:!1}),o(n,h)},$$slots:{default:!0}}),o(r,p)},$$slots:{default:!0}});var Z=l(U,6);a(Z,{code:"YWNjZWxlcmF0ZSUyMGNvbmZpZw==",highlighted:"accelerate config",lang:"bash",wrap:!1});var _=l(Z,4);a(_,{code:"YWNjZWxlcmF0ZSUyMGNvbmZpZyUyMGRlZmF1bHQ=",highlighted:"accelerate config default",lang:"bash",wrap:!1});var j=l(_,4);a(j,{code:"ZnJvbSUyMGFjY2VsZXJhdGUudXRpbHMlMjBpbXBvcnQlMjB3cml0ZV9iYXNpY19jb25maWclMEElMEF3cml0ZV9iYXNpY19jb25maWcoKQ==",highlighted:`<span class="hljs-keyword">from</span> accelerate.utils <span class="hljs-keyword">import</span> write_basic_config

write_basic_config()`,lang:"py",wrap:!1});var W=l(j,6);M(W,{title:"脚本参数",local:"脚本参数",headingTag:"h2"});var X=l(W,6);a(X,{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMHRyYWluX2NvbnRyb2xuZXQucHklMjAlNUMlMEElMjAlMjAtLW1peGVkX3ByZWNpc2lvbiUzRCUyMmZwMTYlMjI=",highlighted:`accelerate launch train_controlnet.py \\
  --mixed_precision=<span class="hljs-string">&quot;fp16&quot;</span>`,lang:"bash",wrap:!1});var R=l(X,6);M(R,{title:"Min-SNR加权策略",local:"min-snr加权策略",headingTag:"h3"});var I=l(R,6);a(I,{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMHRyYWluX2NvbnRyb2xuZXQucHklMjAlNUMlMEElMjAlMjAtLXNucl9nYW1tYSUzRDUuMA==",highlighted:`accelerate launch train_controlnet.py \\
  --snr_gamma=5.0`,lang:"bash",wrap:!1});var G=l(I,2);M(G,{title:"训练脚本",local:"训练脚本",headingTag:"h2"});var N=l(G,8);a(N,{code:"Y29uZGl0aW9uaW5nX2ltYWdlX3RyYW5zZm9ybXMlMjAlM0QlMjB0cmFuc2Zvcm1zLkNvbXBvc2UoJTBBJTIwJTIwJTIwJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdHJhbnNmb3Jtcy5SZXNpemUoYXJncy5yZXNvbHV0aW9uJTJDJTIwaW50ZXJwb2xhdGlvbiUzRHRyYW5zZm9ybXMuSW50ZXJwb2xhdGlvbk1vZGUuQklMSU5FQVIpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdHJhbnNmb3Jtcy5DZW50ZXJDcm9wKGFyZ3MucmVzb2x1dGlvbiklMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0cmFuc2Zvcm1zLlRvVGVuc29yKCklMkMlMEElMjAlMjAlMjAlMjAlNUQlMEEp",highlighted:`conditioning_image_transforms = transforms.Compose(
    [
        transforms.Resize(args.resolution, interpolation=transforms.InterpolationMode.BILINEAR),
        transforms.CenterCrop(args.resolution),
        transforms.ToTensor(),
    ]
)`,lang:"py",wrap:!1});var v=l(N,4);a(v,{code:"aWYlMjBhcmdzLmNvbnRyb2xuZXRfbW9kZWxfbmFtZV9vcl9wYXRoJTNBJTBBJTIwJTIwJTIwJTIwbG9nZ2VyLmluZm8oJTIyTG9hZGluZyUyMGV4aXN0aW5nJTIwY29udHJvbG5ldCUyMHdlaWdodHMlMjIpJTBBJTIwJTIwJTIwJTIwY29udHJvbG5ldCUyMCUzRCUyMENvbnRyb2xOZXRNb2RlbC5mcm9tX3ByZXRyYWluZWQoYXJncy5jb250cm9sbmV0X21vZGVsX25hbWVfb3JfcGF0aCklMEFlbHNlJTNBJTBBJTIwJTIwJTIwJTIwbG9nZ2VyLmluZm8oJTIySW5pdGlhbGl6aW5nJTIwY29udHJvbG5ldCUyMHdlaWdodHMlMjBmcm9tJTIwdW5ldCUyMiklMEElMjAlMjAlMjAlMjBjb250cm9sbmV0JTIwJTNEJTIwQ29udHJvbE5ldE1vZGVsLmZyb21fdW5ldCh1bmV0KQ==",highlighted:`<span class="hljs-keyword">if</span> args.controlnet_model_name_or_path:
    logger.info(<span class="hljs-string">&quot;Loading existing controlnet weights&quot;</span>)
    controlnet = ControlNetModel.from_pretrained(args.controlnet_model_name_or_path)
<span class="hljs-keyword">else</span>:
    logger.info(<span class="hljs-string">&quot;Initializing controlnet weights from unet&quot;</span>)
    controlnet = ControlNetModel.from_unet(unet)`,lang:"py",wrap:!1});var B=l(v,4);a(B,{code:"cGFyYW1zX3RvX29wdGltaXplJTIwJTNEJTIwY29udHJvbG5ldC5wYXJhbWV0ZXJzKCklMEFvcHRpbWl6ZXIlMjAlM0QlMjBvcHRpbWl6ZXJfY2xhc3MoJTBBJTIwJTIwJTIwJTIwcGFyYW1zX3RvX29wdGltaXplJTJDJTBBJTIwJTIwJTIwJTIwbHIlM0RhcmdzLmxlYXJuaW5nX3JhdGUlMkMlMEElMjAlMjAlMjAlMjBiZXRhcyUzRChhcmdzLmFkYW1fYmV0YTElMkMlMjBhcmdzLmFkYW1fYmV0YTIpJTJDJTBBJTIwJTIwJTIwJTIwd2VpZ2h0X2RlY2F5JTNEYXJncy5hZGFtX3dlaWdodF9kZWNheSUyQyUwQSUyMCUyMCUyMCUyMGVwcyUzRGFyZ3MuYWRhbV9lcHNpbG9uJTJDJTBBKQ==",highlighted:`params_to_optimize = controlnet.parameters()
optimizer = optimizer_class(
    params_to_optimize,
    lr=args.learning_rate,
    betas=(args.adam_beta1, args.adam_beta2),
    weight_decay=args.adam_weight_decay,
    eps=args.adam_epsilon,
)`,lang:"py",wrap:!1});var V=l(B,4);a(V,{code:"ZW5jb2Rlcl9oaWRkZW5fc3RhdGVzJTIwJTNEJTIwdGV4dF9lbmNvZGVyKGJhdGNoJTVCJTIyaW5wdXRfaWRzJTIyJTVEKSU1QjAlNUQlMEFjb250cm9sbmV0X2ltYWdlJTIwJTNEJTIwYmF0Y2glNUIlMjJjb25kaXRpb25pbmdfcGl4ZWxfdmFsdWVzJTIyJTVELnRvKGR0eXBlJTNEd2VpZ2h0X2R0eXBlKSUwQSUwQWRvd25fYmxvY2tfcmVzX3NhbXBsZXMlMkMlMjBtaWRfYmxvY2tfcmVzX3NhbXBsZSUyMCUzRCUyMGNvbnRyb2xuZXQoJTBBJTIwJTIwJTIwJTIwbm9pc3lfbGF0ZW50cyUyQyUwQSUyMCUyMCUyMCUyMHRpbWVzdGVwcyUyQyUwQSUyMCUyMCUyMCUyMGVuY29kZXJfaGlkZGVuX3N0YXRlcyUzRGVuY29kZXJfaGlkZGVuX3N0YXRlcyUyQyUwQSUyMCUyMCUyMCUyMGNvbnRyb2xuZXRfY29uZCUzRGNvbnRyb2xuZXRfaW1hZ2UlMkMlMEElMjAlMjAlMjAlMjByZXR1cm5fZGljdCUzREZhbHNlJTJDJTBBKQ==",highlighted:`encoder_hidden_states = text_encoder(batch[<span class="hljs-string">&quot;input_ids&quot;</span>])[<span class="hljs-number">0</span>]
controlnet_image = batch[<span class="hljs-string">&quot;conditioning_pixel_values&quot;</span>].to(dtype=weight_dtype)

down_block_res_samples, mid_block_res_sample = controlnet(
    noisy_latents,
    timesteps,
    encoder_hidden_states=encoder_hidden_states,
    controlnet_cond=controlnet_image,
    return_dict=<span class="hljs-literal">False</span>,
)`,lang:"py",wrap:!1});var Y=l(V,4);M(Y,{title:"启动训练",local:"启动训练",headingTag:"h2"});var E=l(Y,10);a(E,{code:"d2dldCUyMGh0dHBzJTNBJTJGJTJGaHVnZ2luZ2ZhY2UuY28lMkZkYXRhc2V0cyUyRmh1Z2dpbmdmYWNlJTJGZG9jdW1lbnRhdGlvbi1pbWFnZXMlMkZyZXNvbHZlJTJGbWFpbiUyRmRpZmZ1c2VycyUyRmNvbnRyb2xuZXRfdHJhaW5pbmclMkZjb25kaXRpb25pbmdfaW1hZ2VfMS5wbmclMEF3Z2V0JTIwaHR0cHMlM0ElMkYlMkZodWdnaW5nZmFjZS5jbyUyRmRhdGFzZXRzJTJGaHVnZ2luZ2ZhY2UlMkZkb2N1bWVudGF0aW9uLWltYWdlcyUyRnJlc29sdmUlMkZtYWluJTJGZGlmZnVzZXJzJTJGY29udHJvbG5ldF90cmFpbmluZyUyRmNvbmRpdGlvbmluZ19pbWFnZV8yLnBuZw==",highlighted:`wget https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/controlnet_training/conditioning_image_1.png
wget https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/controlnet_training/conditioning_image_2.png`,lang:"bash",wrap:!1});var C=l(E,4);y(C,{id:"gpu-select",options:["16GB","12GB","8GB"],children:(r,m)=>{var p=cl(),d=i(p);b(d,{id:"gpu-select",option:"16GB",children:(s,h)=>{var e=nl(),t=l(i(e),2);a(t,{code:"cGlwJTIwaW5zdGFsbCUyMGJpdHNhbmRieXRlcw==",highlighted:"pip install bitsandbytes",lang:"py",wrap:!1});var u=l(t,4);a(u,{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMHRyYWluX2NvbnRyb2xuZXQucHklMjAlNUMlMEElMjAlMjAtLWdyYWRpZW50X2NoZWNrcG9pbnRpbmclMjAlNUMlMEElMjAlMjAtLXVzZV84Yml0X2FkYW0lMjAlNUM=",highlighted:`accelerate launch train_controlnet.py \\
  --gradient_checkpointing \\
  --use_8bit_adam \\`,lang:"bash",wrap:!1}),o(s,e)},$$slots:{default:!0}});var J=l(d,2);b(J,{id:"gpu-select",option:"12GB",children:(s,h)=>{var e=sl(),t=l(i(e),2);a(t,{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMHRyYWluX2NvbnRyb2xuZXQucHklMjAlNUMlMEElMjAlMjAtLXVzZV84Yml0X2FkYW0lMjAlNUMlMEElMjAlMjAtLWdyYWRpZW50X2NoZWNrcG9pbnRpbmclMjAlNUMlMEElMjAlMjAtLWVuYWJsZV94Zm9ybWVyc19tZW1vcnlfZWZmaWNpZW50X2F0dGVudGlvbiUyMCU1QyUwQSUyMCUyMC0tc2V0X2dyYWRzX3RvX25vbmUlMjAlNUM=",highlighted:`accelerate launch train_controlnet.py \\
  --use_8bit_adam \\
  --gradient_checkpointing \\
  --enable_xformers_memory_efficient_attention \\
  --set_grads_to_none \\`,lang:"bash",wrap:!1}),o(s,e)},$$slots:{default:!0}});var n=l(J,2);b(n,{id:"gpu-select",option:"8GB",children:(s,h)=>{var e=il(),t=l(i(e),4);a(t,{code:"YWNjZWxlcmF0ZSUyMGNvbmZpZw==",highlighted:"accelerate config",lang:"bash",wrap:!1});var u=l(t,4);a(u,{code:"Y29tcHV0ZV9lbnZpcm9ubWVudCUzQSUyMExPQ0FMX01BQ0hJTkUlMEFkZWVwc3BlZWRfY29uZmlnJTNBJTBBJTIwJTIwZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTNBJTIwNCUwQSUyMCUyMG9mZmxvYWRfb3B0aW1pemVyX2RldmljZSUzQSUyMGNwdSUwQSUyMCUyMG9mZmxvYWRfcGFyYW1fZGV2aWNlJTNBJTIwY3B1JTBBJTIwJTIwemVybzNfaW5pdF9mbGFnJTNBJTIwZmFsc2UlMEElMjAlMjB6ZXJvX3N0YWdlJTNBJTIwMiUwQWRpc3RyaWJ1dGVkX3R5cGUlM0ElMjBERUVQU1BFRUQ=",highlighted:`compute_environment: LOCAL_MACHINE
deepspeed_config:
  gradient_accumulation_steps: 4
  offload_optimizer_device: cpu
  offload_param_device: cpu
  zero3_init_flag: <span class="hljs-literal">false</span>
  zero_stage: 2
distributed_type: DEEPSPEED`,lang:"bash",wrap:!1}),Q(6),o(s,e)},$$slots:{default:!0}}),o(r,p)},$$slots:{default:!0}});var F=l(C,2);y(F,{id:"training-inference",options:["PyTorch","Flax"],children:(r,m)=>{var p=H(),d=i(p);b(d,{id:"training-inference",option:"PyTorch",children:(n,s)=>{a(n,{code:"ZXhwb3J0JTIwTU9ERUxfRElSJTNEJTIyc3RhYmxlLWRpZmZ1c2lvbi12MS01JTJGc3RhYmxlLWRpZmZ1c2lvbi12MS01JTIyJTBBZXhwb3J0JTIwT1VUUFVUX0RJUiUzRCUyMnBhdGglMkZ0byUyRnNhdmUlMkZtb2RlbCUyMiUwQSUwQWFjY2VsZXJhdGUlMjBsYXVuY2glMjB0cmFpbl9jb250cm9sbmV0LnB5JTIwJTVDJTBBJTIwLS1wcmV0cmFpbmVkX21vZGVsX25hbWVfb3JfcGF0aCUzRCUyNE1PREVMX0RJUiUyMCU1QyUwQSUyMC0tb3V0cHV0X2RpciUzRCUyNE9VVFBVVF9ESVIlMjAlNUMlMEElMjAtLWRhdGFzZXRfbmFtZSUzRGZ1c2luZyUyRmZpbGw1MGslMjAlNUMlMEElMjAtLXJlc29sdXRpb24lM0Q1MTIlMjAlNUMlMEElMjAtLWxlYXJuaW5nX3JhdGUlM0QxZS01JTIwJTVDJTBBJTIwLS12YWxpZGF0aW9uX2ltYWdlJTIwJTIyLiUyRmNvbmRpdGlvbmluZ19pbWFnZV8xLnBuZyUyMiUyMCUyMi4lMkZjb25kaXRpb25pbmdfaW1hZ2VfMi5wbmclMjIlMjAlNUMlMEElMjAtLXZhbGlkYXRpb25fcHJvbXB0JTIwJTIycmVkJTIwY2lyY2xlJTIwd2l0aCUyMGJsdWUlMjBiYWNrZ3JvdW5kJTIyJTIwJTIyY3lhbiUyMGNpcmNsZSUyMHdpdGglMjBicm93biUyMGZsb3JhbCUyMGJhY2tncm91bmQlMjIlMjAlNUMlMEElMjAtLXRyYWluX2JhdGNoX3NpemUlM0QxJTIwJTVDJTBBJTIwLS1ncmFkaWVudF9hY2N1bXVsYXRpb25fc3RlcHMlM0Q0JTIwJTVDJTBBJTIwLS1wdXNoX3RvX2h1Yg==",highlighted:`<span class="hljs-built_in">export</span> MODEL_DIR=<span class="hljs-string">&quot;stable-diffusion-v1-5/stable-diffusion-v1-5&quot;</span>
<span class="hljs-built_in">export</span> OUTPUT_DIR=<span class="hljs-string">&quot;path/to/save/model&quot;</span>

accelerate launch train_controlnet.py \\
 --pretrained_model_name_or_path=<span class="hljs-variable">$MODEL_DIR</span> \\
 --output_dir=<span class="hljs-variable">$OUTPUT_DIR</span> \\
 --dataset_name=fusing/fill50k \\
 --resolution=512 \\
 --learning_rate=1e-5 \\
 --validation_image <span class="hljs-string">&quot;./conditioning_image_1.png&quot;</span> <span class="hljs-string">&quot;./conditioning_image_2.png&quot;</span> \\
 --validation_prompt <span class="hljs-string">&quot;red circle with blue background&quot;</span> <span class="hljs-string">&quot;cyan circle with brown floral background&quot;</span> \\
 --train_batch_size=1 \\
 --gradient_accumulation_steps=4 \\
 --push_to_hub`,lang:"bash",wrap:!1})},$$slots:{default:!0}});var J=l(d,2);b(J,{id:"training-inference",option:"Flax",children:(n,s)=>{var h=rl(),e=l(i(h),2);a(e,{code:"cGlwJTIwaW5zdGFsbCUyMHRlbnNvcmZsb3clMjB0ZW5zb3Jib2FyZC1wbHVnaW4tcHJvZmlsZSUwQXRlbnNvcmJvYXJkJTIwLS1sb2dkaXIlMjBydW5zJTJGZmlsbC1jaXJjbGUtMTAwc3RlcHMtMjAyMzA0MTFfMTY1NjEyJTJG",highlighted:`pip install tensorflow tensorboard-plugin-profile
tensorboard --logdir runs/fill-circle-100steps-20230411_165612/`,lang:"bash",wrap:!1});var t=l(e,6);a(t,{code:"cHl0aG9uMyUyMHRyYWluX2NvbnRyb2xuZXRfZmxheC5weSUyMCU1QyUwQSUyMC0tcHJldHJhaW5lZF9tb2RlbF9uYW1lX29yX3BhdGglM0QlMjRNT0RFTF9ESVIlMjAlNUMlMEElMjAtLW91dHB1dF9kaXIlM0QlMjRPVVRQVVRfRElSJTIwJTVDJTBBJTIwLS1kYXRhc2V0X25hbWUlM0RmdXNpbmclMkZmaWxsNTBrJTIwJTVDJTBBJTIwLS1yZXNvbHV0aW9uJTNENTEyJTIwJTVDJTBBJTIwLS1sZWFybmluZ19yYXRlJTNEMWUtNSUyMCU1QyUwQSUyMC0tdmFsaWRhdGlvbl9pbWFnZSUyMCUyMi4lMkZjb25kaXRpb25pbmdfaW1hZ2VfMS5wbmclMjIlMjAlMjIuJTJGY29uZGl0aW9uaW5nX2ltYWdlXzIucG5nJTIyJTIwJTVDJTBBJTIwLS12YWxpZGF0aW9uX3Byb21wdCUyMCUyMnJlZCUyMGNpcmNsZSUyMHdpdGglMjBibHVlJTIwYmFja2dyb3VuZCUyMiUyMCUyMmN5YW4lMjBjaXJjbGUlMjB3aXRoJTIwYnJvd24lMjBmbG9yYWwlMjBiYWNrZ3JvdW5kJTIyJTIwJTVDJTBBJTIwLS12YWxpZGF0aW9uX3N0ZXBzJTNEMTAwMCUyMCU1QyUwQSUyMC0tdHJhaW5fYmF0Y2hfc2l6ZSUzRDIlMjAlNUMlMEElMjAtLXJldmlzaW9uJTNEJTIybm9uLWVtYSUyMiUyMCU1QyUwQSUyMC0tZnJvbV9wdCUyMCU1QyUwQSUyMC0tcmVwb3J0X3RvJTNEJTIyd2FuZGIlMjIlMjAlNUMlMEElMjAtLXRyYWNrZXJfcHJvamVjdF9uYW1lJTNEJTI0SFVCX01PREVMX0lEJTIwJTVDJTBBJTIwLS1udW1fdHJhaW5fZXBvY2hzJTNEMTElMjAlNUMlMEElMjAtLXB1c2hfdG9faHViJTIwJTVDJTBBJTIwLS1odWJfbW9kZWxfaWQlM0QlMjRIVUJfTU9ERUxfSUQ=",highlighted:`python3 train_controlnet_flax.py \\
 --pretrained_model_name_or_path=<span class="hljs-variable">$MODEL_DIR</span> \\
 --output_dir=<span class="hljs-variable">$OUTPUT_DIR</span> \\
 --dataset_name=fusing/fill50k \\
 --resolution=512 \\
 --learning_rate=1e-5 \\
 --validation_image <span class="hljs-string">&quot;./conditioning_image_1.png&quot;</span> <span class="hljs-string">&quot;./conditioning_image_2.png&quot;</span> \\
 --validation_prompt <span class="hljs-string">&quot;red circle with blue background&quot;</span> <span class="hljs-string">&quot;cyan circle with brown floral background&quot;</span> \\
 --validation_steps=1000 \\
 --train_batch_size=2 \\
 --revision=<span class="hljs-string">&quot;non-ema&quot;</span> \\
 --from_pt \\
 --report_to=<span class="hljs-string">&quot;wandb&quot;</span> \\
 --tracker_project_name=<span class="hljs-variable">$HUB_MODEL_ID</span> \\
 --num_train_epochs=11 \\
 --push_to_hub \\
 --hub_model_id=<span class="hljs-variable">$HUB_MODEL_ID</span>`,lang:"bash",wrap:!1}),o(n,h)},$$slots:{default:!0}}),o(r,p)},$$slots:{default:!0}});var x=l(F,4);a(x,{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMFN0YWJsZURpZmZ1c2lvbkNvbnRyb2xOZXRQaXBlbGluZSUyQyUyMENvbnRyb2xOZXRNb2RlbCUwQWZyb20lMjBkaWZmdXNlcnMudXRpbHMlMjBpbXBvcnQlMjBsb2FkX2ltYWdlJTBBaW1wb3J0JTIwdG9yY2glMEElMEFjb250cm9sbmV0JTIwJTNEJTIwQ29udHJvbE5ldE1vZGVsLmZyb21fcHJldHJhaW5lZCglMjJwYXRoJTJGdG8lMkZjb250cm9sbmV0JTIyJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2KSUwQXBpcGVsaW5lJTIwJTNEJTIwU3RhYmxlRGlmZnVzaW9uQ29udHJvbE5ldFBpcGVsaW5lLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJwYXRoJTJGdG8lMkZiYXNlJTJGbW9kZWwlMjIlMkMlMjBjb250cm9sbmV0JTNEY29udHJvbG5ldCUyQyUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUwQSkudG8oJTIyY3VkYSUyMiklMEElMEFjb250cm9sX2ltYWdlJTIwJTNEJTIwbG9hZF9pbWFnZSglMjIuJTJGY29uZGl0aW9uaW5nX2ltYWdlXzEucG5nJTIyKSUwQXByb21wdCUyMCUzRCUyMCUyMnBhbGUlMjBnb2xkZW4lMjByb2QlMjBjaXJjbGUlMjB3aXRoJTIwb2xkJTIwbGFjZSUyMGJhY2tncm91bmQlMjIlMEElMEFnZW5lcmF0b3IlMjAlM0QlMjB0b3JjaC5tYW51YWxfc2VlZCgwKSUwQWltYWdlJTIwJTNEJTIwcGlwZWxpbmUocHJvbXB0JTJDJTIwbnVtX2luZmVyZW5jZV9zdGVwcyUzRDIwJTJDJTIwZ2VuZXJhdG9yJTNEZ2VuZXJhdG9yJTJDJTIwaW1hZ2UlM0Rjb250cm9sX2ltYWdlKS5pbWFnZXMlNUIwJTVEJTBBaW1hZ2Uuc2F2ZSglMjIuJTJGb3V0cHV0LnBuZyUyMik=",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionControlNetPipeline, ControlNetModel
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image
<span class="hljs-keyword">import</span> torch

controlnet = ControlNetModel.from_pretrained(<span class="hljs-string">&quot;path/to/controlnet&quot;</span>, torch_dtype=torch.float16)
pipeline = StableDiffusionControlNetPipeline.from_pretrained(
    <span class="hljs-string">&quot;path/to/base/model&quot;</span>, controlnet=controlnet, torch_dtype=torch.float16
).to(<span class="hljs-string">&quot;cuda&quot;</span>)

control_image = load_image(<span class="hljs-string">&quot;./conditioning_image_1.png&quot;</span>)
prompt = <span class="hljs-string">&quot;pale golden rod circle with old lace background&quot;</span>

generator = torch.manual_seed(<span class="hljs-number">0</span>)
image = pipeline(prompt, num_inference_steps=<span class="hljs-number">20</span>, generator=generator, image=control_image).images[<span class="hljs-number">0</span>]
image.save(<span class="hljs-string">&quot;./output.png&quot;</span>)`,lang:"py",wrap:!1});var k=l(x,2);M(k,{title:"Stable Diffusion XL",local:"stable-diffusion-xl",headingTag:"h2"});var S=l(k,6);M(S,{title:"后续步骤",local:"后续步骤",headingTag:"h2"});var L=l(S,6);$(L,{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/zh/training/controlnet.md"}),Q(2),o(A,g),al()}export{Jl as component};