diffusers/pr_14313/zh/_app/immutable/nodes/21.CUC_8dEA.js

import"../chunks/DsnmJJEf.js";import{i as L,h as D,C as q,H as n,b as e,a as M,E as P,s as $}from"../chunks/Dv-EnAnp.js";import{p as K,o as O,s as l,f as d,a as i,b as ll,c,n as S}from"../chunks/F-c_tsrl.js";import{H as h}from"../chunks/BYc-qY3j.js";const el='{"title":"ControlNet","local":"controlnet","sections":[{"title":"脚本参数","local":"脚本参数","sections":[{"title":"Min-SNR加权策略","local":"min-snr加权策略","sections":[],"depth":3}],"depth":2},{"title":"训练脚本","local":"训练脚本","sections":[],"depth":2},{"title":"启动训练","local":"启动训练","sections":[],"depth":2},{"title":"Stable Diffusion XL","local":"stable-diffusion-xl","sections":[],"depth":2},{"title":"后续步骤","local":"后续步骤","sections":[],"depth":2}],"depth":1}';var al=c('<meta name="hf:doc:metadata"/>'),tl=c("<p>16GB显卡可使用bitsandbytes 8-bit优化器和梯度检查点：</p> <!> <p>训练命令添加以下参数：</p> <!>",1),ol=c("<p>12GB显卡需组合使用bitsandbytes 8-bit优化器、梯度检查点、xFormers，并将梯度置为None而非0：</p> <!>",1),nl=c('<p>8GB显卡需使用 <a href="https://www.deepspeed.ai/" rel="nofollow">DeepSpeed</a> 将张量卸载到CPU或NVME：</p> <p>运行以下命令配置环境：</p> <!> <p>选择DeepSpeed stage 2，结合fp16混合精度和参数卸载到CPU的方案。注意这会增加约25GB内存占用。配置示例如下：</p> <!> <p>建议将优化器替换为DeepSpeed特化版 <a href="https://deepspeed.readthedocs.io/en/latest/optimizers.html#adam-cpu" rel="nofollow"><code>deepspeed.ops.adam.DeepSpeedCPUAdam</code></a>，注意CUDA工具链版本需与PyTorch匹配。</p> <p>当前bitsandbytes与DeepSpeed存在兼容性问题。</p> <p>无需额外添加训练参数。</p>',1),sl=c("<!> <!> <!>",1),il=c('<p></p> <!> <!> <p><a href="https://hf.co/papers/2302.05543" rel="nofollow">ControlNet</a> 是一种基于预训练模型的适配器架构。它通过额外输入的条件图像（如边缘检测图、深度图、人体姿态图等），实现对生成图像的精细化控制。</p> <p>在显存有限的GPU上训练时，建议启用训练命令中的 <code>gradient_checkpointing</code>（梯度检查点）、<code>gradient_accumulation_steps</code>（梯度累积步数）和 <code>mixed_precision</code>（混合精度）参数。还可使用 <a href="../optimization/xformers">xFormers</a> 的内存高效注意力机制进一步降低显存占用。</p> <p>本指南将解析 <a href="https://github.com/huggingface/diffusers/blob/main/examples/controlnet/train_controlnet.py" rel="nofollow">train_controlnet.py</a> 训练脚本，帮助您理解其逻辑并适配自定义需求。</p> <p>运行脚本前，请确保从源码安装库：</p> <!> <p>然后进入包含训练脚本的示例目录，安装所需依赖：</p> <!> <blockquote class="tip"><p>🤗 Accelerate 是一个支持多GPU/TPU训练和混合精度的库，它能根据硬件环境自动配置训练方案。参阅 🤗 Accelerate <a href="https://huggingface.co/docs/accelerate/quicktour" rel="nofollow">快速入门</a> 了解更多。</p></blockquote> <p>初始化🤗 Accelerate环境：</p> <!> <p>若要创建默认配置（不进行交互式选择）：</p> <!> <p>若环境不支持交互式shell（如notebook），可使用：</p> <!> <p>最后，如需训练自定义数据集，请参阅 <a href="create_dataset">创建训练数据集</a> 指南了解数据准备方法。</p> <blockquote class="tip"><p>下文重点解析脚本中的关键模块，但不会覆盖所有实现细节。如需深入了解，建议直接阅读 <a href="https://github.com/huggingface/diffusers/blob/main/examples/controlnet/train_controlnet.py" rel="nofollow">脚本源码</a>，如有疑问欢迎反馈。</p></blockquote> <!> <p>训练脚本提供了丰富的可配置参数，所有参数及其说明详见 <a href="https://github.com/huggingface/diffusers/blob/64603389da01082055a901f2883c4810d1144edb/examples/controlnet/train_controlnet.py#L231" rel="nofollow"><code>parse_args()</code></a> 函数。虽然该函数已为每个参数提供默认值（如训练批大小、学习率等），但您可以通过命令行参数覆盖这些默认值。</p> <p>例如，使用fp16混合精度加速训练, 可使用<code>--mixed_precision</code>参数</p> <!> <p>基础参数说明可参考 <a href="text2image#script-parameters">文生图</a> 训练指南，此处重点介绍ControlNet相关参数：</p> <ul><li><code>--max_train_samples</code>: 训练样本数量，减少该值可加快训练，但对超大数据集需配合 <code>--streaming</code> 参数使用</li> <li><code>--gradient_accumulation_steps</code>: 梯度累积步数，通过分步计算实现显存受限情况下的更大批次训练</li></ul> <!> <p><a href="https://huggingface.co/papers/2303.09556" rel="nofollow">Min-SNR</a> 加权策略通过重新平衡损失函数加速模型收敛。虽然训练脚本支持预测 <code>epsilon</code>（噪声）或 <code>v_prediction</code>，但Min-SNR对两种预测类型均兼容。</p> <p>推荐值设为5.0：</p> <!> <!> <p>与参数说明类似，训练流程的通用解析可参考 <a href="text2image#training-script">文生图</a> 指南。此处重点分析ControlNet特有的实现。</p> <p>脚本中的 <a href="https://github.com/huggingface/diffusers/blob/64603389da01082055a901f2883c4810d1144edb/examples/controlnet/train_controlnet.py#L582" rel="nofollow"><code>make_train_dataset</code></a> 函数负责数据预处理，除常规的文本标注分词和图像变换外，还包含条件图像的特效处理：</p> <blockquote class="tip"><p>在TPU上流式加载数据集时，🤗 Datasets库可能成为性能瓶颈（因其未针对图像数据优化）。建议考虑 <a href="https://webdataset.github.io/webdataset/" rel="nofollow">WebDataset</a>、<a href="https://github.com/pytorch/data" rel="nofollow">TorchData</a> 或 <a href="https://www.tensorflow.org/datasets/tfless_tfds" rel="nofollow">TensorFlow Datasets</a> 等高效数据格式。</p></blockquote> <!> <p>在 <a href="https://github.com/huggingface/diffusers/blob/64603389da01082055a901f2883c4810d1144edb/examples/controlnet/train_controlnet.py#L713" rel="nofollow"><code>main()</code></a> 函数中，代码会加载分词器、文本编码器、调度器和模型。此处也是ControlNet模型的加载点（支持从现有权重加载或从UNet随机初始化）：</p> <!> <p><a href="https://github.com/huggingface/diffusers/blob/64603389da01082055a901f2883c4810d1144edb/examples/controlnet/train_controlnet.py#L871" rel="nofollow">优化器</a> 专门针对ControlNet参数进行更新：</p> <!> <p>在 <a href="https://github.com/huggingface/diffusers/blob/64603389da01082055a901f2883c4810d1144edb/examples/controlnet/train_controlnet.py#L943" rel="nofollow">训练循环</a> 中，条件文本嵌入和图像被输入到ControlNet的下采样和中层模块：</p> <!> <p>若想深入理解训练循环机制，可参阅 <a href="../using-diffusers/write_own_pipeline">理解管道、模型与调度器</a> 教程，该教程详细解析了去噪过程的基本原理。</p> <!> <p>现在可以启动训练脚本了！🚀</p> <p>本指南使用 <a href="https://huggingface.co/datasets/fusing/fill50k" rel="nofollow">fusing/fill50k</a> 数据集，当然您也可以按照 <a href="create_dataset">创建训练数据集</a> 指南准备自定义数据。</p> <p>设置环境变量 <code>MODEL_NAME</code> 为Hub模型ID或本地路径，<code>OUTPUT_DIR</code> 为模型保存路径。</p> <p>下载训练用的条件图像：</p> <!> <p>根据GPU型号，可能需要启用特定优化。默认配置需要约38GB显存。若使用多GPU训练，请在 <code>accelerate launch</code> 命令中添加 <code>--multi_gpu</code> 参数。</p> <!> <!> <p>训练完成后即可进行推理：</p> <!> <!> <p>Stable Diffusion XL (SDXL) 是新一代文生图模型，通过添加第二文本编码器支持生成更高分辨率图像。使用 <a href="https://github.com/huggingface/diffusers/blob/main/examples/controlnet/train_controlnet_sdxl.py" rel="nofollow"><code>train_controlnet_sdxl.py</code></a> 脚本可为SDXL训练ControlNet适配器。</p> <p>SDXL训练脚本的详细解析请参阅 <a href="sdxl">SDXL训练</a> 指南。</p> <!> <p>恭喜完成ControlNet训练！如需进一步了解模型应用，以下指南可能有所帮助：</p> <ul><li>学习如何 <a href="../using-diffusers/controlnet">使用ControlNet</a> 进行多样化任务的推理</li></ul> <!> <p></p>',1);function hl(A,z){K(z,!1),O(()=>{new URLSearchParams(window.location.search).get("fw")}),L();var J=il();D("1n8wm8g",a=>{var p=al();$(p,"content",el),i(a,p)});var u=l(d(J),2);q(u,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var g=l(u,2);n(g,{title:"ControlNet",local:"controlnet",headingTag:"h1"});var y=l(g,10);e(y,{code:"Z2l0JTIwY2xvbmUlMjBodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZodWdnaW5nZmFjZSUyRmRpZmZ1c2VycyUwQWNkJTIwZGlmZnVzZXJzJTBBcGlwJTIwaW5zdGFsbCUyMC4=",highlighted:`git <span class="hljs-built_in">clone</span> https://github.com/huggingface/diffusers
<span class="hljs-built_in">cd</span> diffusers
pip install .`,lang:"bash",wrap:!1});var f=l(y,4);M(f,{id:"installation",options:["PyTorch"],children:(a,p)=>{h(a,{id:"installation",option:"PyTorch",children:(s,b)=>{e(s,{code:"Y2QlMjBleGFtcGxlcyUyRmNvbnRyb2xuZXQlMEFwaXAlMjBpbnN0YWxsJTIwLXIlMjByZXF1aXJlbWVudHMudHh0",highlighted:`<span class="hljs-built_in">cd</span> examples/controlnet
pip install -r requirements.txt`,lang:"bash",wrap:!1})},$$slots:{default:!0}})},$$slots:{default:!0}});var w=l(f,6);e(w,{code:"YWNjZWxlcmF0ZSUyMGNvbmZpZw==",highlighted:"accelerate config",lang:"bash",wrap:!1});var T=l(w,4);e(T,{code:"YWNjZWxlcmF0ZSUyMGNvbmZpZyUyMGRlZmF1bHQ=",highlighted:"accelerate config default",lang:"bash",wrap:!1});var Z=l(T,4);e(Z,{code:"ZnJvbSUyMGFjY2VsZXJhdGUudXRpbHMlMjBpbXBvcnQlMjB3cml0ZV9iYXNpY19jb25maWclMEElMEF3cml0ZV9iYXNpY19jb25maWcoKQ==",highlighted:`<span class="hljs-keyword">from</span> accelerate.utils <span class="hljs-keyword">import</span> write_basic_config

write_basic_config()`,lang:"py",wrap:!1});var U=l(Z,6);n(U,{title:"脚本参数",local:"脚本参数",headingTag:"h2"});var _=l(U,6);e(_,{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMHRyYWluX2NvbnRyb2xuZXQucHklMjAlNUMlMEElMjAlMjAtLW1peGVkX3ByZWNpc2lvbiUzRCUyMmZwMTYlMjI=",highlighted:`accelerate launch train_controlnet.py \\
  --mixed_precision=<span class="hljs-string">&quot;fp16&quot;</span>`,lang:"bash",wrap:!1});var j=l(_,6);n(j,{title:"Min-SNR加权策略",local:"min-snr加权策略",headingTag:"h3"});var W=l(j,6);e(W,{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMHRyYWluX2NvbnRyb2xuZXQucHklMjAlNUMlMEElMjAlMjAtLXNucl9nYW1tYSUzRDUuMA==",highlighted:`accelerate launch train_controlnet.py \\
  --snr_gamma=5.0`,lang:"bash",wrap:!1});var G=l(W,2);n(G,{title:"训练脚本",local:"训练脚本",headingTag:"h2"});var X=l(G,8);e(X,{code:"Y29uZGl0aW9uaW5nX2ltYWdlX3RyYW5zZm9ybXMlMjAlM0QlMjB0cmFuc2Zvcm1zLkNvbXBvc2UoJTBBJTIwJTIwJTIwJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdHJhbnNmb3Jtcy5SZXNpemUoYXJncy5yZXNvbHV0aW9uJTJDJTIwaW50ZXJwb2xhdGlvbiUzRHRyYW5zZm9ybXMuSW50ZXJwb2xhdGlvbk1vZGUuQklMSU5FQVIpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdHJhbnNmb3Jtcy5DZW50ZXJDcm9wKGFyZ3MucmVzb2x1dGlvbiklMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0cmFuc2Zvcm1zLlRvVGVuc29yKCklMkMlMEElMjAlMjAlMjAlMjAlNUQlMEEp",highlighted:`conditioning_image_transforms = transforms.Compose(
    [
        transforms.Resize(args.resolution, interpolation=transforms.InterpolationMode.BILINEAR),
        transforms.CenterCrop(args.resolution),
        transforms.ToTensor(),
    ]
)`,lang:"py",wrap:!1});var R=l(X,4);e(R,{code:"aWYlMjBhcmdzLmNvbnRyb2xuZXRfbW9kZWxfbmFtZV9vcl9wYXRoJTNBJTBBJTIwJTIwJTIwJTIwbG9nZ2VyLmluZm8oJTIyTG9hZGluZyUyMGV4aXN0aW5nJTIwY29udHJvbG5ldCUyMHdlaWdodHMlMjIpJTBBJTIwJTIwJTIwJTIwY29udHJvbG5ldCUyMCUzRCUyMENvbnRyb2xOZXRNb2RlbC5mcm9tX3ByZXRyYWluZWQoYXJncy5jb250cm9sbmV0X21vZGVsX25hbWVfb3JfcGF0aCklMEFlbHNlJTNBJTBBJTIwJTIwJTIwJTIwbG9nZ2VyLmluZm8oJTIySW5pdGlhbGl6aW5nJTIwY29udHJvbG5ldCUyMHdlaWdodHMlMjBmcm9tJTIwdW5ldCUyMiklMEElMjAlMjAlMjAlMjBjb250cm9sbmV0JTIwJTNEJTIwQ29udHJvbE5ldE1vZGVsLmZyb21fdW5ldCh1bmV0KQ==",highlighted:`<span class="hljs-keyword">if</span> args.controlnet_model_name_or_path:
    logger.info(<span class="hljs-string">&quot;Loading existing controlnet weights&quot;</span>)
    controlnet = ControlNetModel.from_pretrained(args.controlnet_model_name_or_path)
<span class="hljs-keyword">else</span>:
    logger.info(<span class="hljs-string">&quot;Initializing controlnet weights from unet&quot;</span>)
    controlnet = ControlNetModel.from_unet(unet)`,lang:"py",wrap:!1});var I=l(R,4);e(I,{code:"cGFyYW1zX3RvX29wdGltaXplJTIwJTNEJTIwY29udHJvbG5ldC5wYXJhbWV0ZXJzKCklMEFvcHRpbWl6ZXIlMjAlM0QlMjBvcHRpbWl6ZXJfY2xhc3MoJTBBJTIwJTIwJTIwJTIwcGFyYW1zX3RvX29wdGltaXplJTJDJTBBJTIwJTIwJTIwJTIwbHIlM0RhcmdzLmxlYXJuaW5nX3JhdGUlMkMlMEElMjAlMjAlMjAlMjBiZXRhcyUzRChhcmdzLmFkYW1fYmV0YTElMkMlMjBhcmdzLmFkYW1fYmV0YTIpJTJDJTBBJTIwJTIwJTIwJTIwd2VpZ2h0X2RlY2F5JTNEYXJncy5hZGFtX3dlaWdodF9kZWNheSUyQyUwQSUyMCUyMCUyMCUyMGVwcyUzRGFyZ3MuYWRhbV9lcHNpbG9uJTJDJTBBKQ==",highlighted:`params_to_optimize = controlnet.parameters()
optimizer = optimizer_class(
    params_to_optimize,
    lr=args.learning_rate,
    betas=(args.adam_beta1, args.adam_beta2),
    weight_decay=args.adam_weight_decay,
    eps=args.adam_epsilon,
)`,lang:"py",wrap:!1});var N=l(I,4);e(N,{code:"ZW5jb2Rlcl9oaWRkZW5fc3RhdGVzJTIwJTNEJTIwdGV4dF9lbmNvZGVyKGJhdGNoJTVCJTIyaW5wdXRfaWRzJTIyJTVEKSU1QjAlNUQlMEFjb250cm9sbmV0X2ltYWdlJTIwJTNEJTIwYmF0Y2glNUIlMjJjb25kaXRpb25pbmdfcGl4ZWxfdmFsdWVzJTIyJTVELnRvKGR0eXBlJTNEd2VpZ2h0X2R0eXBlKSUwQSUwQWRvd25fYmxvY2tfcmVzX3NhbXBsZXMlMkMlMjBtaWRfYmxvY2tfcmVzX3NhbXBsZSUyMCUzRCUyMGNvbnRyb2xuZXQoJTBBJTIwJTIwJTIwJTIwbm9pc3lfbGF0ZW50cyUyQyUwQSUyMCUyMCUyMCUyMHRpbWVzdGVwcyUyQyUwQSUyMCUyMCUyMCUyMGVuY29kZXJfaGlkZGVuX3N0YXRlcyUzRGVuY29kZXJfaGlkZGVuX3N0YXRlcyUyQyUwQSUyMCUyMCUyMCUyMGNvbnRyb2xuZXRfY29uZCUzRGNvbnRyb2xuZXRfaW1hZ2UlMkMlMEElMjAlMjAlMjAlMjByZXR1cm5fZGljdCUzREZhbHNlJTJDJTBBKQ==",highlighted:`encoder_hidden_states = text_encoder(batch[<span class="hljs-string">&quot;input_ids&quot;</span>])[<span class="hljs-number">0</span>]
controlnet_image = batch[<span class="hljs-string">&quot;conditioning_pixel_values&quot;</span>].to(dtype=weight_dtype)

down_block_res_samples, mid_block_res_sample = controlnet(
    noisy_latents,
    timesteps,
    encoder_hidden_states=encoder_hidden_states,
    controlnet_cond=controlnet_image,
    return_dict=<span class="hljs-literal">False</span>,
)`,lang:"py",wrap:!1});var v=l(N,4);n(v,{title:"启动训练",local:"启动训练",headingTag:"h2"});var B=l(v,10);e(B,{code:"d2dldCUyMGh0dHBzJTNBJTJGJTJGaHVnZ2luZ2ZhY2UuY28lMkZkYXRhc2V0cyUyRmh1Z2dpbmdmYWNlJTJGZG9jdW1lbnRhdGlvbi1pbWFnZXMlMkZyZXNvbHZlJTJGbWFpbiUyRmRpZmZ1c2VycyUyRmNvbnRyb2xuZXRfdHJhaW5pbmclMkZjb25kaXRpb25pbmdfaW1hZ2VfMS5wbmclMEF3Z2V0JTIwaHR0cHMlM0ElMkYlMkZodWdnaW5nZmFjZS5jbyUyRmRhdGFzZXRzJTJGaHVnZ2luZ2ZhY2UlMkZkb2N1bWVudGF0aW9uLWltYWdlcyUyRnJlc29sdmUlMkZtYWluJTJGZGlmZnVzZXJzJTJGY29udHJvbG5ldF90cmFpbmluZyUyRmNvbmRpdGlvbmluZ19pbWFnZV8yLnBuZw==",highlighted:`wget https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/controlnet_training/conditioning_image_1.png
wget https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/controlnet_training/conditioning_image_2.png`,lang:"bash",wrap:!1});var Y=l(B,4);M(Y,{id:"gpu-select",options:["16GB","12GB","8GB"],children:(a,p)=>{var s=sl(),b=d(s);h(b,{id:"gpu-select",option:"16GB",children:(r,F)=>{var t=tl(),o=l(d(t),2);e(o,{code:"cGlwJTIwaW5zdGFsbCUyMGJpdHNhbmRieXRlcw==",highlighted:"pip install bitsandbytes",lang:"py",wrap:!1});var m=l(o,4);e(m,{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMHRyYWluX2NvbnRyb2xuZXQucHklMjAlNUMlMEElMjAlMjAtLWdyYWRpZW50X2NoZWNrcG9pbnRpbmclMjAlNUMlMEElMjAlMjAtLXVzZV84Yml0X2FkYW0lMjAlNUM=",highlighted:`accelerate launch train_controlnet.py \\
  --gradient_checkpointing \\
  --use_8bit_adam \\`,lang:"bash",wrap:!1}),i(r,t)},$$slots:{default:!0}});var x=l(b,2);h(x,{id:"gpu-select",option:"12GB",children:(r,F)=>{var t=ol(),o=l(d(t),2);e(o,{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMHRyYWluX2NvbnRyb2xuZXQucHklMjAlNUMlMEElMjAlMjAtLXVzZV84Yml0X2FkYW0lMjAlNUMlMEElMjAlMjAtLWdyYWRpZW50X2NoZWNrcG9pbnRpbmclMjAlNUMlMEElMjAlMjAtLWVuYWJsZV94Zm9ybWVyc19tZW1vcnlfZWZmaWNpZW50X2F0dGVudGlvbiUyMCU1QyUwQSUyMCUyMC0tc2V0X2dyYWRzX3RvX25vbmUlMjAlNUM=",highlighted:`accelerate launch train_controlnet.py \\
  --use_8bit_adam \\
  --gradient_checkpointing \\
  --enable_xformers_memory_efficient_attention \\
  --set_grads_to_none \\`,lang:"bash",wrap:!1}),i(r,t)},$$slots:{default:!0}});var H=l(x,2);h(H,{id:"gpu-select",option:"8GB",children:(r,F)=>{var t=nl(),o=l(d(t),4);e(o,{code:"YWNjZWxlcmF0ZSUyMGNvbmZpZw==",highlighted:"accelerate config",lang:"bash",wrap:!1});var m=l(o,4);e(m,{code:"Y29tcHV0ZV9lbnZpcm9ubWVudCUzQSUyMExPQ0FMX01BQ0hJTkUlMEFkZWVwc3BlZWRfY29uZmlnJTNBJTBBJTIwJTIwZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTNBJTIwNCUwQSUyMCUyMG9mZmxvYWRfb3B0aW1pemVyX2RldmljZSUzQSUyMGNwdSUwQSUyMCUyMG9mZmxvYWRfcGFyYW1fZGV2aWNlJTNBJTIwY3B1JTBBJTIwJTIwemVybzNfaW5pdF9mbGFnJTNBJTIwZmFsc2UlMEElMjAlMjB6ZXJvX3N0YWdlJTNBJTIwMiUwQWRpc3RyaWJ1dGVkX3R5cGUlM0ElMjBERUVQU1BFRUQ=",highlighted:`compute_environment: LOCAL_MACHINE
deepspeed_config:
  gradient_accumulation_steps: 4
  offload_optimizer_device: cpu
  offload_param_device: cpu
  zero3_init_flag: <span class="hljs-literal">false</span>
  zero_stage: 2
distributed_type: DEEPSPEED`,lang:"bash",wrap:!1}),S(6),i(r,t)},$$slots:{default:!0}}),i(a,s)},$$slots:{default:!0}});var V=l(Y,2);M(V,{id:"training-inference",options:["PyTorch"],children:(a,p)=>{h(a,{id:"training-inference",option:"PyTorch",children:(s,b)=>{e(s,{code:"ZXhwb3J0JTIwTU9ERUxfRElSJTNEJTIyc3RhYmxlLWRpZmZ1c2lvbi12MS01JTJGc3RhYmxlLWRpZmZ1c2lvbi12MS01JTIyJTBBZXhwb3J0JTIwT1VUUFVUX0RJUiUzRCUyMnBhdGglMkZ0byUyRnNhdmUlMkZtb2RlbCUyMiUwQSUwQWFjY2VsZXJhdGUlMjBsYXVuY2glMjB0cmFpbl9jb250cm9sbmV0LnB5JTIwJTVDJTBBJTIwLS1wcmV0cmFpbmVkX21vZGVsX25hbWVfb3JfcGF0aCUzRCUyNE1PREVMX0RJUiUyMCU1QyUwQSUyMC0tb3V0cHV0X2RpciUzRCUyNE9VVFBVVF9ESVIlMjAlNUMlMEElMjAtLWRhdGFzZXRfbmFtZSUzRGZ1c2luZyUyRmZpbGw1MGslMjAlNUMlMEElMjAtLXJlc29sdXRpb24lM0Q1MTIlMjAlNUMlMEElMjAtLWxlYXJuaW5nX3JhdGUlM0QxZS01JTIwJTVDJTBBJTIwLS12YWxpZGF0aW9uX2ltYWdlJTIwJTIyLiUyRmNvbmRpdGlvbmluZ19pbWFnZV8xLnBuZyUyMiUyMCUyMi4lMkZjb25kaXRpb25pbmdfaW1hZ2VfMi5wbmclMjIlMjAlNUMlMEElMjAtLXZhbGlkYXRpb25fcHJvbXB0JTIwJTIycmVkJTIwY2lyY2xlJTIwd2l0aCUyMGJsdWUlMjBiYWNrZ3JvdW5kJTIyJTIwJTIyY3lhbiUyMGNpcmNsZSUyMHdpdGglMjBicm93biUyMGZsb3JhbCUyMGJhY2tncm91bmQlMjIlMjAlNUMlMEElMjAtLXRyYWluX2JhdGNoX3NpemUlM0QxJTIwJTVDJTBBJTIwLS1ncmFkaWVudF9hY2N1bXVsYXRpb25fc3RlcHMlM0Q0JTIwJTVDJTBBJTIwLS1wdXNoX3RvX2h1Yg==",highlighted:`<span class="hljs-built_in">export</span> MODEL_DIR=<span class="hljs-string">&quot;stable-diffusion-v1-5/stable-diffusion-v1-5&quot;</span>
<span class="hljs-built_in">export</span> OUTPUT_DIR=<span class="hljs-string">&quot;path/to/save/model&quot;</span>

accelerate launch train_controlnet.py \\
 --pretrained_model_name_or_path=<span class="hljs-variable">$MODEL_DIR</span> \\
 --output_dir=<span class="hljs-variable">$OUTPUT_DIR</span> \\
 --dataset_name=fusing/fill50k \\
 --resolution=512 \\
 --learning_rate=1e-5 \\
 --validation_image <span class="hljs-string">&quot;./conditioning_image_1.png&quot;</span> <span class="hljs-string">&quot;./conditioning_image_2.png&quot;</span> \\
 --validation_prompt <span class="hljs-string">&quot;red circle with blue background&quot;</span> <span class="hljs-string">&quot;cyan circle with brown floral background&quot;</span> \\
 --train_batch_size=1 \\
 --gradient_accumulation_steps=4 \\
 --push_to_hub`,lang:"bash",wrap:!1})},$$slots:{default:!0}})},$$slots:{default:!0}});var E=l(V,4);e(E,{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMFN0YWJsZURpZmZ1c2lvbkNvbnRyb2xOZXRQaXBlbGluZSUyQyUyMENvbnRyb2xOZXRNb2RlbCUwQWZyb20lMjBkaWZmdXNlcnMudXRpbHMlMjBpbXBvcnQlMjBsb2FkX2ltYWdlJTBBaW1wb3J0JTIwdG9yY2glMEElMEFjb250cm9sbmV0JTIwJTNEJTIwQ29udHJvbE5ldE1vZGVsLmZyb21fcHJldHJhaW5lZCglMjJwYXRoJTJGdG8lMkZjb250cm9sbmV0JTIyJTJDJTIwZHR5cGUlM0R0b3JjaC5mbG9hdDE2KSUwQXBpcGVsaW5lJTIwJTNEJTIwU3RhYmxlRGlmZnVzaW9uQ29udHJvbE5ldFBpcGVsaW5lLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJwYXRoJTJGdG8lMkZiYXNlJTJGbW9kZWwlMjIlMkMlMjBjb250cm9sbmV0JTNEY29udHJvbG5ldCUyQyUyMGR0eXBlJTNEdG9yY2guZmxvYXQxNiUwQSkudG8oJTIyY3VkYSUyMiklMEElMEFjb250cm9sX2ltYWdlJTIwJTNEJTIwbG9hZF9pbWFnZSglMjIuJTJGY29uZGl0aW9uaW5nX2ltYWdlXzEucG5nJTIyKSUwQXByb21wdCUyMCUzRCUyMCUyMnBhbGUlMjBnb2xkZW4lMjByb2QlMjBjaXJjbGUlMjB3aXRoJTIwb2xkJTIwbGFjZSUyMGJhY2tncm91bmQlMjIlMEElMEFnZW5lcmF0b3IlMjAlM0QlMjB0b3JjaC5tYW51YWxfc2VlZCgwKSUwQWltYWdlJTIwJTNEJTIwcGlwZWxpbmUocHJvbXB0JTJDJTIwbnVtX2luZmVyZW5jZV9zdGVwcyUzRDIwJTJDJTIwZ2VuZXJhdG9yJTNEZ2VuZXJhdG9yJTJDJTIwaW1hZ2UlM0Rjb250cm9sX2ltYWdlKS5pbWFnZXMlNUIwJTVEJTBBaW1hZ2Uuc2F2ZSglMjIuJTJGb3V0cHV0LnBuZyUyMik=",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionControlNetPipeline, ControlNetModel
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image
<span class="hljs-keyword">import</span> torch

controlnet = ControlNetModel.from_pretrained(<span class="hljs-string">&quot;path/to/controlnet&quot;</span>, dtype=torch.float16)
pipeline = StableDiffusionControlNetPipeline.from_pretrained(
    <span class="hljs-string">&quot;path/to/base/model&quot;</span>, controlnet=controlnet, dtype=torch.float16
).to(<span class="hljs-string">&quot;cuda&quot;</span>)

control_image = load_image(<span class="hljs-string">&quot;./conditioning_image_1.png&quot;</span>)
prompt = <span class="hljs-string">&quot;pale golden rod circle with old lace background&quot;</span>

generator = torch.manual_seed(<span class="hljs-number">0</span>)
image = pipeline(prompt, num_inference_steps=<span class="hljs-number">20</span>, generator=generator, image=control_image).images[<span class="hljs-number">0</span>]
image.save(<span class="hljs-string">&quot;./output.png&quot;</span>)`,lang:"py",wrap:!1});var k=l(E,2);n(k,{title:"Stable Diffusion XL",local:"stable-diffusion-xl",headingTag:"h2"});var C=l(k,6);n(C,{title:"后续步骤",local:"后续步骤",headingTag:"h2"});var Q=l(C,6);P(Q,{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/zh/training/controlnet.md"}),S(2),i(A,J),ll()}export{hl as component};