diffusers/pr_14407/zh/_app/immutable/nodes/45.DhC_X9D5.js

import"../chunks/DsnmJJEf.js";import{i as Y,h as A,C as z,H as M,a,b as u,E as S,s as q}from"../chunks/DdZvggmf.js";import{p as L,o as D,s as l,f as r,a as o,b as P,c as d,n as J}from"../chunks/BbekZcyp.js";import{H as y}from"../chunks/BcnRgdDK.js";const $='{"title":"Kandinsky 2.2","local":"kandinsky-22","sections":[{"title":"脚本参数","local":"脚本参数","sections":[{"title":"Min-SNR 加权","local":"min-snr-加权","sections":[],"depth":3}],"depth":2},{"title":"训练脚本","local":"训练脚本","sections":[],"depth":2},{"title":"启动脚本","local":"启动脚本","sections":[],"depth":2},{"title":"后续步骤","local":"后续步骤","sections":[],"depth":2}],"depth":1}';var K=d('<meta name="hf:doc:metadata"/>'),O=d(`<p><a href="https://github.com/huggingface/diffusers/blob/6e68c71503682c8693cb5b06a4da4911dfd655ee/examples/kandinsky2_2/text_to_image/train_text_to_image_prior.py#L441" rel="nofollow"><code>main()</code></a> 函数包含代码 f
或准备数据集和训练模型。</p> <p>您会立即注意到的主要区别之一是，训练脚本除了调度器和分词器外，还加载了一个 <code>CLIPImageProcessor</code> 用于预处理图像，以及一个 <code>CLIPVisionModelWithProjection</code> 模型用于编码图像：</p> <!> <p>Kandinsky 使用一个 <code>PriorTransformer</code> 来生成图像嵌入，因此您需要设置优化器来学习先验模型的参数。</p> <!> <p>接下来，输入标题被分词，图像由 <code>CLIPImageProcessor</code> <a href="https://github.com/huggingface/diffusers/blob/6e68c71503682c8693cb5b06a4da4911dfd655ee/examples/kandinsky2_2/text_to_image/train_text_to_image_prior.py#L632" rel="nofollow">预处理</a>：</p> <!> <p>最后，<a href="https://github.com/huggingface/diffusers/blob/6e68c71503682c8693cb5b06a4da4911dfd655ee/examples/kandinsky2_2/text_to_image/train_text_to_image_prior.py#L718" rel="nofollow">训练循环</a> 将输入图像转换为潜在表示，向图像嵌入添加噪声，并进行预测：</p> <!> <p>如果您想了解更多关于训练循环的工作原理，请查看 <a href="../using-diffusers/write_own_pipeline">理解管道、模型和调度器</a> 教程，该教程分解了去噪过程的基本模式。</p>`,1),ll=d('<p>The [<code>main()</code>](<a href="https://github.com/huggingface/di" rel="nofollow">https://github.com/huggingface/di</a> ffusers/blob/6e68c71503682c8693cb5b06a4da4911dfd655ee/examples/kandinsky2_2/text_to_image/train_text_to_image_decoder.py#L440) 函数包含准备数据集和训练模型的代码。</p> <p>与之前的模型不同，解码器初始化一个 <code>VQModel</code> 来将潜在变量解码为图像，并使用一个 <code>UNet2DConditionModel</code>：</p> <!> <p>接下来，脚本包括几个图像变换和一个用于对图像应用变换并返回像素值的<a href="https://github.com/huggingface/diffusers/blob/6e68c71503682c8693cb5b06a4da4911dfd655ee/examples/kandinsky2_2/text_to_image/train_text_to_image_decoder.py#L622" rel="nofollow">预处理</a>函数：</p> <!> <p>最后，<a href="https://github.com/huggingface/diffusers/blob/6e68c71503682c8693cb5b06a4da4911dfd655ee/examples/kandinsky2_2/text_to_image/train_text_to_image_decoder.py#L706" rel="nofollow">训练循环</a>处理将图像转换为潜在变量、添加噪声和预测噪声残差。</p> <p>如果您想了解更多关于训练循环如何工作的信息，请查看<a href="../using-diffusers/write_own_pipeline">理解管道、模型和调度器</a>教程，该教程分解了去噪过程的基本模式。</p> <!>',1),_=d("<!> <!>",1),al=d('<!> <blockquote class="tip"><p>可以随意将 <code>kandinsky-community/kandinsky-2-2-decoder</code> 替换为您自己训练的 decoder 检查点！</p></blockquote>',1),el=d("<!> <p>对于 decoder 模型，您还可以从保存的检查点进行推理，这对于查看中间结果很有用。在这种情况下，将检查点加载到 UNet 中：</p> <!>",1),sl=d(`<p></p> <!> <!> <blockquote class="warning"><p>此脚本是实验性的，容易过拟合并遇到灾难性遗忘等问题。尝试探索不同的超参数以在您的数据集上获得最佳结果。</p></blockquote> <p>Kandinsky 2.2 是一个多语言文本到图像模型，能够生成更逼真的图像。该模型包括一个图像先验模型，用于从文本提示创建图像嵌入，以及一个解码器模型，基于先验模型的嵌入生成图像。这就是为什么在 Diffusers 中您会找到两个独立的脚本用于 Kandinsky 2.2，一个用于训练先验模型，另一个用于训练解码器模型。您可以分别训练这两个模型，但为了获得最佳结果，您应该同时训练先验和解码器模型。</p> <p>根据您的 GPU，您可能需要启用 <code>gradient_checkpointing</code>（⚠️ 不支持先验模型！）、<code>mixed_precision</code> 和 <code>gradient_accumulation_steps</code> 来帮助将模型装入内存并加速训练。您可以通过启用 <a href="../optimization/xformers">xFormers</a> 的内存高效注意力来进一步减少内存使用（版本 <a href="https://github.com/huggingface/diffusers/issues/2234#issuecomment-1416931212" rel="nofollow">v0.0.16</a> 在某些 GPU 上训练时失败，因此您可能需要安装开发版本）。</p> <p>本指南探讨了 <a href="https://github.com/huggingface/diffusers/blob/main/examples/kandinsky2_2/text_to_image/train_text_to_image_prior.py" rel="nofollow">train_text_to_image_prior.py</a> 和 <a href="https://github.com/huggingface/diffusers/blob/main/examples/kandinsky2_2/text_to_image/train_text_to_image_decoder.py" rel="nofollow">train_text_to_image_decoder.py</a> 脚本，以帮助您更熟悉它，以及如何根据您的用例进行调整。</p> <p>在运行脚本之前，请确保从源代码安装库：</p> <!> <p>然后导航到包含训练脚本的示例文件夹，并安装脚本所需的依赖项：</p> <!> <blockquote class="tip"><p>🤗 Accelerate 是一个帮助您在多个 GPU/TPU 上或使用混合精度进行训练的库。它会根据您的硬件和环境自动配置训练设置。查看 🤗 Accelerate 的 <a href="https://huggingface.co/docs/accelerate/quicktour" rel="nofollow">快速入门</a> 了解更多。</p></blockquote> <p>初始化一个 🤗 Accelerate 环境：</p> <!> <p>要设置一个默认的 🤗 Accelerate 环境而不选择任何配置：</p> <!> <p>或者，如果您的环境不支持交互式 shell，比如 notebook，您可以使用：</p> <!> <p>最后，如果您想在自己的数据集上训练模型，请查看 <a href="create_dataset">创建用于训练的数据集</a> 指南，了解如何创建与训练脚本兼容的数据集。</p> <blockquote class="tip"><p>以下部分重点介绍了训练脚本中对于理解如何修改它很重要的部分，但并未详细涵盖脚本的每个方面。如果您有兴趣了解更多，请随时阅读脚本，并让我们知道您有任何疑问或顾虑。</p></blockquote> <!> <p>训练脚本提供了许多参数来帮助您自定义训练运行。所有参数及其描述都可以在 <a href="https://github.com/huggingface/diffusers/blob/6e68c71503682c8693cb5b06a4da4911dfd655ee/examples/kandinsky2_2/text_to_image/train_text_to_image_prior.py#L190" rel="nofollow"><code>parse_args()</code></a> 函数中找到。训练脚本为每个参数提供了默认值，例如训练批次大小和学习率，但如果您愿意，也可以在训练命令中设置自己的值。</p> <p>例如，要使用 fp16 格式的混合精度加速训练，请在训练命令中添加 <code>--mixed_precision</code> 参数：</p> <!> <p>大多数参数与 <a href="text2image#script-parameters">文本到图像</a> 训练指南中的参数相同，所以让我们直接进入 Kandinsky 训练脚本的 walkthrough！</p> <!> <p><a href="https://huggingface.co/papers/2303.09556" rel="nofollow">Min-SNR</a> 加权策略可以通过重新平衡损失来帮助训练，实现更快的收敛。训练脚本支持预测 <code>epsilon</code>（噪声）或 <code>v_prediction</code>，但 Min-SNR 与两种预测类型都兼容。</p> <p>添加 <code>--snr_gamma</code> 参数并将其设置为推荐值 5.0：</p> <!> <!> <p>训练脚本也类似于 <a href="text2image#training-script">文本到图像</a> 训练指南，但已修改以支持训练 prior 和 decoder 模型。本指南重点介绍 Kandinsky 2.2 训练脚本中独特的代码。</p> <!> <!> <p>一旦您完成了所有更改或接受默认配置，就可以启动训练脚本了！🚀</p> <p>您将在<a href="https://huggingface.co/datasets/lambdalabs/naruto-blip-captions" rel="nofollow">Naruto BLIP 字幕</a>数据集上进行训练，以生成您自己的Naruto角色，但您也可以通过遵循<a href="create_dataset">创建用于训练的数据集</a>指南来创建和训练您自己的数据集。将环境变量 <code>DATASET_NAME</code> 设置为Hub上数据集的名称，或者如果您在自己的文件上训练，将环境变量 <code>TRAIN_DIR</code> 设置为数据集的路径。</p> <p>如果您在多个GPU上训练，请在 <code>accelerate launch</code> 命令中添加 <code>--multi_gpu</code> 参数。</p> <blockquote class="tip"><p>要使用Weights & Biases监控训练进度，请在训练命令中添加 <code>--report_to=wandb</code> 参数。您还需要
建议在训练命令中添加 <code>--validation_prompt</code> 以跟踪结果。这对于调试模型和查看中间结果非常有用。</p></blockquote> <!> <p>训练完成后，您可以使用新训练的模型进行推理！</p> <!> <!> <p>恭喜您训练了一个 Kandinsky 2.2 模型！要了解更多关于如何使用您的新模型的信息，以下指南可能会有所帮助：</p> <ul><li>阅读 <a href="../using-diffusers/kandinsky">Kandinsky</a> 指南，学习如何将其用于各种不同的任务（文本到图像、图像到图像、修复、插值），以及如何与 ControlNet 结合使用。</li> <li>查看 <a href="dreambooth">DreamBooth</a> 和 <a href="lora">LoRA</a> 训练指南，学习如何使用少量示例图像训练个性化的 Kandinsky 模型。这两种训练技术甚至可以结合使用！</li></ul> <!> <p></p>`,1);function il(x,F){L(F,!1),D(()=>{new URLSearchParams(window.location.search).get("fw")}),Y();var g=sl();A("11w7mfc",n=>{var m=K();q(m,"content",$),o(n,m)});var Z=l(r(g),2);z(Z,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var w=l(Z,2);M(w,{title:"Kandinsky 2.2",local:"kandinsky-22",headingTag:"h1"});var f=l(w,12);a(f,{code:"Z2l0JTIwY2xvbmUlMjBodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZodWdnaW5nZmFjZSUyRmRpZmZ1c2VycyUwQWNkJTIwZGlmZnVzZXJzJTBBcGlwJTIwaW5zdGFsbCUyMC4=",highlighted:`git <span class="hljs-built_in">clone</span> https://github.com/huggingface/diffusers
<span class="hljs-built_in">cd</span> diffusers
pip install .`,lang:"bash",wrap:!1});var j=l(f,4);a(j,{code:"Y2QlMjBleGFtcGxlcyUyRmthbmRpbnNreTJfMiUyRnRleHRfdG9faW1hZ2UlMEFwaXAlMjBpbnN0YWxsJTIwLXIlMjByZXF1aXJlbWVudHMudHh0",highlighted:`<span class="hljs-built_in">cd</span> examples/kandinsky2_2/text_to_image
pip install -r requirements.txt`,lang:"bash",wrap:!1});var T=l(j,6);a(T,{code:"YWNjZWxlcmF0ZSUyMGNvbmZpZw==",highlighted:"accelerate config",lang:"bash",wrap:!1});var W=l(T,4);a(W,{code:"YWNjZWxlcmF0ZSUyMGNvbmZpZyUyMGRlZmF1bHQ=",highlighted:"accelerate config default",lang:"bash",wrap:!1});var R=l(W,4);a(R,{code:"ZnJvbSUyMGFjY2VsZXJhdGUudXRpbHMlMjBpbXBvcnQlMjB3cml0ZV9iYXNpY19jb25maWclMEElMEF3cml0ZV9iYXNpY19jb25maWcoKQ==",highlighted:`<span class="hljs-keyword">from</span> accelerate.utils <span class="hljs-keyword">import</span> write_basic_config

write_basic_config()`,lang:"py",wrap:!1});var G=l(R,6);M(G,{title:"脚本参数",local:"脚本参数",headingTag:"h2"});var k=l(G,6);a(k,{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMHRyYWluX3RleHRfdG9faW1hZ2VfcHJpb3IucHklMjAlNUMlMEElMjAlMjAtLW1peGVkX3ByZWNpc2lvbiUzRCUyMmZwMTYlMjI=",highlighted:`accelerate launch train_text_to_image_prior.py \\
  --mixed_precision=<span class="hljs-string">&quot;fp16&quot;</span>`,lang:"bash",wrap:!1});var V=l(k,4);M(V,{title:"Min-SNR 加权",local:"min-snr-加权",headingTag:"h3"});var I=l(V,6);a(I,{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMHRyYWluX3RleHRfdG9faW1hZ2VfcHJpb3IucHklMjAlNUMlMEElMjAlMjAtLXNucl9nYW1tYSUzRDUuMA==",highlighted:`accelerate launch train_text_to_image_prior.py \\
  --snr_gamma=5.0`,lang:"bash",wrap:!1});var X=l(I,2);M(X,{title:"训练脚本",local:"训练脚本",headingTag:"h2"});var C=l(X,4);u(C,{id:"script",options:["prior model","decoder model"],children:(n,m)=>{var p=_(),i=r(p);y(i,{id:"script",option:"prior model",children:(e,b)=>{var s=O(),t=l(r(s),4);a(t,{code:"bm9pc2Vfc2NoZWR1bGVyJTIwJTNEJTIwRERQTVNjaGVkdWxlcihiZXRhX3NjaGVkdWxlJTNEJTIyc3F1YXJlZGNvc19jYXBfdjIlMjIlMkMlMjBwcmVkaWN0aW9uX3R5cGUlM0QlMjJzYW1wbGUlMjIpJTBBaW1hZ2VfcHJvY2Vzc29yJTIwJTNEJTIwQ0xJUEltYWdlUHJvY2Vzc29yLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjBhcmdzLnByZXRyYWluZWRfcHJpb3JfbW9kZWxfbmFtZV9vcl9wYXRoJTJDJTIwc3ViZm9sZGVyJTNEJTIyaW1hZ2VfcHJvY2Vzc29yJTIyJTBBKSUwQXRva2VuaXplciUyMCUzRCUyMENMSVBUb2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKGFyZ3MucHJldHJhaW5lZF9wcmlvcl9tb2RlbF9uYW1lX29yX3BhdGglMkMlMjBzdWJmb2xkZXIlM0QlMjJ0b2tlbml6ZXIlMjIpJTBBJTBBd2l0aCUyMENvbnRleHRNYW5hZ2VycyhkZWVwc3BlZWRfemVyb19pbml0X2Rpc2FibGVkX2NvbnRleHRfbWFuYWdlcigpKSUzQSUwQSUyMCUyMCUyMCUyMGltYWdlX2VuY29kZXIlMjAlM0QlMjBDTElQVmlzaW9uTW9kZWxXaXRoUHJvamVjdGlvbi5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYXJncy5wcmV0cmFpbmVkX3ByaW9yX21vZGVsX25hbWVfb3JfcGF0aCUyQyUyMHN1YmZvbGRlciUzRCUyMmltYWdlX2VuY29kZXIlMjIlMkMlMjBkdHlwZSUzRHdlaWdodF9kdHlwZSUwQSUyMCUyMCUyMCUyMCkuZXZhbCgpJTBBJTIwJTIwJTIwJTIwdGV4dF9lbmNvZGVyJTIwJTNEJTIwQ0xJUFRleHRNb2RlbFdpdGhQcm9qZWN0aW9uLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhcmdzLnByZXRyYWluZWRfcHJpb3JfbW9kZWxfbmFtZV9vcl9wYXRoJTJDJTIwc3ViZm9sZGVyJTNEJTIydGV4dF9lbmNvZGVyJTIyJTJDJTIwZHR5cGUlM0R3ZWlnaHRfZHR5cGUlMEElMjAlMjAlMjAlMjApLmV2YWwoKQ==",highlighted:`noise_scheduler = DDPMScheduler(beta_schedule=<span class="hljs-string">&quot;squaredcos_cap_v2&quot;</span>, prediction_type=<span class="hljs-string">&quot;sample&quot;</span>)
image_processor = CLIPImageProcessor.from_pretrained(
    args.pretrained_prior_model_name_or_path, subfolder=<span class="hljs-string">&quot;image_processor&quot;</span>
)
tokenizer = CLIPTokenizer.from_pretrained(args.pretrained_prior_model_name_or_path, subfolder=<span class="hljs-string">&quot;tokenizer&quot;</span>)

<span class="hljs-keyword">with</span> ContextManagers(deepspeed_zero_init_disabled_context_manager()):
    image_encoder = CLIPVisionModelWithProjection.from_pretrained(
        args.pretrained_prior_model_name_or_path, subfolder=<span class="hljs-string">&quot;image_encoder&quot;</span>, dtype=weight_dtype
    ).<span class="hljs-built_in">eval</span>()
    text_encoder = CLIPTextModelWithProjection.from_pretrained(
        args.pretrained_prior_model_name_or_path, subfolder=<span class="hljs-string">&quot;text_encoder&quot;</span>, dtype=weight_dtype
    ).<span class="hljs-built_in">eval</span>()`,lang:"py",wrap:!1});var c=l(t,4);a(c,{code:"cHJpb3IlMjAlM0QlMjBQcmlvclRyYW5zZm9ybWVyLmZyb21fcHJldHJhaW5lZChhcmdzLnByZXRyYWluZWRfcHJpb3JfbW9kZWxfbmFtZV9vcl9wYXRoJTJDJTIwc3ViZm9sZGVyJTNEJTIycHJpb3IlMjIpJTBBcHJpb3IudHJhaW4oKSUwQW9wdGltaXplciUyMCUzRCUyMG9wdGltaXplcl9jbHMoJTBBJTIwJTIwJTIwJTIwcHJpb3IucGFyYW1ldGVycygpJTJDJTBBJTIwJTIwJTIwJTIwbHIlM0RhcmdzLmxlYXJuaW5nX3JhdGUlMkMlMEElMjAlMjAlMjAlMjBiZXRhcyUzRChhcmdzLmFkYW1fYmV0YTElMkMlMjBhcmdzLmFkYW1fYmV0YTIpJTJDJTBBJTIwJTIwJTIwJTIwd2VpZ2h0X2RlY2F5JTNEYXJncy5hZGFtX3dlaWdodF9kZWNheSUyQyUwQSUyMCUyMCUyMCUyMGVwcyUzRGFyZ3MuYWRhbV9lcHNpbG9uJTJDJTBBKQ==",highlighted:`prior = PriorTransformer.from_pretrained(args.pretrained_prior_model_name_or_path, subfolder=<span class="hljs-string">&quot;prior&quot;</span>)
prior.train()
optimizer = optimizer_cls(
    prior.parameters(),
    lr=args.learning_rate,
    betas=(args.adam_beta1, args.adam_beta2),
    weight_decay=args.adam_weight_decay,
    eps=args.adam_epsilon,
)`,lang:"py",wrap:!1});var U=l(c,4);a(U,{code:"ZGVmJTIwcHJlcHJvY2Vzc190cmFpbihleGFtcGxlcyklM0ElMEElMjAlMjAlMjAlMjBpbWFnZXMlMjAlM0QlMjAlNUJpbWFnZS5jb252ZXJ0KCUyMlJHQiUyMiklMjBmb3IlMjBpbWFnZSUyMGluJTIwZXhhbXBsZXMlNUJpbWFnZV9jb2x1bW4lNUQlNUQlMEElMjAlMjAlMjAlMjBleGFtcGxlcyU1QiUyMmNsaXBfcGl4ZWxfdmFsdWVzJTIyJTVEJTIwJTNEJTIwaW1hZ2VfcHJvY2Vzc29yKGltYWdlcyUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnBpeGVsX3ZhbHVlcyUwQSUyMCUyMCUyMCUyMGV4YW1wbGVzJTVCJTIydGV4dF9pbnB1dF9pZHMlMjIlNUQlMkMlMjBleGFtcGxlcyU1QiUyMnRleHRfbWFzayUyMiU1RCUyMCUzRCUyMHRva2VuaXplX2NhcHRpb25zKGV4YW1wbGVzKSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMGV4YW1wbGVz",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_train</span>(<span class="hljs-params">examples</span>):
    images = [image.convert(<span class="hljs-string">&quot;RGB&quot;</span>) <span class="hljs-keyword">for</span> image <span class="hljs-keyword">in</span> examples[image_column]]
    examples[<span class="hljs-string">&quot;clip_pixel_values&quot;</span>] = image_processor(images, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).pixel_values
    examples[<span class="hljs-string">&quot;text_input_ids&quot;</span>], examples[<span class="hljs-string">&quot;text_mask&quot;</span>] = tokenize_captions(examples)
    <span class="hljs-keyword">return</span> examples`,lang:"py",wrap:!1});var E=l(U,4);a(E,{code:"bW9kZWxfcHJlZCUyMCUzRCUyMHByaW9yKCUwQSUyMCUyMCUyMCUyMG5vaXN5X2xhdGVudHMlMkMlMEElMjAlMjAlMjAlMjB0aW1lc3RlcCUzRHRpbWVzdGVwcyUyQyUwQSUyMCUyMCUyMCUyMHByb2pfZW1iZWRkaW5nJTNEcHJvbXB0X2VtYmVkcyUyQyUwQSUyMCUyMCUyMCUyMGVuY29kZXJfaGlkZGVuX3N0YXRlcyUzRHRleHRfZW5jb2Rlcl9oaWRkZW5fc3RhdGVzJTJDJTBBJTIwJTIwJTIwJTIwYXR0ZW50aW9uX21hc2slM0R0ZXh0X21hc2slMkMlMEEpLnByZWRpY3RlZF9pbWFnZV9lbWJlZGRpbmc=",highlighted:`model_pred = prior(
    noisy_latents,
    timestep=timesteps,
    proj_embedding=prompt_embeds,
    encoder_hidden_states=text_encoder_hidden_states,
    attention_mask=text_mask,
).predicted_image_embedding`,lang:"py",wrap:!1}),J(2),o(e,s)},$$slots:{default:!0}});var h=l(i,2);y(h,{id:"script",option:"decoder model",children:(e,b)=>{var s=ll(),t=l(r(s),4);a(t,{code:"d2l0aCUyMENvbnRleHRNYW5hZ2VycyhkZWVwc3BlZWRfemVyb19pbml0X2Rpc2FibGVkX2NvbnRleHRfbWFuYWdlcigpKSUzQSUwQSUyMCUyMCUyMCUyMHZhZSUyMCUzRCUyMFZRTW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGFyZ3MucHJldHJhaW5lZF9kZWNvZGVyX21vZGVsX25hbWVfb3JfcGF0aCUyQyUyMHN1YmZvbGRlciUzRCUyMm1vdnElMjIlMkMlMjBkdHlwZSUzRHdlaWdodF9kdHlwZSUwQSUyMCUyMCUyMCUyMCkuZXZhbCgpJTBBJTIwJTIwJTIwJTIwaW1hZ2VfZW5jb2RlciUyMCUzRCUyMENMSVBWaXNpb25Nb2RlbFdpdGhQcm9qZWN0aW9uLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhcmdzLnByZXRyYWluZWRfcHJpb3JfbW9kZWxfbmFtZV9vcl9wYXRoJTJDJTIwc3ViZm9sZGVyJTNEJTIyaW1hZ2VfZW5jb2RlciUyMiUyQyUyMGR0eXBlJTNEd2VpZ2h0X2R0eXBlJTBBJTIwJTIwJTIwJTIwKS5ldmFsKCklMEF1bmV0JTIwJTNEJTIwVU5ldDJEQ29uZGl0aW9uTW9kZWwuZnJvbV9wcmV0cmFpbmVkKGFyZ3MucHJldHJhaW5lZF9kZWNvZGVyX21vZGVsX25hbWVfb3JfcGF0aCUyQyUyMHN1YmZvbGRlciUzRCUyMnVuZXQlMjIp",highlighted:`<span class="hljs-keyword">with</span> ContextManagers(deepspeed_zero_init_disabled_context_manager()):
    vae = VQModel.from_pretrained(
        args.pretrained_decoder_model_name_or_path, subfolder=<span class="hljs-string">&quot;movq&quot;</span>, dtype=weight_dtype
    ).<span class="hljs-built_in">eval</span>()
    image_encoder = CLIPVisionModelWithProjection.from_pretrained(
        args.pretrained_prior_model_name_or_path, subfolder=<span class="hljs-string">&quot;image_encoder&quot;</span>, dtype=weight_dtype
    ).<span class="hljs-built_in">eval</span>()
unet = UNet2DConditionModel.from_pretrained(args.pretrained_decoder_model_name_or_path, subfolder=<span class="hljs-string">&quot;unet&quot;</span>)`,lang:"py",wrap:!1});var c=l(t,4);a(c,{code:"ZGVmJTIwcHJlcHJvY2Vzc190cmFpbihleGFtcGxlcyklM0ElMEElMjAlMjAlMjAlMjBpbWFnZXMlMjAlM0QlMjAlNUJpbWFnZS5jb252ZXJ0KCUyMlJHQiUyMiklMjBmb3IlMjBpbWFnZSUyMGluJTIwZXhhbXBsZXMlNUJpbWFnZV9jb2x1bW4lNUQlNUQlMEElMjAlMjAlMjAlMjBleGFtcGxlcyU1QiUyMnBpeGVsX3ZhbHVlcyUyMiU1RCUyMCUzRCUyMCU1QnRyYWluX3RyYW5zZm9ybXMoaW1hZ2UpJTIwZm9yJTIwaW1hZ2UlMjBpbiUyMGltYWdlcyU1RCUwQSUyMCUyMCUyMCUyMGV4YW1wbGVzJTVCJTIyY2xpcF9waXhlbF92YWx1ZXMlMjIlNUQlMjAlM0QlMjBpbWFnZV9wcm9jZXNzb3IoaW1hZ2VzJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikucGl4ZWxfdmFsdWVzJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwZXhhbXBsZXM=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_train</span>(<span class="hljs-params">examples</span>):
    images = [image.convert(<span class="hljs-string">&quot;RGB&quot;</span>) <span class="hljs-keyword">for</span> image <span class="hljs-keyword">in</span> examples[image_column]]
    examples[<span class="hljs-string">&quot;pixel_values&quot;</span>] = [train_transforms(image) <span class="hljs-keyword">for</span> image <span class="hljs-keyword">in</span> images]
    examples[<span class="hljs-string">&quot;clip_pixel_values&quot;</span>] = image_processor(images, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).pixel_values
    <span class="hljs-keyword">return</span> examples`,lang:"py",wrap:!1});var U=l(c,6);a(U,{code:"bW9kZWxfcHJlZCUyMCUzRCUyMHVuZXQobm9pc3lfbGF0ZW50cyUyQyUyMHRpbWVzdGVwcyUyQyUyME5vbmUlMkMlMjBhZGRlZF9jb25kX2t3YXJncyUzRGFkZGVkX2NvbmRfa3dhcmdzKS5zYW1wbGUlNUIlM0ElMkMlMjAlM0E0JTVE",highlighted:'model_pred = unet(noisy_latents, timesteps, <span class="hljs-literal">None</span>, added_cond_kwargs=added_cond_kwargs).sample[:, :<span class="hljs-number">4</span>]',lang:"py",wrap:!1}),o(e,s)},$$slots:{default:!0}}),o(n,p)},$$slots:{default:!0}});var B=l(C,2);M(B,{title:"启动脚本",local:"启动脚本",headingTag:"h2"});var v=l(B,10);u(v,{id:"training-inference",options:["prior model","decoder model"],children:(n,m)=>{var p=_(),i=r(p);y(i,{id:"training-inference",option:"prior model",children:(e,b)=>{a(e,{code:"ZXhwb3J0JTIwREFUQVNFVF9OQU1FJTNEJTIybGFtYmRhbGFicyUyRm5hcnV0by1ibGlwLWNhcHRpb25zJTIyJTBBJTBBYWNjZWxlcmF0ZSUyMGxhdW5jaCUyMC0tbWl4ZWRfcHJlY2lzaW9uJTNEJTIyZnAxNiUyMiUyMCUyMHRyYWluX3RleHRfdG9faW1hZ2VfcHJpb3IucHklMjAlNUMlMEElMjAlMjAtLWRhdGFzZXRfbmFtZSUzRCUyNERBVEFTRVRfTkFNRSUyMCU1QyUwQSUyMCUyMC0tcmVzb2x1dGlvbiUzRDc2OCUyMCU1QyUwQSUyMCUyMC0tdHJhaW5fYmF0Y2hfc2l6ZSUzRDElMjAlNUMlMEElMjAlMjAtLWdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUzRDQlMjAlNUMlMEElMjAlMjAtLW1heF90cmFpbl9zdGVwcyUzRDE1MDAwJTIwJTVDJTBBJTIwJTIwLS1sZWFybmluZ19yYXRlJTNEMWUtMDUlMjAlNUMlMEElMjAlMjAtLW1heF9ncmFkX25vcm0lM0QxJTIwJTVDJTBBJTIwJTIwLS1jaGVja3BvaW50c190b3RhbF9saW1pdCUzRDMlMjAlNUMlMEElMjAlMjAtLWxyX3NjaGVkdWxlciUzRCUyMmNvbnN0YW50JTIyJTIwJTVDJTBBJTIwJTIwLS1scl93YXJtdXBfc3RlcHMlM0QwJTIwJTVDJTBBJTIwJTIwLS12YWxpZGF0aW9uX3Byb21wdHMlM0QlMjJBJTIwcm9ib3QlMjBuYXJ1dG8lMkMlMjA0ayUyMHBob3RvJTIyJTIwJTVDJTBBJTIwJTIwLS1yZXBvcnRfdG8lM0QlMjJ3YW5kYiUyMiUyMCU1QyUwQSUyMCUyMC0tcHVzaF90b19odWIlMjAlNUMlMEElMjAlMjAtLW91dHB1dF9kaXIlM0QlMjJrYW5kaTItcHJpb3ItbmFydXRvLW1vZGVsJTIy",highlighted:`<span class="hljs-built_in">export</span> DATASET_NAME=<span class="hljs-string">&quot;lambdalabs/naruto-blip-captions&quot;</span>

accelerate launch --mixed_precision=<span class="hljs-string">&quot;fp16&quot;</span>  train_text_to_image_prior.py \\
  --dataset_name=<span class="hljs-variable">$DATASET_NAME</span> \\
  --resolution=768 \\
  --train_batch_size=1 \\
  --gradient_accumulation_steps=4 \\
  --max_train_steps=15000 \\
  --learning_rate=1e-05 \\
  --max_grad_norm=1 \\
  --checkpoints_total_limit=3 \\
  --lr_scheduler=<span class="hljs-string">&quot;constant&quot;</span> \\
  --lr_warmup_steps=0 \\
  --validation_prompts=<span class="hljs-string">&quot;A robot naruto, 4k photo&quot;</span> \\
  --report_to=<span class="hljs-string">&quot;wandb&quot;</span> \\
  --push_to_hub \\
  --output_dir=<span class="hljs-string">&quot;kandi2-prior-naruto-model&quot;</span>`,lang:"bash",wrap:!1})},$$slots:{default:!0}});var h=l(i,2);y(h,{id:"training-inference",option:"decoder model",children:(e,b)=>{a(e,{code:"ZXhwb3J0JTIwREFUQVNFVF9OQU1FJTNEJTIybGFtYmRhbGFicyUyRm5hcnV0by1ibGlwLWNhcHRpb25zJTIyJTBBJTBBYWNjZWxlcmF0ZSUyMGxhdW5jaCUyMC0tbWl4ZWRfcHJlY2lzaW9uJTNEJTIyZnAxNiUyMiUyMCUyMHRyYWluX3RleHRfdG9faW1hZ2VfZGVjb2Rlci5weSUyMCU1QyUwQSUyMCUyMC0tZGF0YXNldF9uYW1lJTNEJTI0REFUQVNFVF9OQU1FJTIwJTVDJTBBJTIwJTIwLS1yZXNvbHV0aW9uJTNENzY4JTIwJTVDJTBBJTIwJTIwLS10cmFpbl9iYXRjaF9zaXplJTNEMSUyMCU1QyUwQSUyMCUyMC0tZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTNENCUyMCU1QyUwQSUyMCUyMC0tZ3JhZGllbnRfY2hlY2twb2ludGluZyUyMCU1QyUwQSUyMCUyMC0tbWF4X3RyYWluX3N0ZXBzJTNEMTUwMDAlMjAlNUMlMEElMjAlMjAtLWxlYXJuaW5nX3JhdGUlM0QxZS0wNSUyMCU1QyUwQSUyMCUyMC0tbWF4X2dyYWRfbm9ybSUzRDElMjAlNUMlMEElMjAlMjAtLWNoZWNrcG9pbnRzX3RvdGFsX2xpbWl0JTNEMyUyMCU1QyUwQSUyMCUyMC0tbHJfc2NoZWR1bGVyJTNEJTIyY29uc3RhbnQlMjIlMjAlNUMlMEElMjAlMjAtLWxyX3dhcm11cF9zdGVwcyUzRDAlMjAlNUMlMEElMjAlMjAtLXZhbGlkYXRpb25fcHJvbXB0cyUzRCUyMkElMjByb2JvdCUyMG5hcnV0byUyQyUyMDRrJTIwcGhvdG8lMjIlMjAlNUMlMEElMjAlMjAtLXJlcG9ydF90byUzRCUyMndhbmRiJTIyJTIwJTVDJTBBJTIwJTIwLS1wdXNoX3RvX2h1YiUyMCU1QyUwQSUyMCUyMC0tb3V0cHV0X2RpciUzRCUyMmthbmRpMi1kZWNvZGVyLW5hcnV0by1tb2RlbCUyMg==",highlighted:`<span class="hljs-built_in">export</span> DATASET_NAME=<span class="hljs-string">&quot;lambdalabs/naruto-blip-captions&quot;</span>

accelerate launch --mixed_precision=<span class="hljs-string">&quot;fp16&quot;</span>  train_text_to_image_decoder.py \\
  --dataset_name=<span class="hljs-variable">$DATASET_NAME</span> \\
  --resolution=768 \\
  --train_batch_size=1 \\
  --gradient_accumulation_steps=4 \\
  --gradient_checkpointing \\
  --max_train_steps=15000 \\
  --learning_rate=1e-05 \\
  --max_grad_norm=1 \\
  --checkpoints_total_limit=3 \\
  --lr_scheduler=<span class="hljs-string">&quot;constant&quot;</span> \\
  --lr_warmup_steps=0 \\
  --validation_prompts=<span class="hljs-string">&quot;A robot naruto, 4k photo&quot;</span> \\
  --report_to=<span class="hljs-string">&quot;wandb&quot;</span> \\
  --push_to_hub \\
  --output_dir=<span class="hljs-string">&quot;kandi2-decoder-naruto-model&quot;</span>`,lang:"bash",wrap:!1})},$$slots:{default:!0}}),o(n,p)},$$slots:{default:!0}});var N=l(v,4);u(N,{id:"training-inference",options:["prior model","decoder model"],children:(n,m)=>{var p=_(),i=r(p);y(i,{id:"training-inference",option:"prior model",children:(e,b)=>{var s=al(),t=r(s);a(t,{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEF1dG9QaXBlbGluZUZvclRleHQySW1hZ2UlMkMlMjBEaWZmdXNpb25QaXBlbGluZSUwQWltcG9ydCUyMHRvcmNoJTBBJTBBcHJpb3JfcGlwZWxpbmUlMjAlM0QlMjBEaWZmdXNpb25QaXBlbGluZS5mcm9tX3ByZXRyYWluZWQob3V0cHV0X2RpciUyQyUyMGR0eXBlJTNEdG9yY2guZmxvYXQxNiklMEFwcmlvcl9jb21wb25lbnRzJTIwJTNEJTIwJTdCJTIycHJpb3JfJTIyJTIwJTJCJTIwayUzQSUyMHYlMjBmb3IlMjBrJTJDdiUyMGluJTIwcHJpb3JfcGlwZWxpbmUuY29tcG9uZW50cy5pdGVtcygpJTdEJTBBcGlwZWxpbmUlMjAlM0QlMjBBdXRvUGlwZWxpbmVGb3JUZXh0MkltYWdlLmZyb21fcHJldHJhaW5lZCglMjJrYW5kaW5za3ktY29tbXVuaXR5JTJGa2FuZGluc2t5LTItMi1kZWNvZGVyJTIyJTJDJTIwKipwcmlvcl9jb21wb25lbnRzJTJDJTIwZHR5cGUlM0R0b3JjaC5mbG9hdDE2KSUwQSUwQXBpcGUuZW5hYmxlX21vZGVsX2NwdV9vZmZsb2FkKCklMEFwcm9tcHQlM0QlMjJBJTIwcm9ib3QlMjBuYXJ1dG8lMkMlMjA0ayUyMHBob3RvJTIyJTBBaW1hZ2UlMjAlM0QlMjBwaXBlbGluZShwcm9tcHQlM0Rwcm9tcHQlMkMlMjBuZWdhdGl2ZV9wcm9tcHQlM0RuZWdhdGl2ZV9wcm9tcHQpLmltYWdlcyU1QjAlNUQ=",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoPipelineForText2Image, DiffusionPipeline
<span class="hljs-keyword">import</span> torch

prior_pipeline = DiffusionPipeline.from_pretrained(output_dir, dtype=torch.float16)
prior_components = {<span class="hljs-string">&quot;prior_&quot;</span> + k: v <span class="hljs-keyword">for</span> k,v <span class="hljs-keyword">in</span> prior_pipeline.components.items()}
pipeline = AutoPipelineForText2Image.from_pretrained(<span class="hljs-string">&quot;kandinsky-community/kandinsky-2-2-decoder&quot;</span>, **prior_components, dtype=torch.float16)

pipe.enable_model_cpu_offload()
prompt=<span class="hljs-string">&quot;A robot naruto, 4k photo&quot;</span>
image = pipeline(prompt=prompt, negative_prompt=negative_prompt).images[<span class="hljs-number">0</span>]`,lang:"py",wrap:!1}),J(2),o(e,s)},$$slots:{default:!0}});var h=l(i,2);y(h,{id:"training-inference",option:"decoder model",children:(e,b)=>{var s=el(),t=r(s);a(t,{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEF1dG9QaXBlbGluZUZvclRleHQySW1hZ2UlMEFpbXBvcnQlMjB0b3JjaCUwQSUwQXBpcGVsaW5lJTIwJTNEJTIwQXV0b1BpcGVsaW5lRm9yVGV4dDJJbWFnZS5mcm9tX3ByZXRyYWluZWQoJTIycGF0aCUyRnRvJTJGc2F2ZWQlMkZtb2RlbCUyMiUyQyUyMGR0eXBlJTNEdG9yY2guZmxvYXQxNiklMEFwaXBlbGluZS5lbmFibGVfbW9kZWxfY3B1X29mZmxvYWQoKSUwQSUwQXByb21wdCUzRCUyMkElMjByb2JvdCUyMG5hcnV0byUyQyUyMDRrJTIwcGhvdG8lMjIlMEFpbWFnZSUyMCUzRCUyMHBpcGVsaW5lKHByb21wdCUzRHByb21wdCkuaW1hZ2VzJTVCMCU1RA==",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoPipelineForText2Image
<span class="hljs-keyword">import</span> torch

pipeline = AutoPipelineForText2Image.from_pretrained(<span class="hljs-string">&quot;path/to/saved/model&quot;</span>, dtype=torch.float16)
pipeline.enable_model_cpu_offload()

prompt=<span class="hljs-string">&quot;A robot naruto, 4k photo&quot;</span>
image = pipeline(prompt=prompt).images[<span class="hljs-number">0</span>]`,lang:"py",wrap:!1});var c=l(t,4);a(c,{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEF1dG9QaXBlbGluZUZvclRleHQySW1hZ2UlMkMlMjBVTmV0MkRDb25kaXRpb25Nb2RlbCUwQSUwQXVuZXQlMjAlM0QlMjBVTmV0MkRDb25kaXRpb25Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTIycGF0aCUyRnRvJTJGc2F2ZWQlMkZtb2RlbCUyMiUyMCUyQiUyMCUyMiUyRmNoZWNrcG9pbnQtJTNDTiUzRSUyRnVuZXQlMjIpJTBBJTBBcGlwZWxpbmUlMjAlM0QlMjBBdXRvUGlwZWxpbmVGb3JUZXh0MkltYWdlLmZyb21fcHJldHJhaW5lZCglMjJrYW5kaW5za3ktY29tbXVuaXR5JTJGa2FuZGluc2t5LTItMi1kZWNvZGVyJTIyJTJDJTIwdW5ldCUzRHVuZXQlMkMlMjBkdHlwZSUzRHRvcmNoLmZsb2F0MTYpJTBBcGlwZWxpbmUuZW5hYmxlX21vZGVsX2NwdV9vZmZsb2FkKCklMEElMEFpbWFnZSUyMCUzRCUyMHBpcGVsaW5lKHByb21wdCUzRCUyMkElMjByb2JvdCUyMG5hcnV0byUyQyUyMDRrJTIwcGhvdG8lMjIpLmltYWdlcyU1QjAlNUQ=",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoPipelineForText2Image, UNet2DConditionModel

unet = UNet2DConditionModel.from_pretrained(<span class="hljs-string">&quot;path/to/saved/model&quot;</span> + <span class="hljs-string">&quot;/checkpoint-&lt;N&gt;/unet&quot;</span>)

pipeline = AutoPipelineForText2Image.from_pretrained(<span class="hljs-string">&quot;kandinsky-community/kandinsky-2-2-decoder&quot;</span>, unet=unet, dtype=torch.float16)
pipeline.enable_model_cpu_offload()

image = pipeline(prompt=<span class="hljs-string">&quot;A robot naruto, 4k photo&quot;</span>).images[<span class="hljs-number">0</span>]`,lang:"py",wrap:!1}),o(e,s)},$$slots:{default:!0}}),o(n,p)},$$slots:{default:!0}});var Q=l(N,2);M(Q,{title:"后续步骤",local:"后续步骤",headingTag:"h2"});var H=l(Q,6);S(H,{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/zh/training/kandinsky.md"}),J(2),o(x,g),P()}export{il as component};