diffusers/pr_14180/en/_app/immutable/nodes/272.x_MPpH5q.js

import"../chunks/DsnmJJEf.js";import{i as j,h as b,C as w,H as e,a as t,E as Z,s as U}from"../chunks/CmJXCtRL.js";import{p as _,o as I,s as a,f as k,a as m,b as G,d as y,n as X}from"../chunks/DK803DsY.js";const B='{"title":"Create a dataset for training","local":"create-a-dataset-for-training","sections":[{"title":"Provide a dataset as a folder","local":"provide-a-dataset-as-a-folder","sections":[],"depth":2},{"title":"Upload your data to the Hub","local":"upload-your-data-to-the-hub","sections":[],"depth":2},{"title":"Next steps","local":"next-steps","sections":[],"depth":2}],"depth":1}';var R=y('<meta name="hf:doc:metadata"/>'),Y=y('<p></p> <!> <!> <p>There are many datasets on the <a href="https://huggingface.co/datasets?task_categories=task_categories:text-to-image&amp;sort=downloads" rel="nofollow">Hub</a> to train a model on, but if you can’t find one you’re interested in or want to use your own, you can create a dataset with the 🤗 <a href="https://huggingface.co/docs/datasets" rel="nofollow">Datasets</a> library. The dataset structure depends on the task you want to train your model on. The most basic dataset structure is a directory of images for tasks like unconditional image generation. Another dataset structure may be a directory of images and a text file containing their corresponding text captions for tasks like text-to-image generation.</p> <p>This guide will show you two ways to create a dataset to finetune on:</p> <ul><li>provide a folder of images to the <code>--train_data_dir</code> argument</li> <li>upload a dataset to the Hub and pass the dataset repository id to the <code>--dataset_name</code> argument</li></ul> <blockquote class="tip"><p>💡 Learn more about how to create an image dataset for training in the <a href="https://huggingface.co/docs/datasets/image_dataset" rel="nofollow">Create an image dataset</a> guide.</p></blockquote> <!> <p>For unconditional generation, you can provide your own dataset as a folder of images. The training script uses the <a href="https://huggingface.co/docs/datasets/en/image_dataset#imagefolder" rel="nofollow"><code>ImageFolder</code></a> builder from 🤗 Datasets to automatically build a dataset from the folder. Your directory structure should look like:</p> <!> <p>Pass the path to the dataset directory to the <code>--train_data_dir</code> argument, and then you can start training:</p> <!> <!> <blockquote class="tip"><p>💡 For more details and context about creating and uploading a dataset to the Hub, take a look at the <a href="https://huggingface.co/blog/image-search-datasets" rel="nofollow">Image search with 🤗 Datasets</a> post.</p></blockquote> <p>Start by creating a dataset with the <a href="https://huggingface.co/docs/datasets/image_load#imagefolder" rel="nofollow"><code>ImageFolder</code></a> feature, which creates an <code>image</code> column containing the PIL-encoded images.</p> <p>You can use the <code>data_dir</code> or <code>data_files</code> parameters to specify the location of the dataset. The <code>data_files</code> parameter supports mapping specific files to dataset splits like <code>train</code> or <code>test</code>:</p> <!> <p>Then use the <code>push_to_hub</code> method to upload the dataset to the Hub:</p> <!> <p>Now the dataset is available for training by passing the dataset name to the <code>--dataset_name</code> argument:</p> <!> <!> <p>Now that you’ve created a dataset, you can plug it into the <code>train_data_dir</code> (if your dataset is local) or <code>dataset_name</code> (if your dataset is on the Hub) arguments of a training script.</p> <p>For your next steps, feel free to try and use your dataset to train a model for <a href="unconditional_training">unconditional generation</a> or <a href="text2image">text-to-image generation</a>!</p> <!> <p></p>',1);function F(f,J){_(J,!1),I(()=>{new URLSearchParams(window.location.search).get("fw")}),j();var s=Y();b("kg0lsn",u=>{var g=R();U(g,"content",B),m(u,g)});var o=a(k(s),2);w(o,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var l=a(o,2);e(l,{title:"Create a dataset for training",local:"create-a-dataset-for-training",headingTag:"h1"});var n=a(l,10);e(n,{title:"Provide a dataset as a folder",local:"provide-a-dataset-as-a-folder",headingTag:"h2"});var d=a(n,4);t(d,{code:"ZGF0YV9kaXIlMkZ4eHgucG5nJTBBZGF0YV9kaXIlMkZ4eHkucG5nJTBBZGF0YV9kaXIlMkYlNUIuLi4lNUQlMkZ4eHoucG5n",highlighted:`data_dir/xxx.png
data_dir/xxy.png
data_dir/[...]/xxz.png`,lang:"bash",wrap:!1});var i=a(d,4);t(i,{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMHRyYWluX3VuY29uZGl0aW9uYWwucHklMjAlNUMlMEElMjAlMjAlMjAlMjAtLXRyYWluX2RhdGFfZGlyJTIwJTNDcGF0aC10by10cmFpbi1kaXJlY3RvcnklM0UlMjAlNUMlMEElMjAlMjAlMjAlMjAlM0NvdGhlci1hcmd1bWVudHMlM0U=",highlighted:`accelerate launch train_unconditional.py \\
    --train_data_dir &lt;path-to-train-directory&gt; \\
    &lt;other-arguments&gt;`,lang:"bash",wrap:!1});var r=a(i,2);e(r,{title:"Upload your data to the Hub",local:"upload-your-data-to-the-hub",headingTag:"h2"});var c=a(r,8);t(c,{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBJTIzJTIwZXhhbXBsZSUyMDElM0ElMjBsb2NhbCUyMGZvbGRlciUwQWRhdGFzZXQlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIyaW1hZ2Vmb2xkZXIlMjIlMkMlMjBkYXRhX2RpciUzRCUyMnBhdGhfdG9feW91cl9mb2xkZXIlMjIpJTBBJTBBJTIzJTIwZXhhbXBsZSUyMDIlM0ElMjBsb2NhbCUyMGZpbGVzJTIwKHN1cHBvcnRlZCUyMGZvcm1hdHMlMjBhcmUlMjB0YXIlMkMlMjBnemlwJTJDJTIwemlwJTJDJTIweHolMkMlMjByYXIlMkMlMjB6c3RkKSUwQWRhdGFzZXQlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIyaW1hZ2Vmb2xkZXIlMjIlMkMlMjBkYXRhX2ZpbGVzJTNEJTIycGF0aF90b196aXBfZmlsZSUyMiklMEElMEElMjMlMjBleGFtcGxlJTIwMyUzQSUyMHJlbW90ZSUyMGZpbGVzJTIwKHN1cHBvcnRlZCUyMGZvcm1hdHMlMjBhcmUlMjB0YXIlMkMlMjBnemlwJTJDJTIwemlwJTJDJTIweHolMkMlMjByYXIlMkMlMjB6c3RkKSUwQWRhdGFzZXQlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTBBJTIwJTIwJTIwJTIwJTIyaW1hZ2Vmb2xkZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBkYXRhX2ZpbGVzJTNEJTIyaHR0cHMlM0ElMkYlMkZkb3dubG9hZC5taWNyb3NvZnQuY29tJTJGZG93bmxvYWQlMkYzJTJGRSUyRjElMkYzRTFDM0YyMS1FQ0RCLTQ4NjktODM2OC02REVCQTc3QjkxOUYlMkZrYWdnbGVjYXRzYW5kZG9nc18zMzY3YS56aXAlMjIlMkMlMEEpJTBBJTBBJTIzJTIwZXhhbXBsZSUyMDQlM0ElMjBwcm92aWRpbmclMjBzZXZlcmFsJTIwc3BsaXRzJTBBZGF0YXNldCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMEElMjAlMjAlMjAlMjAlMjJpbWFnZWZvbGRlciUyMiUyQyUyMGRhdGFfZmlsZXMlM0QlN0IlMjJ0cmFpbiUyMiUzQSUyMCU1QiUyMnBhdGglMkZ0byUyRmZpbGUxJTIyJTJDJTIwJTIycGF0aCUyRnRvJTJGZmlsZTIlMjIlNUQlMkMlMjAlMjJ0ZXN0JTIyJTNBJTIwJTVCJTIycGF0aCUyRnRvJTJGZmlsZTMlMjIlMkMlMjAlMjJwYXRoJTJGdG8lMkZmaWxlNCUyMiU1RCU3RCUwQSk=",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-comment"># example 1: local folder</span>
dataset = load_dataset(<span class="hljs-string">&quot;imagefolder&quot;</span>, data_dir=<span class="hljs-string">&quot;path_to_your_folder&quot;</span>)

<span class="hljs-comment"># example 2: local files (supported formats are tar, gzip, zip, xz, rar, zstd)</span>
dataset = load_dataset(<span class="hljs-string">&quot;imagefolder&quot;</span>, data_files=<span class="hljs-string">&quot;path_to_zip_file&quot;</span>)

<span class="hljs-comment"># example 3: remote files (supported formats are tar, gzip, zip, xz, rar, zstd)</span>
dataset = load_dataset(
    <span class="hljs-string">&quot;imagefolder&quot;</span>,
    data_files=<span class="hljs-string">&quot;https://download.microsoft.com/download/3/E/1/3E1C3F21-ECDB-4869-8368-6DEBA77B919F/kagglecatsanddogs_3367a.zip&quot;</span>,
)

<span class="hljs-comment"># example 4: providing several splits</span>
dataset = load_dataset(
    <span class="hljs-string">&quot;imagefolder&quot;</span>, data_files={<span class="hljs-string">&quot;train&quot;</span>: [<span class="hljs-string">&quot;path/to/file1&quot;</span>, <span class="hljs-string">&quot;path/to/file2&quot;</span>], <span class="hljs-string">&quot;test&quot;</span>: [<span class="hljs-string">&quot;path/to/file3&quot;</span>, <span class="hljs-string">&quot;path/to/file4&quot;</span>]}
)`,lang:"python",wrap:!1});var h=a(c,4);t(h,{code:"JTIzJTIwYXNzdW1pbmclMjB5b3UlMjBoYXZlJTIwcmFuJTIwdGhlJTIwaGYlMjBhdXRoJTIwbG9naW4lMjBjb21tYW5kJTIwaW4lMjBhJTIwdGVybWluYWwlMEFkYXRhc2V0LnB1c2hfdG9faHViKCUyMm5hbWVfb2ZfeW91cl9kYXRhc2V0JTIyKSUwQSUwQSUyMyUyMGlmJTIweW91JTIwd2FudCUyMHRvJTIwcHVzaCUyMHRvJTIwYSUyMHByaXZhdGUlMjByZXBvJTJDJTIwc2ltcGx5JTIwcGFzcyUyMHByaXZhdGUlM0RUcnVlJTNBJTBBZGF0YXNldC5wdXNoX3RvX2h1YiglMjJuYW1lX29mX3lvdXJfZGF0YXNldCUyMiUyQyUyMHByaXZhdGUlM0RUcnVlKQ==",highlighted:`<span class="hljs-comment"># assuming you have ran the hf auth login command in a terminal</span>
dataset.push_to_hub(<span class="hljs-string">&quot;name_of_your_dataset&quot;</span>)

<span class="hljs-comment"># if you want to push to a private repo, simply pass private=True:</span>
dataset.push_to_hub(<span class="hljs-string">&quot;name_of_your_dataset&quot;</span>, private=<span class="hljs-literal">True</span>)`,lang:"python",wrap:!1});var p=a(h,4);t(p,{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMC0tbWl4ZWRfcHJlY2lzaW9uJTNEJTIyZnAxNiUyMiUyMCUyMHRyYWluX3RleHRfdG9faW1hZ2UucHklMjAlNUMlMEElMjAlMjAtLXByZXRyYWluZWRfbW9kZWxfbmFtZV9vcl9wYXRoJTNEJTIyc3RhYmxlLWRpZmZ1c2lvbi12MS01JTJGc3RhYmxlLWRpZmZ1c2lvbi12MS01JTIyJTIwJTVDJTBBJTIwJTIwLS1kYXRhc2V0X25hbWUlM0QlMjJuYW1lX29mX3lvdXJfZGF0YXNldCUyMiUyMCU1QyUwQSUyMCUyMCUzQ290aGVyLWFyZ3VtZW50cyUzRQ==",highlighted:`accelerate launch --mixed_precision=<span class="hljs-string">&quot;fp16&quot;</span>  train_text_to_image.py \\
  --pretrained_model_name_or_path=<span class="hljs-string">&quot;stable-diffusion-v1-5/stable-diffusion-v1-5&quot;</span> \\
  --dataset_name=<span class="hljs-string">&quot;name_of_your_dataset&quot;</span> \\
  &lt;other-arguments&gt;`,lang:"bash",wrap:!1});var M=a(p,2);e(M,{title:"Next steps",local:"next-steps",headingTag:"h2"});var T=a(M,6);Z(T,{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/training/create_dataset.md"}),X(2),m(f,s),G()}export{F as component};