Buckets:

hf-doc-build
/

doc-dev

Files

xet

hf-doc-build/doc-dev / hub /main /en /datasets-adding.html

rtrm

about 1 month ago

download

raw

28 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Uploading datasets","local":"uploading-datasets","sections":[{"title":"Upload using the Hub UI","local":"upload-using-the-hub-ui","sections":[{"title":"Create a repository","local":"create-a-repository","sections":[],"depth":3},{"title":"Upload dataset","local":"upload-dataset","sections":[],"depth":3},{"title":"Create a Dataset card","local":"create-a-dataset-card","sections":[],"depth":3}],"depth":2},{"title":"Using the huggingface_hub client library","local":"using-the-huggingfacehub-client-library","sections":[],"depth":2},{"title":"Using other libraries","local":"using-other-libraries","sections":[],"depth":2},{"title":"Using Git","local":"using-git","sections":[],"depth":2},{"title":"File formats","local":"file-formats","sections":[{"title":"Which file format should I use?","local":"which-file-format-should-i-use","sections":[],"depth":3},{"title":"Dataset Viewer","local":"dataset-viewer","sections":[],"depth":3}],"depth":2},{"title":"Large scale datasets","local":"large-scale-datasets","sections":[],"depth":2}],"depth":1}">
	<link href="/docs/hub/main/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/hub/main/en/_app/immutable/entry/start.d0cd5065.js">
	<link rel="modulepreload" href="/docs/hub/main/en/_app/immutable/chunks/scheduler.d6170356.js">
	<link rel="modulepreload" href="/docs/hub/main/en/_app/immutable/chunks/singletons.d032f1eb.js">
	<link rel="modulepreload" href="/docs/hub/main/en/_app/immutable/chunks/paths.752f1c6b.js">
	<link rel="modulepreload" href="/docs/hub/main/en/_app/immutable/entry/app.b6abe3c1.js">
	<link rel="modulepreload" href="/docs/hub/main/en/_app/immutable/chunks/index.fcd4cc08.js">
	<link rel="modulepreload" href="/docs/hub/main/en/_app/immutable/nodes/0.f045427f.js">
	<link rel="modulepreload" href="/docs/hub/main/en/_app/immutable/nodes/12.de84908a.js">
	<link rel="modulepreload" href="/docs/hub/main/en/_app/immutable/chunks/EditOnGithub.da2b595c.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Uploading datasets","local":"uploading-datasets","sections":[{"title":"Upload using the Hub UI","local":"upload-using-the-hub-ui","sections":[{"title":"Create a repository","local":"create-a-repository","sections":[],"depth":3},{"title":"Upload dataset","local":"upload-dataset","sections":[],"depth":3},{"title":"Create a Dataset card","local":"create-a-dataset-card","sections":[],"depth":3}],"depth":2},{"title":"Using the huggingface_hub client library","local":"using-the-huggingfacehub-client-library","sections":[],"depth":2},{"title":"Using other libraries","local":"using-other-libraries","sections":[],"depth":2},{"title":"Using Git","local":"using-git","sections":[],"depth":2},{"title":"File formats","local":"file-formats","sections":[{"title":"Which file format should I use?","local":"which-file-format-should-i-use","sections":[],"depth":3},{"title":"Dataset Viewer","local":"dataset-viewer","sections":[],"depth":3}],"depth":2},{"title":"Large scale datasets","local":"large-scale-datasets","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <h1 class="relative group"><a id="uploading-datasets" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#uploading-datasets"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Uploading datasets</span></h1> <p data-svelte-h="svelte-1754kge">The <a href="https://huggingface.co/datasets" rel="nofollow">Hub</a> is home to an extensive collection of community-curated and research datasets. We encourage you to share your dataset to the Hub to help grow the ML community and accelerate progress for everyone. All contributions are welcome; adding a dataset is just a drag and drop away!</p> <p data-svelte-h="svelte-1fojn2">Start by <a href="https://huggingface.co/join" rel="nofollow">creating a Hugging Face Hub account</a> if you don’t have one yet.</p> <h2 class="relative group"><a id="upload-using-the-hub-ui" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#upload-using-the-hub-ui"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Upload using the Hub UI</span></h2> <p data-svelte-h="svelte-wcztra">The Hub’s web-based interface allows users without any developer experience to upload a dataset.</p> <h3 class="relative group"><a id="create-a-repository" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#create-a-repository"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Create a repository</span></h3> <p data-svelte-h="svelte-193c6le">A repository hosts all your dataset files, including the revision history, making storing more than one dataset version possible.</p> <ol data-svelte-h="svelte-t7tqd0"><li>Click on your profile and select <strong>New Dataset</strong> to create a <a href="https://huggingface.co/new-dataset" rel="nofollow">new dataset repository</a>.</li> <li>Pick a name for your dataset, and choose whether it is a public or private dataset. A public dataset is visible to anyone, whereas a private dataset can only be viewed by you or members of your organization.</li></ol> <div class="flex justify-center" data-svelte-h="svelte-ggmafm"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/create_repo.png"></div> <h3 class="relative group"><a id="upload-dataset" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#upload-dataset"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Upload dataset</span></h3> <ol data-svelte-h="svelte-1371ekm"><li>Once you’ve created a repository, navigate to the <strong>Files and versions</strong> tab to add a file. Select <strong>Add file</strong> to upload your dataset files. We support many text, audio, image and other data extensions such as <code>.csv</code>, <code>.mp3</code>, and <code>.jpg</code> (see the full list of <a href="#file-formats">File formats</a>).</li></ol> <div class="flex justify-center" data-svelte-h="svelte-ejbx7w"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/upload_files.png"></div> <ol start="2" data-svelte-h="svelte-1q9alp6"><li>Drag and drop your dataset files.</li></ol> <div class="flex justify-center" data-svelte-h="svelte-14wj5uc"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/commit_files.png"></div> <ol start="3" data-svelte-h="svelte-9bhhwd"><li>After uploading your dataset files, they are stored in your dataset repository.</li></ol> <div class="flex justify-center" data-svelte-h="svelte-15ihy8q"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/files_stored.png"></div> <h3 class="relative group"><a id="create-a-dataset-card" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#create-a-dataset-card"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Create a Dataset card</span></h3> <p data-svelte-h="svelte-5ezmk1">Adding a Dataset card is super valuable for helping users find your dataset and understand how to use it responsibly.</p> <ol data-svelte-h="svelte-vc122w"><li>Click on <strong>Create Dataset Card</strong> to create a <a href="./datasets-cards">Dataset card</a>. This button creates a <code>README.md</code> file in your repository.</li></ol> <div class="flex justify-center" data-svelte-h="svelte-1o3g2qu"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/dataset_card.png"></div> <ol start="2" data-svelte-h="svelte-tbqb69"><li><p>At the top, you’ll see the <strong>Metadata UI</strong> with several fields to select from such as license, language, and task categories. These are the most important tags for helping users discover your dataset on the Hub (when applicable). When you select an option for a field, it will be automatically added to the top of the dataset card.</p> <p>You can also look at the <a href="https://github.com/huggingface/hub-docs/blob/main/datasetcard.md?plain=1" rel="nofollow">Dataset Card specifications</a>, which has a complete set of allowed tags, including optional like <code>annotations_creators</code>, to help you choose the ones that are useful for your dataset.</p></li></ol> <div class="flex justify-center" data-svelte-h="svelte-stlsmn"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/metadata_ui.png"></div> <ol start="3" data-svelte-h="svelte-1nopxc3"><li><p>Write your dataset documentation in the Dataset Card to introduce your dataset to the community and help users understand what is inside: what are the use cases and limitations, where the data comes from, what are important ethical considerations, and any other relevant details.</p> <p>You can click on the <strong>Import dataset card template</strong> link at the top of the editor to automatically create a dataset card template. For a detailed example of what a good Dataset card should look like, take a look at the <a href="https://huggingface.co/datasets/cnn_dailymail" rel="nofollow">CNN DailyMail Dataset card</a>.</p></li></ol> <h2 class="relative group"><a id="using-the-huggingfacehub-client-library" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#using-the-huggingfacehub-client-library"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Using the huggingface_hub client library</span></h2> <p data-svelte-h="svelte-1jsh2bn">The rich features set in the <code>huggingface_hub</code> library allows you to manage repositories, including creating repos and uploading datasets to the Hub. Visit <a href="/docs/huggingface_hub/index">the client library’s documentation</a> to learn more.</p> <h2 class="relative group"><a id="using-other-libraries" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#using-other-libraries"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Using other libraries</span></h2> <p data-svelte-h="svelte-14ewh22">Some libraries like <a href="/docs/datasets/index">🤗 Datasets</a>, <a href="https://pandas.pydata.org/" rel="nofollow">Pandas</a>, <a href="https://pola.rs" rel="nofollow">Polars</a>, <a href="https://www.dask.org/" rel="nofollow">Dask</a> or <a href="https://duckdb.org/" rel="nofollow">DuckDB</a> can upload files to the Hub.
	See the list of <a href="./datasets-libraries">Libraries supported by the Datasets Hub</a> for more information.</p> <h2 class="relative group"><a id="using-git" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#using-git"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Using Git</span></h2> <p data-svelte-h="svelte-b2l0oe">Since dataset repos are Git repositories, you can use Git to push your data files to the Hub. Follow the guide on <a href="repositories-getting-started">Getting Started with Repositories</a> to learn about using the <code>git</code> CLI to commit and push your datasets.</p> <h2 class="relative group"><a id="file-formats" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#file-formats"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>File formats</span></h2> <p data-svelte-h="svelte-dyuuws">The Hub natively supports multiple file formats:</p> <ul data-svelte-h="svelte-nrshj9"><li>CSV (.csv, .tsv)</li> <li>JSON Lines, JSON (.jsonl, .json)</li> <li>Parquet (.parquet)</li> <li>Arrow streaming format (.arrow)</li> <li>Text (.txt)</li> <li>Images (.png, .jpg, etc.)</li> <li>Audio (.wav, .mp3, etc.)</li> <li><a href="https://github.com/webdataset/webdataset" rel="nofollow">WebDataset</a> (.tar)</li></ul> <p data-svelte-h="svelte-mpmhfr">It supports files compressed using ZIP (.zip), GZIP (.gz), ZSTD (.zst), BZ2 (.bz2), LZ4 (.lz4) and LZMA (.xz).</p> <p data-svelte-h="svelte-185tivl">Image and audio files can also have additional metadata files. See the <a href="./datasets-data-files-configuration#image-and-audio-datasets">Data files Configuration</a> on image and audio datasets, as well as the collections of <a href="https://huggingface.co/datasets-examples" rel="nofollow">example datasets</a> for CSV, TSV and images.</p> <p data-svelte-h="svelte-1tta5o7">You may want to convert your files to these formats to benefit from all the Hub features.
	Other formats and structures may not be recognized by the Hub.</p> <h3 class="relative group"><a id="which-file-format-should-i-use" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#which-file-format-should-i-use"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Which file format should I use?</span></h3> <p data-svelte-h="svelte-1lms0fv">For most types of datasets, Parquet is the recommended format due to its efficient compression, rich typing, and since a variety of tools supports this format with optimized read and batched operations. Alternatively, CSV or JSON Lines/JSON can be used for tabular data (prefer JSON Lines for nested data). Although easy to parse compared to Parquet, these formats are not recommended for data larger than several GBs. For image and audio datasets, uploading raw files is the most practical for most use cases since it’s easy to access individual files. For large scale image and audio datasets streaming, <a href="https://github.com/webdataset/webdataset" rel="nofollow">WebDataset</a> should be preferred over raw image and audio files to avoid the overhead of accessing individual files. Though for more general use cases involving analytics, data filtering or metadata parsing, Parquet is the recommended option for large scale image and audio datasets.</p> <h3 class="relative group"><a id="dataset-viewer" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#dataset-viewer"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Dataset Viewer</span></h3> <p data-svelte-h="svelte-1s5m6to">The <a href="./datasets-viewer">Dataset Viewer</a> is useful to know how the data actually looks like before you download it.
	It is enabled by default for all public datasets. It is also available for private datasets owned by a <a href="https://huggingface.co/pricing" rel="nofollow">PRO user</a> or an <a href="https://huggingface.co/enterprise" rel="nofollow">Enterprise Hub organization</a>.</p> <p data-svelte-h="svelte-189qumh">After uploading your dataset, make sure the Dataset Viewer correctly shows your data, or <a href="./datasets-viewer-configure">Configure the Dataset Viewer</a>.</p> <h2 class="relative group"><a id="large-scale-datasets" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#large-scale-datasets"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Large scale datasets</span></h2> <p data-svelte-h="svelte-ivxrwk">The Hugging Face Hub supports large scale datasets, usually uploaded in Parquet (e.g. via <code>push_to_hub()</code> using <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset.push_to_hub">🤗 Datasets</a>) or <a href="https://github.com/webdataset/webdataset" rel="nofollow">WebDataset</a> format.</p> <p data-svelte-h="svelte-14mep43">You can upload large scale datasets at high speed using the <code>huggingface_hub</code> library.</p> <p data-svelte-h="svelte-wxady5">See <a href="/docs/huggingface_hub/guides/upload#upload-a-folder-by-chunks">how to upload a folder by chunks</a>, the <a href="/docs/huggingface_hub/guides/upload#tips-and-tricks-for-large-uploads">tips and tricks for large uploads</a> and the <a href="./repositories-recommendations">repository limitations and recommendations</a>.</p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/hub-docs/blob/main/docs/hub/datasets-adding.md" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span data-svelte-h="svelte-x0xyl0">></span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_1vatp3t = {
	assets: "/docs/hub/main/en",
	base: "/docs/hub/main/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/hub/main/en/_app/immutable/entry/start.d0cd5065.js"),
	import("/docs/hub/main/en/_app/immutable/entry/app.b6abe3c1.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 12],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 28 kB
Xet hash:: 5b1d51380a62aad898298a3c9fefea26a9c997ad71e8df4a2a582fc9d37aa4d4

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.