Hugging Face's logo

Buckets:
hf-doc-build
/
doc-dev

Files
xet
hf-doc-build/doc-dev / hub /pr_2521 /en /storage-buckets.html
HuggingFaceDocBuilder's picture
HuggingFaceDocBuilder
download
raw
79.6 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Storage Buckets&quot;,&quot;local&quot;:&quot;storage-buckets&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Buckets vs Repositories&quot;,&quot;local&quot;:&quot;buckets-vs-repositories&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Creating a Bucket&quot;,&quot;local&quot;:&quot;creating-a-bucket&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;From the Hub UI&quot;,&quot;local&quot;:&quot;from-the-hub-ui&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;From the CLI&quot;,&quot;local&quot;:&quot;from-the-cli&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;From Python&quot;,&quot;local&quot;:&quot;from-python&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Browsing Buckets on the Hub&quot;,&quot;local&quot;:&quot;browsing-buckets-on-the-hub&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;README rendering&quot;,&quot;local&quot;:&quot;readme-rendering&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Listing from the CLI&quot;,&quot;local&quot;:&quot;listing-from-the-cli&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Managing Files&quot;,&quot;local&quot;:&quot;managing-files&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Uploading files&quot;,&quot;local&quot;:&quot;uploading-files&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Downloading files&quot;,&quot;local&quot;:&quot;downloading-files&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Syncing directories&quot;,&quot;local&quot;:&quot;syncing-directories&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Deleting files&quot;,&quot;local&quot;:&quot;deleting-files&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Copying files between repos and buckets&quot;,&quot;local&quot;:&quot;copying-files-between-repos-and-buckets&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Pre-warming and CDN&quot;,&quot;local&quot;:&quot;pre-warming-and-cdn&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Use Cases&quot;,&quot;local&quot;:&quot;use-cases&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Training checkpoints and logs&quot;,&quot;local&quot;:&quot;training-checkpoints-and-logs&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Data processing pipelines&quot;,&quot;local&quot;:&quot;data-processing-pipelines&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Agentic storage&quot;,&quot;local&quot;:&quot;agentic-storage&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Rolling backups&quot;,&quot;local&quot;:&quot;rolling-backups&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Linking models to buckets&quot;,&quot;local&quot;:&quot;linking-models-to-buckets&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Pricing&quot;,&quot;local&quot;:&quot;pricing&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}">
 <link href="/docs/hub/pr_2521/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/entry/start.d19e5ca7.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/scheduler.409792a1.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/singletons.1ece723a.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/index.0f0d9f26.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/paths.a67d9216.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/entry/app.98ab229a.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/preload-helper.78e52d9f.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/index.92d389ff.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/nodes/0.7799902b.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/each.e59479a4.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/nodes/242.b11f11b3.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/CopyLLMTxtMenu.dcf7fb47.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.0c189c7e.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/CodeBlock.5ed269c1.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Storage Buckets&quot;,&quot;local&quot;:&quot;storage-buckets&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Buckets vs Repositories&quot;,&quot;local&quot;:&quot;buckets-vs-repositories&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Creating a Bucket&quot;,&quot;local&quot;:&quot;creating-a-bucket&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;From the Hub UI&quot;,&quot;local&quot;:&quot;from-the-hub-ui&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;From the CLI&quot;,&quot;local&quot;:&quot;from-the-cli&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;From Python&quot;,&quot;local&quot;:&quot;from-python&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Browsing Buckets on the Hub&quot;,&quot;local&quot;:&quot;browsing-buckets-on-the-hub&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;README rendering&quot;,&quot;local&quot;:&quot;readme-rendering&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Listing from the CLI&quot;,&quot;local&quot;:&quot;listing-from-the-cli&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Managing Files&quot;,&quot;local&quot;:&quot;managing-files&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Uploading files&quot;,&quot;local&quot;:&quot;uploading-files&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Downloading files&quot;,&quot;local&quot;:&quot;downloading-files&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Syncing directories&quot;,&quot;local&quot;:&quot;syncing-directories&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Deleting files&quot;,&quot;local&quot;:&quot;deleting-files&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Copying files between repos and buckets&quot;,&quot;local&quot;:&quot;copying-files-between-repos-and-buckets&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Pre-warming and CDN&quot;,&quot;local&quot;:&quot;pre-warming-and-cdn&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Use Cases&quot;,&quot;local&quot;:&quot;use-cases&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Training checkpoints and logs&quot;,&quot;local&quot;:&quot;training-checkpoints-and-logs&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Data processing pipelines&quot;,&quot;local&quot;:&quot;data-processing-pipelines&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Agentic storage&quot;,&quot;local&quot;:&quot;agentic-storage&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Rolling backups&quot;,&quot;local&quot;:&quot;rolling-backups&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Linking models to buckets&quot;,&quot;local&quot;:&quot;linking-models-to-buckets&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Pricing&quot;,&quot;local&quot;:&quot;pricing&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <h1 class="relative group"><a id="storage-buckets" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#storage-buckets"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Storage Buckets</span></h1> <p data-svelte-h="svelte-1y3ghvg">Storage Buckets are a repo type on the Hugging Face Hub providing S3-like object storage, powered by the <a href="./xet/index">Xet</a> storage backend. Unlike Git-based <a href="./repositories">repositories</a> (models, datasets, Spaces), buckets are <strong>non-versioned</strong> and <strong>mutable</strong>, designed for use cases where you need simple, fast storage such as training checkpoints, logs, intermediate artifacts, or any large collection of files that doesn’t need version control.</p> <p data-svelte-h="svelte-3ej5on">You can interact with buckets using the Hub web interface, the <a href="https://huggingface.co/docs/huggingface_hub/guides/cli#hf-buckets" rel="nofollow"><code>hf</code> CLI</a>, or the <a href="https://huggingface.co/docs/huggingface_hub/guides/buckets" rel="nofollow">Python API</a>.</p> <blockquote class="tip" data-svelte-h="svelte-1a2s3ga"><p>Buckets are available to all users and organizations. See <a href="https://huggingface.co/storage" rel="nofollow">hf.co/storage</a> for pricing details.</p></blockquote> <blockquote class="tip" data-svelte-h="svelte-1lnaxh3"><p>See <a href="./storage-buckets-access">Access Patterns</a> for how to reach bucket data from your tools (mount as a filesystem, <code>hf://</code> paths, volume mounts in Jobs/Spaces), and <a href="./storage-buckets-integrations">Bucket Integrations</a> for ready-to-use snippets in popular data libraries like pandas, Dask, and Spark.</p></blockquote> <h2 class="relative group"><a id="buckets-vs-repositories" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#buckets-vs-repositories"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Buckets vs Repositories</span></h2> <p data-svelte-h="svelte-1v6an01">The Hub offers two types of storage: Git-based <strong>repositories</strong> for versioned, collaborative work and <strong>buckets</strong> for fast, mutable object storage.</p> <table data-svelte-h="svelte-2e3ymm"><thead><tr><th>Feature</th> <th>Repositories (Git-based)</th> <th>Storage Buckets</th></tr></thead> <tbody><tr><td>Versioning</td> <td>Full Git history</td> <td>None (mutable, overwrite-in-place)</td></tr> <tr><td>Types</td> <td>Models, Datasets, Spaces</td> <td>Standalone bucket</td></tr> <tr><td>Primary use case</td> <td>Publishing finished artifacts</td> <td>Working storage / intermediate data</td></tr> <tr><td>Operations</td> <td>Hub API, Git push/pull</td> <td>S3-like <code>sync</code>, <code>cp</code>, <code>rm</code></td></tr> <tr><td>Deduplication</td> <td>Xet chunk-level</td> <td>Xet chunk-level</td></tr> <tr><td>Pull Requests</td> <td>Yes</td> <td>No</td></tr> <tr><td>Model/Dataset Cards</td> <td>Yes</td> <td>No (but plain README rendered)</td></tr></tbody></table> <p data-svelte-h="svelte-1mxx6zj">Use <strong>repositories</strong> when you want version history, collaboration features (PRs, discussions), and library integrations. Use <strong>buckets</strong> when you need fast, mutable storage for data that changes frequently — files can be overwritten or deleted in place.</p> <h2 class="relative group"><a id="creating-a-bucket" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#creating-a-bucket"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Creating a Bucket</span></h2> <h3 class="relative group"><a id="from-the-hub-ui" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#from-the-hub-ui"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>From the Hub UI</span></h3> <ol data-svelte-h="svelte-12deud1"><li>Navigate to <a href="https://huggingface.co/new-bucket" rel="nofollow">huggingface.co/new-bucket</a>:</li></ol> <div class="flex justify-center" data-svelte-h="svelte-ye7dql"><img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/buckets/buckets-create.png"> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/buckets/buckets-create-dark.png"></div> <ol start="2" data-svelte-h="svelte-ry15qa"><li><p>Specify the owner of the bucket: this can be either you or any of the organizations you’re affiliated with.</p></li> <li><p>Enter a bucket name.</p></li> <li><p>Choose whether the bucket should be public or private.</p></li> <li><p>Optionally, preselect <a href="#pre-warming-and-cdn">CDN pre-warming</a> regions to cache your data closer to your compute from the start.</p></li></ol> <p data-svelte-h="svelte-tyc1t">After creating the bucket, you should see the bucket page:</p> <div class="flex justify-center" data-svelte-h="svelte-wruvo7"><img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/buckets/buckets-empty.png"> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/buckets/buckets-empty-dark.png"></div> <h3 class="relative group"><a id="from-the-cli" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#from-the-cli"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>From the CLI</span></h3> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START --><span class="hljs-comment"># Create a bucket under your namespace</span>
hf buckets create my-bucket
<span class="hljs-comment"># Create a private bucket</span>
hf buckets create my-bucket --private
<span class="hljs-comment"># Create a bucket under an organization</span>
hf buckets create my-org/shared-bucket<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="from-python" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#from-python"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>From Python</span></h3> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> create_bucket
<span class="hljs-comment"># Create a bucket under your namespace</span>
create_bucket(<span class="hljs-string">&quot;my-bucket&quot;</span>)
<span class="hljs-comment"># Create a private bucket</span>
create_bucket(<span class="hljs-string">&quot;my-bucket&quot;</span>, private=<span class="hljs-literal">True</span>)
<span class="hljs-comment"># Create a bucket under an organization</span>
create_bucket(<span class="hljs-string">&quot;my-org/shared-bucket&quot;</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-hto4aq">For the full Python API reference including deleting, moving, and listing buckets, see the <a href="https://huggingface.co/docs/huggingface_hub/guides/buckets" rel="nofollow"><code>huggingface_hub</code> Buckets guide</a>.</p> <h2 class="relative group"><a id="browsing-buckets-on-the-hub" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#browsing-buckets-on-the-hub"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Browsing Buckets on the Hub</span></h2> <p data-svelte-h="svelte-57sd5l">Every bucket has a page on the Hub where you can browse its contents, navigate directories, and view file details. Bucket pages are available at <code>https://huggingface.co/buckets/&lt;owner&gt;/&lt;bucket-name&gt;</code>.</p> <div class="flex justify-center" data-svelte-h="svelte-1filgl7"><img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/buckets/buckets-file-browser.png"> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/buckets/buckets-file-browser-dark.png"></div> <h3 class="relative group"><a id="readme-rendering" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#readme-rendering"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>README rendering</span></h3> <p data-svelte-h="svelte-ofbtkt">If a directory in your bucket contains a <code>README.md</code> file, the Hub renders it below the file list on that directory’s page. This works both at the bucket root and inside any subdirectory — useful for documenting what a bucket contains, how data is organized, or how downstream tools should consume it.</p> <h3 class="relative group"><a id="listing-from-the-cli" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#listing-from-the-cli"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Listing from the CLI</span></h3> <p data-svelte-h="svelte-gwueyt">You can also list bucket contents from the CLI:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START --><span class="hljs-comment"># List files in a bucket (with human-readable sizes)</span>
hf buckets list julien-c/my-training-bucket -h
 Feb 17 14:46 art/
 Feb 17 14:58 arxivqa/
 Feb 17 15:02 arxivqa2/
 Feb 17 15:04 arxivqa3/
 Feb 17 14:47 captcha/
 Feb 17 14:53 captcha2/
 Feb 24 17:22 julien/
<span class="hljs-comment"># Recursive listing</span>
hf buckets list julien-c/my-training-bucket/art -h -R
 423.6 MB Feb 17 14:29 art/train-00000-of-00011.parquet
 441.0 MB Feb 17 14:29 art/train-00001-of-00011.parquet
 521.7 MB Feb 17 14:29 art/train-00002-of-00011.parquet
 481.4 MB Feb 17 14:29 art/train-00003-of-00011.parquet
 444.6 MB Feb 17 14:29 art/train-00004-of-00011.parquet
 461.6 MB Feb 17 14:29 art/train-00005-of-00011.parquet
 466.4 MB Feb 17 14:29 art/train-00006-of-00011.parquet
 486.3 MB Feb 17 14:29 art/train-00007-of-00011.parquet
 477.0 MB Feb 17 14:29 art/train-00008-of-00011.parquet
 454.0 MB Feb 17 14:29 art/train-00009-of-00011.parquet
 483.1 MB Feb 17 14:29 art/train-00010-of-00011.parquet
<span class="hljs-comment"># Tree view</span>
hf buckets list julien-c/my-training-bucket --tree -h -R
 ├── art/
423.6 MB Feb 17 14:29 │ ├── train-00000-of-00011.parquet
441.0 MB Feb 17 14:29 │ ├── train-00001-of-00011.parquet
521.7 MB Feb 17 14:29 │ ├── train-00002-of-00011.parquet
481.4 MB Feb 17 14:29 │ ├── train-00003-of-00011.parquet
444.6 MB Feb 17 14:29 │ ├── train-00004-of-00011.parquet
461.6 MB Feb 17 14:29 │ ├── train-00005-of-00011.parquet
466.4 MB Feb 17 14:29 │ ├── train-00006-of-00011.parquet
486.3 MB Feb 17 14:29 │ ├── train-00007-of-00011.parquet
477.0 MB Feb 17 14:29 │ ├── train-00008-of-00011.parquet
454.0 MB Feb 17 14:29 │ ├── train-00009-of-00011.parquet
483.1 MB Feb 17 14:29 │ └── train-00010-of-00011.parquet
 ├── arxivqa/
495.9 MB Feb 17 14:32 │ ├── train-00000-of-00164.parquet
518.3 MB Feb 17 14:32 │ ├── train-00001-of-00164.parquet
495.5 MB Feb 17 14:32 │ ├── train-00002-of-00164.parquet
486.6 MB Feb 17 14:32 │ ├── train-00003-of-00164.parquet
490.4 MB Feb 17 14:32 │ ├── train-00004-of-00164.parquet
...<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="managing-files" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#managing-files"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Managing Files</span></h2> <p data-svelte-h="svelte-1hlq8q6">You can upload and download files directly from the bucket page on the Hub, or use the CLI and Python API for programmatic access. Bucket files are referenced using <code>hf://buckets/</code> paths (e.g., <code>hf://buckets/username/my-bucket/path/to/file</code>). The <code>hf buckets cp</code> command handles individual file transfers while <code>hf buckets sync</code> is better suited for directories. All commands work in both directions — local-to-remote and remote-to-local.</p> <h3 class="relative group"><a id="uploading-files" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#uploading-files"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Uploading files</span></h3> <p data-svelte-h="svelte-mdxwsa">For quick uploads, you can drag and drop files directly on the bucket page in your browser. For programmatic use, <code>hf buckets cp</code> copies individual files into a bucket. The source is a local path and the destination is an <code>hf://buckets/</code> path. You can also pipe data from stdin, which is handy for programmatically generated content.</p> <p data-svelte-h="svelte-as8qnx"><strong>CLI:</strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START --><span class="hljs-comment"># Upload a single file</span>
hf buckets <span class="hljs-built_in">cp</span> ./model.safetensors hf://buckets/username/my-bucket/models/model.safetensors
<span class="hljs-comment"># Upload from stdin</span>
<span class="hljs-built_in">cat</span> config.json | hf buckets <span class="hljs-built_in">cp</span> - hf://buckets/username/my-bucket/config.json<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-cva8x5">In Python, use <code>batch_bucket_files</code> to upload one or more files in a single call. Each entry is a tuple of <code>(local_path, remote_path)</code>.</p> <p data-svelte-h="svelte-1bwqtx5"><strong>Python:</strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> batch_bucket_files
batch_bucket_files(
 <span class="hljs-string">&quot;username/my-bucket&quot;</span>,
 add=[
 (<span class="hljs-string">&quot;./model.safetensors&quot;</span>, <span class="hljs-string">&quot;models/model.safetensors&quot;</span>),
 (<span class="hljs-string">&quot;./config.json&quot;</span>, <span class="hljs-string">&quot;models/config.json&quot;</span>),
 ],
)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-s3h4ee">For more upload options (raw bytes, combined upload+delete, etc.), see the <a href="https://huggingface.co/docs/huggingface_hub/guides/buckets#upload-files" rel="nofollow"><code>huggingface_hub</code> upload guide</a>.</p> <h3 class="relative group"><a id="downloading-files" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#downloading-files"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Downloading files</span></h3> <p data-svelte-h="svelte-d00f12">You can download individual files directly from the bucket page on the Hub by clicking on them. For programmatic access, downloading mirrors the upload syntax — swap the source and destination in <code>hf buckets cp</code>. You can also stream a file to stdout by using <code>-</code> as the destination, which lets you pipe bucket contents directly into other tools.</p> <p data-svelte-h="svelte-as8qnx"><strong>CLI:</strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START --><span class="hljs-comment"># Download a single file</span>
hf buckets <span class="hljs-built_in">cp</span> hf://buckets/username/my-bucket/models/model.safetensors ./model.safetensors
<span class="hljs-comment"># Download to stdout and pipe</span>
hf buckets <span class="hljs-built_in">cp</span> hf://buckets/username/my-bucket/config.json - | jq .<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1152o8g">In Python, use <code>download_bucket_files</code> with a list of <code>(remote_path, local_path)</code> tuples.</p> <p data-svelte-h="svelte-1bwqtx5"><strong>Python:</strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> download_bucket_files
download_bucket_files(
 <span class="hljs-string">&quot;username/my-bucket&quot;</span>,
 files=[
 (<span class="hljs-string">&quot;models/model.safetensors&quot;</span>, <span class="hljs-string">&quot;./local/model.safetensors&quot;</span>),
 (<span class="hljs-string">&quot;config.json&quot;</span>, <span class="hljs-string">&quot;./local/config.json&quot;</span>),
 ],
)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-11zknfw">For faster downloads using pre-fetched metadata, see the <a href="https://huggingface.co/docs/huggingface_hub/guides/buckets#download-files" rel="nofollow"><code>huggingface_hub</code> download guide</a>.</p> <h3 class="relative group"><a id="syncing-directories" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#syncing-directories"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Syncing directories</span></h3> <p data-svelte-h="svelte-1yquxmu">The <code>sync</code> command works like <code>rsync</code> or <code>aws s3 sync</code> — it compares source and destination and only transfers files that have changed. This is the most efficient way to keep a local directory and a bucket in sync. By default, <code>sync</code> only adds and updates files. Pass <code>--delete</code> to also remove files at the destination that no longer exist at the source. Use <code>--dry-run</code> to preview what would happen without actually transferring anything.</p> <p data-svelte-h="svelte-as8qnx"><strong>CLI:</strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START --><span class="hljs-comment"># Upload a local directory to a bucket</span>
hf buckets <span class="hljs-built_in">sync</span> ./data hf://buckets/username/my-bucket/data
<span class="hljs-comment"># Download from a bucket to a local directory</span>
hf buckets <span class="hljs-built_in">sync</span> hf://buckets/username/my-bucket/data ./data
<span class="hljs-comment"># Sync with deletion of extraneous files</span>
hf buckets <span class="hljs-built_in">sync</span> ./data hf://buckets/username/my-bucket/data --delete
<span class="hljs-comment"># Preview what would be synced without executing</span>
hf buckets <span class="hljs-built_in">sync</span> ./data hf://buckets/username/my-bucket/data --dry-run
<span class="hljs-comment"># Plan and apply: review the sync plan before executing</span>
hf buckets <span class="hljs-built_in">sync</span> ./data hf://buckets/username/my-bucket/data --plan sync-plan.jsonl
<span class="hljs-comment"># ... review the plan file, then apply it</span>
hf buckets <span class="hljs-built_in">sync</span> --apply sync-plan.jsonl<!-- HTML_TAG_END --></pre></div> <blockquote class="tip" data-svelte-h="svelte-x6hdq5"><p><code>hf sync</code> is a convenient alias for <code>hf buckets sync</code>.</p></blockquote> <p data-svelte-h="svelte-1bwqtx5"><strong>Python:</strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> sync_bucket
<span class="hljs-comment"># Upload a local directory to a bucket</span>
sync_bucket(<span class="hljs-string">&quot;./data&quot;</span>, <span class="hljs-string">&quot;hf://buckets/username/my-bucket/data&quot;</span>)
<span class="hljs-comment"># Download from a bucket to a local directory</span>
sync_bucket(<span class="hljs-string">&quot;hf://buckets/username/my-bucket/data&quot;</span>, <span class="hljs-string">&quot;./data&quot;</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-e9zpxw">The <code>sync</code> command supports filtering (<code>--include</code>, <code>--exclude</code>), comparison modes (<code>--ignore-times</code>, <code>--existing</code>), and a <strong>plan-and-apply</strong> workflow to review operations before executing them. For the full set of options, see the <a href="https://huggingface.co/docs/huggingface_hub/guides/buckets#sync-directories" rel="nofollow"><code>huggingface_hub</code> sync guide</a>.</p> <h3 class="relative group"><a id="deleting-files" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#deleting-files"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Deleting files</span></h3> <p data-svelte-h="svelte-1n6b02t">Since buckets are non-versioned, deletions are immediate and permanent — there is no way to recover a deleted file. Use <code>--dry-run</code> to double-check before removing files, especially when using <code>--recursive</code>.</p> <p data-svelte-h="svelte-as8qnx"><strong>CLI:</strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START --><span class="hljs-comment"># Remove a single file</span>
hf buckets <span class="hljs-built_in">rm</span> username/my-bucket/old-model.bin
<span class="hljs-comment"># Remove all files under a prefix</span>
hf buckets <span class="hljs-built_in">rm</span> username/my-bucket/logs/ --recursive
<span class="hljs-comment"># Preview what would be deleted</span>
hf buckets <span class="hljs-built_in">rm</span> username/my-bucket/checkpoints/ --recursive --dry-run<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1bwqtx5"><strong>Python:</strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> batch_bucket_files
batch_bucket_files(<span class="hljs-string">&quot;username/my-bucket&quot;</span>, delete=[<span class="hljs-string">&quot;old-model.bin&quot;</span>, <span class="hljs-string">&quot;logs/debug.log&quot;</span>])<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-13jwq4e">For more deletion options (pattern-based filtering, recursive removal, etc.), see the <a href="https://huggingface.co/docs/huggingface_hub/guides/buckets#delete-files" rel="nofollow"><code>huggingface_hub</code> delete guide</a>.</p> <h3 class="relative group"><a id="copying-files-between-repos-and-buckets" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#copying-files-between-repos-and-buckets"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Copying files between repos and buckets</span></h3> <p data-svelte-h="svelte-1s8hmb0">You can copy <a href="./xet/index">Xet</a>-tracked files from any repository (model, dataset, Space) or bucket into a destination bucket without re-uploading the data. The copy is server-side: only the Xet content hashes are migrated, so even very large files are copied instantly.</p> <blockquote class="note" data-svelte-h="svelte-1s206bm"><p>Only Xet-tracked files are copied server-to-server. Small non-Xet files (e.g., config files and READMEs) are automatically downloaded and re-uploaded.</p></blockquote> <p data-svelte-h="svelte-as8qnx"><strong>CLI:</strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START -->hf buckets <span class="hljs-built_in">cp</span> \
 hf://datasets/HuggingFaceFW/fineweb/data \
 hf://buckets/username/fineweb-data<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1bwqtx5"><strong>Python:</strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> HfApi
api = HfApi()
api.copy_files(
 <span class="hljs-string">&quot;hf://datasets/HuggingFaceFW/fineweb/data&quot;</span>,
 <span class="hljs-string">&quot;hf://buckets/username/fineweb-data&quot;</span>,
)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-969wpn">You need read access to the source repository or bucket and write access to the destination bucket.</p> <p data-svelte-h="svelte-2y5mn0">Note that transferring data the other way from a bucket to a repository (model, dataset, Space) without reuploading is not yet available, but is on the roadmap.</p> <h2 class="relative group"><a id="pre-warming-and-cdn" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#pre-warming-and-cdn"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Pre-warming and CDN</span></h2> <p data-svelte-h="svelte-sa1wfw">Buckets live on the Hub’s global storage by default. For workloads where storage location directly affects throughput you can <strong>pre-warm</strong> bucket data to bring it closer to your compute.</p> <p data-svelte-h="svelte-1muw2tc">Pre-warming caches files at edge locations near specific cloud providers and regions, so your jobs read data locally instead of pulling it across regions. This is especially useful for:</p> <ul data-svelte-h="svelte-1fpuer5"><li>Training clusters that need fast access to large datasets or checkpoints</li> <li>Multi-region setups where different parts of a pipeline run in different clouds</li> <li>Distributing large artifacts to many consumers worldwide</li></ul> <p data-svelte-h="svelte-1ideti5">See <a href="https://huggingface.co/storage" rel="nofollow">hf.co/storage</a> for available regions and details on enabling pre-warming.</p> <h2 class="relative group"><a id="use-cases" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#use-cases"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Use Cases</span></h2> <h3 class="relative group"><a id="training-checkpoints-and-logs" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#training-checkpoints-and-logs"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Training checkpoints and logs</span></h3> <p data-svelte-h="svelte-17m47gu">When running training jobs (e.g., via <a href="./jobs">Jobs</a>), save checkpoints and logs to a bucket. Unlike a Git repo, you can overwrite the latest checkpoint without accumulating version history, and <code>sync</code> ensures only changed data is transferred.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START --><span class="hljs-comment"># After each evaluation step, sync checkpoints to a bucket</span>
hf <span class="hljs-built_in">sync</span> ./checkpoints hf://buckets/my-org/training-run-42/checkpoints<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-9dfhz">Because buckets are built on <a href="./xet/index">Xet</a>, successive checkpoints where large parts of the model are frozen benefit from chunk-level deduplication. Only the changed chunks are uploaded.</p> <h3 class="relative group"><a id="data-processing-pipelines" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#data-processing-pipelines"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Data processing pipelines</span></h3> <p data-svelte-h="svelte-51qwqn">Buckets serve as staging areas for data processing workflows. Process raw data, write intermediate outputs to a bucket, then promote the final artifact to a versioned <a href="./datasets">Dataset</a> repository when the pipeline completes. This keeps your versioned repo clean while giving your pipeline fast mutable storage.</p> <p data-svelte-h="svelte-l6l9x8">Note that transferring data from a Bucket to a repository without reuploading is not yet available, but is on the roadmap.</p> <h3 class="relative group"><a id="agentic-storage" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#agentic-storage"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Agentic storage</span></h3> <p data-svelte-h="svelte-1hgf34">AI agents need scratch storage for intermediate results, tool outputs, traces, and working memory. Buckets provide a Hub-native place for this data: fast mutable access without Git overhead, standard Hugging Face permissions, and addressable via <code>hf://buckets/</code> paths across the Hub ecosystem.</p> <h3 class="relative group"><a id="rolling-backups" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#rolling-backups"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Rolling backups</span></h3> <p data-svelte-h="svelte-1xbrqwz">Buckets are well-suited for maintaining rolling backups. With a Git-based <a href="./datasets">Dataset</a> repository, deleting outdated files doesn’t free storage — Git history retains every past version, so you’d need to squash commits or rewrite history to actually reclaim space. With buckets, old files are truly gone once deleted, and you only pay for what’s currently stored.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START --><span class="hljs-comment"># Sync today&#x27;s backup, removing files that no longer exist locally</span>
hf <span class="hljs-built_in">sync</span> ./daily-backup hf://buckets/my-user/backups/latest --delete<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="linking-models-to-buckets" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#linking-models-to-buckets"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Linking models to buckets</span></h3> <p data-svelte-h="svelte-1kjzmio">You can create a two-way link between a model and a bucket by adding the <code>buckets</code> field to the model card metadata. The linked models will then appear on the bucket page, and the bucket will appear as a tag on the model page.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-yaml "><!-- HTML_TAG_START --><span class="hljs-comment"># In the model card YAML frontmatter</span>
<span class="hljs-attr">buckets:</span>
<span class="hljs-bullet">-</span> <span class="hljs-string">my-org/my-bucket</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-o2acgx">See <a href="./model-cards#specifying-a-bucket">Specifying a bucket</a> in the model cards documentation for more details.</p> <h2 class="relative group"><a id="pricing" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#pricing"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Pricing</span></h2> <p data-svelte-h="svelte-es122o">Storage Buckets are billed based on the amount of data stored, with simple per-TB pricing. Enterprise plans benefit from dedup-based billing, where shared chunks across files directly reduce the billed footprint.</p> <p data-svelte-h="svelte-iywhsc">As for other repositories, buckets are free to create and have a free storage allowance. For usage above the <a href="https://huggingface.co/docs/hub/storage-limits" rel="nofollow">free tier</a>, see <a href="https://huggingface.co/storage" rel="nofollow">hf.co/storage</a>. For general billing information, see the <a href="./billing">Billing</a> documentation.</p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/hub-docs/blob/main/docs/hub/storage-buckets.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p>
<script>
 {
 __sveltekit_1bollga = {
 assets: "/docs/hub/pr_2521/en",
 base: "/docs/hub/pr_2521/en",
 env: {}
 };
 const element = document.currentScript.parentElement;
 const data = [null,null];
 Promise.all([
 import("/docs/hub/pr_2521/en/_app/immutable/entry/start.d19e5ca7.js"),
 import("/docs/hub/pr_2521/en/_app/immutable/entry/app.98ab229a.js")
 ]).then(([kit, app]) => {
 kit.start(app, element, {
 node_ids: [0, 242],
 data,
 form: null,
 error: null
 });
 });
 }
 </script>

Xet Storage Details

Size:
79.6 kB
·
Xet hash:
a074ce4fc3622d4db341cde88a1df6965d83c416e96b0b91706a48a7df72130f

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.