Hugging Face's logo

Buckets:
hf-doc-build
/
doc-dev

Files
xet
hf-doc-build/doc-dev / hub /pr_2521 /en /datasets-pyarrow.html
HuggingFaceDocBuilder's picture
HuggingFaceDocBuilder
download
raw
55.3 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;PyArrow&quot;,&quot;local&quot;:&quot;pyarrow&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Load a Table&quot;,&quot;local&quot;:&quot;load-a-table&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Save a Table&quot;,&quot;local&quot;:&quot;save-a-table&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Leverage Xet deduplication for Parquet&quot;,&quot;local&quot;:&quot;leverage-xet-deduplication-for-parquet&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Use Images&quot;,&quot;local&quot;:&quot;use-images&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Embed Images inside Parquet&quot;,&quot;local&quot;:&quot;embed-images-inside-parquet&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Use Audios&quot;,&quot;local&quot;:&quot;use-audios&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Embed Audio inside Parquet&quot;,&quot;local&quot;:&quot;embed-audio-inside-parquet&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2}],&quot;depth&quot;:1}">
 <link href="/docs/hub/pr_2521/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/entry/start.d19e5ca7.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/scheduler.409792a1.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/singletons.1ece723a.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/index.0f0d9f26.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/paths.a67d9216.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/entry/app.98ab229a.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/preload-helper.78e52d9f.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/index.92d389ff.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/nodes/0.7799902b.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/each.e59479a4.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/nodes/60.2a4ffedf.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/CopyLLMTxtMenu.dcf7fb47.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.0c189c7e.js">
 <link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/CodeBlock.5ed269c1.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;PyArrow&quot;,&quot;local&quot;:&quot;pyarrow&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Load a Table&quot;,&quot;local&quot;:&quot;load-a-table&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Save a Table&quot;,&quot;local&quot;:&quot;save-a-table&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Leverage Xet deduplication for Parquet&quot;,&quot;local&quot;:&quot;leverage-xet-deduplication-for-parquet&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Use Images&quot;,&quot;local&quot;:&quot;use-images&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Embed Images inside Parquet&quot;,&quot;local&quot;:&quot;embed-images-inside-parquet&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Use Audios&quot;,&quot;local&quot;:&quot;use-audios&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Embed Audio inside Parquet&quot;,&quot;local&quot;:&quot;embed-audio-inside-parquet&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2}],&quot;depth&quot;:1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <h1 class="relative group"><a id="pyarrow" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#pyarrow"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>PyArrow</span></h1> <p data-svelte-h="svelte-56vgn"><a href="https://github.com/apache/arrow" rel="nofollow">Arrow</a> is a columnar format and a toolbox for fast data interchange and in-memory analytics.
Since PyArrow supports <a href="https://filesystem-spec.readthedocs.io" rel="nofollow">fsspec</a> to read and write remote data, you can use the Hugging Face paths (<a href="/docs/huggingface_hub/guides/hf_file_system#integrations"><code>hf://</code></a>) to read and write data on the Hub.
It is especially useful for <a href="https://parquet.apache.org/" rel="nofollow">Parquet</a> data, since Parquet is the most common file format on Hugging Face.
Indeed, Parquet is particularly efficient thanks to its structure, typing, metadata and compression.</p> <h2 class="relative group"><a id="load-a-table" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#load-a-table"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Load a Table</span></h2> <p data-svelte-h="svelte-1263bgm">You can load data from local files or from remote storage like Hugging Face Datasets. PyArrow supports many formats including CSV, JSON and more importantly Parquet:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> pyarrow.parquet <span class="hljs-keyword">as</span> pq
<span class="hljs-meta">&gt;&gt;&gt; </span>table = pq.read_table(<span class="hljs-string">&quot;path/to/data.parquet&quot;</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1othel7">To load a file from Hugging Face, the path needs to start with <code>hf://</code>. For example, the path to the <a href="https://huggingface.co/datasets/stanfordnlp/imdb" rel="nofollow">stanfordnlp/imdb</a> dataset repository is <code>hf://datasets/stanfordnlp/imdb</code>. The dataset on Hugging Face contains multiple Parquet files. The Parquet file format is designed to make reading and writing data frames efficient, and to make sharing data across data analysis languages easy. Here is how to load the file <code>plain_text/train-00000-of-00001.parquet</code> as a pyarrow Table (it requires <code>pyarrow&gt;=21.0</code>):</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> pyarrow.parquet <span class="hljs-keyword">as</span> pq
<span class="hljs-meta">&gt;&gt;&gt; </span>table = pq.read_table(<span class="hljs-string">&quot;hf://datasets/stanfordnlp/imdb/plain_text/train-00000-of-00001.parquet&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>table
pyarrow.Table
text: string
label: int64
----
text: [[<span class="hljs-string">&quot;I rented I AM CURIOUS-YELLOW from my video store because of all the controversy that surrounded it (... 1542 chars omitted)&quot;</span>, ...],...,[..., <span class="hljs-string">&quot;The story centers around Barry McKenzie who must go to England if he wishes to claim his inheritan (... 221 chars omitted)&quot;</span>]]
label: [[<span class="hljs-number">0</span>,<span class="hljs-number">0</span>,<span class="hljs-number">0</span>,<span class="hljs-number">0</span>,<span class="hljs-number">0</span>,...,<span class="hljs-number">0</span>,<span class="hljs-number">0</span>,<span class="hljs-number">0</span>,<span class="hljs-number">0</span>,<span class="hljs-number">0</span>],...,[<span class="hljs-number">1</span>,<span class="hljs-number">1</span>,<span class="hljs-number">1</span>,<span class="hljs-number">1</span>,<span class="hljs-number">1</span>,...,<span class="hljs-number">1</span>,<span class="hljs-number">1</span>,<span class="hljs-number">1</span>,<span class="hljs-number">1</span>,<span class="hljs-number">1</span>]]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-u7zvq0">If you don’t want to load the full Parquet data, you can get the Parquet metadata or load row group by row group instead:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> pyarrow.parquet <span class="hljs-keyword">as</span> pq
<span class="hljs-meta">&gt;&gt;&gt; </span>pf = pq.ParquetFile(<span class="hljs-string">&quot;hf://datasets/stanfordnlp/imdb/plain_text/train-00000-of-00001.parquet&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>pf.metadata
&lt;pyarrow._parquet.FileMetaData <span class="hljs-built_in">object</span> at <span class="hljs-number">0x1171b4090</span>&gt;
 created_by: parquet-cpp-arrow version <span class="hljs-number">12.0</span><span class="hljs-number">.0</span>
 num_columns: <span class="hljs-number">2</span>
 num_rows: <span class="hljs-number">25000</span>
 num_row_groups: <span class="hljs-number">25</span>
 format_version: <span class="hljs-number">2.6</span>
 serialized_size: <span class="hljs-number">62036</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> pf.num_row_groups:
<span class="hljs-meta">... </span> table = pf.read_row_group(i)
<span class="hljs-meta">... </span> ...<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-smuvdf">For more information on the Hugging Face paths and how they are implemented, please refer to the <a href="/docs/huggingface_hub/guides/hf_file_system">the client library’s documentation on the HfFileSystem</a>.</p> <h2 class="relative group"><a id="save-a-table" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#save-a-table"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Save a Table</span></h2> <p data-svelte-h="svelte-15a6az3">You can save a pyarrow Table using <code>pyarrow.parquet.write_table</code> to a local file or to Hugging Face directly.</p> <p data-svelte-h="svelte-1c59jr5">To save the Table on Hugging Face, you first need to <a href="/docs/huggingface_hub/quick-start#login">Login with your Hugging Face account</a>, for example using:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=" "><!-- HTML_TAG_START --><span class="hljs-attribute">hf auth login</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1cj3kqp">Then you can <a href="/docs/huggingface_hub/quick-start#create-a-repository">create a dataset repository</a>, for example using:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> HfApi
HfApi().create_repo(repo_id=<span class="hljs-string">&quot;username/my_dataset&quot;</span>, repo_type=<span class="hljs-string">&quot;dataset&quot;</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-110fpi">Finally, you can use <a href="/docs/huggingface_hub/guides/hf_file_system#integrations">Hugging Face paths</a> in PyArrow:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> pyarrow.parquet <span class="hljs-keyword">as</span> pq
pq.write_table(table, <span class="hljs-string">&quot;hf://datasets/username/my_dataset/imdb.parquet&quot;</span>, use_content_defined_chunking=<span class="hljs-literal">True</span>)
<span class="hljs-comment"># or write in separate files if the dataset has train/validation/test splits</span>
pq.write_table(table_train, <span class="hljs-string">&quot;hf://datasets/username/my_dataset/train.parquet&quot;</span>, use_content_defined_chunking=<span class="hljs-literal">True</span>)
pq.write_table(table_valid, <span class="hljs-string">&quot;hf://datasets/username/my_dataset/validation.parquet&quot;</span>, use_content_defined_chunking=<span class="hljs-literal">True</span>)
pq.write_table(table_test , <span class="hljs-string">&quot;hf://datasets/username/my_dataset/test.parquet&quot;</span>, use_content_defined_chunking=<span class="hljs-literal">True</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1yqjbbg">Note that Parquet files on Hugging Face are optimized to improve storage efficiency, accelerate downloads and uploads, and enable efficient dataset streaming and editing:</p> <ul data-svelte-h="svelte-sdwshs"><li><a href="https://huggingface.co/blog/parquet-cdc" rel="nofollow">Parquet Content Defined Chunking</a> optimizes Parquet for <a href="https://huggingface.co/docs/hub/en/xet/index" rel="nofollow">Xet</a>, Hugging Face’s storage backend. It accelerates uploads and downloads thanks to chunk-based deduplication and allows efficient file editing</li> <li>Page index accelerates filters when streaming and enables efficient random access, e.g. in the <a href="https://huggingface.co/docs/dataset-viewer" rel="nofollow">Dataset Viewer</a></li></ul> <p data-svelte-h="svelte-1x67i5s">PyArrow requires extra arguments to write optimized Parquet files:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> pyarrow.parquet <span class="hljs-keyword">as</span> pq
pq.write_table(
 table,
 <span class="hljs-string">&quot;hf://datasets/username/my_dataset/imdb.parquet&quot;</span>,
 <span class="hljs-comment"># Optimize for Xet</span>
 use_content_defined_chunking=<span class="hljs-literal">True</span>,
 write_page_index=<span class="hljs-literal">True</span>,
)<!-- HTML_TAG_END --></pre></div> <ul data-svelte-h="svelte-1u1d1hk"><li><code>use_content_defined_chunking=True</code> to enable Parquet Content Defined Chunking, for <a href="https://huggingface.co/blog/parquet-cdc" rel="nofollow">deduplication</a> and <a href="./datasets-editing">editing</a> (it requires <code>pyarrow&gt;=21.0</code>)</li> <li><code>write_page_index=True</code> to include a page index in the Parquet metadata, for <a href="./datasets-streaming">streaming and random access</a></li></ul> <blockquote class="tip" data-svelte-h="svelte-1jam3g9"><p>Content defined chunking (CDC) makes the Parquet writer chunk the data pages in a way that makes duplicate data chunked and compressed identically.
Without CDC, the pages are arbitrarily chunked and therefore duplicate data are impossible to detect because of compression.
Thanks to CDC, Parquet uploads and downloads from Hugging Face are faster, since duplicate data are uploaded or downloaded only once.</p></blockquote> <p data-svelte-h="svelte-10e2td">Find more information about Xet <a href="https://huggingface.co/join/xet" rel="nofollow">here</a>.</p> <h2 class="relative group"><a id="leverage-xet-deduplication-for-parquet" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#leverage-xet-deduplication-for-parquet"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Leverage Xet deduplication for Parquet</span></h2> <p data-svelte-h="svelte-19d2vdb">Optimized Parquet files are written with Content Defined Chunking, which enables deduplication.
This accelerates uploads since chunks of data that already exist on Hugging Face don’t need to be uploaded again, and this saves a lot of I/O.</p> <p data-svelte-h="svelte-1ssf7io">For example, this code uploads the content of <code>table</code> and then for <code>edited_table</code> the upload is faster since it only uploads the chunks that changed:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> pyarrow.parquet <span class="hljs-keyword">as</span> pq
pq.write_table(
 table,
 <span class="hljs-string">&quot;hf://datasets/username/my_dataset/imdb.parquet&quot;</span>,
 <span class="hljs-comment"># Optimize for Xet</span>
 use_content_defined_chunking=<span class="hljs-literal">True</span>,
 write_page_index=<span class="hljs-literal">True</span>,
)
edited_table = ... <span class="hljs-comment"># e.g. with added/modified/removed rows or columns</span>
pq.write_table(
 edited_table,
 <span class="hljs-string">&quot;hf://datasets/username/my_dataset/imdb.parquet&quot;</span>,
 <span class="hljs-comment"># Optimize for Xet</span>
 use_content_defined_chunking=<span class="hljs-literal">True</span>,
 write_page_index=<span class="hljs-literal">True</span>,
)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1psrwcu">Chunks are ~64kB and Parquet saves data column per column, so in practice this is what happens when editing an Optimized Parquet file:</p> <ul data-svelte-h="svelte-1scito"><li>add a new column -&gt; only the chunks of the new column are uploaded</li> <li>add/edit/delete a row -&gt; one chunk per column is uploaded</li></ul> <p data-svelte-h="svelte-nyjwuh">And in addition to this, the chunks of the Parquet footer containing metadata are also uploaded.</p> <h2 class="relative group"><a id="use-images" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#use-images"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Use Images</span></h2> <p data-svelte-h="svelte-i1x932">You can load a folder with a metadata file containing a field for the names or paths to the images, structured like this:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=" "><!-- HTML_TAG_START -->Example <span class="hljs-number">1</span>: Example <span class="hljs-number">2</span>:
folder/ folder/
├── metadata<span class="hljs-selector-class">.parquet</span> ├── metadata<span class="hljs-selector-class">.parquet</span>
├── img000<span class="hljs-selector-class">.png</span> └── images
├── img001<span class="hljs-selector-class">.png</span> ├── img000<span class="hljs-selector-class">.png</span>
... ...
└── imgNNN<span class="hljs-selector-class">.png</span> └── imgNNN.png<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1wo91tc">You can iterate on the images paths like this:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> pathlib <span class="hljs-keyword">import</span> Path
<span class="hljs-keyword">import</span> pyarrow <span class="hljs-keyword">as</span> pq
folder_path = Path(<span class="hljs-string">&quot;path/to/folder&quot;</span>)
table = pq.read_table(folder_path + <span class="hljs-string">&quot;metadata.parquet&quot;</span>)
<span class="hljs-keyword">for</span> file_name <span class="hljs-keyword">in</span> table[<span class="hljs-string">&quot;file_name&quot;</span>].to_pylist():
 image_path = folder_path / file_name
 ...<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-17xhcwl">Since the dataset is in a <a href="https://huggingface.co/docs/hub/en/datasets-image#additional-columns" rel="nofollow">supported structure</a> (a <code>metadata.parquet</code> file with a <code>file_name</code> field), you can save this dataset to Hugging Face and the Dataset Viewer shows both the metadata and images.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> HfApi
api = HfApi()
api.upload_folder(
 folder_path=folder_path,
 repo_id=<span class="hljs-string">&quot;username/my_image_dataset&quot;</span>,
 repo_type=<span class="hljs-string">&quot;dataset&quot;</span>,
)<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="embed-images-inside-parquet" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#embed-images-inside-parquet"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Embed Images inside Parquet</span></h3> <p data-svelte-h="svelte-1ypsa1z">PyArrow has a binary type which allows to have the images bytes in Arrow tables. Therefore it enables saving the dataset as one single Parquet file containing both the images (bytes and path) and the samples metadata:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> pyarrow <span class="hljs-keyword">as</span> pa
<span class="hljs-keyword">import</span> pyarrow.parquet <span class="hljs-keyword">as</span> pq
<span class="hljs-comment"># Embed the image bytes in Arrow</span>
image_array = pa.array([
 {
 <span class="hljs-string">&quot;bytes&quot;</span>: (folder_path / file_name).read_bytes(),
 <span class="hljs-string">&quot;path&quot;</span>: file_name,
 }
 <span class="hljs-keyword">for</span> file_name <span class="hljs-keyword">in</span> table[<span class="hljs-string">&quot;file_name&quot;</span>].to_pylist()
])
table.append_column(<span class="hljs-string">&quot;image&quot;</span>, image_array)
<span class="hljs-comment"># (Optional) Set the HF Image type for the Dataset Viewer and the `datasets` library</span>
features = {<span class="hljs-string">&quot;image&quot;</span>: {<span class="hljs-string">&quot;_type&quot;</span>: <span class="hljs-string">&quot;Image&quot;</span>}} <span class="hljs-comment"># or using datasets.Features(...).to_dict()</span>
schema_metadata = {<span class="hljs-string">&quot;huggingface&quot;</span>: {<span class="hljs-string">&quot;dataset_info&quot;</span>: {<span class="hljs-string">&quot;features&quot;</span>: features}}}
table = table.replace_schema_metadata(schema_metadata)
<span class="hljs-comment"># Save to Parquet</span>
<span class="hljs-comment"># (Optional) with use_content_defined_chunking for faster uploads and downloads</span>
pq.write_table(table, <span class="hljs-string">&quot;data.parquet&quot;</span>, use_content_defined_chunking=<span class="hljs-literal">True</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-snsv43">Setting the Image type in the Arrow schema metadata allows other libraries and the Hugging Face Dataset Viewer to know that “image” contains images and not just binary data.</p> <h2 class="relative group"><a id="use-audios" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#use-audios"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Use Audios</span></h2> <p data-svelte-h="svelte-14ssghx">You can load a folder with a metadata file containing a field for the names or paths to the audios, structured like this:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=" "><!-- HTML_TAG_START -->Example <span class="hljs-number">1</span>: Example <span class="hljs-number">2</span>:
folder/ folder/
├── metadata<span class="hljs-selector-class">.parquet</span> ├── metadata<span class="hljs-selector-class">.parquet</span>
├── rec000<span class="hljs-selector-class">.wav</span> └── audios
├── rec001<span class="hljs-selector-class">.wav</span> ├── rec000<span class="hljs-selector-class">.wav</span>
... ...
└── recNNN<span class="hljs-selector-class">.wav</span> └── recNNN.wav<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-9uthgh">You can iterate on the audios paths like this:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> pathlib <span class="hljs-keyword">import</span> Path
<span class="hljs-keyword">import</span> pyarrow <span class="hljs-keyword">as</span> pq
folder_path = Path(<span class="hljs-string">&quot;path/to/folder&quot;</span>)
table = pq.read_table(folder_path + <span class="hljs-string">&quot;metadata.parquet&quot;</span>)
<span class="hljs-keyword">for</span> file_name <span class="hljs-keyword">in</span> table[<span class="hljs-string">&quot;file_name&quot;</span>].to_pylist():
 audio_path = folder_path / file_name
 ...<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-gmhdi8">Since the dataset is in a <a href="https://huggingface.co/docs/hub/en/datasets-audio#additional-columns" rel="nofollow">supported structure</a> (a <code>metadata.parquet</code> file with a <code>file_name</code> field), you can save it to Hugging Face, and the Hub Dataset Viewer shows both the metadata and audio.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> HfApi
api = HfApi()
api.upload_folder(
 folder_path=folder_path,
 repo_id=<span class="hljs-string">&quot;username/my_audio_dataset&quot;</span>,
 repo_type=<span class="hljs-string">&quot;dataset&quot;</span>,
)<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="embed-audio-inside-parquet" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#embed-audio-inside-parquet"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Embed Audio inside Parquet</span></h3> <p data-svelte-h="svelte-1w1yrqx">PyArrow has a binary type which allows for having audio bytes in Arrow tables. Therefore, it enables saving the dataset as one single Parquet file containing both the audio (bytes and path) and the samples metadata:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> pyarrow <span class="hljs-keyword">as</span> pa
<span class="hljs-keyword">import</span> pyarrow.parquet <span class="hljs-keyword">as</span> pq
<span class="hljs-comment"># Embed the audio bytes in Arrow</span>
audio_array = pa.array([
 {
 <span class="hljs-string">&quot;bytes&quot;</span>: (folder_path / file_name).read_bytes(),
 <span class="hljs-string">&quot;path&quot;</span>: file_name,
 }
 <span class="hljs-keyword">for</span> file_name <span class="hljs-keyword">in</span> table[<span class="hljs-string">&quot;file_name&quot;</span>].to_pylist()
])
table.append_column(<span class="hljs-string">&quot;audio&quot;</span>, audio_array)
<span class="hljs-comment"># (Optional) Set the HF Audio type for the Dataset Viewer and the `datasets` library</span>
features = {<span class="hljs-string">&quot;audio&quot;</span>: {<span class="hljs-string">&quot;_type&quot;</span>: <span class="hljs-string">&quot;Audio&quot;</span>}} <span class="hljs-comment"># or using datasets.Features(...).to_dict()</span>
schema_metadata = {<span class="hljs-string">&quot;huggingface&quot;</span>: {<span class="hljs-string">&quot;dataset_info&quot;</span>: {<span class="hljs-string">&quot;features&quot;</span>: features}}}
table = table.replace_schema_metadata(schema_metadata)
<span class="hljs-comment"># Save to Parquet</span>
<span class="hljs-comment"># (Optional) with use_content_defined_chunking for faster uploads and downloads</span>
pq.write_table(table, <span class="hljs-string">&quot;data.parquet&quot;</span>, use_content_defined_chunking=<span class="hljs-literal">True</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-i7mbpg">Setting the Audio type in the Arrow schema metadata enables other libraries and the Hugging Face Dataset Viewer to recognise that “audio” contains audio data, not just binary data.</p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/hub-docs/blob/main/docs/hub/datasets-pyarrow.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p>
<script>
 {
 __sveltekit_1bollga = {
 assets: "/docs/hub/pr_2521/en",
 base: "/docs/hub/pr_2521/en",
 env: {}
 };
 const element = document.currentScript.parentElement;
 const data = [null,null];
 Promise.all([
 import("/docs/hub/pr_2521/en/_app/immutable/entry/start.d19e5ca7.js"),
 import("/docs/hub/pr_2521/en/_app/immutable/entry/app.98ab229a.js")
 ]).then(([kit, app]) => {
 kit.start(app, element, {
 node_ids: [0, 60],
 data,
 form: null,
 error: null
 });
 });
 }
 </script>

Xet Storage Details

Size:
55.3 kB
·
Xet hash:
0e94b44d4b4d093d7716ef797d1a41bc9efce4ce1756738b10f17fbed0c25632

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.