hub/pr_2521/en/datasets-streaming.html

<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Streaming datasets&quot;,&quot;local&quot;:&quot;streaming-datasets&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Integrated libraries&quot;,&quot;local&quot;:&quot;integrated-libraries&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Using the Hugging Face Client Library&quot;,&quot;local&quot;:&quot;using-the-hugging-face-client-library&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Using cURL&quot;,&quot;local&quot;:&quot;using-curl&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Streaming Parquet&quot;,&quot;local&quot;:&quot;streaming-parquet&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Stream Row Groups&quot;,&quot;local&quot;:&quot;stream-row-groups&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Efficient random access&quot;,&quot;local&quot;:&quot;efficient-random-access&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2}],&quot;depth&quot;:1}">
		<link href="/docs/hub/pr_2521/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
		<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/entry/start.d19e5ca7.js">
		<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/scheduler.409792a1.js">
		<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/singletons.1ece723a.js">
		<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/index.0f0d9f26.js">
		<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/paths.a67d9216.js">
		<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/entry/app.98ab229a.js">
		<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/preload-helper.78e52d9f.js">
		<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/index.92d389ff.js">
		<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/nodes/0.7799902b.js">
		<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/each.e59479a4.js">
		<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/nodes/62.6d1cc9e8.js">
		<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/CopyLLMTxtMenu.dcf7fb47.js">
		<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.0c189c7e.js">
		<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/CodeBlock.5ed269c1.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Streaming datasets&quot;,&quot;local&quot;:&quot;streaming-datasets&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Integrated libraries&quot;,&quot;local&quot;:&quot;integrated-libraries&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Using the Hugging Face Client Library&quot;,&quot;local&quot;:&quot;using-the-hugging-face-client-library&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Using cURL&quot;,&quot;local&quot;:&quot;using-curl&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Streaming Parquet&quot;,&quot;local&quot;:&quot;streaming-parquet&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Stream Row Groups&quot;,&quot;local&quot;:&quot;stream-row-groups&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Efficient random access&quot;,&quot;local&quot;:&quot;efficient-random-access&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2}],&quot;depth&quot;:1}"><!-- HEAD_svelte-u9bgzb_END -->      <p></p>  <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div>  <h1 class="relative group"><a id="streaming-datasets" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#streaming-datasets"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Streaming datasets</span></h1>  <h2 class="relative group"><a id="integrated-libraries" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#integrated-libraries"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Integrated libraries</span></h2> <p data-svelte-h="svelte-vjrxv5">If a dataset on the Hub is compatible with a <a href="./datasets-libraries">supported library</a> that allows streaming from Hugging Face, streaming the dataset can be done in just a few lines. For information on accessing the dataset, you can click on the “Use this dataset” button on the dataset page to see how to do so. For example, <a href="https://huggingface.co/datasets/knkarthick/samsum?library=datasets" rel="nofollow"><code>knkarthick/samsum</code></a> shows how to do so with <code>datasets</code> below.</p> <div class="flex justify-center" data-svelte-h="svelte-1ach5zb"><img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-usage.png"> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-usage-dark.png"></div> <div class="flex justify-center" data-svelte-h="svelte-1rm4drj"><img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-usage-modal.png"> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-usage-modal-dark.png"></div>  <h2 class="relative group"><a id="using-the-hugging-face-client-library" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#using-the-hugging-face-client-library"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Using the Hugging Face Client Library</span></h2> <p data-svelte-h="svelte-j4e5rx">You can use the <a href="/docs/huggingface_hub"><code>huggingface_hub</code></a> library to create, delete, and access files from repositories. For example, to stream the <code>allenai/c4</code> dataset in Python, simply install the library (we recommend using the latest version) and run the following code.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START -->pip install -U huggingface_hub<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> hffs

repo_id = <span class="hljs-string">&quot;allenai/c4&quot;</span>
path_in_repo = <span class="hljs-string">&quot;en/c4-train.00000-of-01024.json.gz&quot;</span>

<span class="hljs-comment"># Stream the file</span>
<span class="hljs-keyword">with</span> hffs.<span class="hljs-built_in">open</span>(<span class="hljs-string">f&quot;datasets/<span class="hljs-subst">{repo_id}</span>/<span class="hljs-subst">{path_in_repo}</span>&quot;</span>, <span class="hljs-string">&quot;r&quot;</span>, compression=<span class="hljs-string">&quot;gzip&quot;</span>) <span class="hljs-keyword">as</span> f:
    <span class="hljs-built_in">print</span>(f.readline())  <span class="hljs-comment"># read only the first line</span>
    <span class="hljs-comment"># {&quot;text&quot;:&quot;Beginners BBQ Class Taking Place in Missoula!...}</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1nzcb3j">See the <a href="https://huggingface.co/docs/huggingface_hub/en/guides/hf_file_system" rel="nofollow"><code>HfFileSystem</code> documentation</a> for more information.</p> <p data-svelte-h="svelte-1p3lfun">You can also integrate this into your own library! For example, you can quickly stream a CSV dataset using Pandas in batches.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd

repo_id = <span class="hljs-string">&quot;YOUR_REPO_ID&quot;</span>
path_in_repo = <span class="hljs-string">&quot;data.csv&quot;</span>

batch_size = <span class="hljs-number">5</span>

<span class="hljs-comment"># Stream the file</span>
<span class="hljs-keyword">with</span> hffs.<span class="hljs-built_in">open</span>(<span class="hljs-string">f&quot;datasets/<span class="hljs-subst">{repo_id}</span>/<span class="hljs-subst">{path_in_repo}</span>&quot;</span>) <span class="hljs-keyword">as</span> f:
    <span class="hljs-keyword">for</span> df <span class="hljs-keyword">in</span> pd.read_csv(f, iterator=<span class="hljs-literal">True</span>, chunksize=batch_size):  <span class="hljs-comment"># read 5 lines at a time</span>
        <span class="hljs-built_in">print</span>(<span class="hljs-built_in">len</span>(df))  <span class="hljs-comment"># 5</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1089vx6">Streaming is especially useful to read big files on Hugging Face progressively or only a small portion.
For example <code>tarfile</code> can iterate on the files of TAR archives, <code>zipfile</code> can read files from ZIP archives and <code>pyarrow</code> can access row groups of Parquet files.</p> <blockquote class="tip" data-svelte-h="svelte-19klf88"><p>There is an equivalent filesystem implementation in Rust available in <a href="https://github.com/apache/opendal" rel="nofollow">OpenDAL</a>.</p></blockquote>  <h2 class="relative group"><a id="using-curl" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#using-curl"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Using cURL</span></h2> <p data-svelte-h="svelte-15nvy23">Since all files on the Hub are available via HTTP, you can stream files using <code>cURL</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START --><span class="hljs-meta">&gt;&gt;&gt; </span>curl -L https://huggingface.co/datasets/fka/awesome-chatgpt-prompts/resolve/main/prompts.csv | <span class="hljs-built_in">head</span> -n 5
<span class="hljs-string">&quot;act&quot;</span>,<span class="hljs-string">&quot;prompt&quot;</span>
<span class="hljs-string">&quot;An Ethereum Developer&quot;</span>,<span class="hljs-string">&quot;Imagine you are an experienced Ethereum developer tasked with creating...
&quot;</span>SEO Prompt<span class="hljs-string">&quot;,&quot;</span>Using WebPilot, create an outline <span class="hljs-keyword">for</span> an article that will be 2,000 words on the ...
<span class="hljs-string">&quot;Linux Terminal&quot;</span>,<span class="hljs-string">&quot;I want you to act as a linux terminal. I will type commands and you will repl...
&quot;</span>English Translator and Improver<span class="hljs-string">&quot;,&quot;</span>I want you to act as an English translator, spelling correct...<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-16j2jzr">Use range requests to access a specific portion of a file:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START --><span class="hljs-meta">&gt;&gt;&gt; </span>curl -r 40-88 -L https://huggingface.co/datasets/fka/awesome-chatgpt-prompts/resolve/main/prompts.csv
Imagine you are an experienced Ethereum developer<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1mdwl33">Stream from private repositories using an <a href="https://huggingface.co/docs/hub/en/security-tokens" rel="nofollow">access token</a>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START --><span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">export</span> HF_TOKEN=hf_xxx
<span class="hljs-meta">&gt;&gt;&gt; </span>curl -H <span class="hljs-string">&quot;Authorization: Bearer <span class="hljs-variable">$HF_TOKEN</span>&quot;</span> -L https://huggingface.co/...<!-- HTML_TAG_END --></pre></div>  <h2 class="relative group"><a id="streaming-parquet" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#streaming-parquet"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Streaming Parquet</span></h2> <p data-svelte-h="svelte-6ictlu">Parquet is a great format for AI datasets. It offers good compression, a columnar structure for efficient processing and projections, and multi-level metadata for fast filtering, and is suitable for datasets of all sizes.</p> <p data-svelte-h="svelte-sv1uun">Parquet files are divided in row groups that are often around 100MB each. This lets data loaders and data processing frameworks stream data progressively, iterating on row groups.</p> <p data-svelte-h="svelte-ronzpj">Inside row groups are individual columns, which are divied into pages. Pages are the compressed block of around 1MB which contain the actual data.</p> <div class="flex justify-center" data-svelte-h="svelte-o2qal4"><img class="block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets_parquet_layout.png"></div>  <h3 class="relative group"><a id="stream-row-groups" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#stream-row-groups"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Stream Row Groups</span></h3> <p data-svelte-h="svelte-q2vmfq">Use PyArrow to stream row groups from Parquet files on Hugging Face:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> pyarrow.parquet <span class="hljs-keyword">as</span> pq

repo_id = <span class="hljs-string">&quot;HuggingFaceFW/finewiki&quot;</span>
path_in_repo = <span class="hljs-string">&quot;data/enwiki/000_00000.parquet&quot;</span>

<span class="hljs-comment"># Stream the Parquet file row group per row group</span>
<span class="hljs-keyword">with</span> pq.ParquetFile(<span class="hljs-string">f&quot;hf://datasets/<span class="hljs-subst">{repo_id}</span>/<span class="hljs-subst">{path_in_repo}</span>&quot;</span>) <span class="hljs-keyword">as</span> pf:
    <span class="hljs-keyword">for</span> row_group_idx <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(pf.num_row_groups):
        row_group_table = pf.read_row_group(row_group_idx)
        df = row_group_table.to_pandas()<!-- HTML_TAG_END --></pre></div> <blockquote class="tip" data-svelte-h="svelte-1xm5wi"><p>PyArrow supports <code>hf://</code> paths out-of-the-box and uses <code>HfFileSystem</code> automatically</p></blockquote> <p data-svelte-h="svelte-v848cc">Find more information in the <a href="./datasets-pyarrow">PyArrow documentation</a>.</p>  <h3 class="relative group"><a id="efficient-random-access" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#efficient-random-access"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Efficient random access</span></h3> <p data-svelte-h="svelte-1qac6o0">Row groups are further divided into columns, and columns into pages. Pages are often around 1MB and are the smallest unit of data in Parquet, since this is where compression is applied. Accessing pages enables loading specific rows without having to load a full row group, and is possible if the Parquet file has a page index. However not every Parquet frameworks support reading at the page level. PyArrow doesn’t for example, but the <code>parquet</code> crate in Rust does:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-rust "><!-- HTML_TAG_START --><span class="hljs-keyword">use</span> std::sync::Arc;
<span class="hljs-keyword">use</span> object_store::path::Path;
<span class="hljs-keyword">use</span> object_store_opendal::OpendalStore;
<span class="hljs-keyword">use</span> opendal::services::Huggingface;
<span class="hljs-keyword">use</span> opendal::Operator;
<span class="hljs-keyword">use</span> parquet::arrow::async_reader::ParquetObjectReader;
<span class="hljs-keyword">use</span> parquet::arrow::ParquetRecordBatchStreamBuilder;
<span class="hljs-keyword">use</span> futures::TryStreamExt;

<span class="hljs-meta">#[tokio::main]</span>
<span class="hljs-keyword">async</span> <span class="hljs-keyword">fn</span> <span class="hljs-title function_">main</span>() <span class="hljs-punctuation">-&gt;</span> <span class="hljs-type">Result</span>&lt;(), <span class="hljs-type">Box</span>&lt;<span class="hljs-keyword">dyn</span> std::error::Error&gt;&gt; {
    <span class="hljs-keyword">let</span> <span class="hljs-variable">repo_id</span> = <span class="hljs-string">&quot;HuggingFaceFW/finewiki&quot;</span>;
    <span class="hljs-keyword">let</span> <span class="hljs-variable">path_in_repo</span> = Path::<span class="hljs-title function_ invoke__">from</span>(<span class="hljs-string">&quot;data/enwiki/000_00000.parquet&quot;</span>);
    <span class="hljs-keyword">let</span> <span class="hljs-variable">offset</span> = <span class="hljs-number">0</span>;
    <span class="hljs-keyword">let</span> <span class="hljs-variable">limit</span> = <span class="hljs-number">10</span>;

    <span class="hljs-keyword">let</span> <span class="hljs-variable">builder</span> = Huggingface::<span class="hljs-title function_ invoke__">default</span>().<span class="hljs-title function_ invoke__">repo_type</span>(<span class="hljs-string">&quot;dataset&quot;</span>).<span class="hljs-title function_ invoke__">repo_id</span>(repo_id);
    <span class="hljs-keyword">let</span> <span class="hljs-variable">operator</span> = Operator::<span class="hljs-title function_ invoke__">new</span>(builder)?.<span class="hljs-title function_ invoke__">finish</span>();
    <span class="hljs-keyword">let</span> <span class="hljs-variable">store</span> = Arc::<span class="hljs-title function_ invoke__">new</span>(OpendalStore::<span class="hljs-title function_ invoke__">new</span>(operator));
    <span class="hljs-keyword">let</span> <span class="hljs-variable">reader</span> = ParquetObjectReader::<span class="hljs-title function_ invoke__">new</span>(store, path_in_repo.<span class="hljs-title function_ invoke__">clone</span>());
    <span class="hljs-keyword">let</span> <span class="hljs-variable">batch_stream</span> =
        ParquetRecordBatchStreamBuilder::<span class="hljs-title function_ invoke__">new</span>(reader).<span class="hljs-keyword">await</span>?
            .<span class="hljs-title function_ invoke__">with_offset</span>(offset <span class="hljs-keyword">as</span> <span class="hljs-type">usize</span>)
            .<span class="hljs-title function_ invoke__">with_limit</span>(limit <span class="hljs-keyword">as</span> <span class="hljs-type">usize</span>)
            .<span class="hljs-title function_ invoke__">build</span>()?;
    <span class="hljs-keyword">let</span> <span class="hljs-variable">results</span> = batch_stream.try_collect::&lt;<span class="hljs-type">Vec</span>&lt;_&gt;&gt;().<span class="hljs-keyword">await</span>?;
    <span class="hljs-built_in">println!</span>(<span class="hljs-string">&quot;Read {} batches&quot;</span>, results.<span class="hljs-title function_ invoke__">len</span>());
    <span class="hljs-title function_ invoke__">Ok</span>(())
}<!-- HTML_TAG_END --></pre></div> <blockquote class="tip" data-svelte-h="svelte-axq32b"><p>In Rust we use OpenDAL’s <code>Huggingface</code> service which is equivalent to <code>HfFileSystem</code> in python</p></blockquote> <p data-svelte-h="svelte-16nd5um">Pass <code>write_page_index=True</code> in PyArrow to include the page index that enables efficient random access.
It notably adds “offset_index_offset” and “offset_index_length” to Parquet columns that you can see in the <a href="https://huggingface.co/blog/cfahlgren1/intro-to-parquet-format" rel="nofollow">Parquet metadata viewer on Hugging Face</a>.
Page indexes also speed up the <a href="https://huggingface.co/docs/dataset-viewer" rel="nofollow">Hugging Face Dataset Viewer</a> and allows it to show data without row group size limit.</p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/hub-docs/blob/main/docs/hub/datasets-streaming.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a>  <p></p> 
			
			<script>
				{
					__sveltekit_1bollga = {
						assets: "/docs/hub/pr_2521/en",
						base: "/docs/hub/pr_2521/en",
						env: {}
					};

					const element = document.currentScript.parentElement;

					const data = [null,null];

					Promise.all([
						import("/docs/hub/pr_2521/en/_app/immutable/entry/start.d19e5ca7.js"),
						import("/docs/hub/pr_2521/en/_app/immutable/entry/app.98ab229a.js")
					]).then(([kit, app]) => {
						kit.start(app, element, {
							node_ids: [0, 62],
							data,
							form: null,
							error: null
						});
					});
				}
			</script>