Buckets:

hf-doc-build
/

doc-dev

Files

xet

hf-doc-build/doc-dev / hub /pr_2521 /en /datasets-spark.html

HuggingFaceDocBuilder

about 1 month ago

download

raw

57 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Spark","local":"spark","sections":[{"title":"Set up","local":"set-up","sections":[{"title":"Installation","local":"installation","sections":[],"depth":3},{"title":"Authentication","local":"authentication","sections":[],"depth":3},{"title":"Enable the “huggingface” Data Source","local":"enable-the-huggingface-data-source","sections":[],"depth":3}],"depth":2},{"title":"Read","local":"read","sections":[{"title":"Options","local":"options","sections":[],"depth":3},{"title":"Run SQL queries","local":"run-sql-queries","sections":[],"depth":3}],"depth":2},{"title":"Write","local":"write","sections":[{"title":"Mode","local":"mode","sections":[],"depth":3},{"title":"Options","local":"options","sections":[],"depth":3}],"depth":2},{"title":"Storage Buckets","local":"storage-buckets","sections":[],"depth":2}],"depth":1}">
	<link href="/docs/hub/pr_2521/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/entry/start.d19e5ca7.js">
	<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/scheduler.409792a1.js">
	<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/singletons.1ece723a.js">
	<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/index.0f0d9f26.js">
	<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/paths.a67d9216.js">
	<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/entry/app.98ab229a.js">
	<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/preload-helper.78e52d9f.js">
	<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/index.92d389ff.js">
	<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/nodes/0.7799902b.js">
	<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/each.e59479a4.js">
	<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/nodes/61.cd1944b2.js">
	<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/CopyLLMTxtMenu.dcf7fb47.js">
	<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.0c189c7e.js">
	<link rel="modulepreload" href="/docs/hub/pr_2521/en/_app/immutable/chunks/CodeBlock.5ed269c1.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Spark","local":"spark","sections":[{"title":"Set up","local":"set-up","sections":[{"title":"Installation","local":"installation","sections":[],"depth":3},{"title":"Authentication","local":"authentication","sections":[],"depth":3},{"title":"Enable the “huggingface” Data Source","local":"enable-the-huggingface-data-source","sections":[],"depth":3}],"depth":2},{"title":"Read","local":"read","sections":[{"title":"Options","local":"options","sections":[],"depth":3},{"title":"Run SQL queries","local":"run-sql-queries","sections":[],"depth":3}],"depth":2},{"title":"Write","local":"write","sections":[{"title":"Mode","local":"mode","sections":[],"depth":3},{"title":"Options","local":"options","sections":[],"depth":3}],"depth":2},{"title":"Storage Buckets","local":"storage-buckets","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <h1 class="relative group"><a id="spark" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#spark"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Spark</span></h1> <p data-svelte-h="svelte-s0r53o">Spark enables real-time, large-scale data processing in a distributed environment.</p> <p data-svelte-h="svelte-ireapg">You can use <code>pyspark_huggingface</code> to access Hugging Face datasets repositories in PySpark via the “huggingface” Data Source.</p> <p data-svelte-h="svelte-17lvzmk">Try out <a href="https://huggingface.co/spaces/Dataset-Tools/Spark-Notebooks" rel="nofollow">Spark Notebooks</a> on Hugging Face Spaces to get Notebooks with PySpark and <code>pyspark_huggingface</code> pre-installed.</p> <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/spark-notebooks-min.png"> <h2 class="relative group"><a id="set-up" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#set-up"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Set up</span></h2> <h3 class="relative group"><a id="installation" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#installation"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Installation</span></h3> <p data-svelte-h="svelte-1i6u7b2">To be able to read and write to Hugging Face Datasets, you need to install the <code>pyspark_huggingface</code> library:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=" "><!-- HTML_TAG_START -->pip <span class="hljs-keyword">install</span> pyspark_huggingface<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1dyqnq0">This will also install required dependencies like <code>huggingface_hub</code> for authentication, and <code>pyarrow</code> for reading and writing datasets.</p> <h3 class="relative group"><a id="authentication" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#authentication"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Authentication</span></h3> <p data-svelte-h="svelte-aqw7kt">You need to authenticate to Hugging Face to read private/gated dataset repositories or to write to your dataset repositories.</p> <p data-svelte-h="svelte-pdivv8">You can use the CLI for example:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=" "><!-- HTML_TAG_START --><span class="hljs-attribute">hf auth login</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-dcl8ev">It’s also possible to provide your Hugging Face token with the <code>HF_TOKEN</code> environment variable or passing the <code>token</code> option to the reader.
	For more details about authentication, check out <a href="https://huggingface.co/docs/huggingface_hub/quick-start#authentication" rel="nofollow">this guide</a>.</p> <h3 class="relative group"><a id="enable-the-huggingface-data-source" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#enable-the-huggingface-data-source"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Enable the “huggingface” Data Source</span></h3> <p data-svelte-h="svelte-8zj7ot">PySpark 4 came with a new Data Source API which allows to use datasets from custom sources.
	If <code>pyspark_huggingface</code> is installed, PySpark auto-imports it and enables the “huggingface” Data Source.</p> <p data-svelte-h="svelte-5pp20">The library also backports the Data Source API for the “huggingface” Data Source for PySpark 3.5, 3.4 and 3.3.
	However in this case <code>pyspark_huggingface</code> should be imported explicitly to activate the backport and enable the “huggingface” Data Dource:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> pyspark_huggingface
	huggingface datasource enabled <span class="hljs-keyword">for</span> pyspark <span class="hljs-number">3.</span>x.x (backport <span class="hljs-keyword">from</span> pyspark <span class="hljs-number">4</span>)<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="read" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#read"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Read</span></h2> <p data-svelte-h="svelte-1k66fhy">The “huggingface” Data Source allows to read datasets from Hugging Face, using <code>pyarrow</code> under the hood to stream Arrow data.
	This is compatible with all the dataset in <a href="https://huggingface.co/docs/hub/datasets-adding#file-formats" rel="nofollow">supported format</a> on Hugging Face, like Parquet datasets.</p> <p data-svelte-h="svelte-yczgy2">For example here is how to load the <a href="https://huggingface.co/stanfordnlp/imdb" rel="nofollow">stanfordnlp/imdb</a> dataset:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> pyspark_huggingface
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> pyspark.sql <span class="hljs-keyword">import</span> SparkSession
	<span class="hljs-meta">>>> </span>spark = SparkSession.builder.appName(<span class="hljs-string">"demo"</span>).getOrCreate()
	<span class="hljs-meta">>>> </span>df = spark.read.<span class="hljs-built_in">format</span>(<span class="hljs-string">"huggingface"</span>).load(<span class="hljs-string">"stanfordnlp/imdb"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1q9wgqv">Here is another example with the <a href="https://huggingface.co/datasets/BAAI/Infinity-Instruct" rel="nofollow">BAAI/Infinity-Instruct</a> dataset.
	It is a gated repository, users have to accept the terms of use before accessing it.
	It also has multiple subsets, namely, “3M” and “7M”. So we need to specify which one to load.</p> <div class="flex justify-center" data-svelte-h="svelte-1ct11n9"><img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-spark-infinity-instruct-7M-min.png"> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-spark-infinity-instruct-7M-dark-min.png"></div> <p data-svelte-h="svelte-phbdhi">We use the <code>.format()</code> function to use the “huggingface” Data Source, and <code>.load()</code> to load the dataset (more precisely the config or subset named “7M” containing 7M samples). Then we compute the number of dialogue per language and filter the dataset.</p> <p data-svelte-h="svelte-19d7o0j">After logging-in to access the gated repository, we can run:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> pyspark_huggingface
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> pyspark.sql <span class="hljs-keyword">import</span> SparkSession
	<span class="hljs-meta">>>> </span>spark = SparkSession.builder.appName(<span class="hljs-string">"demo"</span>).getOrCreate()
	<span class="hljs-meta">>>> </span>df = spark.read.<span class="hljs-built_in">format</span>(<span class="hljs-string">"huggingface"</span>).option(<span class="hljs-string">"config"</span>, <span class="hljs-string">"7M"</span>).load(<span class="hljs-string">"BAAI/Infinity-Instruct"</span>)
	<span class="hljs-meta">>>> </span>df.show()
	+---+----------------------------+-----+----------+--------------------+
	\| <span class="hljs-built_in">id</span>\| conversations\|label\|langdetect\| source\|
	+---+----------------------------+-----+----------+--------------------+
	\| <span class="hljs-number">0</span>\| [{human, <span class="hljs-keyword">def</span> <span class="hljs-title function_">exti</span>...\| \| en\| code_exercises\|
	\| <span class="hljs-number">1</span>\| [{human, See the ...\| \| en\| flan\|
	\| <span class="hljs-number">2</span>\| [{human, This <span class="hljs-keyword">is</span> ...\| \| en\| flan\|
	\| <span class="hljs-number">3</span>\| [{human, If you d...\| \| en\| flan\|
	\| <span class="hljs-number">4</span>\| [{human, In a Uni...\| \| en\| flan\|
	\| <span class="hljs-number">5</span>\| [{human, Read the...\| \| en\| flan\|
	\| <span class="hljs-number">6</span>\| [{human, You are ...\| \| en\| code_bagel\|
	\| <span class="hljs-number">7</span>\| [{human, I want y...\| \| en\| Subjective\|
	\| <span class="hljs-number">8</span>\| [{human, Given th...\| \| en\| flan\|
	\| <span class="hljs-number">9</span>\|[{human, 因果联系原则是法...\| \| zh-cn\| Subjective\|
	\| <span class="hljs-number">10</span>\| [{human, Provide ...\| \| en\|self-oss-instruct...\|
	\| <span class="hljs-number">11</span>\| [{human, The univ...\| \| en\| flan\|
	\| <span class="hljs-number">12</span>\| [{human, Q: I am ...\| \| en\| flan\|
	\| <span class="hljs-number">13</span>\| [{human, What <span class="hljs-keyword">is</span> ...\| \| en\| OpenHermes-<span class="hljs-number">2.5</span>\|
	\| <span class="hljs-number">14</span>\| [{human, In react...\| \| en\| flan\|
	\| <span class="hljs-number">15</span>\| [{human, Write Py...\| \| en\| code_exercises\|
	\| <span class="hljs-number">16</span>\| [{human, Find the...\| \| en\| MetaMath\|
	\| <span class="hljs-number">17</span>\| [{human, Three of...\| \| en\| MetaMath\|
	\| <span class="hljs-number">18</span>\| [{human, Chandra ...\| \| en\| MetaMath\|
	\| <span class="hljs-number">19</span>\|[{human, 用经济学知识分析...\| \| zh-cn\| Subjective\|
	+---+----------------------------+-----+----------+--------------------+<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-rdc7zs">This loads the dataset in a streaming fashion, and the output DataFrame has one partition per data file in the dataset to enable efficient distributed processing.</p> <p data-svelte-h="svelte-1du45l">To compute the number of dialogues per language we run this code that uses the <code>columns</code> option and a <code>groupBy()</code> operation.
	The <code>columns</code> option is useful to only load the data we need, since PySpark doesn’t enable predicate push-down with the Data Source API.
	There is also a <code>filters</code> option to only load data with values within a certain range.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>df_langdetect_only = (
	<span class="hljs-meta">... </span> spark.read.<span class="hljs-built_in">format</span>(<span class="hljs-string">"huggingface"</span>)
	<span class="hljs-meta">... </span> .option(<span class="hljs-string">"config"</span>, <span class="hljs-string">"7M"</span>)
	<span class="hljs-meta">... </span> .option(<span class="hljs-string">"columns"</span>, <span class="hljs-string">'["langdetect"]'</span>)
	<span class="hljs-meta">... </span> .load(<span class="hljs-string">"BAAI/Infinity-Instruct"</span>)
	<span class="hljs-meta">... </span>)
	<span class="hljs-meta">>>> </span>df_langdetect_only.groupBy(<span class="hljs-string">"langdetect"</span>).count().show()
	+----------+-------+
	\|langdetect\| count\|
	+----------+-------+
	\| en\|<span class="hljs-number">6697793</span>\|
	\| zh-cn\| <span class="hljs-number">751313</span>\|
	+----------+-------+<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-e2gcj5">To filter the dataset and only keep dialogues in Chinese:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>df_chinese_only = (
	<span class="hljs-meta">... </span> spark.read.<span class="hljs-built_in">format</span>(<span class="hljs-string">"huggingface"</span>)
	<span class="hljs-meta">... </span> .option(<span class="hljs-string">"config"</span>, <span class="hljs-string">"7M"</span>)
	<span class="hljs-meta">... </span> .option(<span class="hljs-string">"filters"</span>, <span class="hljs-string">'[("langdetect", "=", "zh-cn")]'</span>)
	<span class="hljs-meta">... </span> .load(<span class="hljs-string">"BAAI/Infinity-Instruct"</span>)
	<span class="hljs-meta">... </span>)
	<span class="hljs-meta">>>> </span>df_chinese_only.show()
	+---+----------------------------+-----+----------+----------+
	\| <span class="hljs-built_in">id</span>\| conversations\|label\|langdetect\| source\|
	+---+----------------------------+-----+----------+----------+
	\| <span class="hljs-number">9</span>\|[{human, 因果联系原则是法...\| \| zh-cn\|Subjective\|
	\| <span class="hljs-number">19</span>\|[{human, 用经济学知识分析...\| \| zh-cn\|Subjective\|
	\| <span class="hljs-number">38</span>\| [{human, 某个考试共有A、...\| \| zh-cn\|Subjective\|
	\| <span class="hljs-number">39</span>\|[{human, 撰写一篇关于斐波...\| \| zh-cn\|Subjective\|
	\| <span class="hljs-number">57</span>\|[{human, 总结世界历史上的...\| \| zh-cn\|Subjective\|
	\| <span class="hljs-number">61</span>\|[{human, 生成一则广告词。...\| \| zh-cn\|Subjective\|
	\| <span class="hljs-number">66</span>\|[{human, 描述一个有效的团...\| \| zh-cn\|Subjective\|
	\| <span class="hljs-number">94</span>\|[{human, 如果比利和蒂芙尼...\| \| zh-cn\|Subjective\|
	\|<span class="hljs-number">102</span>\|[{human, 生成一句英文名言...\| \| zh-cn\|Subjective\|
	\|<span class="hljs-number">106</span>\|[{human, 写一封感谢信，感...\| \| zh-cn\|Subjective\|
	\|<span class="hljs-number">118</span>\| [{human, 生成一个故事。}...\| \| zh-cn\|Subjective\|
	\|<span class="hljs-number">174</span>\|[{human, 高胆固醇水平的后...\| \| zh-cn\|Subjective\|
	\|<span class="hljs-number">180</span>\|[{human, 基于以下角色信息...\| \| zh-cn\|Subjective\|
	\|<span class="hljs-number">192</span>\|[{human, 请写一篇文章，概...\| \| zh-cn\|Subjective\|
	\|<span class="hljs-number">221</span>\|[{human, 以诗歌形式表达对...\| \| zh-cn\|Subjective\|
	\|<span class="hljs-number">228</span>\|[{human, 根据给定的指令，...\| \| zh-cn\|Subjective\|
	\|<span class="hljs-number">236</span>\|[{human, 打开一个新的生成...\| \| zh-cn\|Subjective\|
	\|<span class="hljs-number">260</span>\|[{human, 生成一个有关未来...\| \| zh-cn\|Subjective\|
	\|<span class="hljs-number">268</span>\|[{human, 如果有一定数量的...\| \| zh-cn\|Subjective\|
	\|<span class="hljs-number">273</span>\| [{human, 题目：小明有<span class="hljs-number">5</span>个...\| \| zh-cn\|Subjective\|
	+---+----------------------------+-----+----------+----------+<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1au61ak">It is also possible to apply filters or remove columns on the loaded DataFrame, but it is more efficient to do it while loading, especially on Parquet datasets.
	Indeed, Parquet contains metadata at the file and row group level, which allows to skip entire parts of the dataset that don’t contain samples that satisfy the criteria. Columns in Parquet can also be loaded independently, which allows to skip the excluded columns and avoid loading unnecessary data.</p> <h3 class="relative group"><a id="options" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#options"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Options</span></h3> <p data-svelte-h="svelte-1s464mf">Here is the list of available options you can pass to <code>read..option()</code>:</p> <ul data-svelte-h="svelte-1tfwhkx"><li><code>config</code> (string): select a dataset subset/config</li> <li><code>split</code> (string): select a dataset split (default is “train”)</li> <li><code>token</code> (string): your Hugging Face token</li></ul> <p data-svelte-h="svelte-xfbniv">Instead of specifying a config or split, you can select which files to load manually:</p> <ul data-svelte-h="svelte-uz1zj2"><li><code>data_dir</code> (string): select a directory</li> <li><code>data_files</code> (string): select one or many files, e.g. <code>"data/*.parquet"</code> or <code>'["part1.parquet", "par2.parquet"]'</code></li></ul> <p data-svelte-h="svelte-luhsa2">For Parquet datasets:</p> <ul data-svelte-h="svelte-cja3or"><li><code>columns</code> (string): select a subset of columns to load, e.g. <code>'["id"]'</code></li> <li><code>filters</code> (string): to skip files and row groups that don’t match a criteria, e.g. <code>'[("source", "=", "code_exercises")]'</code>. Filters are passed to <a href="https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetDataset.html" rel="nofollow">pyarrow.parquet.ParquetDataset</a>.</li></ul> <p data-svelte-h="svelte-1reyf4q">Any other option is passed as an argument to [datasets.load_dataset] (<a href="https://huggingface.co/docs/datasets/en/package_reference/loading_methods#datasets.load_dataset" rel="nofollow">https://huggingface.co/docs/datasets/en/package_reference/loading_methods#datasets.load_dataset</a>)</p> <h3 class="relative group"><a id="run-sql-queries" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#run-sql-queries"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Run SQL queries</span></h3> <p data-svelte-h="svelte-13i529n">Once you have your PySpark Dataframe ready, you can run SQL queries using <code>spark.sql</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> pyspark_huggingface
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> pyspark.sql <span class="hljs-keyword">import</span> SparkSession
	<span class="hljs-meta">>>> </span>spark = SparkSession.builder.appName(<span class="hljs-string">"demo"</span>).getOrCreate()
	<span class="hljs-meta">>>> </span>df = (
	<span class="hljs-meta">... </span> spark.read.<span class="hljs-built_in">format</span>(<span class="hljs-string">"huggingface"</span>)
	<span class="hljs-meta">... </span> .option(<span class="hljs-string">"config"</span>, <span class="hljs-string">"7M"</span>)
	<span class="hljs-meta">... </span> .option(<span class="hljs-string">"columns"</span>, <span class="hljs-string">'["source"]'</span>)
	<span class="hljs-meta">... </span> .load(<span class="hljs-string">"BAAI/Infinity-Instruct"</span>)
	<span class="hljs-meta">... </span>)
	<span class="hljs-meta">>>> </span>spark.sql(<span class="hljs-string">"SELECT source, count(*) AS total FROM {df} GROUP BY source ORDER BY total DESC"</span>, df=df).show()
	+--------------------+-------+
	\| source\| total\|
	+--------------------+-------+
	\| flan\|<span class="hljs-number">2435840</span>\|
	\| Subjective\|<span class="hljs-number">1342427</span>\|
	\| OpenHermes-<span class="hljs-number">2.5</span>\| <span class="hljs-number">855478</span>\|
	\| MetaMath\| <span class="hljs-number">690138</span>\|
	\| code_exercises\| <span class="hljs-number">590958</span>\|
	\|Orca-math-word-pr...\| <span class="hljs-number">398168</span>\|
	\| code_bagel\| <span class="hljs-number">386649</span>\|
	\| MathInstruct\| <span class="hljs-number">329254</span>\|
	\|python-code-datas...\| <span class="hljs-number">88632</span>\|
	\|instructional_cod...\| <span class="hljs-number">82920</span>\|
	\| CodeFeedback\| <span class="hljs-number">79513</span>\|
	\|self-oss-instruct...\| <span class="hljs-number">50467</span>\|
	\|Evol-Instruct-Cod...\| <span class="hljs-number">43354</span>\|
	\|CodeExercise-Pyth...\| <span class="hljs-number">27159</span>\|
	\|code_instructions...\| <span class="hljs-number">23130</span>\|
	\| Code-Instruct-700k\| <span class="hljs-number">10860</span>\|
	\|Glaive-code-assis...\| <span class="hljs-number">9281</span>\|
	\|python_code_instr...\| <span class="hljs-number">2581</span>\|
	\|Python-Code-23k-S...\| <span class="hljs-number">2297</span>\|
	+--------------------+-------+<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-fpkb7l">Again, specifying the <code>columns</code> option is not necessary, but is useful to avoid loading unnecessary data and make the query faster.</p> <h2 class="relative group"><a id="write" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#write"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Write</span></h2> <p data-svelte-h="svelte-140g2zh">You can write a PySpark Dataframe to Hugging Face with the “huggingface” Data Source.
	It uploads Parquet files in parallel in a distributed manner, and only commits the files once they’re all uploaded.
	It works like this:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> pyspark_huggingface
	<span class="hljs-meta">>>> </span>df.write.<span class="hljs-built_in">format</span>(<span class="hljs-string">"huggingface"</span>).save(<span class="hljs-string">"username/dataset_name"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1d35p3o">Here is how we can use this function to write the filtered version of the <a href="https://huggingface.co/datasets/BAAI/Infinity-Instruct" rel="nofollow">BAAI/Infinity-Instruct</a> dataset back to Hugging Face.</p> <p data-svelte-h="svelte-vthxjd">First you need to <a href="https://huggingface.co/new-dataset" rel="nofollow">create a dataset repository</a>, e.g. <code>username/Infinity-Instruct-Chinese-Only</code> (you can set it to private if you want).
	Then, make sure you are authenticated and you can use the “huggingface” Data Source, set the <code>mode</code> to “overwrite” (or “append” if you want to extend an existing dataset), and push to Hugging Face with <code>.save()</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>df_chinese_only.write.<span class="hljs-built_in">format</span>(<span class="hljs-string">"huggingface"</span>).mode(<span class="hljs-string">"overwrite"</span>).save(<span class="hljs-string">"username/Infinity-Instruct-Chinese-Only"</span>)<!-- HTML_TAG_END --></pre></div> <div class="flex justify-center" data-svelte-h="svelte-tmflad"><img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-spark-infinity-instruct-chinese-only-min.png"> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-spark-infinity-instruct-chinese-only-dark-min.png"></div> <h3 class="relative group"><a id="mode" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#mode"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Mode</span></h3> <p data-svelte-h="svelte-zuvewl">Two modes are available when pushing a dataset to Hugging Face:</p> <ul data-svelte-h="svelte-e9isj"><li>“overwrite”: overwrite the dataset if it already exists</li> <li>“append”: append the dataset to an existing dataset</li></ul> <h3 class="relative group"><a id="options" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#options"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Options</span></h3> <p data-svelte-h="svelte-1acq8iy">Here is the list of available options you can pass to <code>write.option()</code>:</p> <ul data-svelte-h="svelte-sthc9l"><li><code>token</code> (string): your Hugging Face token</li></ul> <p data-svelte-h="svelte-cukjdh">Contributions are welcome to add more options here, in particular <code>subset</code> and <code>split</code>.</p> <h2 class="relative group"><a id="storage-buckets" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#storage-buckets"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Storage Buckets</span></h2> <p data-svelte-h="svelte-gbiz71">It is common to process raw data in <a href="/docs/hub/storage-buckets">Storage Buckets</a> and experiment there before publishing AI-ready data in Dataset repositories.</p> <p data-svelte-h="svelte-7ttk7a">Access Storage Buckets the same way as Dataset repositories but with the <code>buckets/</code> prefix and with the <code>data_dir</code> or <code>data_files</code> options:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>df = spark.read.<span class="hljs-built_in">format</span>(<span class="hljs-string">"huggingface"</span>).option(<span class="hljs-string">"data_dir"</span>, <span class="hljs-string">"data"</span>).load(<span class="hljs-string">"buckets/username/my-bucket"</span>)
	<span class="hljs-meta">>>> </span><span class="hljs-comment"># OR with a glob pattern</span>
	<span class="hljs-meta">>>> </span><span class="hljs-comment"># df = spark.read.format("huggingface").option("data_files", "data/*.parquet").load("buckets/username/my-bucket")</span>
	<span class="hljs-meta">>>> </span>df.write.<span class="hljs-built_in">format</span>(<span class="hljs-string">"huggingface"</span>).option(<span class="hljs-string">"data_dir"</span>, <span class="hljs-string">"new-data"</span>).save(<span class="hljs-string">"buckets/username/my-bucket"</span>)<!-- HTML_TAG_END --></pre></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/hub-docs/blob/main/docs/hub/datasets-spark.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_1bollga = {
	assets: "/docs/hub/pr_2521/en",
	base: "/docs/hub/pr_2521/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/hub/pr_2521/en/_app/immutable/entry/start.d19e5ca7.js"),
	import("/docs/hub/pr_2521/en/_app/immutable/entry/app.98ab229a.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 61],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 57 kB
Xet hash:: 7303aa617af3123c85aebe04caf1dc3312d99250bc88f5d78921763e5936297d

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.