datasets/pr_8021/en/_app/immutable/nodes/5.64cd9f43.js

import{s as pt,n as ht,o as mt}from"../chunks/scheduler.d75c11ed.js";import{S as _t,i as gt,e as d,s as i,c as f,h as bt,a as r,d as a,b as l,f as ut,g as c,j as n,k as ee,l as vt,m as s,n as u,t as p,o as h,p as m}from"../chunks/index.4ec9dfe9.js";import{C as yt,H as Y,E as kt}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.ee0f129e.js";function wt(Ue){let o,te,K,ae,v,se,y,ie,k,Re='Nearly every deep learning workflow begins with loading a dataset, which makes it one of the most important steps. With 🤗 Datasets, there are more than 900 datasets available to help you get started with your NLP task. All you have to do is call: <a href="/docs/datasets/pr_8021/en/package_reference/loading_methods#datasets.load_dataset">load_dataset()</a> to take your first step. This function is a true workhorse in every sense because it builds and loads every dataset you use.',le,w,de,T,je="Let’s begin with a basic Explain Like I’m Five.",re,$,Oe="A dataset is a directory that contains:",ne,x,Ge="<li>Some data files in generic formats (JSON, CSV, Parquet, text, etc.)</li> <li>A dataset card named <code>README.md</code> that contains documentation about the dataset as well as a YAML header to define the datasets tags and configurations</li>",oe,C,Ne=`The <a href="/docs/datasets/pr_8021/en/package_reference/loading_methods#datasets.load_dataset">load_dataset()</a> function fetches the requested dataset locally or from the Hugging Face Hub.
The Hub is a central repository where all the Hugging Face datasets and models are stored.`,fe,L,We=`If the dataset only contains data files, then <a href="/docs/datasets/pr_8021/en/package_reference/loading_methods#datasets.load_dataset">load_dataset()</a> automatically infers how to load the data files from their extensions (json, csv, parquet, txt, etc.).
Under the hood, 🤗 Datasets will use an appropriate <a href="/docs/datasets/pr_8021/en/package_reference/builder_classes#datasets.DatasetBuilder">DatasetBuilder</a> based on the data files format. There exist one builder per data file format in 🤗 Datasets:`,ce,M,ze='<li><a href="/docs/datasets/pr_8021/en/package_reference/loading_methods#datasets.packaged_modules.text.Text">datasets.packaged_modules.text.Text</a> for text</li> <li><a href="/docs/datasets/pr_8021/en/package_reference/loading_methods#datasets.packaged_modules.csv.Csv">datasets.packaged_modules.csv.Csv</a> for CSV and TSV</li> <li><a href="/docs/datasets/pr_8021/en/package_reference/loading_methods#datasets.packaged_modules.json.Json">datasets.packaged_modules.json.Json</a> for JSON and JSONL</li> <li><a href="/docs/datasets/pr_8021/en/package_reference/loading_methods#datasets.packaged_modules.parquet.Parquet">datasets.packaged_modules.parquet.Parquet</a> for Parquet</li> <li><a href="/docs/datasets/pr_8021/en/package_reference/loading_methods#datasets.packaged_modules.arrow.Arrow">datasets.packaged_modules.arrow.Arrow</a> for Arrow (streaming file format)</li> <li><a href="/docs/datasets/pr_8021/en/package_reference/loading_methods#datasets.packaged_modules.sql.Sql">datasets.packaged_modules.sql.Sql</a> for SQL databases</li> <li><a href="/docs/datasets/pr_8021/en/package_reference/loading_methods#datasets.packaged_modules.imagefolder.ImageFolder">datasets.packaged_modules.imagefolder.ImageFolder</a> for image folders</li> <li><a href="/docs/datasets/pr_8021/en/package_reference/loading_methods#datasets.packaged_modules.audiofolder.AudioFolder">datasets.packaged_modules.audiofolder.AudioFolder</a> for audio folders</li>',ue,_,Je='<p>Read the <a href="./upload_dataset">Share</a> section to learn more about how to share a dataset.</p>',pe,B,Ve=`🤗 Datasets downloads the dataset files from the original URL, generates the dataset and caches it in an Arrow table on your drive.
If you’ve downloaded the dataset before, then 🤗 Datasets will reload it from the cache to save you the trouble of downloading it again.`,he,H,Ze="Now that you have a high-level understanding about how datasets are built, let’s take a closer look at the nuts and bolts of how all this works.",me,D,_e,P,Qe='When you load a dataset for the first time, 🤗 Datasets takes the raw data file and builds it into a table of rows and typed columns. There are two main classes responsible for building a dataset: <a href="/docs/datasets/pr_8021/en/package_reference/builder_classes#datasets.BuilderConfig">BuilderConfig</a> and <a href="/docs/datasets/pr_8021/en/package_reference/builder_classes#datasets.DatasetBuilder">DatasetBuilder</a>.',ge,g,Ye='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/builderconfig.png"/>',be,S,ve,q,Ke='<a href="/docs/datasets/pr_8021/en/package_reference/builder_classes#datasets.BuilderConfig">BuilderConfig</a> is the configuration class of <a href="/docs/datasets/pr_8021/en/package_reference/builder_classes#datasets.DatasetBuilder">DatasetBuilder</a>. The <a href="/docs/datasets/pr_8021/en/package_reference/builder_classes#datasets.BuilderConfig">BuilderConfig</a> contains the following basic attributes about a dataset:',ye,A,Xe="<thead><tr><th>Attribute</th> <th>Description</th></tr></thead> <tbody><tr><td><code>name</code></td> <td>Short name of the dataset.</td></tr> <tr><td><code>version</code></td> <td>Dataset version identifier.</td></tr> <tr><td><code>data_dir</code></td> <td>Stores the path to a local folder containing the data files.</td></tr> <tr><td><code>data_files</code></td> <td>Stores paths to local data files.</td></tr> <tr><td><code>description</code></td> <td>Description of the dataset.</td></tr></tbody>",ke,I,et='If you want to add additional attributes to your dataset such as the class labels, you can subclass the base <a href="/docs/datasets/pr_8021/en/package_reference/builder_classes#datasets.BuilderConfig">BuilderConfig</a> class. There are two ways to populate the attributes of a <a href="/docs/datasets/pr_8021/en/package_reference/builder_classes#datasets.BuilderConfig">BuilderConfig</a> class or subclass:',we,E,tt='<li><p>Provide a list of predefined <a href="/docs/datasets/pr_8021/en/package_reference/builder_classes#datasets.BuilderConfig">BuilderConfig</a> class (or subclass) instances in the datasets <code>DatasetBuilder.BUILDER_CONFIGS()</code> attribute.</p></li> <li><p>When you call <a href="/docs/datasets/pr_8021/en/package_reference/loading_methods#datasets.load_dataset">load_dataset()</a>, any keyword arguments that are not specific to the method will be used to set the associated attributes of the <a href="/docs/datasets/pr_8021/en/package_reference/builder_classes#datasets.BuilderConfig">BuilderConfig</a> class. This will override the predefined attributes if a specific configuration was selected.</p></li>',Te,F,at='You can also set the <a href="/docs/datasets/pr_8021/en/package_reference/builder_classes#datasets.BuilderConfig">DatasetBuilder.BUILDER_CONFIG_CLASS</a> to any custom subclass of <a href="/docs/datasets/pr_8021/en/package_reference/builder_classes#datasets.BuilderConfig">BuilderConfig</a>.',$e,U,xe,R,st='<a href="/docs/datasets/pr_8021/en/package_reference/builder_classes#datasets.DatasetBuilder">DatasetBuilder</a> accesses all the attributes inside <a href="/docs/datasets/pr_8021/en/package_reference/builder_classes#datasets.BuilderConfig">BuilderConfig</a> to build the actual dataset.',Ce,b,it='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/datasetbuilder.png"/>',Le,j,lt='There are three main methods in <a href="/docs/datasets/pr_8021/en/package_reference/builder_classes#datasets.DatasetBuilder">DatasetBuilder</a>:',Me,O,dt='<li><p><code>DatasetBuilder._info()</code> is in charge of defining the dataset attributes. When you call <code>dataset.info</code>, 🤗 Datasets returns the information stored here. Likewise, the <a href="/docs/datasets/pr_8021/en/package_reference/main_classes#datasets.Features">Features</a> are also specified here. Remember, the <a href="/docs/datasets/pr_8021/en/package_reference/main_classes#datasets.Features">Features</a> are like the skeleton of the dataset. It provides the names and types of each column.</p></li> <li><p><code>DatasetBuilder._split_generator</code> downloads or retrieves the requested data files, organizes them into splits, and defines specific arguments for the generation process. This method has a <a href="/docs/datasets/pr_8021/en/package_reference/builder_classes#datasets.DownloadManager">DownloadManager</a> that downloads files or fetches them from your local filesystem. Within the <a href="/docs/datasets/pr_8021/en/package_reference/builder_classes#datasets.DownloadManager">DownloadManager</a>, there is a <a href="/docs/datasets/pr_8021/en/package_reference/builder_classes#datasets.DownloadManager.download_and_extract">DownloadManager.download_and_extract()</a> method that accepts a dictionary of URLs to the original data files, and downloads the requested files. Accepted inputs include: a single URL or path, or a list/dictionary of URLs or paths. Any compressed file types like TAR, GZIP and ZIP archives will be automatically extracted.</p> <p>Once the files are downloaded, <a href="/docs/datasets/pr_8021/en/package_reference/builder_classes#datasets.SplitGenerator">SplitGenerator</a> organizes them into splits. The <a href="/docs/datasets/pr_8021/en/package_reference/builder_classes#datasets.SplitGenerator">SplitGenerator</a> contains the name of the split, and any keyword arguments that are provided to the <code>DatasetBuilder._generate_examples</code> method. The keyword arguments can be specific to each split, and typically comprise at least the local path to the data files for each split.</p></li> <li><p><code>DatasetBuilder._generate_examples</code> reads and parses the data files for a split. Then it yields dataset examples according to the format specified in the <code>features</code> from <code>DatasetBuilder._info()</code>. The input of <code>DatasetBuilder._generate_examples</code> is actually the <code>filepath</code> provided in the keyword arguments of the last method.</p> <p>The dataset is generated with a Python generator, which doesn’t load all the data in memory. As a result, the generator can handle large datasets. However, before the generated samples are flushed to the dataset file on disk, they are stored in an <code>ArrowWriter</code> buffer. This means the generated samples are written by batch. If your dataset samples consumes a lot of memory (images or videos), then make sure to specify a low value for the <code>DEFAULT_WRITER_BATCH_SIZE</code> attribute in <a href="/docs/datasets/pr_8021/en/package_reference/builder_classes#datasets.DatasetBuilder">DatasetBuilder</a>. We recommend not exceeding a size of 200 MB.</p></li>',Be,G,He,N,rt='To ensure a dataset is complete, <a href="/docs/datasets/pr_8021/en/package_reference/loading_methods#datasets.load_dataset">load_dataset()</a> will perform a series of tests on the downloaded files to make sure everything is there. This way, you don’t encounter any surprises when your requested dataset doesn’t get generated as expected. <a href="/docs/datasets/pr_8021/en/package_reference/loading_methods#datasets.load_dataset">load_dataset()</a> verifies:',De,W,nt="<li>The number of splits in the generated <code>DatasetDict</code>.</li> <li>The number of samples in each split of the generated <code>DatasetDict</code>.</li> <li>The list of downloaded files.</li> <li>The SHA256 checksums of the downloaded files (disabled by default).</li>",Pe,z,ot="If the dataset doesn’t pass the verifications, it is likely that the dataset author made some changes in the data files.",Se,J,ft=`In this case, an error is raised to alert that the dataset has changed.
To ignore the error, one needs to specify <code>verification_mode=&quot;no_checks&quot;</code> in <a href="/docs/datasets/pr_8021/en/package_reference/loading_methods#datasets.load_dataset">load_dataset()</a>.
Anytime you see a verification error, feel free to open a discussion or pull request in the corresponding dataset “Community” tab, so that the integrity checks for that dataset are updated.`,qe,V,Ae,Z,ct='The dataset repositories on the Hub are scanned for malware, see more information <a href="https://huggingface.co/docs/hub/security#malware-scanning" rel="nofollow">here</a>.',Ie,Q,Ee,X,Fe;return v=new yt({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),y=new Y({props:{title:"Build and load",local:"build-and-load",headingTag:"h1"}}),w=new Y({props:{title:"ELI5: load_dataset",local:"eli5-loaddataset",headingTag:"h2"}}),D=new Y({props:{title:"Building a dataset",local:"building-a-dataset",headingTag:"h2"}}),S=new Y({props:{title:"BuilderConfig",local:"datasets-builderconfig",headingTag:"h3"}}),U=new Y({props:{title:"DatasetBuilder",local:"datasets-datasetbuilder",headingTag:"h3"}}),G=new Y({props:{title:"Maintaining integrity",local:"maintaining-integrity",headingTag:"h2"}}),V=new Y({props:{title:"Security",local:"security",headingTag:"h2"}}),Q=new kt({props:{source:"https://github.com/huggingface/datasets/blob/main/docs/source/about_dataset_load.mdx"}}),{c(){o=d("meta"),te=i(),K=d("p"),ae=i(),f(v.$$.fragment),se=i(),f(y.$$.fragment),ie=i(),k=d("p"),k.innerHTML=Re,le=i(),f(w.$$.fragment),de=i(),T=d("p"),T.textContent=je,re=i(),$=d("p"),$.textContent=Oe,ne=i(),x=d("ul"),x.innerHTML=Ge,oe=i(),C=d("p"),C.innerHTML=Ne,fe=i(),L=d("p"),L.innerHTML=We,ce=i(),M=d("ul"),M.innerHTML=ze,ue=i(),_=d("blockquote"),_.innerHTML=Je,pe=i(),B=d("p"),B.textContent=Ve,he=i(),H=d("p"),H.textContent=Ze,me=i(),f(D.$$.fragment),_e=i(),P=d("p"),P.innerHTML=Qe,ge=i(),g=d("div"),g.innerHTML=Ye,be=i(),f(S.$$.fragment),ve=i(),q=d("p"),q.innerHTML=Ke,ye=i(),A=d("table"),A.innerHTML=Xe,ke=i(),I=d("p"),I.innerHTML=et,we=i(),E=d("ul"),E.innerHTML=tt,Te=i(),F=d("p"),F.innerHTML=at,$e=i(),f(U.$$.fragment),xe=i(),R=d("p"),R.innerHTML=st,Ce=i(),b=d("div"),b.innerHTML=it,Le=i(),j=d("p"),j.innerHTML=lt,Me=i(),O=d("ol"),O.innerHTML=dt,Be=i(),f(G.$$.fragment),He=i(),N=d("p"),N.innerHTML=rt,De=i(),W=d("ul"),W.innerHTML=nt,Pe=i(),z=d("p"),z.textContent=ot,Se=i(),J=d("p"),J.innerHTML=ft,qe=i(),f(V.$$.fragment),Ae=i(),Z=d("p"),Z.innerHTML=ct,Ie=i(),f(Q.$$.fragment),Ee=i(),X=d("p"),this.h()},l(e){const t=bt("svelte-u9bgzb",document.head);o=r(t,"META",{name:!0,content:!0}),t.forEach(a),te=l(e),K=r(e,"P",{}),ut(K).forEach(a),ae=l(e),c(v.$$.fragment,e),se=l(e),c(y.$$.fragment,e),ie=l(e),k=r(e,"P",{"data-svelte-h":!0}),n(k)!=="svelte-8n54nx"&&(k.innerHTML=Re),le=l(e),c(w.$$.fragment,e),de=l(e),T=r(e,"P",{"data-svelte-h":!0}),n(T)!=="svelte-1et77y"&&(T.textContent=je),re=l(e),$=r(e,"P",{"data-svelte-h":!0}),n($)!=="svelte-gersvb"&&($.textContent=Oe),ne=l(e),x=r(e,"UL",{"data-svelte-h":!0}),n(x)!=="svelte-vbiom9"&&(x.innerHTML=Ge),oe=l(e),C=r(e,"P",{"data-svelte-h":!0}),n(C)!=="svelte-70lh2v"&&(C.innerHTML=Ne),fe=l(e),L=r(e,"P",{"data-svelte-h":!0}),n(L)!=="svelte-f9tbpx"&&(L.innerHTML=We),ce=l(e),M=r(e,"UL",{"data-svelte-h":!0}),n(M)!=="svelte-1tlhev0"&&(M.innerHTML=ze),ue=l(e),_=r(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),n(_)!=="svelte-1ragf3s"&&(_.innerHTML=Je),pe=l(e),B=r(e,"P",{"data-svelte-h":!0}),n(B)!=="svelte-iduykv"&&(B.textContent=Ve),he=l(e),H=r(e,"P",{"data-svelte-h":!0}),n(H)!=="svelte-1jl5e53"&&(H.textContent=Ze),me=l(e),c(D.$$.fragment,e),_e=l(e),P=r(e,"P",{"data-svelte-h":!0}),n(P)!=="svelte-1y1475o"&&(P.innerHTML=Qe),ge=l(e),g=r(e,"DIV",{class:!0,"data-svelte-h":!0}),n(g)!=="svelte-1adyav4"&&(g.innerHTML=Ye),be=l(e),c(S.$$.fragment,e),ve=l(e),q=r(e,"P",{"data-svelte-h":!0}),n(q)!=="svelte-1utq1tj"&&(q.innerHTML=Ke),ye=l(e),A=r(e,"TABLE",{"data-svelte-h":!0}),n(A)!=="svelte-fm8v6i"&&(A.innerHTML=Xe),ke=l(e),I=r(e,"P",{"data-svelte-h":!0}),n(I)!=="svelte-11eflli"&&(I.innerHTML=et),we=l(e),E=r(e,"UL",{"data-svelte-h":!0}),n(E)!=="svelte-15fnxlq"&&(E.innerHTML=tt),Te=l(e),F=r(e,"P",{"data-svelte-h":!0}),n(F)!=="svelte-1wjr0sd"&&(F.innerHTML=at),$e=l(e),c(U.$$.fragment,e),xe=l(e),R=r(e,"P",{"data-svelte-h":!0}),n(R)!=="svelte-vin3sf"&&(R.innerHTML=st),Ce=l(e),b=r(e,"DIV",{class:!0,"data-svelte-h":!0}),n(b)!=="svelte-11mt2nw"&&(b.innerHTML=it),Le=l(e),j=r(e,"P",{"data-svelte-h":!0}),n(j)!=="svelte-h1taeq"&&(j.innerHTML=lt),Me=l(e),O=r(e,"OL",{"data-svelte-h":!0}),n(O)!=="svelte-an1z4k"&&(O.innerHTML=dt),Be=l(e),c(G.$$.fragment,e),He=l(e),N=r(e,"P",{"data-svelte-h":!0}),n(N)!=="svelte-l1qwol"&&(N.innerHTML=rt),De=l(e),W=r(e,"UL",{"data-svelte-h":!0}),n(W)!=="svelte-1gxjcyf"&&(W.innerHTML=nt),Pe=l(e),z=r(e,"P",{"data-svelte-h":!0}),n(z)!=="svelte-ukpxjx"&&(z.textContent=ot),Se=l(e),J=r(e,"P",{"data-svelte-h":!0}),n(J)!=="svelte-qismhe"&&(J.innerHTML=ft),qe=l(e),c(V.$$.fragment,e),Ae=l(e),Z=r(e,"P",{"data-svelte-h":!0}),n(Z)!=="svelte-p38c5d"&&(Z.innerHTML=ct),Ie=l(e),c(Q.$$.fragment,e),Ee=l(e),X=r(e,"P",{}),ut(X).forEach(a),this.h()},h(){ee(o,"name","hf:doc:metadata"),ee(o,"content",Tt),ee(_,"class","tip"),ee(g,"class","flex justify-center"),ee(b,"class","flex justify-center")},m(e,t){vt(document.head,o),s(e,te,t),s(e,K,t),s(e,ae,t),u(v,e,t),s(e,se,t),u(y,e,t),s(e,ie,t),s(e,k,t),s(e,le,t),u(w,e,t),s(e,de,t),s(e,T,t),s(e,re,t),s(e,$,t),s(e,ne,t),s(e,x,t),s(e,oe,t),s(e,C,t),s(e,fe,t),s(e,L,t),s(e,ce,t),s(e,M,t),s(e,ue,t),s(e,_,t),s(e,pe,t),s(e,B,t),s(e,he,t),s(e,H,t),s(e,me,t),u(D,e,t),s(e,_e,t),s(e,P,t),s(e,ge,t),s(e,g,t),s(e,be,t),u(S,e,t),s(e,ve,t),s(e,q,t),s(e,ye,t),s(e,A,t),s(e,ke,t),s(e,I,t),s(e,we,t),s(e,E,t),s(e,Te,t),s(e,F,t),s(e,$e,t),u(U,e,t),s(e,xe,t),s(e,R,t),s(e,Ce,t),s(e,b,t),s(e,Le,t),s(e,j,t),s(e,Me,t),s(e,O,t),s(e,Be,t),u(G,e,t),s(e,He,t),s(e,N,t),s(e,De,t),s(e,W,t),s(e,Pe,t),s(e,z,t),s(e,Se,t),s(e,J,t),s(e,qe,t),u(V,e,t),s(e,Ae,t),s(e,Z,t),s(e,Ie,t),u(Q,e,t),s(e,Ee,t),s(e,X,t),Fe=!0},p:ht,i(e){Fe||(p(v.$$.fragment,e),p(y.$$.fragment,e),p(w.$$.fragment,e),p(D.$$.fragment,e),p(S.$$.fragment,e),p(U.$$.fragment,e),p(G.$$.fragment,e),p(V.$$.fragment,e),p(Q.$$.fragment,e),Fe=!0)},o(e){h(v.$$.fragment,e),h(y.$$.fragment,e),h(w.$$.fragment,e),h(D.$$.fragment,e),h(S.$$.fragment,e),h(U.$$.fragment,e),h(G.$$.fragment,e),h(V.$$.fragment,e),h(Q.$$.fragment,e),Fe=!1},d(e){e&&(a(te),a(K),a(ae),a(se),a(ie),a(k),a(le),a(de),a(T),a(re),a($),a(ne),a(x),a(oe),a(C),a(fe),a(L),a(ce),a(M),a(ue),a(_),a(pe),a(B),a(he),a(H),a(me),a(_e),a(P),a(ge),a(g),a(be),a(ve),a(q),a(ye),a(A),a(ke),a(I),a(we),a(E),a(Te),a(F),a($e),a(xe),a(R),a(Ce),a(b),a(Le),a(j),a(Me),a(O),a(Be),a(He),a(N),a(De),a(W),a(Pe),a(z),a(Se),a(J),a(qe),a(Ae),a(Z),a(Ie),a(Ee),a(X)),a(o),m(v,e),m(y,e),m(w,e),m(D,e),m(S,e),m(U,e),m(G,e),m(V,e),m(Q,e)}}}const Tt='{"title":"Build and load","local":"build-and-load","sections":[{"title":"ELI5: load_dataset","local":"eli5-loaddataset","sections":[],"depth":2},{"title":"Building a dataset","local":"building-a-dataset","sections":[{"title":"BuilderConfig","local":"datasets-builderconfig","sections":[],"depth":3},{"title":"DatasetBuilder","local":"datasets-datasetbuilder","sections":[],"depth":3}],"depth":2},{"title":"Maintaining integrity","local":"maintaining-integrity","sections":[],"depth":2},{"title":"Security","local":"security","sections":[],"depth":2}],"depth":1}';function $t(Ue){return mt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Mt extends _t{constructor(o){super(),gt(this,o,$t,wt,pt,{})}}export{Mt as component};