Buckets:
| import{s as Vn,o as Xn,n as Y}from"../chunks/scheduler.bdbef820.js";import{S as En,i as Ln,g as i,s as o,r as m,A as Pn,h as c,f as a,c as d,j as N,u as g,x,k as C,y as p,a as s,v as f,d as u,t as h,w as _}from"../chunks/index.c0aea24a.js";import{D as O}from"../chunks/Docstring.20f7ed2e.js";import{C as I}from"../chunks/CodeBlock.6ccca92e.js";import{E as B}from"../chunks/ExampleCodeBlock.da027134.js";import{H as T,E as An}from"../chunks/EditOnGithub.725ee0c1.js";function Bn(D){let r,k="Load a dataset from the Hugging Face Hub:",b,l,v;return l=new I({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBZHMlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJ3JvdHRlbl90b21hdG9lcyclMkMlMjBzcGxpdCUzRCd0cmFpbicpJTBBJTBBZGF0YV9maWxlcyUyMCUzRCUyMCU3Qid0cmFpbiclM0ElMjAndHJhaW4uY3N2JyUyQyUyMCd0ZXN0JyUzQSUyMCd0ZXN0LmNzdiclN0QlMEFkcyUyMCUzRCUyMGxvYWRfZGF0YXNldCgnbmFtZXNwYWNlJTJGeW91cl9kYXRhc2V0X25hbWUnJTJDJTIwZGF0YV9maWxlcyUzRGRhdGFfZmlsZXMp",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset | |
| <span class="hljs-meta">>>> </span>ds = load_dataset(<span class="hljs-string">'rotten_tomatoes'</span>, split=<span class="hljs-string">'train'</span>) | |
| <span class="hljs-comment"># Map data files to splits</span> | |
| <span class="hljs-meta">>>> </span>data_files = {<span class="hljs-string">'train'</span>: <span class="hljs-string">'train.csv'</span>, <span class="hljs-string">'test'</span>: <span class="hljs-string">'test.csv'</span>} | |
| <span class="hljs-meta">>>> </span>ds = load_dataset(<span class="hljs-string">'namespace/your_dataset_name'</span>, data_files=data_files)`,wrap:!1}}),{c(){r=i("p"),r.textContent=k,b=o(),m(l.$$.fragment)},l(t){r=c(t,"P",{"data-svelte-h":!0}),x(r)!=="svelte-cpjyx5"&&(r.textContent=k),b=d(t),g(l.$$.fragment,t)},m(t,$){s(t,r,$),s(t,b,$),f(l,t,$),v=!0},p:Y,i(t){v||(u(l.$$.fragment,t),v=!0)},o(t){h(l.$$.fragment,t),v=!1},d(t){t&&(a(r),a(b)),_(l,t)}}}function Yn(D){let r,k="Load a local dataset:",b,l,v;return l=new I({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBZHMlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJ2NzdiclMkMlMjBkYXRhX2ZpbGVzJTNEJ3BhdGglMkZ0byUyRmxvY2FsJTJGbXlfZGF0YXNldC5jc3YnKSUwQSUwQWZyb20lMjBkYXRhc2V0cyUyMGltcG9ydCUyMGxvYWRfZGF0YXNldCUwQWRzJTIwJTNEJTIwbG9hZF9kYXRhc2V0KCdqc29uJyUyQyUyMGRhdGFfZmlsZXMlM0QncGF0aCUyRnRvJTJGbG9jYWwlMkZteV9kYXRhc2V0Lmpzb24nKSUwQSUwQWZyb20lMjBkYXRhc2V0cyUyMGltcG9ydCUyMGxvYWRfZGF0YXNldCUwQWRzJTIwJTNEJTIwbG9hZF9kYXRhc2V0KCdwYXRoJTJGdG8lMkZsb2NhbCUyRmxvYWRpbmdfc2NyaXB0JTJGbG9hZGluZ19zY3JpcHQucHknJTJDJTIwc3BsaXQlM0QndHJhaW4nKQ==",highlighted:`<span class="hljs-comment"># Load a CSV file</span> | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset | |
| <span class="hljs-meta">>>> </span>ds = load_dataset(<span class="hljs-string">'csv'</span>, data_files=<span class="hljs-string">'path/to/local/my_dataset.csv'</span>) | |
| <span class="hljs-comment"># Load a JSON file</span> | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset | |
| <span class="hljs-meta">>>> </span>ds = load_dataset(<span class="hljs-string">'json'</span>, data_files=<span class="hljs-string">'path/to/local/my_dataset.json'</span>) | |
| <span class="hljs-comment"># Load from a local loading script</span> | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset | |
| <span class="hljs-meta">>>> </span>ds = load_dataset(<span class="hljs-string">'path/to/local/loading_script/loading_script.py'</span>, split=<span class="hljs-string">'train'</span>)`,wrap:!1}}),{c(){r=i("p"),r.textContent=k,b=o(),m(l.$$.fragment)},l(t){r=c(t,"P",{"data-svelte-h":!0}),x(r)!=="svelte-18tmtyu"&&(r.textContent=k),b=d(t),g(l.$$.fragment,t)},m(t,$){s(t,r,$),s(t,b,$),f(l,t,$),v=!0},p:Y,i(t){v||(u(l.$$.fragment,t),v=!0)},o(t){h(l.$$.fragment,t),v=!1},d(t){t&&(a(r),a(b)),_(l,t)}}}function Wn(D){let r,k='Load an <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset">IterableDataset</a>:',b,l,v;return l=new I({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBZHMlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJ3JvdHRlbl90b21hdG9lcyclMkMlMjBzcGxpdCUzRCd0cmFpbiclMkMlMjBzdHJlYW1pbmclM0RUcnVlKQ==",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset | |
| <span class="hljs-meta">>>> </span>ds = load_dataset(<span class="hljs-string">'rotten_tomatoes'</span>, split=<span class="hljs-string">'train'</span>, streaming=<span class="hljs-literal">True</span>)`,wrap:!1}}),{c(){r=i("p"),r.innerHTML=k,b=o(),m(l.$$.fragment)},l(t){r=c(t,"P",{"data-svelte-h":!0}),x(r)!=="svelte-lv2916"&&(r.innerHTML=k),b=d(t),g(l.$$.fragment,t)},m(t,$){s(t,r,$),s(t,b,$),f(l,t,$),v=!0},p:Y,i(t){v||(u(l.$$.fragment,t),v=!0)},o(t){h(l.$$.fragment,t),v=!1},d(t){t&&(a(r),a(b)),_(l,t)}}}function Qn(D){let r,k="Load an image dataset with the <code>ImageFolder</code> dataset builder:",b,l,v;return l=new I({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBZHMlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJ2ltYWdlZm9sZGVyJyUyQyUyMGRhdGFfZGlyJTNEJyUyRnBhdGglMkZ0byUyRmltYWdlcyclMkMlMjBzcGxpdCUzRCd0cmFpbicp",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset | |
| <span class="hljs-meta">>>> </span>ds = load_dataset(<span class="hljs-string">'imagefolder'</span>, data_dir=<span class="hljs-string">'/path/to/images'</span>, split=<span class="hljs-string">'train'</span>)`,wrap:!1}}),{c(){r=i("p"),r.innerHTML=k,b=o(),m(l.$$.fragment)},l(t){r=c(t,"P",{"data-svelte-h":!0}),x(r)!=="svelte-9alpt2"&&(r.innerHTML=k),b=d(t),g(l.$$.fragment,t)},m(t,$){s(t,r,$),s(t,b,$),f(l,t,$),v=!0},p:Y,i(t){v||(u(l.$$.fragment,t),v=!0)},o(t){h(l.$$.fragment,t),v=!1},d(t){t&&(a(r),a(b)),_(l,t)}}}function Kn(D){let r,k="Example:",b,l,v;return l=new I({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9mcm9tX2Rpc2slMEFkcyUyMCUzRCUyMGxvYWRfZnJvbV9kaXNrKCdwYXRoJTJGdG8lMkZkYXRhc2V0JTJGZGlyZWN0b3J5Jyk=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_from_disk | |
| <span class="hljs-meta">>>> </span>ds = load_from_disk(<span class="hljs-string">'path/to/dataset/directory'</span>)`,wrap:!1}}),{c(){r=i("p"),r.textContent=k,b=o(),m(l.$$.fragment)},l(t){r=c(t,"P",{"data-svelte-h":!0}),x(r)!=="svelte-11lpom8"&&(r.textContent=k),b=d(t),g(l.$$.fragment,t)},m(t,$){s(t,r,$),s(t,b,$),f(l,t,$),v=!0},p:Y,i(t){v||(u(l.$$.fragment,t),v=!0)},o(t){h(l.$$.fragment,t),v=!1},d(t){t&&(a(r),a(b)),_(l,t)}}}function es(D){let r,k="Example:",b,l,v;return l=new I({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0X2J1aWxkZXIlMEFkc19idWlsZGVyJTIwJTNEJTIwbG9hZF9kYXRhc2V0X2J1aWxkZXIoJ3JvdHRlbl90b21hdG9lcycpJTBBZHNfYnVpbGRlci5pbmZvLmZlYXR1cmVz",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset_builder | |
| <span class="hljs-meta">>>> </span>ds_builder = load_dataset_builder(<span class="hljs-string">'rotten_tomatoes'</span>) | |
| <span class="hljs-meta">>>> </span>ds_builder.info.features | |
| {<span class="hljs-string">'label'</span>: ClassLabel(num_classes=<span class="hljs-number">2</span>, names=[<span class="hljs-string">'neg'</span>, <span class="hljs-string">'pos'</span>], <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>), | |
| <span class="hljs-string">'text'</span>: Value(dtype=<span class="hljs-string">'string'</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>)}`,wrap:!1}}),{c(){r=i("p"),r.textContent=k,b=o(),m(l.$$.fragment)},l(t){r=c(t,"P",{"data-svelte-h":!0}),x(r)!=="svelte-11lpom8"&&(r.textContent=k),b=d(t),g(l.$$.fragment,t)},m(t,$){s(t,r,$),s(t,b,$),f(l,t,$),v=!0},p:Y,i(t){v||(u(l.$$.fragment,t),v=!0)},o(t){h(l.$$.fragment,t),v=!1},d(t){t&&(a(r),a(b)),_(l,t)}}}function ts(D){let r,k="Example:",b,l,v;return l=new I({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwZ2V0X2RhdGFzZXRfY29uZmlnX25hbWVzJTBBZ2V0X2RhdGFzZXRfY29uZmlnX25hbWVzKCUyMmdsdWUlMjIp",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> get_dataset_config_names | |
| <span class="hljs-meta">>>> </span>get_dataset_config_names(<span class="hljs-string">"glue"</span>) | |
| [<span class="hljs-string">'cola'</span>, | |
| <span class="hljs-string">'sst2'</span>, | |
| <span class="hljs-string">'mrpc'</span>, | |
| <span class="hljs-string">'qqp'</span>, | |
| <span class="hljs-string">'stsb'</span>, | |
| <span class="hljs-string">'mnli'</span>, | |
| <span class="hljs-string">'mnli_mismatched'</span>, | |
| <span class="hljs-string">'mnli_matched'</span>, | |
| <span class="hljs-string">'qnli'</span>, | |
| <span class="hljs-string">'rte'</span>, | |
| <span class="hljs-string">'wnli'</span>, | |
| <span class="hljs-string">'ax'</span>]`,wrap:!1}}),{c(){r=i("p"),r.textContent=k,b=o(),m(l.$$.fragment)},l(t){r=c(t,"P",{"data-svelte-h":!0}),x(r)!=="svelte-11lpom8"&&(r.textContent=k),b=d(t),g(l.$$.fragment,t)},m(t,$){s(t,r,$),s(t,b,$),f(l,t,$),v=!0},p:Y,i(t){v||(u(l.$$.fragment,t),v=!0)},o(t){h(l.$$.fragment,t),v=!1},d(t){t&&(a(r),a(b)),_(l,t)}}}function as(D){let r,k="Example:",b,l,v;return l=new I({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwZ2V0X2RhdGFzZXRfaW5mb3MlMEFnZXRfZGF0YXNldF9pbmZvcygncm90dGVuX3RvbWF0b2VzJyk=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> get_dataset_infos | |
| <span class="hljs-meta">>>> </span>get_dataset_infos(<span class="hljs-string">'rotten_tomatoes'</span>) | |
| {<span class="hljs-string">'default'</span>: DatasetInfo(description=<span class="hljs-string">"Movie Review Dataset. | |
| is a dataset of containing 5,331 positive and 5,331 negative processed | |
| ences from Rotten Tomatoes movie reviews...), ...}</span>`,wrap:!1}}),{c(){r=i("p"),r.textContent=k,b=o(),m(l.$$.fragment)},l(t){r=c(t,"P",{"data-svelte-h":!0}),x(r)!=="svelte-11lpom8"&&(r.textContent=k),b=d(t),g(l.$$.fragment,t)},m(t,$){s(t,r,$),s(t,b,$),f(l,t,$),v=!0},p:Y,i(t){v||(u(l.$$.fragment,t),v=!0)},o(t){h(l.$$.fragment,t),v=!1},d(t){t&&(a(r),a(b)),_(l,t)}}}function ns(D){let r,k="Example:",b,l,v;return l=new I({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwZ2V0X2RhdGFzZXRfc3BsaXRfbmFtZXMlMEFnZXRfZGF0YXNldF9zcGxpdF9uYW1lcygncm90dGVuX3RvbWF0b2VzJyk=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> get_dataset_split_names | |
| <span class="hljs-meta">>>> </span>get_dataset_split_names(<span class="hljs-string">'rotten_tomatoes'</span>) | |
| [<span class="hljs-string">'train'</span>, <span class="hljs-string">'validation'</span>, <span class="hljs-string">'test'</span>]`,wrap:!1}}),{c(){r=i("p"),r.textContent=k,b=o(),m(l.$$.fragment)},l(t){r=c(t,"P",{"data-svelte-h":!0}),x(r)!=="svelte-11lpom8"&&(r.textContent=k),b=d(t),g(l.$$.fragment,t)},m(t,$){s(t,r,$),s(t,b,$),f(l,t,$),v=!0},p:Y,i(t){v||(u(l.$$.fragment,t),v=!0)},o(t){h(l.$$.fragment,t),v=!1},d(t){t&&(a(r),a(b)),_(l,t)}}}function ss(D){let r,k,b,l,v,t,$,sn="Methods for listing and loading datasets:",Ut,de,Tt,w,re,ka,at,on="Load a dataset from the Hugging Face Hub, or a local dataset.",ya,nt,dn='You can find the list of datasets on the <a href="https://huggingface.co/datasets" rel="nofollow">Hub</a> or with <code>huggingface_hub.list_datasets</code>.',Na,st,rn="A dataset is a directory that contains:",Ca,ot,ln="<li>some data files in generic formats (JSON, CSV, Parquet, text, etc.).</li> <li>and optionally a dataset script, if it requires some code to read the data files. This is used to load any kind of formats or structures.</li>",Oa,dt,cn="Note that dataset scripts can also download and read data files from anywhere - in case your data files already exist online.",Da,rt,pn="This function does the following under the hood:",ja,lt,mn=`<li><p>Download and import in the library the dataset script from <code>path</code> if it’s not already cached inside the library.</p> <p>If the dataset has no dataset script, then a generic dataset script is imported instead (JSON, CSV, Parquet, text, etc.)</p> <p>Dataset scripts are small python scripts that define dataset builders. They define the citation, info and format of the dataset, | |
| contain the path or URL to the original data files and the code to load examples from the original data files.</p> <p>You can find the complete list of datasets in the Datasets <a href="https://huggingface.co/datasets" rel="nofollow">Hub</a>.</p></li> <li><p>Run the dataset script which will:</p> <ul><li><p>Download the dataset file from the original URL (see the script) if it’s not already available locally or cached.</p></li> <li><p>Process and cache the dataset in typed Arrow tables for caching.</p> <p>Arrow table are arbitrarily long, typed tables which can store nested objects and be mapped to numpy/pandas/python generic types. | |
| They can be directly accessed from disk, loaded in RAM or even streamed over the web.</p></li></ul></li> <li><p>Return a dataset built from the requested splits in <code>split</code> (default: all).</p></li>`,Ua,it,gn=`It also allows to load a dataset from a local directory or a dataset repository on the Hugging Face Hub without dataset script. | |
| In this case, it automatically loads all the data files from the directory or the dataset repository.`,Ta,ct,fn="Example:",Ma,W,Fa,Q,Ja,K,qa,ee,Mt,M,le,Ia,pt,un=`Loads a dataset that was previously saved using <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset.save_to_disk">save_to_disk()</a> from a dataset directory, or | |
| from a filesystem using any implementation of <code>fsspec.spec.AbstractFileSystem</code>.`,Ra,te,Ft,j,ie,Ga,mt,hn=`Load a dataset builder from the Hugging Face Hub, or a local dataset. A dataset builder can be used to inspect general information that is required to build a dataset (cache directory, config, dataset info, etc.) | |
| without downloading the dataset itself.`,Sa,gt,_n='You can find the list of datasets on the <a href="https://huggingface.co/datasets" rel="nofollow">Hub</a> or with <code>huggingface_hub.list_datasets</code>.',Za,ft,vn="A dataset is a directory that contains:",Ha,ut,bn="<li>some data files in generic formats (JSON, CSV, Parquet, text, etc.)</li> <li>and optionally a dataset script, if it requires some code to read the data files. This is used to load any kind of formats or structures.</li>",za,ht,$n="Note that dataset scripts can also download and read data files from anywhere - in case your data files already exist online.",Va,ae,Jt,F,ce,Xa,_t,xn="Get the list of available config names for a particular dataset.",Ea,ne,qt,J,pe,La,vt,wn="Get the meta information about a dataset, returned as a dict mapping config name to DatasetInfoDict.",Pa,se,It,q,me,Aa,bt,kn="Get the list of available splits for a particular config and dataset.",Ba,oe,Rt,ge,Gt,fe,yn=`Configurations used to load data files. | |
| They are used when loading local files or a dataset repository:`,St,ue,Nn="<li>local files: <code>load_dataset("parquet", data_dir="path/to/data/dir")</code></li> <li>dataset repository: <code>load_dataset("allenai/c4")</code></li>",Zt,he,Cn=`You can pass arguments to <code>load_dataset</code> to configure data loading. | |
| For example you can specify the <code>sep</code> parameter to define the <a href="/docs/datasets/main/en/package_reference/loading_methods#datasets.packaged_modules.csv.CsvConfig">CsvConfig</a> that is used to load the data:`,Ht,_e,zt,ve,Vt,R,be,Ya,$t,On="BuilderConfig for text files.",Xt,$e,xe,Et,we,Lt,G,ke,Wa,xt,Dn="BuilderConfig for CSV.",Pt,ye,Ne,At,Ce,Bt,S,Oe,Qa,wt,jn="BuilderConfig for JSON.",Yt,De,je,Wt,Ue,Qt,Z,Te,Ka,kt,Un="BuilderConfig for Parquet.",Kt,Me,Fe,ea,Je,ta,H,qe,en,yt,Tn="BuilderConfig for Arrow.",aa,Ie,Re,na,Ge,sa,z,Se,tn,Nt,Mn="BuilderConfig for SQL.",oa,Ze,He,da,ze,ra,V,Ve,an,Ct,Fn="BuilderConfig for ImageFolder.",la,Xe,Ee,ia,Le,ca,X,Pe,nn,Ot,Jn="Builder Config for AudioFolder.",pa,Ae,Be,ma,Ye,ga,We,Qe,fa,Ke,ua,Dt,ha;return v=new T({props:{title:"Loading methods",local:"loading-methods",headingTag:"h1"}}),de=new T({props:{title:"Datasets",local:"datasets.load_dataset",headingTag:"h2"}}),re=new O({props:{name:"datasets.load_dataset",anchor:"datasets.load_dataset",parameters:[{name:"path",val:": str"},{name:"name",val:": Optional = None"},{name:"data_dir",val:": Optional = None"},{name:"data_files",val:": Union = None"},{name:"split",val:": Union = None"},{name:"cache_dir",val:": Optional = None"},{name:"features",val:": Optional = None"},{name:"download_config",val:": Optional = None"},{name:"download_mode",val:": Union = None"},{name:"verification_mode",val:": Union = None"},{name:"keep_in_memory",val:": Optional = None"},{name:"save_infos",val:": bool = False"},{name:"revision",val:": Union = None"},{name:"token",val:": Union = None"},{name:"streaming",val:": bool = False"},{name:"num_proc",val:": Optional = None"},{name:"storage_options",val:": Optional = None"},{name:"trust_remote_code",val:": bool = None"},{name:"**config_kwargs",val:""}],parametersDescription:[{anchor:"datasets.load_dataset.path",description:`<strong>path</strong> (<code>str</code>) — | |
| Path or name of the dataset. | |
| Depending on <code>path</code>, the dataset builder that is used comes from a generic dataset script (JSON, CSV, Parquet, text etc.) or from the dataset script (a python file) inside the dataset directory.</p> | |
| <p>For local datasets:</p> | |
| <ul> | |
| <li>if <code>path</code> is a local directory (containing data files only) | |
| -> load a generic dataset builder (csv, json, text etc.) based on the content of the directory | |
| e.g. <code>'./path/to/directory/with/my/csv/data'</code>.</li> | |
| <li>if <code>path</code> is a local dataset script or a directory containing a local dataset script (if the script has the same name as the directory) | |
| -> load the dataset builder from the dataset script | |
| e.g. <code>'./dataset/squad'</code> or <code>'./dataset/squad/squad.py'</code>.</li> | |
| </ul> | |
| <p>For datasets on the Hugging Face Hub (list all available datasets with <code>huggingface_hub.list_datasets</code>)</p> | |
| <ul> | |
| <li>if <code>path</code> is a dataset repository on the HF hub (containing data files only) | |
| -> load a generic dataset builder (csv, text etc.) based on the content of the repository | |
| e.g. <code>'username/dataset_name'</code>, a dataset repository on the HF hub containing your data files.</li> | |
| <li>if <code>path</code> is a dataset repository on the HF hub with a dataset script (if the script has the same name as the directory) | |
| -> load the dataset builder from the dataset script in the dataset repository | |
| e.g. <code>glue</code>, <code>squad</code>, <code>'username/dataset_name'</code>, a dataset repository on the HF hub containing a dataset script <code>'dataset_name.py'</code>.</li> | |
| </ul>`,name:"path"},{anchor:"datasets.load_dataset.name",description:`<strong>name</strong> (<code>str</code>, <em>optional</em>) — | |
| Defining the name of the dataset configuration.`,name:"name"},{anchor:"datasets.load_dataset.data_dir",description:`<strong>data_dir</strong> (<code>str</code>, <em>optional</em>) — | |
| Defining the <code>data_dir</code> of the dataset configuration. If specified for the generic builders (csv, text etc.) or the Hub datasets and <code>data_files</code> is <code>None</code>, | |
| the behavior is equal to passing <code>os.path.join(data_dir, **)</code> as <code>data_files</code> to reference all the files in a directory.`,name:"data_dir"},{anchor:"datasets.load_dataset.data_files",description:`<strong>data_files</strong> (<code>str</code> or <code>Sequence</code> or <code>Mapping</code>, <em>optional</em>) — | |
| Path(s) to source data file(s).`,name:"data_files"},{anchor:"datasets.load_dataset.split",description:`<strong>split</strong> (<code>Split</code> or <code>str</code>) — | |
| Which split of the data to load. | |
| If <code>None</code>, will return a <code>dict</code> with all splits (typically <code>datasets.Split.TRAIN</code> and <code>datasets.Split.TEST</code>). | |
| If given, will return a single Dataset. | |
| Splits can be combined and specified like in tensorflow-datasets.`,name:"split"},{anchor:"datasets.load_dataset.cache_dir",description:`<strong>cache_dir</strong> (<code>str</code>, <em>optional</em>) — | |
| Directory to read/write data. Defaults to <code>"~/.cache/huggingface/datasets"</code>.`,name:"cache_dir"},{anchor:"datasets.load_dataset.features",description:`<strong>features</strong> (<code>Features</code>, <em>optional</em>) — | |
| Set the features type to use for this dataset.`,name:"features"},{anchor:"datasets.load_dataset.download_config",description:`<strong>download_config</strong> (<a href="/docs/datasets/main/en/package_reference/builder_classes#datasets.DownloadConfig">DownloadConfig</a>, <em>optional</em>) — | |
| Specific download configuration parameters.`,name:"download_config"},{anchor:"datasets.load_dataset.download_mode",description:`<strong>download_mode</strong> (<a href="/docs/datasets/main/en/package_reference/builder_classes#datasets.DownloadMode">DownloadMode</a> or <code>str</code>, defaults to <code>REUSE_DATASET_IF_EXISTS</code>) — | |
| Download/generate mode.`,name:"download_mode"},{anchor:"datasets.load_dataset.verification_mode",description:`<strong>verification_mode</strong> (<a href="/docs/datasets/main/en/package_reference/builder_classes#datasets.VerificationMode">VerificationMode</a> or <code>str</code>, defaults to <code>BASIC_CHECKS</code>) — | |
| Verification mode determining the checks to run on the downloaded/processed dataset information (checksums/size/splits/…).</p> | |
| <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"> | |
| <p class="font-medium">Added in 2.9.1</p> | |
| </div>`,name:"verification_mode"},{anchor:"datasets.load_dataset.keep_in_memory",description:`<strong>keep_in_memory</strong> (<code>bool</code>, defaults to <code>None</code>) — | |
| Whether to copy the dataset in-memory. If <code>None</code>, the dataset | |
| will not be copied in-memory unless explicitly enabled by setting <code>datasets.config.IN_MEMORY_MAX_SIZE</code> to | |
| nonzero. See more details in the <a href="../cache#improve-performance">improve performance</a> section.`,name:"keep_in_memory"},{anchor:"datasets.load_dataset.save_infos",description:`<strong>save_infos</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Save the dataset information (checksums/size/splits/…).`,name:"save_infos"},{anchor:"datasets.load_dataset.revision",description:`<strong>revision</strong> (<a href="/docs/datasets/main/en/package_reference/builder_classes#datasets.Version">Version</a> or <code>str</code>, <em>optional</em>) — | |
| Version of the dataset script to load. | |
| As datasets have their own git repository on the Datasets Hub, the default version “main” corresponds to their “main” branch. | |
| You can specify a different version than the default “main” by using a commit SHA or a git tag of the dataset repository.`,name:"revision"},{anchor:"datasets.load_dataset.token",description:`<strong>token</strong> (<code>str</code> or <code>bool</code>, <em>optional</em>) — | |
| Optional string or boolean to use as Bearer token for remote files on the Datasets Hub. | |
| If <code>True</code>, or not specified, will get token from <code>"~/.huggingface"</code>.`,name:"token"},{anchor:"datasets.load_dataset.streaming",description:`<strong>streaming</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| If set to <code>True</code>, don’t download the data files. Instead, it streams the data progressively while | |
| iterating on the dataset. An <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset">IterableDataset</a> or <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDatasetDict">IterableDatasetDict</a> is returned instead in this case.</p> | |
| <p>Note that streaming works for datasets that use data formats that support being iterated over like txt, csv, jsonl for example. | |
| Json files may be downloaded completely. Also streaming from remote zip or gzip files is supported but other compressed formats | |
| like rar and xz are not yet supported. The tgz format doesn’t allow streaming.`,name:"streaming"},{anchor:"datasets.load_dataset.num_proc",description:`<strong>num_proc</strong> (<code>int</code>, <em>optional</em>, defaults to <code>None</code>) — | |
| Number of processes when downloading and generating the dataset locally. | |
| Multiprocessing is disabled by default.</p> | |
| <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"> | |
| <p class="font-medium">Added in 2.7.0</p> | |
| </div>`,name:"num_proc"},{anchor:"datasets.load_dataset.storage_options",description:`<strong>storage_options</strong> (<code>dict</code>, <em>optional</em>, defaults to <code>None</code>) — | |
| <strong>Experimental</strong>. Key/value pairs to be passed on to the dataset file-system backend, if any.</p> | |
| <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"> | |
| <p class="font-medium">Added in 2.11.0</p> | |
| </div>`,name:"storage_options"},{anchor:"datasets.load_dataset.trust_remote_code",description:`<strong>trust_remote_code</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Whether or not to allow for datasets defined on the Hub using a dataset script. This option | |
| should only be set to <code>True</code> for repositories you trust and in which you have read the code, as it will | |
| execute code present on the Hub on your local machine.</p> | |
| <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"> | |
| <p class="font-medium">Added in 2.16.0</p> | |
| </div> | |
| <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"> | |
| <p class="font-medium">Changed in 2.20.0</p> | |
| <p><code>trust_remote_code</code> defaults to <code>False</code> if not specified.</p> | |
| </div>`,name:"trust_remote_code"},{anchor:"datasets.load_dataset.*config_kwargs",description:`*<strong>*config_kwargs</strong> (additional keyword arguments) — | |
| Keyword arguments to be passed to the <code>BuilderConfig</code> | |
| and used in the <a href="/docs/datasets/main/en/package_reference/builder_classes#datasets.DatasetBuilder">DatasetBuilder</a>.`,name:"*config_kwargs"}],source:"https://github.com/huggingface/datasets/blob/main/src/datasets/load.py#L1851",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <ul> | |
| <li>if <code>split</code> is not <code>None</code>: the dataset requested,</li> | |
| <li>if <code>split</code> is <code>None</code>, a <a | |
| href="/docs/datasets/main/en/package_reference/main_classes#datasets.DatasetDict" | |
| >DatasetDict</a> with each split.</li> | |
| </ul> | |
| <p>or <a | |
| href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset" | |
| >IterableDataset</a> or <a | |
| href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDatasetDict" | |
| >IterableDatasetDict</a>: if <code>streaming=True</code></p> | |
| <ul> | |
| <li>if <code>split</code> is not <code>None</code>, the dataset is requested</li> | |
| <li>if <code>split</code> is <code>None</code>, a <code>~datasets.streaming.IterableDatasetDict</code> with each split.</li> | |
| </ul> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><a | |
| href="/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset" | |
| >Dataset</a> or <a | |
| href="/docs/datasets/main/en/package_reference/main_classes#datasets.DatasetDict" | |
| >DatasetDict</a></p> | |
| `}}),W=new B({props:{anchor:"datasets.load_dataset.example",$$slots:{default:[Bn]},$$scope:{ctx:D}}}),Q=new B({props:{anchor:"datasets.load_dataset.example-2",$$slots:{default:[Yn]},$$scope:{ctx:D}}}),K=new B({props:{anchor:"datasets.load_dataset.example-3",$$slots:{default:[Wn]},$$scope:{ctx:D}}}),ee=new B({props:{anchor:"datasets.load_dataset.example-4",$$slots:{default:[Qn]},$$scope:{ctx:D}}}),le=new O({props:{name:"datasets.load_from_disk",anchor:"datasets.load_from_disk",parameters:[{name:"dataset_path",val:": Union"},{name:"keep_in_memory",val:": Optional = None"},{name:"storage_options",val:": Optional = None"}],parametersDescription:[{anchor:"datasets.load_from_disk.dataset_path",description:`<strong>dataset_path</strong> (<code>path-like</code>) — | |
| Path (e.g. <code>"dataset/train"</code>) or remote URI (e.g. <code>"s3://my-bucket/dataset/train"</code>) | |
| of the <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset">Dataset</a> or <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.DatasetDict">DatasetDict</a> directory where the dataset/dataset-dict will be | |
| loaded from.`,name:"dataset_path"},{anchor:"datasets.load_from_disk.keep_in_memory",description:`<strong>keep_in_memory</strong> (<code>bool</code>, defaults to <code>None</code>) — | |
| Whether to copy the dataset in-memory. If <code>None</code>, the dataset | |
| will not be copied in-memory unless explicitly enabled by setting <code>datasets.config.IN_MEMORY_MAX_SIZE</code> to | |
| nonzero. See more details in the <a href="../cache#improve-performance">improve performance</a> section.`,name:"keep_in_memory"},{anchor:"datasets.load_from_disk.storage_options",description:`<strong>storage_options</strong> (<code>dict</code>, <em>optional</em>) — | |
| Key/value pairs to be passed on to the file-system backend, if any.</p> | |
| <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"> | |
| <p class="font-medium">Added in 2.9.0</p> | |
| </div>`,name:"storage_options"}],source:"https://github.com/huggingface/datasets/blob/main/src/datasets/load.py#L2115",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <ul> | |
| <li>If <code>dataset_path</code> is a path of a dataset directory: the dataset requested.</li> | |
| <li>If <code>dataset_path</code> is a path of a dataset dict directory, a <a | |
| href="/docs/datasets/main/en/package_reference/main_classes#datasets.DatasetDict" | |
| >DatasetDict</a> with each split.</li> | |
| </ul> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><a | |
| href="/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset" | |
| >Dataset</a> or <a | |
| href="/docs/datasets/main/en/package_reference/main_classes#datasets.DatasetDict" | |
| >DatasetDict</a></p> | |
| `}}),te=new B({props:{anchor:"datasets.load_from_disk.example",$$slots:{default:[Kn]},$$scope:{ctx:D}}}),ie=new O({props:{name:"datasets.load_dataset_builder",anchor:"datasets.load_dataset_builder",parameters:[{name:"path",val:": str"},{name:"name",val:": Optional = None"},{name:"data_dir",val:": Optional = None"},{name:"data_files",val:": Union = None"},{name:"cache_dir",val:": Optional = None"},{name:"features",val:": Optional = None"},{name:"download_config",val:": Optional = None"},{name:"download_mode",val:": Union = None"},{name:"revision",val:": Union = None"},{name:"token",val:": Union = None"},{name:"storage_options",val:": Optional = None"},{name:"trust_remote_code",val:": Optional = None"},{name:"_require_default_config_name",val:" = True"},{name:"**config_kwargs",val:""}],parametersDescription:[{anchor:"datasets.load_dataset_builder.path",description:`<strong>path</strong> (<code>str</code>) — | |
| Path or name of the dataset. | |
| Depending on <code>path</code>, the dataset builder that is used comes from a generic dataset script (JSON, CSV, Parquet, text etc.) or from the dataset script (a python file) inside the dataset directory.</p> | |
| <p>For local datasets:</p> | |
| <ul> | |
| <li>if <code>path</code> is a local directory (containing data files only) | |
| -> load a generic dataset builder (csv, json, text etc.) based on the content of the directory | |
| e.g. <code>'./path/to/directory/with/my/csv/data'</code>.</li> | |
| <li>if <code>path</code> is a local dataset script or a directory containing a local dataset script (if the script has the same name as the directory) | |
| -> load the dataset builder from the dataset script | |
| e.g. <code>'./dataset/squad'</code> or <code>'./dataset/squad/squad.py'</code>.</li> | |
| </ul> | |
| <p>For datasets on the Hugging Face Hub (list all available datasets with <code>huggingface_hub.list_datasets</code>)</p> | |
| <ul> | |
| <li>if <code>path</code> is a dataset repository on the HF hub (containing data files only) | |
| -> load a generic dataset builder (csv, text etc.) based on the content of the repository | |
| e.g. <code>'username/dataset_name'</code>, a dataset repository on the HF hub containing your data files.</li> | |
| <li>if <code>path</code> is a dataset repository on the HF hub with a dataset script (if the script has the same name as the directory) | |
| -> load the dataset builder from the dataset script in the dataset repository | |
| e.g. <code>glue</code>, <code>squad</code>, <code>'username/dataset_name'</code>, a dataset repository on the HF hub containing a dataset script <code>'dataset_name.py'</code>.</li> | |
| </ul>`,name:"path"},{anchor:"datasets.load_dataset_builder.name",description:`<strong>name</strong> (<code>str</code>, <em>optional</em>) — | |
| Defining the name of the dataset configuration.`,name:"name"},{anchor:"datasets.load_dataset_builder.data_dir",description:`<strong>data_dir</strong> (<code>str</code>, <em>optional</em>) — | |
| Defining the <code>data_dir</code> of the dataset configuration. If specified for the generic builders (csv, text etc.) or the Hub datasets and <code>data_files</code> is <code>None</code>, | |
| the behavior is equal to passing <code>os.path.join(data_dir, **)</code> as <code>data_files</code> to reference all the files in a directory.`,name:"data_dir"},{anchor:"datasets.load_dataset_builder.data_files",description:`<strong>data_files</strong> (<code>str</code> or <code>Sequence</code> or <code>Mapping</code>, <em>optional</em>) — | |
| Path(s) to source data file(s).`,name:"data_files"},{anchor:"datasets.load_dataset_builder.cache_dir",description:`<strong>cache_dir</strong> (<code>str</code>, <em>optional</em>) — | |
| Directory to read/write data. Defaults to <code>"~/.cache/huggingface/datasets"</code>.`,name:"cache_dir"},{anchor:"datasets.load_dataset_builder.features",description:`<strong>features</strong> (<a href="/docs/datasets/main/en/package_reference/main_classes#datasets.Features">Features</a>, <em>optional</em>) — | |
| Set the features type to use for this dataset.`,name:"features"},{anchor:"datasets.load_dataset_builder.download_config",description:`<strong>download_config</strong> (<a href="/docs/datasets/main/en/package_reference/builder_classes#datasets.DownloadConfig">DownloadConfig</a>, <em>optional</em>) — | |
| Specific download configuration parameters.`,name:"download_config"},{anchor:"datasets.load_dataset_builder.download_mode",description:`<strong>download_mode</strong> (<a href="/docs/datasets/main/en/package_reference/builder_classes#datasets.DownloadMode">DownloadMode</a> or <code>str</code>, defaults to <code>REUSE_DATASET_IF_EXISTS</code>) — | |
| Download/generate mode.`,name:"download_mode"},{anchor:"datasets.load_dataset_builder.revision",description:`<strong>revision</strong> (<a href="/docs/datasets/main/en/package_reference/builder_classes#datasets.Version">Version</a> or <code>str</code>, <em>optional</em>) — | |
| Version of the dataset script to load. | |
| As datasets have their own git repository on the Datasets Hub, the default version “main” corresponds to their “main” branch. | |
| You can specify a different version than the default “main” by using a commit SHA or a git tag of the dataset repository.`,name:"revision"},{anchor:"datasets.load_dataset_builder.token",description:`<strong>token</strong> (<code>str</code> or <code>bool</code>, <em>optional</em>) — | |
| Optional string or boolean to use as Bearer token for remote files on the Datasets Hub. | |
| If <code>True</code>, or not specified, will get token from <code>"~/.huggingface"</code>.`,name:"token"},{anchor:"datasets.load_dataset_builder.storage_options",description:`<strong>storage_options</strong> (<code>dict</code>, <em>optional</em>, defaults to <code>None</code>) — | |
| <strong>Experimental</strong>. Key/value pairs to be passed on to the dataset file-system backend, if any.</p> | |
| <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"> | |
| <p class="font-medium">Added in 2.11.0</p> | |
| </div>`,name:"storage_options"},{anchor:"datasets.load_dataset_builder.trust_remote_code",description:`<strong>trust_remote_code</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Whether or not to allow for datasets defined on the Hub using a dataset script. This option | |
| should only be set to <code>True</code> for repositories you trust and in which you have read the code, as it will | |
| execute code present on the Hub on your local machine.</p> | |
| <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"> | |
| <p class="font-medium">Added in 2.16.0</p> | |
| </div> | |
| <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"> | |
| <p class="font-medium">Changed in 2.20.0</p> | |
| <p><code>trust_remote_code</code> defaults to <code>False</code> if not specified.</p> | |
| </div>`,name:"trust_remote_code"},{anchor:"datasets.load_dataset_builder.*config_kwargs",description:`*<strong>*config_kwargs</strong> (additional keyword arguments) — | |
| Keyword arguments to be passed to the <a href="/docs/datasets/main/en/package_reference/builder_classes#datasets.BuilderConfig">BuilderConfig</a> | |
| and used in the <a href="/docs/datasets/main/en/package_reference/builder_classes#datasets.DatasetBuilder">DatasetBuilder</a>.`,name:"*config_kwargs"}],source:"https://github.com/huggingface/datasets/blob/main/src/datasets/load.py#L1680",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><a | |
| href="/docs/datasets/main/en/package_reference/builder_classes#datasets.DatasetBuilder" | |
| >DatasetBuilder</a></p> | |
| `}}),ae=new B({props:{anchor:"datasets.load_dataset_builder.example",$$slots:{default:[es]},$$scope:{ctx:D}}}),ce=new O({props:{name:"datasets.get_dataset_config_names",anchor:"datasets.get_dataset_config_names",parameters:[{name:"path",val:": str"},{name:"revision",val:": Union = None"},{name:"download_config",val:": Optional = None"},{name:"download_mode",val:": Union = None"},{name:"dynamic_modules_path",val:": Optional = None"},{name:"data_files",val:": Union = None"},{name:"**download_kwargs",val:""}],parametersDescription:[{anchor:"datasets.get_dataset_config_names.path",description:`<strong>path</strong> (<code>str</code>) — path to the dataset processing script with the dataset builder. Can be either:</p> | |
| <ul> | |
| <li>a local path to processing script or the directory containing the script (if the script has the same name as the directory), | |
| e.g. <code>'./dataset/squad'</code> or <code>'./dataset/squad/squad.py'</code></li> | |
| <li>a dataset identifier on the Hugging Face Hub (list all available datasets and ids with <code>huggingface_hub.list_datasets</code>), | |
| e.g. <code>'squad'</code>, <code>'glue'</code> or <code>'openai/webtext'</code></li> | |
| </ul>`,name:"path"},{anchor:"datasets.get_dataset_config_names.revision",description:`<strong>revision</strong> (<code>Union[str, datasets.Version]</code>, <em>optional</em>) — | |
| If specified, the dataset module will be loaded from the datasets repository at this version. | |
| By default:</p> | |
| <ul> | |
| <li>it is set to the local version of the lib.</li> | |
| <li>it will also try to load it from the main branch if it’s not available at the local version of the lib. | |
| Specifying a version that is different from your local version of the lib might cause compatibility issues.</li> | |
| </ul>`,name:"revision"},{anchor:"datasets.get_dataset_config_names.download_config",description:`<strong>download_config</strong> (<a href="/docs/datasets/main/en/package_reference/builder_classes#datasets.DownloadConfig">DownloadConfig</a>, <em>optional</em>) — | |
| Specific download configuration parameters.`,name:"download_config"},{anchor:"datasets.get_dataset_config_names.download_mode",description:`<strong>download_mode</strong> (<a href="/docs/datasets/main/en/package_reference/builder_classes#datasets.DownloadMode">DownloadMode</a> or <code>str</code>, defaults to <code>REUSE_DATASET_IF_EXISTS</code>) — | |
| Download/generate mode.`,name:"download_mode"},{anchor:"datasets.get_dataset_config_names.dynamic_modules_path",description:`<strong>dynamic_modules_path</strong> (<code>str</code>, defaults to <code>~/.cache/huggingface/modules/datasets_modules</code>) — | |
| Optional path to the directory in which the dynamic modules are saved. It must have been initialized with <code>init_dynamic_modules</code>. | |
| By default the datasets are stored inside the <code>datasets_modules</code> module.`,name:"dynamic_modules_path"},{anchor:"datasets.get_dataset_config_names.data_files",description:`<strong>data_files</strong> (<code>Union[Dict, List, str]</code>, <em>optional</em>) — | |
| Defining the data_files of the dataset configuration.`,name:"data_files"},{anchor:"datasets.get_dataset_config_names.*download_kwargs",description:`*<strong>*download_kwargs</strong> (additional keyword arguments) — | |
| Optional attributes for <a href="/docs/datasets/main/en/package_reference/builder_classes#datasets.DownloadConfig">DownloadConfig</a> which will override the attributes in <code>download_config</code> if supplied, | |
| for example <code>token</code>.`,name:"*download_kwargs"}],source:"https://github.com/huggingface/datasets/blob/main/src/datasets/inspect.py#L108"}}),ne=new B({props:{anchor:"datasets.get_dataset_config_names.example",$$slots:{default:[ts]},$$scope:{ctx:D}}}),pe=new O({props:{name:"datasets.get_dataset_infos",anchor:"datasets.get_dataset_infos",parameters:[{name:"path",val:": str"},{name:"data_files",val:": Union = None"},{name:"download_config",val:": Optional = None"},{name:"download_mode",val:": Union = None"},{name:"revision",val:": Union = None"},{name:"token",val:": Union = None"},{name:"**config_kwargs",val:""}],parametersDescription:[{anchor:"datasets.get_dataset_infos.path",description:`<strong>path</strong> (<code>str</code>) — path to the dataset processing script with the dataset builder. Can be either:</p> | |
| <ul> | |
| <li>a local path to processing script or the directory containing the script (if the script has the same name as the directory), | |
| e.g. <code>'./dataset/squad'</code> or <code>'./dataset/squad/squad.py'</code></li> | |
| <li>a dataset identifier on the Hugging Face Hub (list all available datasets and ids with <code>huggingface_hub.list_datasets</code>), | |
| e.g. <code>'squad'</code>, <code>'glue'</code> or\`<code>'openai/webtext'</code></li> | |
| </ul>`,name:"path"},{anchor:"datasets.get_dataset_infos.revision",description:`<strong>revision</strong> (<code>Union[str, datasets.Version]</code>, <em>optional</em>) — | |
| If specified, the dataset module will be loaded from the datasets repository at this version. | |
| By default:</p> | |
| <ul> | |
| <li>it is set to the local version of the lib.</li> | |
| <li>it will also try to load it from the main branch if it’s not available at the local version of the lib. | |
| Specifying a version that is different from your local version of the lib might cause compatibility issues.</li> | |
| </ul>`,name:"revision"},{anchor:"datasets.get_dataset_infos.download_config",description:`<strong>download_config</strong> (<a href="/docs/datasets/main/en/package_reference/builder_classes#datasets.DownloadConfig">DownloadConfig</a>, <em>optional</em>) — | |
| Specific download configuration parameters.`,name:"download_config"},{anchor:"datasets.get_dataset_infos.download_mode",description:`<strong>download_mode</strong> (<a href="/docs/datasets/main/en/package_reference/builder_classes#datasets.DownloadMode">DownloadMode</a> or <code>str</code>, defaults to <code>REUSE_DATASET_IF_EXISTS</code>) — | |
| Download/generate mode.`,name:"download_mode"},{anchor:"datasets.get_dataset_infos.data_files",description:`<strong>data_files</strong> (<code>Union[Dict, List, str]</code>, <em>optional</em>) — | |
| Defining the data_files of the dataset configuration.`,name:"data_files"},{anchor:"datasets.get_dataset_infos.token",description:`<strong>token</strong> (<code>str</code> or <code>bool</code>, <em>optional</em>) — | |
| Optional string or boolean to use as Bearer token for remote files on the Datasets Hub. | |
| If <code>True</code>, or not specified, will get token from <code>"~/.huggingface"</code>.`,name:"token"},{anchor:"datasets.get_dataset_infos.*config_kwargs",description:`*<strong>*config_kwargs</strong> (additional keyword arguments) — | |
| Optional attributes for builder class which will override the attributes if supplied.`,name:"*config_kwargs"}],source:"https://github.com/huggingface/datasets/blob/main/src/datasets/inspect.py#L41"}}),se=new B({props:{anchor:"datasets.get_dataset_infos.example",$$slots:{default:[as]},$$scope:{ctx:D}}}),me=new O({props:{name:"datasets.get_dataset_split_names",anchor:"datasets.get_dataset_split_names",parameters:[{name:"path",val:": str"},{name:"config_name",val:": Optional = None"},{name:"data_files",val:": Union = None"},{name:"download_config",val:": Optional = None"},{name:"download_mode",val:": Union = None"},{name:"revision",val:": Union = None"},{name:"token",val:": Union = None"},{name:"**config_kwargs",val:""}],parametersDescription:[{anchor:"datasets.get_dataset_split_names.path",description:`<strong>path</strong> (<code>str</code>) — path to the dataset processing script with the dataset builder. Can be either:</p> | |
| <ul> | |
| <li>a local path to processing script or the directory containing the script (if the script has the same name as the directory), | |
| e.g. <code>'./dataset/squad'</code> or <code>'./dataset/squad/squad.py'</code></li> | |
| <li>a dataset identifier on the Hugging Face Hub (list all available datasets and ids with <code>huggingface_hub.list_datasets</code>), | |
| e.g. <code>'squad'</code>, <code>'glue'</code> or <code>'openai/webtext'</code></li> | |
| </ul>`,name:"path"},{anchor:"datasets.get_dataset_split_names.config_name",description:`<strong>config_name</strong> (<code>str</code>, <em>optional</em>) — | |
| Defining the name of the dataset configuration.`,name:"config_name"},{anchor:"datasets.get_dataset_split_names.data_files",description:`<strong>data_files</strong> (<code>str</code> or <code>Sequence</code> or <code>Mapping</code>, <em>optional</em>) — | |
| Path(s) to source data file(s).`,name:"data_files"},{anchor:"datasets.get_dataset_split_names.download_config",description:`<strong>download_config</strong> (<a href="/docs/datasets/main/en/package_reference/builder_classes#datasets.DownloadConfig">DownloadConfig</a>, <em>optional</em>) — | |
| Specific download configuration parameters.`,name:"download_config"},{anchor:"datasets.get_dataset_split_names.download_mode",description:`<strong>download_mode</strong> (<a href="/docs/datasets/main/en/package_reference/builder_classes#datasets.DownloadMode">DownloadMode</a> or <code>str</code>, defaults to <code>REUSE_DATASET_IF_EXISTS</code>) — | |
| Download/generate mode.`,name:"download_mode"},{anchor:"datasets.get_dataset_split_names.revision",description:`<strong>revision</strong> (<a href="/docs/datasets/main/en/package_reference/builder_classes#datasets.Version">Version</a> or <code>str</code>, <em>optional</em>) — | |
| Version of the dataset script to load. | |
| As datasets have their own git repository on the Datasets Hub, the default version “main” corresponds to their “main” branch. | |
| You can specify a different version than the default “main” by using a commit SHA or a git tag of the dataset repository.`,name:"revision"},{anchor:"datasets.get_dataset_split_names.token",description:`<strong>token</strong> (<code>str</code> or <code>bool</code>, <em>optional</em>) — | |
| Optional string or boolean to use as Bearer token for remote files on the Datasets Hub. | |
| If <code>True</code>, or not specified, will get token from <code>"~/.huggingface"</code>.`,name:"token"},{anchor:"datasets.get_dataset_split_names.*config_kwargs",description:`*<strong>*config_kwargs</strong> (additional keyword arguments) — | |
| Optional attributes for builder class which will override the attributes if supplied.`,name:"*config_kwargs"}],source:"https://github.com/huggingface/datasets/blob/main/src/datasets/inspect.py#L307"}}),oe=new B({props:{anchor:"datasets.get_dataset_split_names.example",$$slots:{default:[ns]},$$scope:{ctx:D}}}),ge=new T({props:{title:"From files",local:"from-files",headingTag:"h2"}}),_e=new I({props:{code:"bG9hZF9kYXRhc2V0KCUyMmNzdiUyMiUyQyUyMGRhdGFfZGlyJTNEJTIycGF0aCUyRnRvJTJGZGF0YSUyRmRpciUyMiUyQyUyMHNlcCUzRCUyMiU1Q3QlMjIp",highlighted:'load_dataset(<span class="hljs-string">"csv"</span>, data_dir=<span class="hljs-string">"path/to/data/dir"</span>, sep=<span class="hljs-string">"\\t"</span>)',wrap:!1}}),ve=new T({props:{title:"Text",local:"datasets.packaged_modules.text.TextConfig",headingTag:"h3"}}),be=new O({props:{name:"class datasets.packaged_modules.text.TextConfig",anchor:"datasets.packaged_modules.text.TextConfig",parameters:[{name:"name",val:": str = 'default'"},{name:"version",val:": Union = 0.0.0"},{name:"data_dir",val:": Optional = None"},{name:"data_files",val:": Union = None"},{name:"description",val:": Optional = None"},{name:"features",val:": Optional = None"},{name:"encoding",val:": str = 'utf-8'"},{name:"encoding_errors",val:": Optional = None"},{name:"chunksize",val:": int = 10485760"},{name:"keep_linebreaks",val:": bool = False"},{name:"sample_by",val:": str = 'line'"}],source:"https://github.com/huggingface/datasets/blob/main/src/datasets/packaged_modules/text/text.py#L16"}}),xe=new O({props:{name:"class datasets.packaged_modules.text.Text",anchor:"datasets.packaged_modules.text.Text",parameters:[{name:"cache_dir",val:": Optional = None"},{name:"dataset_name",val:": Optional = None"},{name:"config_name",val:": Optional = None"},{name:"hash",val:": Optional = None"},{name:"base_path",val:": Optional = None"},{name:"info",val:": Optional = None"},{name:"features",val:": Optional = None"},{name:"token",val:": Union = None"},{name:"repo_id",val:": Optional = None"},{name:"data_files",val:": Union = None"},{name:"data_dir",val:": Optional = None"},{name:"storage_options",val:": Optional = None"},{name:"writer_batch_size",val:": Optional = None"},{name:"**config_kwargs",val:""}],source:"https://github.com/huggingface/datasets/blob/main/src/datasets/packaged_modules/text/text.py#L28"}}),we=new T({props:{title:"CSV",local:"datasets.packaged_modules.csv.CsvConfig",headingTag:"h3"}}),ke=new O({props:{name:"class datasets.packaged_modules.csv.CsvConfig",anchor:"datasets.packaged_modules.csv.CsvConfig",parameters:[{name:"name",val:": str = 'default'"},{name:"version",val:": Union = 0.0.0"},{name:"data_dir",val:": Optional = None"},{name:"data_files",val:": Union = None"},{name:"description",val:": Optional = None"},{name:"sep",val:": str = ','"},{name:"delimiter",val:": Optional = None"},{name:"header",val:": Union = 'infer'"},{name:"names",val:": Optional = None"},{name:"column_names",val:": Optional = None"},{name:"index_col",val:": Union = None"},{name:"usecols",val:": Union = None"},{name:"prefix",val:": Optional = None"},{name:"mangle_dupe_cols",val:": bool = True"},{name:"engine",val:": Optional = None"},{name:"converters",val:": Dict = None"},{name:"true_values",val:": Optional = None"},{name:"false_values",val:": Optional = None"},{name:"skipinitialspace",val:": bool = False"},{name:"skiprows",val:": Union = None"},{name:"nrows",val:": Optional = None"},{name:"na_values",val:": Union = None"},{name:"keep_default_na",val:": bool = True"},{name:"na_filter",val:": bool = True"},{name:"verbose",val:": bool = False"},{name:"skip_blank_lines",val:": bool = True"},{name:"thousands",val:": Optional = None"},{name:"decimal",val:": str = '.'"},{name:"lineterminator",val:": Optional = None"},{name:"quotechar",val:`: str = '"'`},{name:"quoting",val:": int = 0"},{name:"escapechar",val:": Optional = None"},{name:"comment",val:": Optional = None"},{name:"encoding",val:": Optional = None"},{name:"dialect",val:": Optional = None"},{name:"error_bad_lines",val:": bool = True"},{name:"warn_bad_lines",val:": bool = True"},{name:"skipfooter",val:": int = 0"},{name:"doublequote",val:": bool = True"},{name:"memory_map",val:": bool = False"},{name:"float_precision",val:": Optional = None"},{name:"chunksize",val:": int = 10000"},{name:"features",val:": Optional = None"},{name:"encoding_errors",val:": Optional = 'strict'"},{name:"on_bad_lines",val:": Literal = 'error'"},{name:"date_format",val:": Optional = None"}],source:"https://github.com/huggingface/datasets/blob/main/src/datasets/packaged_modules/csv/csv.py#L24"}}),Ne=new O({props:{name:"class datasets.packaged_modules.csv.Csv",anchor:"datasets.packaged_modules.csv.Csv",parameters:[{name:"cache_dir",val:": Optional = None"},{name:"dataset_name",val:": Optional = None"},{name:"config_name",val:": Optional = None"},{name:"hash",val:": Optional = None"},{name:"base_path",val:": Optional = None"},{name:"info",val:": Optional = None"},{name:"features",val:": Optional = None"},{name:"token",val:": Union = None"},{name:"repo_id",val:": Optional = None"},{name:"data_files",val:": Union = None"},{name:"data_dir",val:": Optional = None"},{name:"storage_options",val:": Optional = None"},{name:"writer_batch_size",val:": Optional = None"},{name:"**config_kwargs",val:""}],source:"https://github.com/huggingface/datasets/blob/main/src/datasets/packaged_modules/csv/csv.py#L145"}}),Ce=new T({props:{title:"JSON",local:"datasets.packaged_modules.json.JsonConfig",headingTag:"h3"}}),Oe=new O({props:{name:"class datasets.packaged_modules.json.JsonConfig",anchor:"datasets.packaged_modules.json.JsonConfig",parameters:[{name:"name",val:": str = 'default'"},{name:"version",val:": Union = 0.0.0"},{name:"data_dir",val:": Optional = None"},{name:"data_files",val:": Union = None"},{name:"description",val:": Optional = None"},{name:"features",val:": Optional = None"},{name:"encoding",val:": str = 'utf-8'"},{name:"encoding_errors",val:": Optional = None"},{name:"field",val:": Optional = None"},{name:"use_threads",val:": bool = True"},{name:"block_size",val:": Optional = None"},{name:"chunksize",val:": int = 10485760"},{name:"newlines_in_values",val:": Optional = None"}],source:"https://github.com/huggingface/datasets/blob/main/src/datasets/packaged_modules/json/json.py#L41"}}),je=new O({props:{name:"class datasets.packaged_modules.json.Json",anchor:"datasets.packaged_modules.json.Json",parameters:[{name:"cache_dir",val:": Optional = None"},{name:"dataset_name",val:": Optional = None"},{name:"config_name",val:": Optional = None"},{name:"hash",val:": Optional = None"},{name:"base_path",val:": Optional = None"},{name:"info",val:": Optional = None"},{name:"features",val:": Optional = None"},{name:"token",val:": Union = None"},{name:"repo_id",val:": Optional = None"},{name:"data_files",val:": Union = None"},{name:"data_dir",val:": Optional = None"},{name:"storage_options",val:": Optional = None"},{name:"writer_batch_size",val:": Optional = None"},{name:"**config_kwargs",val:""}],source:"https://github.com/huggingface/datasets/blob/main/src/datasets/packaged_modules/json/json.py#L58"}}),Ue=new T({props:{title:"Parquet",local:"datasets.packaged_modules.parquet.ParquetConfig",headingTag:"h3"}}),Te=new O({props:{name:"class datasets.packaged_modules.parquet.ParquetConfig",anchor:"datasets.packaged_modules.parquet.ParquetConfig",parameters:[{name:"name",val:": str = 'default'"},{name:"version",val:": Union = 0.0.0"},{name:"data_dir",val:": Optional = None"},{name:"data_files",val:": Union = None"},{name:"description",val:": Optional = None"},{name:"batch_size",val:": Optional = None"},{name:"columns",val:": Optional = None"},{name:"features",val:": Optional = None"}],source:"https://github.com/huggingface/datasets/blob/main/src/datasets/packaged_modules/parquet/parquet.py#L15"}}),Fe=new O({props:{name:"class datasets.packaged_modules.parquet.Parquet",anchor:"datasets.packaged_modules.parquet.Parquet",parameters:[{name:"cache_dir",val:": Optional = None"},{name:"dataset_name",val:": Optional = None"},{name:"config_name",val:": Optional = None"},{name:"hash",val:": Optional = None"},{name:"base_path",val:": Optional = None"},{name:"info",val:": Optional = None"},{name:"features",val:": Optional = None"},{name:"token",val:": Union = None"},{name:"repo_id",val:": Optional = None"},{name:"data_files",val:": Union = None"},{name:"data_dir",val:": Optional = None"},{name:"storage_options",val:": Optional = None"},{name:"writer_batch_size",val:": Optional = None"},{name:"**config_kwargs",val:""}],source:"https://github.com/huggingface/datasets/blob/main/src/datasets/packaged_modules/parquet/parquet.py#L27"}}),Je=new T({props:{title:"Arrow",local:"datasets.packaged_modules.arrow.ArrowConfig",headingTag:"h3"}}),qe=new O({props:{name:"class datasets.packaged_modules.arrow.ArrowConfig",anchor:"datasets.packaged_modules.arrow.ArrowConfig",parameters:[{name:"name",val:": str = 'default'"},{name:"version",val:": Union = 0.0.0"},{name:"data_dir",val:": Optional = None"},{name:"data_files",val:": Union = None"},{name:"description",val:": Optional = None"},{name:"features",val:": Optional = None"}],source:"https://github.com/huggingface/datasets/blob/main/src/datasets/packaged_modules/arrow/arrow.py#L14"}}),Re=new O({props:{name:"class datasets.packaged_modules.arrow.Arrow",anchor:"datasets.packaged_modules.arrow.Arrow",parameters:[{name:"cache_dir",val:": Optional = None"},{name:"dataset_name",val:": Optional = None"},{name:"config_name",val:": Optional = None"},{name:"hash",val:": Optional = None"},{name:"base_path",val:": Optional = None"},{name:"info",val:": Optional = None"},{name:"features",val:": Optional = None"},{name:"token",val:": Union = None"},{name:"repo_id",val:": Optional = None"},{name:"data_files",val:": Union = None"},{name:"data_dir",val:": Optional = None"},{name:"storage_options",val:": Optional = None"},{name:"writer_batch_size",val:": Optional = None"},{name:"**config_kwargs",val:""}],source:"https://github.com/huggingface/datasets/blob/main/src/datasets/packaged_modules/arrow/arrow.py#L24"}}),Ge=new T({props:{title:"SQL",local:"datasets.packaged_modules.sql.SqlConfig",headingTag:"h3"}}),Se=new O({props:{name:"class datasets.packaged_modules.sql.SqlConfig",anchor:"datasets.packaged_modules.sql.SqlConfig",parameters:[{name:"name",val:": str = 'default'"},{name:"version",val:": Union = 0.0.0"},{name:"data_dir",val:": Optional = None"},{name:"data_files",val:": Union = None"},{name:"description",val:": Optional = None"},{name:"sql",val:": Union = None"},{name:"con",val:": Union = None"},{name:"index_col",val:": Union = None"},{name:"coerce_float",val:": bool = True"},{name:"params",val:": Union = None"},{name:"parse_dates",val:": Union = None"},{name:"columns",val:": Optional = None"},{name:"chunksize",val:": Optional = 10000"},{name:"features",val:": Optional = None"}],source:"https://github.com/huggingface/datasets/blob/main/src/datasets/packaged_modules/sql/sql.py#L23"}}),He=new O({props:{name:"class datasets.packaged_modules.sql.Sql",anchor:"datasets.packaged_modules.sql.Sql",parameters:[{name:"cache_dir",val:": Optional = None"},{name:"dataset_name",val:": Optional = None"},{name:"config_name",val:": Optional = None"},{name:"hash",val:": Optional = None"},{name:"base_path",val:": Optional = None"},{name:"info",val:": Optional = None"},{name:"features",val:": Optional = None"},{name:"token",val:": Union = None"},{name:"repo_id",val:": Optional = None"},{name:"data_files",val:": Union = None"},{name:"data_dir",val:": Optional = None"},{name:"storage_options",val:": Optional = None"},{name:"writer_batch_size",val:": Optional = None"},{name:"**config_kwargs",val:""}],source:"https://github.com/huggingface/datasets/blob/main/src/datasets/packaged_modules/sql/sql.py#L91"}}),ze=new T({props:{title:"Images",local:"datasets.packaged_modules.imagefolder.ImageFolderConfig",headingTag:"h3"}}),Ve=new O({props:{name:"class datasets.packaged_modules.imagefolder.ImageFolderConfig",anchor:"datasets.packaged_modules.imagefolder.ImageFolderConfig",parameters:[{name:"name",val:": str = 'default'"},{name:"version",val:": Union = 0.0.0"},{name:"data_dir",val:": Optional = None"},{name:"data_files",val:": Union = None"},{name:"description",val:": Optional = None"},{name:"features",val:": Optional = None"},{name:"drop_labels",val:": bool = None"},{name:"drop_metadata",val:": bool = None"}],source:"https://github.com/huggingface/datasets/blob/main/src/datasets/packaged_modules/imagefolder/imagefolder.py#L11"}}),Ee=new O({props:{name:"class datasets.packaged_modules.imagefolder.ImageFolder",anchor:"datasets.packaged_modules.imagefolder.ImageFolder",parameters:[{name:"cache_dir",val:": Optional = None"},{name:"dataset_name",val:": Optional = None"},{name:"config_name",val:": Optional = None"},{name:"hash",val:": Optional = None"},{name:"base_path",val:": Optional = None"},{name:"info",val:": Optional = None"},{name:"features",val:": Optional = None"},{name:"token",val:": Union = None"},{name:"repo_id",val:": Optional = None"},{name:"data_files",val:": Union = None"},{name:"data_dir",val:": Optional = None"},{name:"storage_options",val:": Optional = None"},{name:"writer_batch_size",val:": Optional = None"},{name:"**config_kwargs",val:""}],source:"https://github.com/huggingface/datasets/blob/main/src/datasets/packaged_modules/imagefolder/imagefolder.py#L21"}}),Le=new T({props:{title:"Audio",local:"datasets.packaged_modules.audiofolder.AudioFolderConfig",headingTag:"h3"}}),Pe=new O({props:{name:"class datasets.packaged_modules.audiofolder.AudioFolderConfig",anchor:"datasets.packaged_modules.audiofolder.AudioFolderConfig",parameters:[{name:"name",val:": str = 'default'"},{name:"version",val:": Union = 0.0.0"},{name:"data_dir",val:": Optional = None"},{name:"data_files",val:": Union = None"},{name:"description",val:": Optional = None"},{name:"features",val:": Optional = None"},{name:"drop_labels",val:": bool = None"},{name:"drop_metadata",val:": bool = None"}],source:"https://github.com/huggingface/datasets/blob/main/src/datasets/packaged_modules/audiofolder/audiofolder.py#L11"}}),Be=new O({props:{name:"class datasets.packaged_modules.audiofolder.AudioFolder",anchor:"datasets.packaged_modules.audiofolder.AudioFolder",parameters:[{name:"cache_dir",val:": Optional = None"},{name:"dataset_name",val:": Optional = None"},{name:"config_name",val:": Optional = None"},{name:"hash",val:": Optional = None"},{name:"base_path",val:": Optional = None"},{name:"info",val:": Optional = None"},{name:"features",val:": Optional = None"},{name:"token",val:": Union = None"},{name:"repo_id",val:": Optional = None"},{name:"data_files",val:": Union = None"},{name:"data_dir",val:": Optional = None"},{name:"storage_options",val:": Optional = None"},{name:"writer_batch_size",val:": Optional = None"},{name:"**config_kwargs",val:""}],source:"https://github.com/huggingface/datasets/blob/main/src/datasets/packaged_modules/audiofolder/audiofolder.py#L21"}}),Ye=new T({props:{title:"WebDataset",local:"datasets.packaged_modules.webdataset.WebDataset",headingTag:"h3"}}),Qe=new O({props:{name:"class datasets.packaged_modules.webdataset.WebDataset",anchor:"datasets.packaged_modules.webdataset.WebDataset",parameters:[{name:"cache_dir",val:": Optional = None"},{name:"dataset_name",val:": Optional = None"},{name:"config_name",val:": Optional = None"},{name:"hash",val:": Optional = None"},{name:"base_path",val:": Optional = None"},{name:"info",val:": Optional = None"},{name:"features",val:": Optional = None"},{name:"token",val:": Union = None"},{name:"repo_id",val:": Optional = None"},{name:"data_files",val:": Union = None"},{name:"data_dir",val:": Optional = None"},{name:"storage_options",val:": Optional = None"},{name:"writer_batch_size",val:": Optional = None"},{name:"**config_kwargs",val:""}],source:"https://github.com/huggingface/datasets/blob/main/src/datasets/packaged_modules/webdataset/webdataset.py#L18"}}),Ke=new An({props:{source:"https://github.com/huggingface/datasets/blob/main/docs/source/package_reference/loading_methods.mdx"}}),{c(){r=i("meta"),k=o(),b=i("p"),l=o(),m(v.$$.fragment),t=o(),$=i("p"),$.textContent=sn,Ut=o(),m(de.$$.fragment),Tt=o(),w=i("div"),m(re.$$.fragment),ka=o(),at=i("p"),at.textContent=on,ya=o(),nt=i("p"),nt.innerHTML=dn,Na=o(),st=i("p"),st.textContent=rn,Ca=o(),ot=i("ul"),ot.innerHTML=ln,Oa=o(),dt=i("p"),dt.textContent=cn,Da=o(),rt=i("p"),rt.textContent=pn,ja=o(),lt=i("ol"),lt.innerHTML=mn,Ua=o(),it=i("p"),it.textContent=gn,Ta=o(),ct=i("p"),ct.textContent=fn,Ma=o(),m(W.$$.fragment),Fa=o(),m(Q.$$.fragment),Ja=o(),m(K.$$.fragment),qa=o(),m(ee.$$.fragment),Mt=o(),M=i("div"),m(le.$$.fragment),Ia=o(),pt=i("p"),pt.innerHTML=un,Ra=o(),m(te.$$.fragment),Ft=o(),j=i("div"),m(ie.$$.fragment),Ga=o(),mt=i("p"),mt.textContent=hn,Sa=o(),gt=i("p"),gt.innerHTML=_n,Za=o(),ft=i("p"),ft.textContent=vn,Ha=o(),ut=i("ul"),ut.innerHTML=bn,za=o(),ht=i("p"),ht.textContent=$n,Va=o(),m(ae.$$.fragment),Jt=o(),F=i("div"),m(ce.$$.fragment),Xa=o(),_t=i("p"),_t.textContent=xn,Ea=o(),m(ne.$$.fragment),qt=o(),J=i("div"),m(pe.$$.fragment),La=o(),vt=i("p"),vt.textContent=wn,Pa=o(),m(se.$$.fragment),It=o(),q=i("div"),m(me.$$.fragment),Aa=o(),bt=i("p"),bt.textContent=kn,Ba=o(),m(oe.$$.fragment),Rt=o(),m(ge.$$.fragment),Gt=o(),fe=i("p"),fe.textContent=yn,St=o(),ue=i("ul"),ue.innerHTML=Nn,Zt=o(),he=i("p"),he.innerHTML=Cn,Ht=o(),m(_e.$$.fragment),zt=o(),m(ve.$$.fragment),Vt=o(),R=i("div"),m(be.$$.fragment),Ya=o(),$t=i("p"),$t.textContent=On,Xt=o(),$e=i("div"),m(xe.$$.fragment),Et=o(),m(we.$$.fragment),Lt=o(),G=i("div"),m(ke.$$.fragment),Wa=o(),xt=i("p"),xt.textContent=Dn,Pt=o(),ye=i("div"),m(Ne.$$.fragment),At=o(),m(Ce.$$.fragment),Bt=o(),S=i("div"),m(Oe.$$.fragment),Qa=o(),wt=i("p"),wt.textContent=jn,Yt=o(),De=i("div"),m(je.$$.fragment),Wt=o(),m(Ue.$$.fragment),Qt=o(),Z=i("div"),m(Te.$$.fragment),Ka=o(),kt=i("p"),kt.textContent=Un,Kt=o(),Me=i("div"),m(Fe.$$.fragment),ea=o(),m(Je.$$.fragment),ta=o(),H=i("div"),m(qe.$$.fragment),en=o(),yt=i("p"),yt.textContent=Tn,aa=o(),Ie=i("div"),m(Re.$$.fragment),na=o(),m(Ge.$$.fragment),sa=o(),z=i("div"),m(Se.$$.fragment),tn=o(),Nt=i("p"),Nt.textContent=Mn,oa=o(),Ze=i("div"),m(He.$$.fragment),da=o(),m(ze.$$.fragment),ra=o(),V=i("div"),m(Ve.$$.fragment),an=o(),Ct=i("p"),Ct.textContent=Fn,la=o(),Xe=i("div"),m(Ee.$$.fragment),ia=o(),m(Le.$$.fragment),ca=o(),X=i("div"),m(Pe.$$.fragment),nn=o(),Ot=i("p"),Ot.textContent=Jn,pa=o(),Ae=i("div"),m(Be.$$.fragment),ma=o(),m(Ye.$$.fragment),ga=o(),We=i("div"),m(Qe.$$.fragment),fa=o(),m(Ke.$$.fragment),ua=o(),Dt=i("p"),this.h()},l(e){const n=Pn("svelte-u9bgzb",document.head);r=c(n,"META",{name:!0,content:!0}),n.forEach(a),k=d(e),b=c(e,"P",{}),N(b).forEach(a),l=d(e),g(v.$$.fragment,e),t=d(e),$=c(e,"P",{"data-svelte-h":!0}),x($)!=="svelte-spmsxp"&&($.textContent=sn),Ut=d(e),g(de.$$.fragment,e),Tt=d(e),w=c(e,"DIV",{class:!0});var y=N(w);g(re.$$.fragment,y),ka=d(y),at=c(y,"P",{"data-svelte-h":!0}),x(at)!=="svelte-x6t4fm"&&(at.textContent=on),ya=d(y),nt=c(y,"P",{"data-svelte-h":!0}),x(nt)!=="svelte-1j3dvna"&&(nt.innerHTML=dn),Na=d(y),st=c(y,"P",{"data-svelte-h":!0}),x(st)!=="svelte-gersvb"&&(st.textContent=rn),Ca=d(y),ot=c(y,"UL",{"data-svelte-h":!0}),x(ot)!=="svelte-o5t8md"&&(ot.innerHTML=ln),Oa=d(y),dt=c(y,"P",{"data-svelte-h":!0}),x(dt)!=="svelte-1uisk85"&&(dt.textContent=cn),Da=d(y),rt=c(y,"P",{"data-svelte-h":!0}),x(rt)!=="svelte-1osovlg"&&(rt.textContent=pn),ja=d(y),lt=c(y,"OL",{"data-svelte-h":!0}),x(lt)!=="svelte-9lyw3k"&&(lt.innerHTML=mn),Ua=d(y),it=c(y,"P",{"data-svelte-h":!0}),x(it)!=="svelte-zu0m02"&&(it.textContent=gn),Ta=d(y),ct=c(y,"P",{"data-svelte-h":!0}),x(ct)!=="svelte-11lpom8"&&(ct.textContent=fn),Ma=d(y),g(W.$$.fragment,y),Fa=d(y),g(Q.$$.fragment,y),Ja=d(y),g(K.$$.fragment,y),qa=d(y),g(ee.$$.fragment,y),y.forEach(a),Mt=d(e),M=c(e,"DIV",{class:!0});var E=N(M);g(le.$$.fragment,E),Ia=d(E),pt=c(E,"P",{"data-svelte-h":!0}),x(pt)!=="svelte-1w09pe6"&&(pt.innerHTML=un),Ra=d(E),g(te.$$.fragment,E),E.forEach(a),Ft=d(e),j=c(e,"DIV",{class:!0});var U=N(j);g(ie.$$.fragment,U),Ga=d(U),mt=c(U,"P",{"data-svelte-h":!0}),x(mt)!=="svelte-j91dbc"&&(mt.textContent=hn),Sa=d(U),gt=c(U,"P",{"data-svelte-h":!0}),x(gt)!=="svelte-1j3dvna"&&(gt.innerHTML=_n),Za=d(U),ft=c(U,"P",{"data-svelte-h":!0}),x(ft)!=="svelte-gersvb"&&(ft.textContent=vn),Ha=d(U),ut=c(U,"UL",{"data-svelte-h":!0}),x(ut)!=="svelte-9ksyn3"&&(ut.innerHTML=bn),za=d(U),ht=c(U,"P",{"data-svelte-h":!0}),x(ht)!=="svelte-1uisk85"&&(ht.textContent=$n),Va=d(U),g(ae.$$.fragment,U),U.forEach(a),Jt=d(e),F=c(e,"DIV",{class:!0});var L=N(F);g(ce.$$.fragment,L),Xa=d(L),_t=c(L,"P",{"data-svelte-h":!0}),x(_t)!=="svelte-2twqqm"&&(_t.textContent=xn),Ea=d(L),g(ne.$$.fragment,L),L.forEach(a),qt=d(e),J=c(e,"DIV",{class:!0});var P=N(J);g(pe.$$.fragment,P),La=d(P),vt=c(P,"P",{"data-svelte-h":!0}),x(vt)!=="svelte-vg9yfc"&&(vt.textContent=wn),Pa=d(P),g(se.$$.fragment,P),P.forEach(a),It=d(e),q=c(e,"DIV",{class:!0});var A=N(q);g(me.$$.fragment,A),Aa=d(A),bt=c(A,"P",{"data-svelte-h":!0}),x(bt)!=="svelte-hrdkoe"&&(bt.textContent=kn),Ba=d(A),g(oe.$$.fragment,A),A.forEach(a),Rt=d(e),g(ge.$$.fragment,e),Gt=d(e),fe=c(e,"P",{"data-svelte-h":!0}),x(fe)!=="svelte-psk543"&&(fe.textContent=yn),St=d(e),ue=c(e,"UL",{"data-svelte-h":!0}),x(ue)!=="svelte-mua0wr"&&(ue.innerHTML=Nn),Zt=d(e),he=c(e,"P",{"data-svelte-h":!0}),x(he)!=="svelte-1kwvb5j"&&(he.innerHTML=Cn),Ht=d(e),g(_e.$$.fragment,e),zt=d(e),g(ve.$$.fragment,e),Vt=d(e),R=c(e,"DIV",{class:!0});var et=N(R);g(be.$$.fragment,et),Ya=d(et),$t=c(et,"P",{"data-svelte-h":!0}),x($t)!=="svelte-jpahu0"&&($t.textContent=On),et.forEach(a),Xt=d(e),$e=c(e,"DIV",{class:!0});var jt=N($e);g(xe.$$.fragment,jt),jt.forEach(a),Et=d(e),g(we.$$.fragment,e),Lt=d(e),G=c(e,"DIV",{class:!0});var tt=N(G);g(ke.$$.fragment,tt),Wa=d(tt),xt=c(tt,"P",{"data-svelte-h":!0}),x(xt)!=="svelte-993wsc"&&(xt.textContent=Dn),tt.forEach(a),Pt=d(e),ye=c(e,"DIV",{class:!0});var qn=N(ye);g(Ne.$$.fragment,qn),qn.forEach(a),At=d(e),g(Ce.$$.fragment,e),Bt=d(e),S=c(e,"DIV",{class:!0});var _a=N(S);g(Oe.$$.fragment,_a),Qa=d(_a),wt=c(_a,"P",{"data-svelte-h":!0}),x(wt)!=="svelte-12lz3xe"&&(wt.textContent=jn),_a.forEach(a),Yt=d(e),De=c(e,"DIV",{class:!0});var In=N(De);g(je.$$.fragment,In),In.forEach(a),Wt=d(e),g(Ue.$$.fragment,e),Qt=d(e),Z=c(e,"DIV",{class:!0});var va=N(Z);g(Te.$$.fragment,va),Ka=d(va),kt=c(va,"P",{"data-svelte-h":!0}),x(kt)!=="svelte-1kivgjw"&&(kt.textContent=Un),va.forEach(a),Kt=d(e),Me=c(e,"DIV",{class:!0});var Rn=N(Me);g(Fe.$$.fragment,Rn),Rn.forEach(a),ea=d(e),g(Je.$$.fragment,e),ta=d(e),H=c(e,"DIV",{class:!0});var ba=N(H);g(qe.$$.fragment,ba),en=d(ba),yt=c(ba,"P",{"data-svelte-h":!0}),x(yt)!=="svelte-1ah41nl"&&(yt.textContent=Tn),ba.forEach(a),aa=d(e),Ie=c(e,"DIV",{class:!0});var Gn=N(Ie);g(Re.$$.fragment,Gn),Gn.forEach(a),na=d(e),g(Ge.$$.fragment,e),sa=d(e),z=c(e,"DIV",{class:!0});var $a=N(z);g(Se.$$.fragment,$a),tn=d($a),Nt=c($a,"P",{"data-svelte-h":!0}),x(Nt)!=="svelte-18rutfs"&&(Nt.textContent=Mn),$a.forEach(a),oa=d(e),Ze=c(e,"DIV",{class:!0});var Sn=N(Ze);g(He.$$.fragment,Sn),Sn.forEach(a),da=d(e),g(ze.$$.fragment,e),ra=d(e),V=c(e,"DIV",{class:!0});var xa=N(V);g(Ve.$$.fragment,xa),an=d(xa),Ct=c(xa,"P",{"data-svelte-h":!0}),x(Ct)!=="svelte-myiztp"&&(Ct.textContent=Fn),xa.forEach(a),la=d(e),Xe=c(e,"DIV",{class:!0});var Zn=N(Xe);g(Ee.$$.fragment,Zn),Zn.forEach(a),ia=d(e),g(Le.$$.fragment,e),ca=d(e),X=c(e,"DIV",{class:!0});var wa=N(X);g(Pe.$$.fragment,wa),nn=d(wa),Ot=c(wa,"P",{"data-svelte-h":!0}),x(Ot)!=="svelte-obgmtg"&&(Ot.textContent=Jn),wa.forEach(a),pa=d(e),Ae=c(e,"DIV",{class:!0});var Hn=N(Ae);g(Be.$$.fragment,Hn),Hn.forEach(a),ma=d(e),g(Ye.$$.fragment,e),ga=d(e),We=c(e,"DIV",{class:!0});var zn=N(We);g(Qe.$$.fragment,zn),zn.forEach(a),fa=d(e),g(Ke.$$.fragment,e),ua=d(e),Dt=c(e,"P",{}),N(Dt).forEach(a),this.h()},h(){C(r,"name","hf:doc:metadata"),C(r,"content",os),C(w,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(M,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(j,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(F,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(R,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C($e,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(ye,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(S,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(De,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(Me,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(H,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(Ie,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(Ze,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(V,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(Xe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(X,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(Ae,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),C(We,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,n){p(document.head,r),s(e,k,n),s(e,b,n),s(e,l,n),f(v,e,n),s(e,t,n),s(e,$,n),s(e,Ut,n),f(de,e,n),s(e,Tt,n),s(e,w,n),f(re,w,null),p(w,ka),p(w,at),p(w,ya),p(w,nt),p(w,Na),p(w,st),p(w,Ca),p(w,ot),p(w,Oa),p(w,dt),p(w,Da),p(w,rt),p(w,ja),p(w,lt),p(w,Ua),p(w,it),p(w,Ta),p(w,ct),p(w,Ma),f(W,w,null),p(w,Fa),f(Q,w,null),p(w,Ja),f(K,w,null),p(w,qa),f(ee,w,null),s(e,Mt,n),s(e,M,n),f(le,M,null),p(M,Ia),p(M,pt),p(M,Ra),f(te,M,null),s(e,Ft,n),s(e,j,n),f(ie,j,null),p(j,Ga),p(j,mt),p(j,Sa),p(j,gt),p(j,Za),p(j,ft),p(j,Ha),p(j,ut),p(j,za),p(j,ht),p(j,Va),f(ae,j,null),s(e,Jt,n),s(e,F,n),f(ce,F,null),p(F,Xa),p(F,_t),p(F,Ea),f(ne,F,null),s(e,qt,n),s(e,J,n),f(pe,J,null),p(J,La),p(J,vt),p(J,Pa),f(se,J,null),s(e,It,n),s(e,q,n),f(me,q,null),p(q,Aa),p(q,bt),p(q,Ba),f(oe,q,null),s(e,Rt,n),f(ge,e,n),s(e,Gt,n),s(e,fe,n),s(e,St,n),s(e,ue,n),s(e,Zt,n),s(e,he,n),s(e,Ht,n),f(_e,e,n),s(e,zt,n),f(ve,e,n),s(e,Vt,n),s(e,R,n),f(be,R,null),p(R,Ya),p(R,$t),s(e,Xt,n),s(e,$e,n),f(xe,$e,null),s(e,Et,n),f(we,e,n),s(e,Lt,n),s(e,G,n),f(ke,G,null),p(G,Wa),p(G,xt),s(e,Pt,n),s(e,ye,n),f(Ne,ye,null),s(e,At,n),f(Ce,e,n),s(e,Bt,n),s(e,S,n),f(Oe,S,null),p(S,Qa),p(S,wt),s(e,Yt,n),s(e,De,n),f(je,De,null),s(e,Wt,n),f(Ue,e,n),s(e,Qt,n),s(e,Z,n),f(Te,Z,null),p(Z,Ka),p(Z,kt),s(e,Kt,n),s(e,Me,n),f(Fe,Me,null),s(e,ea,n),f(Je,e,n),s(e,ta,n),s(e,H,n),f(qe,H,null),p(H,en),p(H,yt),s(e,aa,n),s(e,Ie,n),f(Re,Ie,null),s(e,na,n),f(Ge,e,n),s(e,sa,n),s(e,z,n),f(Se,z,null),p(z,tn),p(z,Nt),s(e,oa,n),s(e,Ze,n),f(He,Ze,null),s(e,da,n),f(ze,e,n),s(e,ra,n),s(e,V,n),f(Ve,V,null),p(V,an),p(V,Ct),s(e,la,n),s(e,Xe,n),f(Ee,Xe,null),s(e,ia,n),f(Le,e,n),s(e,ca,n),s(e,X,n),f(Pe,X,null),p(X,nn),p(X,Ot),s(e,pa,n),s(e,Ae,n),f(Be,Ae,null),s(e,ma,n),f(Ye,e,n),s(e,ga,n),s(e,We,n),f(Qe,We,null),s(e,fa,n),f(Ke,e,n),s(e,ua,n),s(e,Dt,n),ha=!0},p(e,[n]){const y={};n&2&&(y.$$scope={dirty:n,ctx:e}),W.$set(y);const E={};n&2&&(E.$$scope={dirty:n,ctx:e}),Q.$set(E);const U={};n&2&&(U.$$scope={dirty:n,ctx:e}),K.$set(U);const L={};n&2&&(L.$$scope={dirty:n,ctx:e}),ee.$set(L);const P={};n&2&&(P.$$scope={dirty:n,ctx:e}),te.$set(P);const A={};n&2&&(A.$$scope={dirty:n,ctx:e}),ae.$set(A);const et={};n&2&&(et.$$scope={dirty:n,ctx:e}),ne.$set(et);const jt={};n&2&&(jt.$$scope={dirty:n,ctx:e}),se.$set(jt);const tt={};n&2&&(tt.$$scope={dirty:n,ctx:e}),oe.$set(tt)},i(e){ha||(u(v.$$.fragment,e),u(de.$$.fragment,e),u(re.$$.fragment,e),u(W.$$.fragment,e),u(Q.$$.fragment,e),u(K.$$.fragment,e),u(ee.$$.fragment,e),u(le.$$.fragment,e),u(te.$$.fragment,e),u(ie.$$.fragment,e),u(ae.$$.fragment,e),u(ce.$$.fragment,e),u(ne.$$.fragment,e),u(pe.$$.fragment,e),u(se.$$.fragment,e),u(me.$$.fragment,e),u(oe.$$.fragment,e),u(ge.$$.fragment,e),u(_e.$$.fragment,e),u(ve.$$.fragment,e),u(be.$$.fragment,e),u(xe.$$.fragment,e),u(we.$$.fragment,e),u(ke.$$.fragment,e),u(Ne.$$.fragment,e),u(Ce.$$.fragment,e),u(Oe.$$.fragment,e),u(je.$$.fragment,e),u(Ue.$$.fragment,e),u(Te.$$.fragment,e),u(Fe.$$.fragment,e),u(Je.$$.fragment,e),u(qe.$$.fragment,e),u(Re.$$.fragment,e),u(Ge.$$.fragment,e),u(Se.$$.fragment,e),u(He.$$.fragment,e),u(ze.$$.fragment,e),u(Ve.$$.fragment,e),u(Ee.$$.fragment,e),u(Le.$$.fragment,e),u(Pe.$$.fragment,e),u(Be.$$.fragment,e),u(Ye.$$.fragment,e),u(Qe.$$.fragment,e),u(Ke.$$.fragment,e),ha=!0)},o(e){h(v.$$.fragment,e),h(de.$$.fragment,e),h(re.$$.fragment,e),h(W.$$.fragment,e),h(Q.$$.fragment,e),h(K.$$.fragment,e),h(ee.$$.fragment,e),h(le.$$.fragment,e),h(te.$$.fragment,e),h(ie.$$.fragment,e),h(ae.$$.fragment,e),h(ce.$$.fragment,e),h(ne.$$.fragment,e),h(pe.$$.fragment,e),h(se.$$.fragment,e),h(me.$$.fragment,e),h(oe.$$.fragment,e),h(ge.$$.fragment,e),h(_e.$$.fragment,e),h(ve.$$.fragment,e),h(be.$$.fragment,e),h(xe.$$.fragment,e),h(we.$$.fragment,e),h(ke.$$.fragment,e),h(Ne.$$.fragment,e),h(Ce.$$.fragment,e),h(Oe.$$.fragment,e),h(je.$$.fragment,e),h(Ue.$$.fragment,e),h(Te.$$.fragment,e),h(Fe.$$.fragment,e),h(Je.$$.fragment,e),h(qe.$$.fragment,e),h(Re.$$.fragment,e),h(Ge.$$.fragment,e),h(Se.$$.fragment,e),h(He.$$.fragment,e),h(ze.$$.fragment,e),h(Ve.$$.fragment,e),h(Ee.$$.fragment,e),h(Le.$$.fragment,e),h(Pe.$$.fragment,e),h(Be.$$.fragment,e),h(Ye.$$.fragment,e),h(Qe.$$.fragment,e),h(Ke.$$.fragment,e),ha=!1},d(e){e&&(a(k),a(b),a(l),a(t),a($),a(Ut),a(Tt),a(w),a(Mt),a(M),a(Ft),a(j),a(Jt),a(F),a(qt),a(J),a(It),a(q),a(Rt),a(Gt),a(fe),a(St),a(ue),a(Zt),a(he),a(Ht),a(zt),a(Vt),a(R),a(Xt),a($e),a(Et),a(Lt),a(G),a(Pt),a(ye),a(At),a(Bt),a(S),a(Yt),a(De),a(Wt),a(Qt),a(Z),a(Kt),a(Me),a(ea),a(ta),a(H),a(aa),a(Ie),a(na),a(sa),a(z),a(oa),a(Ze),a(da),a(ra),a(V),a(la),a(Xe),a(ia),a(ca),a(X),a(pa),a(Ae),a(ma),a(ga),a(We),a(fa),a(ua),a(Dt)),a(r),_(v,e),_(de,e),_(re),_(W),_(Q),_(K),_(ee),_(le),_(te),_(ie),_(ae),_(ce),_(ne),_(pe),_(se),_(me),_(oe),_(ge,e),_(_e,e),_(ve,e),_(be),_(xe),_(we,e),_(ke),_(Ne),_(Ce,e),_(Oe),_(je),_(Ue,e),_(Te),_(Fe),_(Je,e),_(qe),_(Re),_(Ge,e),_(Se),_(He),_(ze,e),_(Ve),_(Ee),_(Le,e),_(Pe),_(Be),_(Ye,e),_(Qe),_(Ke,e)}}}const os='{"title":"Loading methods","local":"loading-methods","sections":[{"title":"Datasets","local":"datasets.load_dataset","sections":[],"depth":2},{"title":"From files","local":"from-files","sections":[{"title":"Text","local":"datasets.packaged_modules.text.TextConfig","sections":[],"depth":3},{"title":"CSV","local":"datasets.packaged_modules.csv.CsvConfig","sections":[],"depth":3},{"title":"JSON","local":"datasets.packaged_modules.json.JsonConfig","sections":[],"depth":3},{"title":"Parquet","local":"datasets.packaged_modules.parquet.ParquetConfig","sections":[],"depth":3},{"title":"Arrow","local":"datasets.packaged_modules.arrow.ArrowConfig","sections":[],"depth":3},{"title":"SQL","local":"datasets.packaged_modules.sql.SqlConfig","sections":[],"depth":3},{"title":"Images","local":"datasets.packaged_modules.imagefolder.ImageFolderConfig","sections":[],"depth":3},{"title":"Audio","local":"datasets.packaged_modules.audiofolder.AudioFolderConfig","sections":[],"depth":3},{"title":"WebDataset","local":"datasets.packaged_modules.webdataset.WebDataset","sections":[],"depth":3}],"depth":2}],"depth":1}';function ds(D){return Xn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class gs extends En{constructor(r){super(),Ln(this,r,ds,ss,Vn,{})}}export{gs as component}; | |
Xet Storage Details
- Size:
- 83.6 kB
- Xet hash:
- d68b342ba14c383bf4ec8329ec223c91606e268771f8aaf75c8f9a2c4d389d93
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.