Buckets:

hf-doc-build
/

doc-dev

hf-doc-build/doc-dev / datasets /main /en /_app /immutable /nodes /27.3d44ca30.js

rtrm's picture

about 2 months ago

18.7 kB

	import{s as qs,o as Hs,n as Ls}from"../chunks/scheduler.bdbef820.js";import{S as Ys,i as Bs,g as p,s as n,r,A as Ps,h as o,f as a,c as l,j as zs,u as d,x as f,k as Ws,y as Ss,a as e,v as c,d as m,t as g,w as h}from"../chunks/index.c0aea24a.js";import{T as Ds}from"../chunks/Tip.31005f7d.js";import{C as _}from"../chunks/CodeBlock.6ccca92e.js";import{H as V,E as Ns}from"../chunks/EditOnGithub.725ee0c1.js";function Vs(E){let i,w="For security reasons, 🤗 Datasets do not allow running dataset loading scripts by default, and you have to pass <code>trust_remote_code=True</code> to load datasets that require running a dataset script.";return{c(){i=p("p"),i.innerHTML=w},l(u){i=o(u,"P",{"data-svelte-h":!0}),f(i)!=="svelte-1adalis"&&(i.innerHTML=w)},m(u,D){e(u,i,D)},p:Ls,d(u){u&&a(i)}}}function Es(E){let i,w,u,D,y,Q,$,Ms='Finding high-quality datasets that are reproducible and accessible can be difficult. One of 🤗 Datasets main goals is to provide a simple way to load a dataset of any format or type. The easiest way to get started is to discover an existing dataset on the <a href="https://huggingface.co/datasets" rel="nofollow">Hugging Face Hub</a> - a community-driven collection of datasets for tasks in NLP, computer vision, and audio - and use 🤗 Datasets to download and generate the dataset.',A,b,Ts='This tutorial uses the <a href="https://huggingface.co/datasets/rotten_tomatoes" rel="nofollow">rotten_tomatoes</a> and <a href="https://huggingface.co/datasets/PolyAI/minds14" rel="nofollow">MInDS-14</a> datasets, but feel free to load any dataset you want and follow along. Head over to the Hub now and find a dataset for your task!',K,x,O,M,Js='Before you take the time to download a dataset, it’s often helpful to quickly get some general information about a dataset. A dataset’s information is stored inside <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.DatasetInfo">DatasetInfo</a> and can include information such as the dataset description, features, and dataset size.',ss,T,Rs='Use the <a href="/docs/datasets/main/en/package_reference/loading_methods#datasets.load_dataset_builder">load_dataset_builder()</a> function to load a dataset builder and inspect a dataset’s attributes without committing to downloading it:',ts,J,as,R,ks='If you’re happy with the dataset, then load it with <a href="/docs/datasets/main/en/package_reference/loading_methods#datasets.load_dataset">load_dataset()</a>:',es,k,ns,Z,ls,v,Zs='A split is a specific subset of a dataset like <code>train</code> and <code>test</code>. List a dataset’s split names with the <a href="/docs/datasets/main/en/package_reference/loading_methods#datasets.get_dataset_split_names">get_dataset_split_names()</a> function:',is,X,ps,G,vs='Then you can load a specific split with the <code>split</code> parameter. Loading a dataset <code>split</code> returns a <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset">Dataset</a> object:',os,I,rs,U,Xs='If you don’t specify a <code>split</code>, 🤗 Datasets returns a <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.DatasetDict">DatasetDict</a> object instead:',ds,F,cs,C,ms,z,Gs='Some datasets contain several sub-datasets. For example, the <a href="https://huggingface.co/datasets/PolyAI/minds14" rel="nofollow">MInDS-14</a> dataset has several sub-datasets, each one containing audio data in a different language. These sub-datasets are known as <em>configurations</em> or <em>subsets</em>, and you must explicitly select one when loading the dataset. If you don’t provide a configuration name, 🤗 Datasets will raise a <code>ValueError</code> and remind you to choose a configuration.',gs,W,Is='Use the <a href="/docs/datasets/main/en/package_reference/loading_methods#datasets.get_dataset_config_names">get_dataset_config_names()</a> function to retrieve a list of all the possible configurations available to your dataset:',hs,q,fs,H,Us="Then load the configuration you want:",us,L,js,Y,_s,B,Fs="Certain datasets repositories contain a loading script with the Python code used to generate the dataset. All files and code uploaded to the Hub are scanned for malware (refer to the Hub security documentation for more information), but you should still review the dataset loading scripts and authors to avoid executing malicious code on your machine. You should set <code>trust_remote_code=True</code> to use a dataset with a loading script, or you will get an error:",ws,P,ys,j,$s,S,bs,N,xs;return y=new V({props:{title:"Load a dataset from the Hub",local:"load-a-dataset-from-the-hub",headingTag:"h1"}}),x=new V({props:{title:"Load a dataset",local:"load-a-dataset",headingTag:"h2"}}),J=new _({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0X2J1aWxkZXIlMEFkc19idWlsZGVyJTIwJTNEJTIwbG9hZF9kYXRhc2V0X2J1aWxkZXIoJTIycm90dGVuX3RvbWF0b2VzJTIyKSUwQSUwQWRzX2J1aWxkZXIuaW5mby5kZXNjcmlwdGlvbiUwQSUwQWRzX2J1aWxkZXIuaW5mby5mZWF0dXJlcw==",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset_builder
	<span class="hljs-meta">>>> </span>ds_builder = load_dataset_builder(<span class="hljs-string">"rotten_tomatoes"</span>)

	<span class="hljs-comment"># Inspect dataset description</span>
	<span class="hljs-meta">>>> </span>ds_builder.info.description
	Movie Review Dataset. This <span class="hljs-keyword">is</span> a dataset of containing <span class="hljs-number">5</span>,<span class="hljs-number">331</span> positive <span class="hljs-keyword">and</span> <span class="hljs-number">5</span>,<span class="hljs-number">331</span> negative processed sentences <span class="hljs-keyword">from</span> Rotten Tomatoes movie reviews. This data was first used <span class="hljs-keyword">in</span> Bo Pang <span class="hljs-keyword">and</span> Lillian Lee, \`\`Seeing stars: Exploiting <span class="hljs-keyword">class</span> <span class="hljs-title class_">relationships</span> <span class="hljs-keyword">for</span> sentiment categorization <span class="hljs-keyword">with</span> respect to rating scales.<span class="hljs-string">''</span>, Proceedings of the ACL, <span class="hljs-number">2005.</span>

	<span class="hljs-comment"># Inspect dataset features</span>
	<span class="hljs-meta">>>> </span>ds_builder.info.features
	{<span class="hljs-string">'label'</span>: ClassLabel(num_classes=<span class="hljs-number">2</span>, names=[<span class="hljs-string">'neg'</span>, <span class="hljs-string">'pos'</span>], <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
	<span class="hljs-string">'text'</span>: Value(dtype=<span class="hljs-string">'string'</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>)}`,wrap:!1}}),k=new _({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBZGF0YXNldCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJyb3R0ZW5fdG9tYXRvZXMlMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKQ==",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

	<span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">"rotten_tomatoes"</span>, split=<span class="hljs-string">"train"</span>)`,wrap:!1}}),Z=new V({props:{title:"Splits",local:"splits",headingTag:"h2"}}),X=new _({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwZ2V0X2RhdGFzZXRfc3BsaXRfbmFtZXMlMEElMEFnZXRfZGF0YXNldF9zcGxpdF9uYW1lcyglMjJyb3R0ZW5fdG9tYXRvZXMlMjIp",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> get_dataset_split_names

	<span class="hljs-meta">>>> </span>get_dataset_split_names(<span class="hljs-string">"rotten_tomatoes"</span>)
	[<span class="hljs-string">'train'</span>, <span class="hljs-string">'validation'</span>, <span class="hljs-string">'test'</span>]`,wrap:!1}}),I=new _({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBZGF0YXNldCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJyb3R0ZW5fdG9tYXRvZXMlMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKSUwQWRhdGFzZXQ=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

	<span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">"rotten_tomatoes"</span>, split=<span class="hljs-string">"train"</span>)
	<span class="hljs-meta">>>> </span>dataset
	Dataset({
	features: [<span class="hljs-string">'text'</span>, <span class="hljs-string">'label'</span>],
	num_rows: <span class="hljs-number">8530</span>
	})`,wrap:!1}}),F=new _({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBZGF0YXNldCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJyb3R0ZW5fdG9tYXRvZXMlMjIp",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

	<span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">"rotten_tomatoes"</span>)
	DatasetDict({
	train: Dataset({
	features: [<span class="hljs-string">'text'</span>, <span class="hljs-string">'label'</span>],
	num_rows: <span class="hljs-number">8530</span>
	})
	validation: Dataset({
	features: [<span class="hljs-string">'text'</span>, <span class="hljs-string">'label'</span>],
	num_rows: <span class="hljs-number">1066</span>
	})
	test: Dataset({
	features: [<span class="hljs-string">'text'</span>, <span class="hljs-string">'label'</span>],
	num_rows: <span class="hljs-number">1066</span>
	})
	})`,wrap:!1}}),C=new V({props:{title:"Configurations",local:"configurations",headingTag:"h2"}}),q=new _({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwZ2V0X2RhdGFzZXRfY29uZmlnX25hbWVzJTBBJTBBY29uZmlncyUyMCUzRCUyMGdldF9kYXRhc2V0X2NvbmZpZ19uYW1lcyglMjJQb2x5QUklMkZtaW5kczE0JTIyKSUwQXByaW50KGNvbmZpZ3Mp",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> get_dataset_config_names

	<span class="hljs-meta">>>> </span>configs = get_dataset_config_names(<span class="hljs-string">"PolyAI/minds14"</span>)
	<span class="hljs-meta">>>> </span><span class="hljs-built_in">print</span>(configs)
	[<span class="hljs-string">'cs-CZ'</span>, <span class="hljs-string">'de-DE'</span>, <span class="hljs-string">'en-AU'</span>, <span class="hljs-string">'en-GB'</span>, <span class="hljs-string">'en-US'</span>, <span class="hljs-string">'es-ES'</span>, <span class="hljs-string">'fr-FR'</span>, <span class="hljs-string">'it-IT'</span>, <span class="hljs-string">'ko-KR'</span>, <span class="hljs-string">'nl-NL'</span>, <span class="hljs-string">'pl-PL'</span>, <span class="hljs-string">'pt-PT'</span>, <span class="hljs-string">'ru-RU'</span>, <span class="hljs-string">'zh-CN'</span>, <span class="hljs-string">'all'</span>]`,wrap:!1}}),L=new _({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBbWluZHNGUiUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJQb2x5QUklMkZtaW5kczE0JTIyJTJDJTIwJTIyZnItRlIlMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKQ==",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

	<span class="hljs-meta">>>> </span>mindsFR = load_dataset(<span class="hljs-string">"PolyAI/minds14"</span>, <span class="hljs-string">"fr-FR"</span>, split=<span class="hljs-string">"train"</span>)`,wrap:!1}}),Y=new V({props:{title:"Remote code",local:"remote-code",headingTag:"h2"}}),P=new _({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwZ2V0X2RhdGFzZXRfY29uZmlnX25hbWVzJTJDJTIwZ2V0X2RhdGFzZXRfc3BsaXRfbmFtZXMlMkMlMjBsb2FkX2RhdGFzZXQlMEElMEFjNCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJjNCUyMiUyQyUyMCUyMmVuJTIyJTJDJTIwc3BsaXQlM0QlMjJ0cmFpbiUyMiUyQyUyMHRydXN0X3JlbW90ZV9jb2RlJTNEVHJ1ZSklMEFnZXRfZGF0YXNldF9jb25maWdfbmFtZXMoJTIyYzQlMjIlMkMlMjB0cnVzdF9yZW1vdGVfY29kZSUzRFRydWUpJTBBZ2V0X2RhdGFzZXRfc3BsaXRfbmFtZXMoJTIyYzQlMjIlMkMlMjAlMjJlbiUyMiUyQyUyMHRydXN0X3JlbW90ZV9jb2RlJTNEVHJ1ZSk=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> get_dataset_config_names, get_dataset_split_names, load_dataset

	<span class="hljs-meta">>>> </span>c4 = load_dataset(<span class="hljs-string">"c4"</span>, <span class="hljs-string">"en"</span>, split=<span class="hljs-string">"train"</span>, trust_remote_code=<span class="hljs-literal">True</span>)
	<span class="hljs-meta">>>> </span>get_dataset_config_names(<span class="hljs-string">"c4"</span>, trust_remote_code=<span class="hljs-literal">True</span>)
	[<span class="hljs-string">'en'</span>, <span class="hljs-string">'realnewslike'</span>, <span class="hljs-string">'en.noblocklist'</span>, <span class="hljs-string">'en.noclean'</span>]
	<span class="hljs-meta">>>> </span>get_dataset_split_names(<span class="hljs-string">"c4"</span>, <span class="hljs-string">"en"</span>, trust_remote_code=<span class="hljs-literal">True</span>)
	[<span class="hljs-string">'train'</span>, <span class="hljs-string">'validation'</span>]`,wrap:!1}}),j=new Ds({props:{warning:"true",$$slots:{default:[Vs]},$$scope:{ctx:E}}}),S=new Ns({props:{source:"https://github.com/huggingface/datasets/blob/main/docs/source/load_hub.mdx"}}),{c(){i=p("meta"),w=n(),u=p("p"),D=n(),r(y.$$.fragment),Q=n(),$=p("p"),$.innerHTML=Ms,A=n(),b=p("p"),b.innerHTML=Ts,K=n(),r(x.$$.fragment),O=n(),M=p("p"),M.innerHTML=Js,ss=n(),T=p("p"),T.innerHTML=Rs,ts=n(),r(J.$$.fragment),as=n(),R=p("p"),R.innerHTML=ks,es=n(),r(k.$$.fragment),ns=n(),r(Z.$$.fragment),ls=n(),v=p("p"),v.innerHTML=Zs,is=n(),r(X.$$.fragment),ps=n(),G=p("p"),G.innerHTML=vs,os=n(),r(I.$$.fragment),rs=n(),U=p("p"),U.innerHTML=Xs,ds=n(),r(F.$$.fragment),cs=n(),r(C.$$.fragment),ms=n(),z=p("p"),z.innerHTML=Gs,gs=n(),W=p("p"),W.innerHTML=Is,hs=n(),r(q.$$.fragment),fs=n(),H=p("p"),H.textContent=Us,us=n(),r(L.$$.fragment),js=n(),r(Y.$$.fragment),_s=n(),B=p("p"),B.innerHTML=Fs,ws=n(),r(P.$$.fragment),ys=n(),r(j.$$.fragment),$s=n(),r(S.$$.fragment),bs=n(),N=p("p"),this.h()},l(s){const t=Ps("svelte-u9bgzb",document.head);i=o(t,"META",{name:!0,content:!0}),t.forEach(a),w=l(s),u=o(s,"P",{}),zs(u).forEach(a),D=l(s),d(y.$$.fragment,s),Q=l(s),$=o(s,"P",{"data-svelte-h":!0}),f($)!=="svelte-9qr6ee"&&($.innerHTML=Ms),A=l(s),b=o(s,"P",{"data-svelte-h":!0}),f(b)!=="svelte-1nd3fhb"&&(b.innerHTML=Ts),K=l(s),d(x.$$.fragment,s),O=l(s),M=o(s,"P",{"data-svelte-h":!0}),f(M)!=="svelte-iyvhco"&&(M.innerHTML=Js),ss=l(s),T=o(s,"P",{"data-svelte-h":!0}),f(T)!=="svelte-1p7opk8"&&(T.innerHTML=Rs),ts=l(s),d(J.$$.fragment,s),as=l(s),R=o(s,"P",{"data-svelte-h":!0}),f(R)!=="svelte-1lqsd4u"&&(R.innerHTML=ks),es=l(s),d(k.$$.fragment,s),ns=l(s),d(Z.$$.fragment,s),ls=l(s),v=o(s,"P",{"data-svelte-h":!0}),f(v)!=="svelte-z32per"&&(v.innerHTML=Zs),is=l(s),d(X.$$.fragment,s),ps=l(s),G=o(s,"P",{"data-svelte-h":!0}),f(G)!=="svelte-1i9yshe"&&(G.innerHTML=vs),os=l(s),d(I.$$.fragment,s),rs=l(s),U=o(s,"P",{"data-svelte-h":!0}),f(U)!=="svelte-1gkubwl"&&(U.innerHTML=Xs),ds=l(s),d(F.$$.fragment,s),cs=l(s),d(C.$$.fragment,s),ms=l(s),z=o(s,"P",{"data-svelte-h":!0}),f(z)!=="svelte-2wgw8k"&&(z.innerHTML=Gs),gs=l(s),W=o(s,"P",{"data-svelte-h":!0}),f(W)!=="svelte-1r3luk1"&&(W.innerHTML=Is),hs=l(s),d(q.$$.fragment,s),fs=l(s),H=o(s,"P",{"data-svelte-h":!0}),f(H)!=="svelte-avfjb"&&(H.textContent=Us),us=l(s),d(L.$$.fragment,s),js=l(s),d(Y.$$.fragment,s),_s=l(s),B=o(s,"P",{"data-svelte-h":!0}),f(B)!=="svelte-ffeuwm"&&(B.innerHTML=Fs),ws=l(s),d(P.$$.fragment,s),ys=l(s),d(j.$$.fragment,s),$s=l(s),d(S.$$.fragment,s),bs=l(s),N=o(s,"P",{}),zs(N).forEach(a),this.h()},h(){Ws(i,"name","hf:doc:metadata"),Ws(i,"content",Qs)},m(s,t){Ss(document.head,i),e(s,w,t),e(s,u,t),e(s,D,t),c(y,s,t),e(s,Q,t),e(s,$,t),e(s,A,t),e(s,b,t),e(s,K,t),c(x,s,t),e(s,O,t),e(s,M,t),e(s,ss,t),e(s,T,t),e(s,ts,t),c(J,s,t),e(s,as,t),e(s,R,t),e(s,es,t),c(k,s,t),e(s,ns,t),c(Z,s,t),e(s,ls,t),e(s,v,t),e(s,is,t),c(X,s,t),e(s,ps,t),e(s,G,t),e(s,os,t),c(I,s,t),e(s,rs,t),e(s,U,t),e(s,ds,t),c(F,s,t),e(s,cs,t),c(C,s,t),e(s,ms,t),e(s,z,t),e(s,gs,t),e(s,W,t),e(s,hs,t),c(q,s,t),e(s,fs,t),e(s,H,t),e(s,us,t),c(L,s,t),e(s,js,t),c(Y,s,t),e(s,_s,t),e(s,B,t),e(s,ws,t),c(P,s,t),e(s,ys,t),c(j,s,t),e(s,$s,t),c(S,s,t),e(s,bs,t),e(s,N,t),xs=!0},p(s,[t]){const Cs={};t&2&&(Cs.$$scope={dirty:t,ctx:s}),j.$set(Cs)},i(s){xs\|\|(m(y.$$.fragment,s),m(x.$$.fragment,s),m(J.$$.fragment,s),m(k.$$.fragment,s),m(Z.$$.fragment,s),m(X.$$.fragment,s),m(I.$$.fragment,s),m(F.$$.fragment,s),m(C.$$.fragment,s),m(q.$$.fragment,s),m(L.$$.fragment,s),m(Y.$$.fragment,s),m(P.$$.fragment,s),m(j.$$.fragment,s),m(S.$$.fragment,s),xs=!0)},o(s){g(y.$$.fragment,s),g(x.$$.fragment,s),g(J.$$.fragment,s),g(k.$$.fragment,s),g(Z.$$.fragment,s),g(X.$$.fragment,s),g(I.$$.fragment,s),g(F.$$.fragment,s),g(C.$$.fragment,s),g(q.$$.fragment,s),g(L.$$.fragment,s),g(Y.$$.fragment,s),g(P.$$.fragment,s),g(j.$$.fragment,s),g(S.$$.fragment,s),xs=!1},d(s){s&&(a(w),a(u),a(D),a(Q),a($),a(A),a(b),a(K),a(O),a(M),a(ss),a(T),a(ts),a(as),a(R),a(es),a(ns),a(ls),a(v),a(is),a(ps),a(G),a(os),a(rs),a(U),a(ds),a(cs),a(ms),a(z),a(gs),a(W),a(hs),a(fs),a(H),a(us),a(js),a(_s),a(B),a(ws),a(ys),a($s),a(bs),a(N)),a(i),h(y,s),h(x,s),h(J,s),h(k,s),h(Z,s),h(X,s),h(I,s),h(F,s),h(C,s),h(q,s),h(L,s),h(Y,s),h(P,s),h(j,s),h(S,s)}}}const Qs='{"title":"Load a dataset from the Hub","local":"load-a-dataset-from-the-hub","sections":[{"title":"Load a dataset","local":"load-a-dataset","sections":[],"depth":2},{"title":"Splits","local":"splits","sections":[],"depth":2},{"title":"Configurations","local":"configurations","sections":[],"depth":2},{"title":"Remote code","local":"remote-code","sections":[],"depth":2}],"depth":1}';function As(E){return Hs(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class et extends Ys{constructor(i){super(),Bs(this,i,As,Es,qs,{})}}export{et as component};

Xet Storage Details

Size:: 18.7 kB
Xet hash:: 74760f980d6a18e3c3a39eab6216b0367e0ea72e2243fb4e6ffa99b54483158b

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.