transformers/pr_35939/en/_app/immutable/nodes/10.f856bbb4.js

import{s as Bs,o as Ws,n as ye}from"../chunks/scheduler.25b97de1.js";import{S as Is,i as Gs,g as m,s as r,r as c,A as Es,h as d,f as t,c as p,j as vs,u as h,x as g,k as Zs,y as Xs,a,v as y,d as f,t as M,w as u}from"../chunks/index.d9030fc9.js";import{T as ls}from"../chunks/Tip.baa67368.js";import{C as $}from"../chunks/CodeBlock.e6cd0d95.js";import{H as he,E as Vs}from"../chunks/EditOnGithub.91d95064.js";import{H as Rs,a as Cs}from"../chunks/HfOption.1e589c90.js";function zs(w){let l,o='The randomly created model is initialized with “empty” tensors, which take space in memory without filling it. The random values are whatever was in this chunk of memory at the time. To improve loading speed, the <a href="https://github.com/huggingface/transformers/blob/c9f6e5e35156e068b227dd9b15521767f6afd4d2/src/transformers/modeling_utils.py#L2710" rel="nofollow"><code>_fast_init</code></a> parameter is set to <code>True</code> by default to skip the random initialization for all weights that are correctly loaded.';return{c(){l=m("p"),l.innerHTML=o},l(n){l=d(n,"P",{"data-svelte-h":!0}),g(l)!=="svelte-14vsd0r"&&(l.innerHTML=o)},m(n,j){a(n,l,j)},p:ye,d(n){n&&t(l)}}}function Ls(w){let l,o="Make sure you have Accelerate v0.9.0 or later and PyTorch v1.9.0 or later installed.";return{c(){l=m("p"),l.textContent=o},l(n){l=d(n,"P",{"data-svelte-h":!0}),g(l)!=="svelte-1th3v5y"&&(l.textContent=o)},m(n,j){a(n,l,j)},p:ye,d(n){n&&t(l)}}}function As(w){let l,o="Due to how PyTorch is designed, the <code>torch_dtype</code> parameter only supports floating data types.";return{c(){l=m("p"),l.innerHTML=o},l(n){l=d(n,"P",{"data-svelte-h":!0}),g(l)!=="svelte-10yedw6"&&(l.innerHTML=o)},m(n,j){a(n,l,j)},p:ye,d(n){n&&t(l)}}}function Ss(w){let l,o;return l=new $({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBZ2VtbWElMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIyZ29vZ2xlJTJGZ2VtbWEtN2IlMjIlMkMlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

gemma = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;google/gemma-7b&quot;</span>, torch_dtype=torch.float16)`,wrap:!1}}),{c(){c(l.$$.fragment)},l(n){h(l.$$.fragment,n)},m(n,j){y(l,n,j),o=!0},p:ye,i(n){o||(f(l.$$.fragment,n),o=!0)},o(n){M(l.$$.fragment,n),o=!1},d(n){u(l,n)}}}function Fs(w){let l,o;return l=new $({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBZ2VtbWElMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIyZ29vZ2xlJTJGZ2VtbWEtN2IlMjIlMkMlMjB0b3JjaF9kdHlwZSUzRCUyMmF1dG8lMjIp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

gemma = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;google/gemma-7b&quot;</span>, torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>)`,wrap:!1}}),{c(){c(l.$$.fragment)},l(n){h(l.$$.fragment,n)},m(n,j){y(l,n,j),o=!0},p:ye,i(n){o||(f(l.$$.fragment,n),o=!0)},o(n){M(l.$$.fragment,n),o=!1},d(n){u(l,n)}}}function Hs(w){let l,o,n,j;return l=new Cs({props:{id:"dtype",option:"specific dtype",$$slots:{default:[Ss]},$$scope:{ctx:w}}}),n=new Cs({props:{id:"dtype",option:"auto dtype",$$slots:{default:[Fs]},$$scope:{ctx:w}}}),{c(){c(l.$$.fragment),o=r(),c(n.$$.fragment)},l(i){h(l.$$.fragment,i),o=p(i),h(n.$$.fragment,i)},m(i,b){y(l,i,b),a(i,o,b),y(n,i,b),j=!0},p(i,b){const x={};b&2&&(x.$$scope={dirty:b,ctx:i}),l.$set(x);const de={};b&2&&(de.$$scope={dirty:b,ctx:i}),n.$set(de)},i(i){j||(f(l.$$.fragment,i),f(n.$$.fragment,i),j=!0)},o(i){M(l.$$.fragment,i),M(n.$$.fragment,i),j=!1},d(i){i&&t(o),u(l,i),u(n,i)}}}function Qs(w){let l,o,n,j,i,b,x,de="A barrier to accessing very large pretrained models is the amount of memory required. When loading a pretrained PyTorch model, you usually:",fe,k,ns="<li>Create a model with random weights.</li> <li>Load your pretrained weights.</li> <li>Put those pretrained weights in the model.</li>",Me,v,rs="The first two steps both require a full version of the model in memory and if the model weighs several GBs, you may not have enough memory for two copies of it. This problem is amplified in distributed training environments because each process loads a pretrained model and stores two copies in memory.",ue,J,ge,Z,ps="This guide will show you how Transformers can help you load large pretrained models despite their memory requirements.",je,C,we,B,os='From Transformers v4.18.0, a checkpoint larger than 10GB is automatically sharded by the <a href="/docs/transformers/pr_35939/en/main_classes/model#transformers.PreTrainedModel.save_pretrained">save_pretrained()</a> method. It is split into several smaller partial checkpoints and creates an index file that maps parameter names to the files they’re stored in.',$e,W,is="The maximum shard size is controlled with the <code>max_shard_size</code> parameter, but by default it is 5GB, because it is easier to run on free-tier GPU instances without running out of memory.",be,I,ms='For example, let’s shard <a href="https://hf.co/BioMistral/BioMistral-7B" rel="nofollow">BioMistral/BioMistral-7B</a>.',xe,G,Je,E,ds='The sharded checkpoint is reloaded with the <a href="/docs/transformers/pr_35939/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method.',Te,X,_e,V,cs="The main advantage of sharded checkpoints for big models is that each shard is loaded after the previous one, which caps the memory usage to only the model size and the largest shard size.",Ue,R,hs='You could also directly load a sharded checkpoint inside a model without the <a href="/docs/transformers/pr_35939/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method (similar to PyTorch’s <code>load_state_dict()</code> method for a full checkpoint). In this case, use the <a href="/docs/transformers/pr_35939/en/main_classes/model#transformers.modeling_utils.load_sharded_checkpoint">load_sharded_checkpoint()</a> method.',ke,z,ve,L,Ze,A,ys="The index file determines which keys are in the checkpoint and where the corresponding weights are stored. This file is loaded like any other JSON file and you can get a dictionary from it.",Ce,S,Be,F,fs="The <code>metadata</code> key provides the total model size.",We,H,Ie,Q,Ms="The <code>weight_map</code> key maps each parameter name (typically <code>state_dict</code> in a PyTorch model) to the shard it’s stored in.",Ge,N,Ee,Y,Xe,T,Ve,q,us='From Transformers v4.20.0, the <a href="/docs/transformers/pr_35939/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method is supercharged with Accelerate’s <a href="https://hf.co/docs/accelerate/usage_guides/big_modeling" rel="nofollow">Big Model Inference</a> feature to efficiently handle really big models! Big Model Inference creates a <em>model skeleton</em> on PyTorch’s <a href="https://pytorch.org/docs/main/meta.html" rel="nofollow"><strong>meta</strong></a> device. The randomly initialized parameters are only created when the pretrained weights are loaded. This way, you aren’t keeping two copies of the model in memory at the same time (one for the randomly initialized model and one for the pretrained weights), and the maximum memory consumed is only the full model size.',Re,P,gs='To enable Big Model Inference in Transformers, set <code>low_cpu_mem_usage=True</code> in the <a href="/docs/transformers/pr_35939/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method.',ze,D,Le,K,js='Accelerate automatically dispatches the model weights across all available devices, starting with the fastest device (GPU) first and then offloading to the slower devices (CPU and even hard drive). This is enabled by setting <code>device_map=&quot;auto&quot;</code> in the <a href="/docs/transformers/pr_35939/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method. When you pass the <code>device_map</code> parameter, <code>low_cpu_mem_usage</code> is automatically set to <code>True</code> so you don’t need to specify it.',Ae,O,Se,ee,ws="You can also write your own <code>device_map</code> by mapping each layer to a device. It should map all model parameters to a device, but you don’t have to detail where all the submodules of a layer go if the entire layer is on the same device.",Fe,se,He,te,$s="Access <code>hf_device_map</code> attribute to see how Accelerate split the model across devices.",Qe,ae,Ne,le,Ye,ne,qe,re,bs="PyTorch model weights are normally instantiated as torch.float32 and it can be an issue if you try to load a model as a different data type. For example, you’d need twice as much memory to load the weights in torch.float32 and then again to load them in your desired data type, like torch.float16.",Pe,_,De,pe,xs="To avoid wasting memory like this, explicitly set the <code>torch_dtype</code> parameter to the desired data type or set <code>torch_dtype=&quot;auto&quot;</code> to load the weights with the most optimal memory pattern (the data type is automatically derived from the model weights).",Ke,U,Oe,oe,Js="You can also set the data type to use for models instantiated from scratch.",es,ie,ss,me,ts,ce,as;return i=new he({props:{title:"Instantiate a big model",local:"instantiate-a-big-model",headingTag:"h1"}}),J=new ls({props:{warning:!1,$$slots:{default:[zs]},$$scope:{ctx:w}}}),C=new he({props:{title:"Sharded checkpoints",local:"sharded-checkpoints",headingTag:"h2"}}),G=new $({props:{code:"d2l0aCUyMHRlbXBmaWxlLlRlbXBvcmFyeURpcmVjdG9yeSgpJTIwYXMlMjB0bXBfZGlyJTNBJTBBJTIwJTIwJTIwJTIwbW9kZWwuc2F2ZV9wcmV0cmFpbmVkKHRtcF9kaXIlMkMlMjBtYXhfc2hhcmRfc2l6ZSUzRCUyMjVHQiUyMiklMEElMjAlMjAlMjAlMjBwcmludChzb3J0ZWQob3MubGlzdGRpcih0bXBfZGlyKSkp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> tempfile.TemporaryDirectory() <span class="hljs-keyword">as</span> tmp_dir:
<span class="hljs-meta">... </span>    model.save_pretrained(tmp_dir, max_shard_size=<span class="hljs-string">&quot;5GB&quot;</span>)
<span class="hljs-meta">... </span>    <span class="hljs-built_in">print</span>(<span class="hljs-built_in">sorted</span>(os.listdir(tmp_dir)))
[<span class="hljs-string">&#x27;config.json&#x27;</span>, <span class="hljs-string">&#x27;generation_config.json&#x27;</span>, <span class="hljs-string">&#x27;model-00001-of-00006.safetensors&#x27;</span>, <span class="hljs-string">&#x27;model-00002-of-00006.safetensors&#x27;</span>, <span class="hljs-string">&#x27;model-00003-of-00006.safetensors&#x27;</span>, <span class="hljs-string">&#x27;model-00004-of-00006.safetensors&#x27;</span>, <span class="hljs-string">&#x27;model-00005-of-00006.safetensors&#x27;</span>, <span class="hljs-string">&#x27;model-00006-of-00006.safetensors&#x27;</span>, <span class="hljs-string">&#x27;model.safetensors.index.json&#x27;</span>]`,wrap:!1}}),X=new $({props:{code:"d2l0aCUyMHRlbXBmaWxlLlRlbXBvcmFyeURpcmVjdG9yeSgpJTIwYXMlMjB0bXBfZGlyJTNBJTBBJTIwJTIwJTIwJTIwbW9kZWwuc2F2ZV9wcmV0cmFpbmVkKHRtcF9kaXIlMkMlMjBtYXhfc2hhcmRfc2l6ZSUzRCUyMjVHQiUyMiklMEElMjAlMjAlMjAlMjBuZXdfbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWwuZnJvbV9wcmV0cmFpbmVkKHRtcF9kaXIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> tempfile.TemporaryDirectory() <span class="hljs-keyword">as</span> tmp_dir:
<span class="hljs-meta">... </span>    model.save_pretrained(tmp_dir, max_shard_size=<span class="hljs-string">&quot;5GB&quot;</span>)
<span class="hljs-meta">... </span>    new_model = AutoModel.from_pretrained(tmp_dir)`,wrap:!1}}),z=new $({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5tb2RlbGluZ191dGlscyUyMGltcG9ydCUyMGxvYWRfc2hhcmRlZF9jaGVja3BvaW50JTBBJTBBd2l0aCUyMHRlbXBmaWxlLlRlbXBvcmFyeURpcmVjdG9yeSgpJTIwYXMlMjB0bXBfZGlyJTNBJTBBJTIwJTIwJTIwJTIwbW9kZWwuc2F2ZV9wcmV0cmFpbmVkKHRtcF9kaXIlMkMlMjBtYXhfc2hhcmRfc2l6ZSUzRCUyMjVHQiUyMiklMEElMjAlMjAlMjAlMjBsb2FkX3NoYXJkZWRfY2hlY2twb2ludChtb2RlbCUyQyUyMHRtcF9kaXIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers.modeling_utils <span class="hljs-keyword">import</span> load_sharded_checkpoint

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> tempfile.TemporaryDirectory() <span class="hljs-keyword">as</span> tmp_dir:
<span class="hljs-meta">... </span>    model.save_pretrained(tmp_dir, max_shard_size=<span class="hljs-string">&quot;5GB&quot;</span>)
<span class="hljs-meta">... </span>    load_sharded_checkpoint(model, tmp_dir)`,wrap:!1}}),L=new he({props:{title:"Shard metadata",local:"shard-metadata",headingTag:"h3"}}),S=new $({props:{code:"aW1wb3J0JTIwanNvbiUwQSUwQXdpdGglMjB0ZW1wZmlsZS5UZW1wb3JhcnlEaXJlY3RvcnkoKSUyMGFzJTIwdG1wX2RpciUzQSUwQSUyMCUyMCUyMCUyMG1vZGVsLnNhdmVfcHJldHJhaW5lZCh0bXBfZGlyJTJDJTIwbWF4X3NoYXJkX3NpemUlM0QlMjI1R0IlMjIpJTBBJTIwJTIwJTIwJTIwd2l0aCUyMG9wZW4ob3MucGF0aC5qb2luKHRtcF9kaXIlMkMlMjAlMjJtb2RlbC5zYWZldGVuc29ycy5pbmRleC5qc29uJTIyKSUyQyUyMCUyMnIlMjIpJTIwYXMlMjBmJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaW5kZXglMjAlM0QlMjBqc29uLmxvYWQoZiklMEElMEFwcmludChpbmRleC5rZXlzKCkp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> json

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> tempfile.TemporaryDirectory() <span class="hljs-keyword">as</span> tmp_dir:
<span class="hljs-meta">... </span>    model.save_pretrained(tmp_dir, max_shard_size=<span class="hljs-string">&quot;5GB&quot;</span>)
<span class="hljs-meta">... </span>    <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(os.path.join(tmp_dir, <span class="hljs-string">&quot;model.safetensors.index.json&quot;</span>), <span class="hljs-string">&quot;r&quot;</span>) <span class="hljs-keyword">as</span> f:
<span class="hljs-meta">... </span>        index = json.load(f)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(index.keys())
dict_keys([<span class="hljs-string">&#x27;metadata&#x27;</span>, <span class="hljs-string">&#x27;weight_map&#x27;</span>])`,wrap:!1}}),H=new $({props:{code:"aW5kZXglNUIlMjJtZXRhZGF0YSUyMiU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>index[<span class="hljs-string">&quot;metadata&quot;</span>]
{<span class="hljs-string">&#x27;total_size&#x27;</span>: <span class="hljs-number">28966928384</span>}`,wrap:!1}}),N=new $({props:{code:"aW5kZXglNUIlMjJ3ZWlnaHRfbWFwJTIyJTVE",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>index[<span class="hljs-string">&quot;weight_map&quot;</span>]
{<span class="hljs-string">&#x27;lm_head.weight&#x27;</span>: <span class="hljs-string">&#x27;model-00006-of-00006.safetensors&#x27;</span>,
 <span class="hljs-string">&#x27;model.embed_tokens.weight&#x27;</span>: <span class="hljs-string">&#x27;model-00001-of-00006.safetensors&#x27;</span>,
 <span class="hljs-string">&#x27;model.layers.0.input_layernorm.weight&#x27;</span>: <span class="hljs-string">&#x27;model-00001-of-00006.safetensors&#x27;</span>,
 <span class="hljs-string">&#x27;model.layers.0.mlp.down_proj.weight&#x27;</span>: <span class="hljs-string">&#x27;model-00001-of-00006.safetensors&#x27;</span>,
 ...
}`,wrap:!1}}),Y=new he({props:{title:"Accelerate’s Big Model Inference",local:"accelerates-big-model-inference",headingTag:"h2"}}),T=new ls({props:{warning:!1,$$slots:{default:[Ls]},$$scope:{ctx:w}}}),D=new $({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBZ2VtbWElMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIyZ29vZ2xlJTJGZ2VtbWEtN2IlMjIlMkMlMjBsb3dfY3B1X21lbV91c2FnZSUzRFRydWUp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

gemma = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;google/gemma-7b&quot;</span>, low_cpu_mem_usage=<span class="hljs-literal">True</span>)`,wrap:!1}}),O=new $({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBJTIzJTIwdGhlc2UlMjBsb2FkaW5nJTIwbWV0aG9kcyUyMGFyZSUyMGVxdWl2YWxlbnQlMEFnZW1tYSUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJnb29nbGUlMkZnZW1tYS03YiUyMiUyQyUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyKSUwQWdlbW1hJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmdlbW1hLTdiJTIyJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMjBsb3dfY3B1X21lbV91c2FnZSUzRFRydWUp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

<span class="hljs-comment"># these loading methods are equivalent</span>
gemma = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;google/gemma-7b&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>)
gemma = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;google/gemma-7b&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>, low_cpu_mem_usage=<span class="hljs-literal">True</span>)`,wrap:!1}}),se=new $({props:{code:"ZGV2aWNlX21hcCUyMCUzRCUyMCU3QiUyMm1vZGVsLmxheWVycy4xJTIyJTNBJTIwMCUyQyUyMCUyMm1vZGVsLmxheWVycy4xNCUyMiUzQSUyMDElMkMlMjAlMjJtb2RlbC5sYXllcnMuMzElMjIlM0ElMjAlMjJjcHUlMjIlMkMlMjAlMjJsbV9oZWFkJTIyJTNBJTIwJTIyZGlzayUyMiU3RA==",highlighted:'device_map = {<span class="hljs-string">&quot;model.layers.1&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;model.layers.14&quot;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&quot;model.layers.31&quot;</span>: <span class="hljs-string">&quot;cpu&quot;</span>, <span class="hljs-string">&quot;lm_head&quot;</span>: <span class="hljs-string">&quot;disk&quot;</span>}',wrap:!1}}),ae=new $({props:{code:"Z2VtbWEuaGZfZGV2aWNlX21hcA==",highlighted:"gemma.hf_device_map",wrap:!1}}),le=new $({props:{code:"JTdCJ21vZGVsLmVtYmVkX3Rva2VucyclM0ElMjAwJTJDJTBBJTIwJ21vZGVsLmxheWVycy4wJyUzQSUyMDAlMkMlMEElMjAnbW9kZWwubGF5ZXJzLjEnJTNBJTIwMCUyQyUwQSUyMCdtb2RlbC5sYXllcnMuMiclM0ElMjAwJTJDJTBBJTIwJ21vZGVsLmxheWVycy4zJyUzQSUyMDAlMkMlMEElMjAnbW9kZWwubGF5ZXJzLjQnJTNBJTIwMCUyQyUwQSUyMCdtb2RlbC5sYXllcnMuNSclM0ElMjAwJTJDJTBBJTIwJ21vZGVsLmxheWVycy42JyUzQSUyMDAlMkMlMEElMjAnbW9kZWwubGF5ZXJzLjcnJTNBJTIwMCUyQyUwQSUyMCdtb2RlbC5sYXllcnMuOCclM0ElMjAwJTJDJTBBJTIwJ21vZGVsLmxheWVycy45JyUzQSUyMDAlMkMlMEElMjAnbW9kZWwubGF5ZXJzLjEwJyUzQSUyMDAlMkMlMEElMjAnbW9kZWwubGF5ZXJzLjExJyUzQSUyMDAlMkMlMEElMjAnbW9kZWwubGF5ZXJzLjEyJyUzQSUyMDAlMkMlMEElMjAnbW9kZWwubGF5ZXJzLjEzJyUzQSUyMDAlMkMlMEElMjAnbW9kZWwubGF5ZXJzLjE0JyUzQSUyMCdjcHUnJTJDJTBBJTIwJ21vZGVsLmxheWVycy4xNSclM0ElMjAnY3B1JyUyQyUwQSUyMCdtb2RlbC5sYXllcnMuMTYnJTNBJTIwJ2NwdSclMkMlMEElMjAnbW9kZWwubGF5ZXJzLjE3JyUzQSUyMCdjcHUnJTJDJTBBJTIwJ21vZGVsLmxheWVycy4xOCclM0ElMjAnY3B1JyUyQyUwQSUyMCdtb2RlbC5sYXllcnMuMTknJTNBJTIwJ2NwdSclMkMlMEElMjAnbW9kZWwubGF5ZXJzLjIwJyUzQSUyMCdjcHUnJTJDJTBBJTIwJ21vZGVsLmxheWVycy4yMSclM0ElMjAnY3B1JyUyQyUwQSUyMCdtb2RlbC5sYXllcnMuMjInJTNBJTIwJ2NwdSclMkMlMEElMjAnbW9kZWwubGF5ZXJzLjIzJyUzQSUyMCdjcHUnJTJDJTBBJTIwJ21vZGVsLmxheWVycy4yNCclM0ElMjAnY3B1JyUyQyUwQSUyMCdtb2RlbC5sYXllcnMuMjUnJTNBJTIwJ2NwdSclMkMlMEElMjAnbW9kZWwubGF5ZXJzLjI2JyUzQSUyMCdjcHUnJTJDJTBBJTIwJ21vZGVsLmxheWVycy4yNyclM0ElMjAnY3B1JyUyQyUwQSUyMCdtb2RlbC5sYXllcnMuMjgnJTNBJTIwJ2NwdSclMkMlMEElMjAnbW9kZWwubGF5ZXJzLjI5JyUzQSUyMCdjcHUnJTJDJTBBJTIwJ21vZGVsLmxheWVycy4zMCclM0ElMjAnY3B1JyUyQyUwQSUyMCdtb2RlbC5sYXllcnMuMzEnJTNBJTIwJ2NwdSclMkMlMEElMjAnbW9kZWwubm9ybSclM0ElMjAnY3B1JyUyQyUwQSUyMCdsbV9oZWFkJyUzQSUyMCdjcHUnJTdE",highlighted:`{<span class="hljs-string">&#x27;model.embed_tokens&#x27;</span>: <span class="hljs-number">0</span>,
 <span class="hljs-string">&#x27;model.layers.0&#x27;</span>: <span class="hljs-number">0</span>,
 <span class="hljs-string">&#x27;model.layers.1&#x27;</span>: <span class="hljs-number">0</span>,
 <span class="hljs-string">&#x27;model.layers.2&#x27;</span>: <span class="hljs-number">0</span>,
 <span class="hljs-string">&#x27;model.layers.3&#x27;</span>: <span class="hljs-number">0</span>,
 <span class="hljs-string">&#x27;model.layers.4&#x27;</span>: <span class="hljs-number">0</span>,
 <span class="hljs-string">&#x27;model.layers.5&#x27;</span>: <span class="hljs-number">0</span>,
 <span class="hljs-string">&#x27;model.layers.6&#x27;</span>: <span class="hljs-number">0</span>,
 <span class="hljs-string">&#x27;model.layers.7&#x27;</span>: <span class="hljs-number">0</span>,
 <span class="hljs-string">&#x27;model.layers.8&#x27;</span>: <span class="hljs-number">0</span>,
 <span class="hljs-string">&#x27;model.layers.9&#x27;</span>: <span class="hljs-number">0</span>,
 <span class="hljs-string">&#x27;model.layers.10&#x27;</span>: <span class="hljs-number">0</span>,
 <span class="hljs-string">&#x27;model.layers.11&#x27;</span>: <span class="hljs-number">0</span>,
 <span class="hljs-string">&#x27;model.layers.12&#x27;</span>: <span class="hljs-number">0</span>,
 <span class="hljs-string">&#x27;model.layers.13&#x27;</span>: <span class="hljs-number">0</span>,
 <span class="hljs-string">&#x27;model.layers.14&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>,
 <span class="hljs-string">&#x27;model.layers.15&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>,
 <span class="hljs-string">&#x27;model.layers.16&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>,
 <span class="hljs-string">&#x27;model.layers.17&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>,
 <span class="hljs-string">&#x27;model.layers.18&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>,
 <span class="hljs-string">&#x27;model.layers.19&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>,
 <span class="hljs-string">&#x27;model.layers.20&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>,
 <span class="hljs-string">&#x27;model.layers.21&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>,
 <span class="hljs-string">&#x27;model.layers.22&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>,
 <span class="hljs-string">&#x27;model.layers.23&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>,
 <span class="hljs-string">&#x27;model.layers.24&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>,
 <span class="hljs-string">&#x27;model.layers.25&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>,
 <span class="hljs-string">&#x27;model.layers.26&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>,
 <span class="hljs-string">&#x27;model.layers.27&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>,
 <span class="hljs-string">&#x27;model.layers.28&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>,
 <span class="hljs-string">&#x27;model.layers.29&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>,
 <span class="hljs-string">&#x27;model.layers.30&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>,
 <span class="hljs-string">&#x27;model.layers.31&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>,
 <span class="hljs-string">&#x27;model.norm&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>,
 <span class="hljs-string">&#x27;lm_head&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>}`,wrap:!1}}),ne=new he({props:{title:"Model data type",local:"model-data-type",headingTag:"h2"}}),_=new ls({props:{warning:!0,$$slots:{default:[As]},$$scope:{ctx:w}}}),U=new Rs({props:{id:"dtype",options:["specific dtype","auto dtype"],$$slots:{default:[Hs]},$$scope:{ctx:w}}}),ie=new $({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b0NvbmZpZyUyQyUyMEF1dG9Nb2RlbCUwQSUwQW15X2NvbmZpZyUyMCUzRCUyMEF1dG9Db25maWcuZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmdlbW1hLTJiJTIyJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2KSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsLmZyb21fY29uZmlnKG15X2NvbmZpZyk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoConfig, AutoModel

my_config = AutoConfig.from_pretrained(<span class="hljs-string">&quot;google/gemma-2b&quot;</span>, torch_dtype=torch.float16)
model = AutoModel.from_config(my_config)`,wrap:!1}}),me=new Vs({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/big_models.md"}}),{c(){l=m("meta"),o=r(),n=m("p"),j=r(),c(i.$$.fragment),b=r(),x=m("p"),x.textContent=de,fe=r(),k=m("ol"),k.innerHTML=ns,Me=r(),v=m("p"),v.textContent=rs,ue=r(),c(J.$$.fragment),ge=r(),Z=m("p"),Z.textContent=ps,je=r(),c(C.$$.fragment),we=r(),B=m("p"),B.innerHTML=os,$e=r(),W=m("p"),W.innerHTML=is,be=r(),I=m("p"),I.innerHTML=ms,xe=r(),c(G.$$.fragment),Je=r(),E=m("p"),E.innerHTML=ds,Te=r(),c(X.$$.fragment),_e=r(),V=m("p"),V.textContent=cs,Ue=r(),R=m("p"),R.innerHTML=hs,ke=r(),c(z.$$.fragment),ve=r(),c(L.$$.fragment),Ze=r(),A=m("p"),A.textContent=ys,Ce=r(),c(S.$$.fragment),Be=r(),F=m("p"),F.innerHTML=fs,We=r(),c(H.$$.fragment),Ie=r(),Q=m("p"),Q.innerHTML=Ms,Ge=r(),c(N.$$.fragment),Ee=r(),c(Y.$$.fragment),Xe=r(),c(T.$$.fragment),Ve=r(),q=m("p"),q.innerHTML=us,Re=r(),P=m("p"),P.innerHTML=gs,ze=r(),c(D.$$.fragment),Le=r(),K=m("p"),K.innerHTML=js,Ae=r(),c(O.$$.fragment),Se=r(),ee=m("p"),ee.innerHTML=ws,Fe=r(),c(se.$$.fragment),He=r(),te=m("p"),te.innerHTML=$s,Qe=r(),c(ae.$$.fragment),Ne=r(),c(le.$$.fragment),Ye=r(),c(ne.$$.fragment),qe=r(),re=m("p"),re.textContent=bs,Pe=r(),c(_.$$.fragment),De=r(),pe=m("p"),pe.innerHTML=xs,Ke=r(),c(U.$$.fragment),Oe=r(),oe=m("p"),oe.textContent=Js,es=r(),c(ie.$$.fragment),ss=r(),c(me.$$.fragment),ts=r(),ce=m("p"),this.h()},l(e){const s=Es("svelte-u9bgzb",document.head);l=d(s,"META",{name:!0,content:!0}),s.forEach(t),o=p(e),n=d(e,"P",{}),vs(n).forEach(t),j=p(e),h(i.$$.fragment,e),b=p(e),x=d(e,"P",{"data-svelte-h":!0}),g(x)!=="svelte-1qkl6uj"&&(x.textContent=de),fe=p(e),k=d(e,"OL",{"data-svelte-h":!0}),g(k)!=="svelte-18g3fgx"&&(k.innerHTML=ns),Me=p(e),v=d(e,"P",{"data-svelte-h":!0}),g(v)!=="svelte-1m58d3z"&&(v.textContent=rs),ue=p(e),h(J.$$.fragment,e),ge=p(e),Z=d(e,"P",{"data-svelte-h":!0}),g(Z)!=="svelte-1vqz77w"&&(Z.textContent=ps),je=p(e),h(C.$$.fragment,e),we=p(e),B=d(e,"P",{"data-svelte-h":!0}),g(B)!=="svelte-1pc368i"&&(B.innerHTML=os),$e=p(e),W=d(e,"P",{"data-svelte-h":!0}),g(W)!=="svelte-1cy3m1z"&&(W.innerHTML=is),be=p(e),I=d(e,"P",{"data-svelte-h":!0}),g(I)!=="svelte-1jmx6m5"&&(I.innerHTML=ms),xe=p(e),h(G.$$.fragment,e),Je=p(e),E=d(e,"P",{"data-svelte-h":!0}),g(E)!=="svelte-149lg4j"&&(E.innerHTML=ds),Te=p(e),h(X.$$.fragment,e),_e=p(e),V=d(e,"P",{"data-svelte-h":!0}),g(V)!=="svelte-11n2c68"&&(V.textContent=cs),Ue=p(e),R=d(e,"P",{"data-svelte-h":!0}),g(R)!=="svelte-ohc2po"&&(R.innerHTML=hs),ke=p(e),h(z.$$.fragment,e),ve=p(e),h(L.$$.fragment,e),Ze=p(e),A=d(e,"P",{"data-svelte-h":!0}),g(A)!=="svelte-16iznko"&&(A.textContent=ys),Ce=p(e),h(S.$$.fragment,e),Be=p(e),F=d(e,"P",{"data-svelte-h":!0}),g(F)!=="svelte-n8xk5n"&&(F.innerHTML=fs),We=p(e),h(H.$$.fragment,e),Ie=p(e),Q=d(e,"P",{"data-svelte-h":!0}),g(Q)!=="svelte-1mrv1h2"&&(Q.innerHTML=Ms),Ge=p(e),h(N.$$.fragment,e),Ee=p(e),h(Y.$$.fragment,e),Xe=p(e),h(T.$$.fragment,e),Ve=p(e),q=d(e,"P",{"data-svelte-h":!0}),g(q)!=="svelte-1auiukw"&&(q.innerHTML=us),Re=p(e),P=d(e,"P",{"data-svelte-h":!0}),g(P)!=="svelte-100u8u2"&&(P.innerHTML=gs),ze=p(e),h(D.$$.fragment,e),Le=p(e),K=d(e,"P",{"data-svelte-h":!0}),g(K)!=="svelte-1oqj7mj"&&(K.innerHTML=js),Ae=p(e),h(O.$$.fragment,e),Se=p(e),ee=d(e,"P",{"data-svelte-h":!0}),g(ee)!=="svelte-15c7psq"&&(ee.innerHTML=ws),Fe=p(e),h(se.$$.fragment,e),He=p(e),te=d(e,"P",{"data-svelte-h":!0}),g(te)!=="svelte-1b9v8ra"&&(te.innerHTML=$s),Qe=p(e),h(ae.$$.fragment,e),Ne=p(e),h(le.$$.fragment,e),Ye=p(e),h(ne.$$.fragment,e),qe=p(e),re=d(e,"P",{"data-svelte-h":!0}),g(re)!=="svelte-wrnzpy"&&(re.textContent=bs),Pe=p(e),h(_.$$.fragment,e),De=p(e),pe=d(e,"P",{"data-svelte-h":!0}),g(pe)!=="svelte-62b14u"&&(pe.innerHTML=xs),Ke=p(e),h(U.$$.fragment,e),Oe=p(e),oe=d(e,"P",{"data-svelte-h":!0}),g(oe)!=="svelte-hi16m2"&&(oe.textContent=Js),es=p(e),h(ie.$$.fragment,e),ss=p(e),h(me.$$.fragment,e),ts=p(e),ce=d(e,"P",{}),vs(ce).forEach(t),this.h()},h(){Zs(l,"name","hf:doc:metadata"),Zs(l,"content",Ns)},m(e,s){Xs(document.head,l),a(e,o,s),a(e,n,s),a(e,j,s),y(i,e,s),a(e,b,s),a(e,x,s),a(e,fe,s),a(e,k,s),a(e,Me,s),a(e,v,s),a(e,ue,s),y(J,e,s),a(e,ge,s),a(e,Z,s),a(e,je,s),y(C,e,s),a(e,we,s),a(e,B,s),a(e,$e,s),a(e,W,s),a(e,be,s),a(e,I,s),a(e,xe,s),y(G,e,s),a(e,Je,s),a(e,E,s),a(e,Te,s),y(X,e,s),a(e,_e,s),a(e,V,s),a(e,Ue,s),a(e,R,s),a(e,ke,s),y(z,e,s),a(e,ve,s),y(L,e,s),a(e,Ze,s),a(e,A,s),a(e,Ce,s),y(S,e,s),a(e,Be,s),a(e,F,s),a(e,We,s),y(H,e,s),a(e,Ie,s),a(e,Q,s),a(e,Ge,s),y(N,e,s),a(e,Ee,s),y(Y,e,s),a(e,Xe,s),y(T,e,s),a(e,Ve,s),a(e,q,s),a(e,Re,s),a(e,P,s),a(e,ze,s),y(D,e,s),a(e,Le,s),a(e,K,s),a(e,Ae,s),y(O,e,s),a(e,Se,s),a(e,ee,s),a(e,Fe,s),y(se,e,s),a(e,He,s),a(e,te,s),a(e,Qe,s),y(ae,e,s),a(e,Ne,s),y(le,e,s),a(e,Ye,s),y(ne,e,s),a(e,qe,s),a(e,re,s),a(e,Pe,s),y(_,e,s),a(e,De,s),a(e,pe,s),a(e,Ke,s),y(U,e,s),a(e,Oe,s),a(e,oe,s),a(e,es,s),y(ie,e,s),a(e,ss,s),y(me,e,s),a(e,ts,s),a(e,ce,s),as=!0},p(e,[s]){const Ts={};s&2&&(Ts.$$scope={dirty:s,ctx:e}),J.$set(Ts);const _s={};s&2&&(_s.$$scope={dirty:s,ctx:e}),T.$set(_s);const Us={};s&2&&(Us.$$scope={dirty:s,ctx:e}),_.$set(Us);const ks={};s&2&&(ks.$$scope={dirty:s,ctx:e}),U.$set(ks)},i(e){as||(f(i.$$.fragment,e),f(J.$$.fragment,e),f(C.$$.fragment,e),f(G.$$.fragment,e),f(X.$$.fragment,e),f(z.$$.fragment,e),f(L.$$.fragment,e),f(S.$$.fragment,e),f(H.$$.fragment,e),f(N.$$.fragment,e),f(Y.$$.fragment,e),f(T.$$.fragment,e),f(D.$$.fragment,e),f(O.$$.fragment,e),f(se.$$.fragment,e),f(ae.$$.fragment,e),f(le.$$.fragment,e),f(ne.$$.fragment,e),f(_.$$.fragment,e),f(U.$$.fragment,e),f(ie.$$.fragment,e),f(me.$$.fragment,e),as=!0)},o(e){M(i.$$.fragment,e),M(J.$$.fragment,e),M(C.$$.fragment,e),M(G.$$.fragment,e),M(X.$$.fragment,e),M(z.$$.fragment,e),M(L.$$.fragment,e),M(S.$$.fragment,e),M(H.$$.fragment,e),M(N.$$.fragment,e),M(Y.$$.fragment,e),M(T.$$.fragment,e),M(D.$$.fragment,e),M(O.$$.fragment,e),M(se.$$.fragment,e),M(ae.$$.fragment,e),M(le.$$.fragment,e),M(ne.$$.fragment,e),M(_.$$.fragment,e),M(U.$$.fragment,e),M(ie.$$.fragment,e),M(me.$$.fragment,e),as=!1},d(e){e&&(t(o),t(n),t(j),t(b),t(x),t(fe),t(k),t(Me),t(v),t(ue),t(ge),t(Z),t(je),t(we),t(B),t($e),t(W),t(be),t(I),t(xe),t(Je),t(E),t(Te),t(_e),t(V),t(Ue),t(R),t(ke),t(ve),t(Ze),t(A),t(Ce),t(Be),t(F),t(We),t(Ie),t(Q),t(Ge),t(Ee),t(Xe),t(Ve),t(q),t(Re),t(P),t(ze),t(Le),t(K),t(Ae),t(Se),t(ee),t(Fe),t(He),t(te),t(Qe),t(Ne),t(Ye),t(qe),t(re),t(Pe),t(De),t(pe),t(Ke),t(Oe),t(oe),t(es),t(ss),t(ts),t(ce)),t(l),u(i,e),u(J,e),u(C,e),u(G,e),u(X,e),u(z,e),u(L,e),u(S,e),u(H,e),u(N,e),u(Y,e),u(T,e),u(D,e),u(O,e),u(se,e),u(ae,e),u(le,e),u(ne,e),u(_,e),u(U,e),u(ie,e),u(me,e)}}}const Ns='{"title":"Instantiate a big model","local":"instantiate-a-big-model","sections":[{"title":"Sharded checkpoints","local":"sharded-checkpoints","sections":[{"title":"Shard metadata","local":"shard-metadata","sections":[],"depth":3}],"depth":2},{"title":"Accelerate’s Big Model Inference","local":"accelerates-big-model-inference","sections":[],"depth":2},{"title":"Model data type","local":"model-data-type","sections":[],"depth":2}],"depth":1}';function Ys(w){return Ws(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class st extends Is{constructor(l){super(),Gs(this,l,Ys,Qs,Bs,{})}}export{st as component};