transformers/pr_34752/zh/_app/immutable/nodes/5.d3ff11aa.js

import{s as qt,o as Qt,n as Yt}from"../chunks/scheduler.9991993c.js";import{S as Nt,i as Kt,g as p,s as a,r as c,A as Dt,h as i,f as s,c as n,j as Ft,u as o,x as r,k as At,y as Ot,a as l,v as d,d as h,t as f,w as y}from"../chunks/index.7fc9a5e7.js";import{T as te}from"../chunks/Tip.9de92fc6.js";import{C as w}from"../chunks/CodeBlock.e11cba92.js";import{H as kt,E as ee}from"../chunks/EditOnGithub.84ab7f0e.js";function se(N){let m,$="请注意，随机创建的模型使用“空”张量进行初始化，这些张量占用内存空间但不填充它（因此随机值是给定时间内该内存块中的任何内容）。在第3步之后，对未初始化的权重执行适合模型/参数种类的随机初始化（例如正态分布），以尽可能提高速度！";return{c(){m=p("p"),m.textContent=$},l(M){m=i(M,"P",{"data-svelte-h":!0}),r(m)!=="svelte-1s8s2m3"&&(m.textContent=$)},m(M,Q){l(M,m,Q)},p:Yt,d(M){M&&s(m)}}}function le(N){let m,$,M,Q,_,K,g,Ct="当你想使用一个非常大的预训练模型时，一个挑战是尽量减少对内存的使用。通常从PyTorch开始的工作流程如下：",D,u,vt="<li>用随机权重创建你的模型。</li> <li>加载你的预训练权重。</li> <li>将这些预训练权重放入你的随机模型中。</li>",O,b,Jt="步骤1和2都需要完整版本的模型在内存中，这在大多数情况下不是问题，但如果你的模型开始达到几个GB的大小，这两个副本可能会让你超出内存的限制。更糟糕的是，如果你使用<code>torch.distributed</code>来启动分布式训练，每个进程都会加载预训练模型并将这两个副本存储在内存中。",tt,j,et,T,Bt="在本指南中，我们将探讨 Transformers 提供的解决方案来处理这个问题。请注意，这是一个积极开发的领域，因此这里解释的API在将来可能会略有变化。",st,U,lt,x,Zt="自4.18.0版本起，占用空间超过10GB的模型检查点将自动分成较小的片段。在使用<code>model.save_pretrained(save_dir)</code>时，您最终会得到几个部分<code>checkpoints</code>（每个的大小都小于10GB）以及一个索引，该索引将参数名称映射到存储它们的文件。",at,k,Gt="您可以使用<code>max_shard_size</code>参数来控制分片之前的最大大小。为了示例的目的，我们将使用具有较小分片大小的普通大小的模型：让我们以传统的BERT模型为例。",nt,C,pt,v,Rt='如果您使用 <a href="%E6%A8%A1%E5%9E%8B%E9%A2%84%E8%AE%AD%E7%BB%83%E4%BF%9D%E5%AD%98"><code>PreTrainedModel.save_pretrained</code></a> 进行保存，您将得到一个新的文件夹，其中包含两个文件：模型的配置和权重：',it,J,mt,B,Xt="现在让我们使用最大分片大小为200MB：",rt,Z,ct,G,Ht='在模型配置文件最上方，我们可以看到三个不同的权重文件，以及一个<code>index.json</code>索引文件。这样的<code>checkpoint</code>可以使用<a href="/docs/transformers/pr_34752/zh/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a>方法完全重新加载：',ot,R,dt,X,It="对于大型模型来说，这样做的主要优点是在上述工作流程的步骤2中，每个<code>checkpoint</code>的分片在前一个分片之后加载，从而将内存中的内存使用限制在模型大小加上最大分片的大小。",ht,H,Wt="在后台，索引文件用于确定<code>checkpoint</code>中包含哪些键以及相应的权重存储在哪里。我们可以像加载任何json一样加载该索引，并获得一个字典：",ft,I,yt,W,Lt="目前元数据仅包括模型的总大小。我们计划在将来添加其他信息：",Mt,L,jt,S,St="权重映射是该索引的主要部分，它将每个参数的名称（通常在PyTorch模型的<code>state_dict</code>中找到）映射到存储该参数的文件：",wt,E,$t,P,Et='如果您想直接在模型内部加载这样的分片<code>checkpoint</code>，而不使用 [<code>PreTrainedModel.from_pretrained</code>](就像您会为完整<code>checkpoint</code>执行 <code>model.load_state_dict()</code> 一样)，您应该使用 <a href="/docs/transformers/pr_34752/zh/main_classes/model#transformers.modeling_utils.load_sharded_checkpoint">modeling_utils.load_sharded_checkpoint()</a>：',_t,V,gt,z,ut,F,Pt="分片<code>checkpoints</code>在上述工作流的第2步中降低了内存使用，但为了在低内存环境中使用该模型，我们建议使用基于 Accelerate 库的工具。",bt,A,Vt='请阅读以下指南以获取更多信息：<a href="./main_classes/model#large-model-loading">使用 Accelerate 进行大模型加载</a>',Tt,q,Ut,Y,xt;return _=new kt({props:{title:"实例化大型模型",local:"实例化大型模型",headingTag:"h1"}}),j=new te({props:{$$slots:{default:[se]},$$scope:{ctx:N}}}),U=new kt({props:{title:"分片checkpoints",local:"分片checkpoints",headingTag:"h2"}}),C=new w({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbCUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZCglMjJnb29nbGUtYmVydCUyRmJlcnQtYmFzZS1jYXNlZCUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModel

model = AutoModel.from_pretrained(<span class="hljs-string">&quot;google-bert/bert-base-cased&quot;</span>)`,wrap:!1}}),J=new w({props:{code:"aW1wb3J0JTIwb3MlMEFpbXBvcnQlMjB0ZW1wZmlsZSUwQSUwQXdpdGglMjB0ZW1wZmlsZS5UZW1wb3JhcnlEaXJlY3RvcnkoKSUyMGFzJTIwdG1wX2RpciUzQSUwQSUyMCUyMCUyMCUyMG1vZGVsLnNhdmVfcHJldHJhaW5lZCh0bXBfZGlyKSUwQSUyMCUyMCUyMCUyMHByaW50KHNvcnRlZChvcy5saXN0ZGlyKHRtcF9kaXIpKSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> os
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tempfile

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> tempfile.TemporaryDirectory() <span class="hljs-keyword">as</span> tmp_dir:
<span class="hljs-meta">... </span>    model.save_pretrained(tmp_dir)
<span class="hljs-meta">... </span>    <span class="hljs-built_in">print</span>(<span class="hljs-built_in">sorted</span>(os.listdir(tmp_dir)))
[<span class="hljs-string">&#x27;config.json&#x27;</span>, <span class="hljs-string">&#x27;pytorch_model.bin&#x27;</span>]`,wrap:!1}}),Z=new w({props:{code:"d2l0aCUyMHRlbXBmaWxlLlRlbXBvcmFyeURpcmVjdG9yeSgpJTIwYXMlMjB0bXBfZGlyJTNBJTBBJTIwJTIwJTIwJTIwbW9kZWwuc2F2ZV9wcmV0cmFpbmVkKHRtcF9kaXIlMkMlMjBtYXhfc2hhcmRfc2l6ZSUzRCUyMjIwME1CJTIyKSUwQSUyMCUyMCUyMCUyMHByaW50KHNvcnRlZChvcy5saXN0ZGlyKHRtcF9kaXIpKSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> tempfile.TemporaryDirectory() <span class="hljs-keyword">as</span> tmp_dir:
<span class="hljs-meta">... </span>    model.save_pretrained(tmp_dir, max_shard_size=<span class="hljs-string">&quot;200MB&quot;</span>)
<span class="hljs-meta">... </span>    <span class="hljs-built_in">print</span>(<span class="hljs-built_in">sorted</span>(os.listdir(tmp_dir)))
[<span class="hljs-string">&#x27;config.json&#x27;</span>, <span class="hljs-string">&#x27;pytorch_model-00001-of-00003.bin&#x27;</span>, <span class="hljs-string">&#x27;pytorch_model-00002-of-00003.bin&#x27;</span>, <span class="hljs-string">&#x27;pytorch_model-00003-of-00003.bin&#x27;</span>, <span class="hljs-string">&#x27;pytorch_model.bin.index.json&#x27;</span>]`,wrap:!1}}),R=new w({props:{code:"d2l0aCUyMHRlbXBmaWxlLlRlbXBvcmFyeURpcmVjdG9yeSgpJTIwYXMlMjB0bXBfZGlyJTNBJTBBJTIwJTIwJTIwJTIwbW9kZWwuc2F2ZV9wcmV0cmFpbmVkKHRtcF9kaXIlMkMlMjBtYXhfc2hhcmRfc2l6ZSUzRCUyMjIwME1CJTIyKSUwQSUyMCUyMCUyMCUyMG5ld19tb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQodG1wX2Rpcik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> tempfile.TemporaryDirectory() <span class="hljs-keyword">as</span> tmp_dir:
<span class="hljs-meta">... </span>    model.save_pretrained(tmp_dir, max_shard_size=<span class="hljs-string">&quot;200MB&quot;</span>)
<span class="hljs-meta">... </span>    new_model = AutoModel.from_pretrained(tmp_dir)`,wrap:!1}}),I=new w({props:{code:"aW1wb3J0JTIwanNvbiUwQSUwQXdpdGglMjB0ZW1wZmlsZS5UZW1wb3JhcnlEaXJlY3RvcnkoKSUyMGFzJTIwdG1wX2RpciUzQSUwQSUyMCUyMCUyMCUyMG1vZGVsLnNhdmVfcHJldHJhaW5lZCh0bXBfZGlyJTJDJTIwbWF4X3NoYXJkX3NpemUlM0QlMjIyMDBNQiUyMiklMEElMjAlMjAlMjAlMjB3aXRoJTIwb3Blbihvcy5wYXRoLmpvaW4odG1wX2RpciUyQyUyMCUyMnB5dG9yY2hfbW9kZWwuYmluLmluZGV4Lmpzb24lMjIpJTJDJTIwJTIyciUyMiklMjBhcyUyMGYlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpbmRleCUyMCUzRCUyMGpzb24ubG9hZChmKSUwQSUwQXByaW50KGluZGV4LmtleXMoKSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> json

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> tempfile.TemporaryDirectory() <span class="hljs-keyword">as</span> tmp_dir:
<span class="hljs-meta">... </span>    model.save_pretrained(tmp_dir, max_shard_size=<span class="hljs-string">&quot;200MB&quot;</span>)
<span class="hljs-meta">... </span>    <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(os.path.join(tmp_dir, <span class="hljs-string">&quot;pytorch_model.bin.index.json&quot;</span>), <span class="hljs-string">&quot;r&quot;</span>) <span class="hljs-keyword">as</span> f:
<span class="hljs-meta">... </span>        index = json.load(f)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(index.keys())
dict_keys([<span class="hljs-string">&#x27;metadata&#x27;</span>, <span class="hljs-string">&#x27;weight_map&#x27;</span>])`,wrap:!1}}),L=new w({props:{code:"aW5kZXglNUIlMjJtZXRhZGF0YSUyMiU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>index[<span class="hljs-string">&quot;metadata&quot;</span>]
{<span class="hljs-string">&#x27;total_size&#x27;</span>: <span class="hljs-number">433245184</span>}`,wrap:!1}}),E=new w({props:{code:"aW5kZXglNUIlMjJ3ZWlnaHRfbWFwJTIyJTVE",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>index[<span class="hljs-string">&quot;weight_map&quot;</span>]
{<span class="hljs-string">&#x27;embeddings.LayerNorm.bias&#x27;</span>: <span class="hljs-string">&#x27;pytorch_model-00001-of-00003.bin&#x27;</span>,
 <span class="hljs-string">&#x27;embeddings.LayerNorm.weight&#x27;</span>: <span class="hljs-string">&#x27;pytorch_model-00001-of-00003.bin&#x27;</span>,
 ...`,wrap:!1}}),V=new w({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5tb2RlbGluZ191dGlscyUyMGltcG9ydCUyMGxvYWRfc2hhcmRlZF9jaGVja3BvaW50JTBBJTBBd2l0aCUyMHRlbXBmaWxlLlRlbXBvcmFyeURpcmVjdG9yeSgpJTIwYXMlMjB0bXBfZGlyJTNBJTBBJTIwJTIwJTIwJTIwbW9kZWwuc2F2ZV9wcmV0cmFpbmVkKHRtcF9kaXIlMkMlMjBtYXhfc2hhcmRfc2l6ZSUzRCUyMjIwME1CJTIyKSUwQSUyMCUyMCUyMCUyMGxvYWRfc2hhcmRlZF9jaGVja3BvaW50KG1vZGVsJTJDJTIwdG1wX2Rpcik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers.modeling_utils <span class="hljs-keyword">import</span> load_sharded_checkpoint

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> tempfile.TemporaryDirectory() <span class="hljs-keyword">as</span> tmp_dir:
<span class="hljs-meta">... </span>    model.save_pretrained(tmp_dir, max_shard_size=<span class="hljs-string">&quot;200MB&quot;</span>)
<span class="hljs-meta">... </span>    load_sharded_checkpoint(model, tmp_dir)`,wrap:!1}}),z=new kt({props:{title:"低内存加载",local:"低内存加载",headingTag:"h2"}}),q=new ee({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/zh/big_models.md"}}),{c(){m=p("meta"),$=a(),M=p("p"),Q=a(),c(_.$$.fragment),K=a(),g=p("p"),g.textContent=Ct,D=a(),u=p("ol"),u.innerHTML=vt,O=a(),b=p("p"),b.innerHTML=Jt,tt=a(),c(j.$$.fragment),et=a(),T=p("p"),T.textContent=Bt,st=a(),c(U.$$.fragment),lt=a(),x=p("p"),x.innerHTML=Zt,at=a(),k=p("p"),k.innerHTML=Gt,nt=a(),c(C.$$.fragment),pt=a(),v=p("p"),v.innerHTML=Rt,it=a(),c(J.$$.fragment),mt=a(),B=p("p"),B.textContent=Xt,rt=a(),c(Z.$$.fragment),ct=a(),G=p("p"),G.innerHTML=Ht,ot=a(),c(R.$$.fragment),dt=a(),X=p("p"),X.innerHTML=It,ht=a(),H=p("p"),H.innerHTML=Wt,ft=a(),c(I.$$.fragment),yt=a(),W=p("p"),W.textContent=Lt,Mt=a(),c(L.$$.fragment),jt=a(),S=p("p"),S.innerHTML=St,wt=a(),c(E.$$.fragment),$t=a(),P=p("p"),P.innerHTML=Et,_t=a(),c(V.$$.fragment),gt=a(),c(z.$$.fragment),ut=a(),F=p("p"),F.innerHTML=Pt,bt=a(),A=p("p"),A.innerHTML=Vt,Tt=a(),c(q.$$.fragment),Ut=a(),Y=p("p"),this.h()},l(t){const e=Dt("svelte-u9bgzb",document.head);m=i(e,"META",{name:!0,content:!0}),e.forEach(s),$=n(t),M=i(t,"P",{}),Ft(M).forEach(s),Q=n(t),o(_.$$.fragment,t),K=n(t),g=i(t,"P",{"data-svelte-h":!0}),r(g)!=="svelte-f980nr"&&(g.textContent=Ct),D=n(t),u=i(t,"OL",{"data-svelte-h":!0}),r(u)!=="svelte-qi8gup"&&(u.innerHTML=vt),O=n(t),b=i(t,"P",{"data-svelte-h":!0}),r(b)!=="svelte-3m51fj"&&(b.innerHTML=Jt),tt=n(t),o(j.$$.fragment,t),et=n(t),T=i(t,"P",{"data-svelte-h":!0}),r(T)!=="svelte-1fsody1"&&(T.textContent=Bt),st=n(t),o(U.$$.fragment,t),lt=n(t),x=i(t,"P",{"data-svelte-h":!0}),r(x)!=="svelte-p12rar"&&(x.innerHTML=Zt),at=n(t),k=i(t,"P",{"data-svelte-h":!0}),r(k)!=="svelte-fnq70n"&&(k.innerHTML=Gt),nt=n(t),o(C.$$.fragment,t),pt=n(t),v=i(t,"P",{"data-svelte-h":!0}),r(v)!=="svelte-1vlh5wl"&&(v.innerHTML=Rt),it=n(t),o(J.$$.fragment,t),mt=n(t),B=i(t,"P",{"data-svelte-h":!0}),r(B)!=="svelte-15702pw"&&(B.textContent=Xt),rt=n(t),o(Z.$$.fragment,t),ct=n(t),G=i(t,"P",{"data-svelte-h":!0}),r(G)!=="svelte-1xfd7w8"&&(G.innerHTML=Ht),ot=n(t),o(R.$$.fragment,t),dt=n(t),X=i(t,"P",{"data-svelte-h":!0}),r(X)!=="svelte-bnpz0s"&&(X.innerHTML=It),ht=n(t),H=i(t,"P",{"data-svelte-h":!0}),r(H)!=="svelte-18it1xo"&&(H.innerHTML=Wt),ft=n(t),o(I.$$.fragment,t),yt=n(t),W=i(t,"P",{"data-svelte-h":!0}),r(W)!=="svelte-26dikn"&&(W.textContent=Lt),Mt=n(t),o(L.$$.fragment,t),jt=n(t),S=i(t,"P",{"data-svelte-h":!0}),r(S)!=="svelte-1lig6ro"&&(S.innerHTML=St),wt=n(t),o(E.$$.fragment,t),$t=n(t),P=i(t,"P",{"data-svelte-h":!0}),r(P)!=="svelte-qpkl24"&&(P.innerHTML=Et),_t=n(t),o(V.$$.fragment,t),gt=n(t),o(z.$$.fragment,t),ut=n(t),F=i(t,"P",{"data-svelte-h":!0}),r(F)!=="svelte-1qqd7tm"&&(F.innerHTML=Pt),bt=n(t),A=i(t,"P",{"data-svelte-h":!0}),r(A)!=="svelte-kz6dhx"&&(A.innerHTML=Vt),Tt=n(t),o(q.$$.fragment,t),Ut=n(t),Y=i(t,"P",{}),Ft(Y).forEach(s),this.h()},h(){At(m,"name","hf:doc:metadata"),At(m,"content",ae)},m(t,e){Ot(document.head,m),l(t,$,e),l(t,M,e),l(t,Q,e),d(_,t,e),l(t,K,e),l(t,g,e),l(t,D,e),l(t,u,e),l(t,O,e),l(t,b,e),l(t,tt,e),d(j,t,e),l(t,et,e),l(t,T,e),l(t,st,e),d(U,t,e),l(t,lt,e),l(t,x,e),l(t,at,e),l(t,k,e),l(t,nt,e),d(C,t,e),l(t,pt,e),l(t,v,e),l(t,it,e),d(J,t,e),l(t,mt,e),l(t,B,e),l(t,rt,e),d(Z,t,e),l(t,ct,e),l(t,G,e),l(t,ot,e),d(R,t,e),l(t,dt,e),l(t,X,e),l(t,ht,e),l(t,H,e),l(t,ft,e),d(I,t,e),l(t,yt,e),l(t,W,e),l(t,Mt,e),d(L,t,e),l(t,jt,e),l(t,S,e),l(t,wt,e),d(E,t,e),l(t,$t,e),l(t,P,e),l(t,_t,e),d(V,t,e),l(t,gt,e),d(z,t,e),l(t,ut,e),l(t,F,e),l(t,bt,e),l(t,A,e),l(t,Tt,e),d(q,t,e),l(t,Ut,e),l(t,Y,e),xt=!0},p(t,[e]){const zt={};e&2&&(zt.$$scope={dirty:e,ctx:t}),j.$set(zt)},i(t){xt||(h(_.$$.fragment,t),h(j.$$.fragment,t),h(U.$$.fragment,t),h(C.$$.fragment,t),h(J.$$.fragment,t),h(Z.$$.fragment,t),h(R.$$.fragment,t),h(I.$$.fragment,t),h(L.$$.fragment,t),h(E.$$.fragment,t),h(V.$$.fragment,t),h(z.$$.fragment,t),h(q.$$.fragment,t),xt=!0)},o(t){f(_.$$.fragment,t),f(j.$$.fragment,t),f(U.$$.fragment,t),f(C.$$.fragment,t),f(J.$$.fragment,t),f(Z.$$.fragment,t),f(R.$$.fragment,t),f(I.$$.fragment,t),f(L.$$.fragment,t),f(E.$$.fragment,t),f(V.$$.fragment,t),f(z.$$.fragment,t),f(q.$$.fragment,t),xt=!1},d(t){t&&(s($),s(M),s(Q),s(K),s(g),s(D),s(u),s(O),s(b),s(tt),s(et),s(T),s(st),s(lt),s(x),s(at),s(k),s(nt),s(pt),s(v),s(it),s(mt),s(B),s(rt),s(ct),s(G),s(ot),s(dt),s(X),s(ht),s(H),s(ft),s(yt),s(W),s(Mt),s(jt),s(S),s(wt),s($t),s(P),s(_t),s(gt),s(ut),s(F),s(bt),s(A),s(Tt),s(Ut),s(Y)),s(m),y(_,t),y(j,t),y(U,t),y(C,t),y(J,t),y(Z,t),y(R,t),y(I,t),y(L,t),y(E,t),y(V,t),y(z,t),y(q,t)}}}const ae='{"title":"实例化大型模型","local":"实例化大型模型","sections":[{"title":"分片checkpoints","local":"分片checkpoints","sections":[],"depth":2},{"title":"低内存加载","local":"低内存加载","sections":[],"depth":2}],"depth":1}';function ne(N){return Qt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class oe extends Nt{constructor(m){super(),Kt(this,m,ne,le,qt,{})}}export{oe as component};