transformers/pr_36049/zh/_app/immutable/nodes/55.2cef21ee.js

import{s as rt,n as mt,o as ct}from"../chunks/scheduler.25b97de1.js";import{S as Mt,i as ut,g as p,s as n,r as _,A as Ut,h as o,f as e,c as a,j as at,u as $,x as r,k as pt,l as dt,y as Tt,a as s,v,d as E,t as Q,w as B}from"../chunks/index.d9030fc9.js";import{C as ot}from"../chunks/CodeBlock.e6cd0d95.js";import{H as it,E as ft}from"../chunks/EditOnGithub.91d95064.js";function ht(X){let i,R,w,k,c,W,M,A="某些模型现已支持内置的<strong>张量并行</strong>（Tensor Parallelism, TP），并通过 PyTorch 实现。张量并行技术将模型切分到多个 GPU 上，从而支持更大的模型尺寸，并对诸如矩阵乘法等计算任务进行并行化。",I,u,D="要启用张量并行，只需在调用 <code>from_pretrained()</code> 时传递参数 <code>tp_plan=&quot;auto&quot;</code>：",x,U,L,d,N="您可以使用 <code>torchrun</code> 命令启动上述脚本，多进程模式会自动将每个进程映射到一张 GPU：",H,T,Z,f,K="目前，PyTorch 张量并行支持以下模型：",V,h,tt='<li><a href="https://huggingface.co/docs/transformers/model_doc/llama#transformers.LlamaModel" rel="nofollow">Llama</a></li>',G,J,lt="如果您希望对其他模型添加张量并行支持，可以通过提交 GitHub Issue 或 Pull Request 来提出请求。",F,y,Y,j,et="对于推理场景（尤其是处理大批量或长序列的输入），张量并行可以显著提升计算速度。",q,C,st='以下是 <a href="https://huggingface.co/docs/transformers/model_doc/llama#transformers.LlamaModel" rel="nofollow">Llama</a> 模型在序列长度为 512 且不同批量大小情况下的单次前向推理的预期加速效果：',O,m,nt='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/Meta-Llama-3-8B-Instruct%2C%20seqlen%20%3D%20512%2C%20python%2C%20w_%20compile.png"/>',P,g,z,b,S;return c=new it({props:{title:"多GPU推理",local:"多gpu推理",headingTag:"h1"}}),U=new ot({props:{code:"aW1wb3J0JTIwb3MlMEFpbXBvcnQlMjB0b3JjaCUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUyQyUyMEF1dG9Ub2tlbml6ZXIlMEElMEFtb2RlbF9pZCUyMCUzRCUyMCUyMm1ldGEtbGxhbWElMkZNZXRhLUxsYW1hLTMtOEItSW5zdHJ1Y3QlMjIlMEElMEElMjMlMjAlRTUlODglOUQlRTUlQTclOEIlRTUlOEMlOTYlRTUlODglODYlRTUlQjglODMlRTUlQkMlOEYlRTclOEUlQUYlRTUlQTIlODMlMEFyYW5rJTIwJTNEJTIwaW50KG9zLmVudmlyb24lNUIlMjJSQU5LJTIyJTVEKSUwQWRldmljZSUyMCUzRCUyMHRvcmNoLmRldmljZShmJTIyY3VkYSUzQSU3QnJhbmslN0QlMjIpJTBBdG9yY2guZGlzdHJpYnV0ZWQuaW5pdF9wcm9jZXNzX2dyb3VwKCUyMm5jY2wlMjIlMkMlMjBkZXZpY2VfaWQlM0RkZXZpY2UpJTBBJTBBJTIzJTIwJUU4JThFJUI3JUU1JThGJTk2JUU2JTk0JUFGJUU2JThDJTgxJUU1JUJDJUEwJUU5JTg3JThGJUU1JUI5JUI2JUU4JUExJThDJUU3JTlBJTg0JUU2JUE4JUExJUU1JTlFJThCJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwbW9kZWxfaWQlMkMlMEElMjAlMjAlMjAlMjB0cF9wbGFuJTNEJTIyYXV0byUyMiUyQyUwQSklMEElMEElMjMlMjAlRTUlODclODYlRTUlQTQlODclRTglQkUlOTMlRTUlODUlQTV0b2tlbnMlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChtb2RlbF9pZCklMEFwcm9tcHQlMjAlM0QlMjAlMjJDYW4lMjBJJTIwaGVscCUyMiUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplcihwcm9tcHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS5pbnB1dF9pZHMudG8oZGV2aWNlKSUwQSUwQSUyMyUyMCVFNSU4OCU4NiVFNSVCOCU4MyVFNSVCQyU4RiVFOCVCRiU5MCVFOCVBMSU4QyUwQW91dHB1dHMlMjAlM0QlMjBtb2RlbChpbnB1dHMp",highlighted:`<span class="hljs-keyword">import</span> os
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model_id = <span class="hljs-string">&quot;meta-llama/Meta-Llama-3-8B-Instruct&quot;</span>

<span class="hljs-comment"># 初始化分布式环境</span>
rank = <span class="hljs-built_in">int</span>(os.environ[<span class="hljs-string">&quot;RANK&quot;</span>])
device = torch.device(<span class="hljs-string">f&quot;cuda:<span class="hljs-subst">{rank}</span>&quot;</span>)
torch.distributed.init_process_group(<span class="hljs-string">&quot;nccl&quot;</span>, device_id=device)

<span class="hljs-comment"># 获取支持张量并行的模型</span>
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    tp_plan=<span class="hljs-string">&quot;auto&quot;</span>,
)

<span class="hljs-comment"># 准备输入tokens</span>
tokenizer = AutoTokenizer.from_pretrained(model_id)
prompt = <span class="hljs-string">&quot;Can I help&quot;</span>
inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).input_ids.to(device)

<span class="hljs-comment"># 分布式运行</span>
outputs = model(inputs)`,wrap:!1}}),T=new ot({props:{code:"dG9yY2hydW4lMjAtLW5wcm9jLXBlci1ub2RlJTIwNCUyMGRlbW8ucHk=",highlighted:'torchrun --nproc-per-<span class="hljs-keyword">node</span> <span class="hljs-title">4</span> demo.py',wrap:!1}}),y=new it({props:{title:"预期性能提升",local:"预期性能提升",headingTag:"h3"}}),g=new ft({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/zh/perf_infer_gpu_multi.md"}}),{c(){i=p("meta"),R=n(),w=p("p"),k=n(),_(c.$$.fragment),W=n(),M=p("p"),M.innerHTML=A,I=n(),u=p("p"),u.innerHTML=D,x=n(),_(U.$$.fragment),L=n(),d=p("p"),d.innerHTML=N,H=n(),_(T.$$.fragment),Z=n(),f=p("p"),f.textContent=K,V=n(),h=p("ul"),h.innerHTML=tt,G=n(),J=p("p"),J.textContent=lt,F=n(),_(y.$$.fragment),Y=n(),j=p("p"),j.textContent=et,q=n(),C=p("p"),C.innerHTML=st,O=n(),m=p("div"),m.innerHTML=nt,P=n(),_(g.$$.fragment),z=n(),b=p("p"),this.h()},l(t){const l=Ut("svelte-u9bgzb",document.head);i=o(l,"META",{name:!0,content:!0}),l.forEach(e),R=a(t),w=o(t,"P",{}),at(w).forEach(e),k=a(t),$(c.$$.fragment,t),W=a(t),M=o(t,"P",{"data-svelte-h":!0}),r(M)!=="svelte-1rnooy8"&&(M.innerHTML=A),I=a(t),u=o(t,"P",{"data-svelte-h":!0}),r(u)!=="svelte-1lcqrkq"&&(u.innerHTML=D),x=a(t),$(U.$$.fragment,t),L=a(t),d=o(t,"P",{"data-svelte-h":!0}),r(d)!=="svelte-1u51d1q"&&(d.innerHTML=N),H=a(t),$(T.$$.fragment,t),Z=a(t),f=o(t,"P",{"data-svelte-h":!0}),r(f)!=="svelte-15hz8rz"&&(f.textContent=K),V=a(t),h=o(t,"UL",{"data-svelte-h":!0}),r(h)!=="svelte-wihcs3"&&(h.innerHTML=tt),G=a(t),J=o(t,"P",{"data-svelte-h":!0}),r(J)!=="svelte-dhflz7"&&(J.textContent=lt),F=a(t),$(y.$$.fragment,t),Y=a(t),j=o(t,"P",{"data-svelte-h":!0}),r(j)!=="svelte-sjssgg"&&(j.textContent=et),q=a(t),C=o(t,"P",{"data-svelte-h":!0}),r(C)!=="svelte-1gd4c5g"&&(C.innerHTML=st),O=a(t),m=o(t,"DIV",{style:!0,"data-svelte-h":!0}),r(m)!=="svelte-lg3kfi"&&(m.innerHTML=nt),P=a(t),$(g.$$.fragment,t),z=a(t),b=o(t,"P",{}),at(b).forEach(e),this.h()},h(){pt(i,"name","hf:doc:metadata"),pt(i,"content",Jt),dt(m,"text-align","center")},m(t,l){Tt(document.head,i),s(t,R,l),s(t,w,l),s(t,k,l),v(c,t,l),s(t,W,l),s(t,M,l),s(t,I,l),s(t,u,l),s(t,x,l),v(U,t,l),s(t,L,l),s(t,d,l),s(t,H,l),v(T,t,l),s(t,Z,l),s(t,f,l),s(t,V,l),s(t,h,l),s(t,G,l),s(t,J,l),s(t,F,l),v(y,t,l),s(t,Y,l),s(t,j,l),s(t,q,l),s(t,C,l),s(t,O,l),s(t,m,l),s(t,P,l),v(g,t,l),s(t,z,l),s(t,b,l),S=!0},p:mt,i(t){S||(E(c.$$.fragment,t),E(U.$$.fragment,t),E(T.$$.fragment,t),E(y.$$.fragment,t),E(g.$$.fragment,t),S=!0)},o(t){Q(c.$$.fragment,t),Q(U.$$.fragment,t),Q(T.$$.fragment,t),Q(y.$$.fragment,t),Q(g.$$.fragment,t),S=!1},d(t){t&&(e(R),e(w),e(k),e(W),e(M),e(I),e(u),e(x),e(L),e(d),e(H),e(Z),e(f),e(V),e(h),e(G),e(J),e(F),e(Y),e(j),e(q),e(C),e(O),e(m),e(P),e(z),e(b)),e(i),B(c,t),B(U,t),B(T,t),B(y,t),B(g,t)}}}const Jt='{"title":"多GPU推理","local":"多gpu推理","sections":[{"title":"预期性能提升","local":"预期性能提升","sections":[],"depth":3}],"depth":1}';function yt(X){return ct(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class bt extends Mt{constructor(i){super(),ut(this,i,yt,ht,rt,{})}}export{bt as component};