text-generation-inference/main/en/_app/immutable/nodes/28.5b400aad.js

import{s as O,n as q,o as K}from"../chunks/scheduler.362310b7.js";import{S as ee,i as te,g as o,s as a,r as U,A as le,h as m,f as l,c as s,j as Y,u as j,x as v,k as D,y as ne,a as n,v as I,d as g,t as Z,w as b}from"../chunks/index.57dfc70d.js";import{C as F}from"../chunks/CodeBlock.d3c47f83.js";import{H as Q,E as ae}from"../chunks/EditOnGithub.9633c464.js";function se(E){let i,G,y,$,r,C,p,P='TGI optimized models are supported on Intel Data Center GPU <a href="https://www.intel.com/content/www/us/en/products/sku/232876/intel-data-center-gpu-max-1100/specifications.html" rel="nofollow">Max1100</a>, <a href="https://www.intel.com/content/www/us/en/products/sku/232873/intel-data-center-gpu-max-1550/specifications.html" rel="nofollow">Max1550</a>, the recommended usage is through Docker.',B,c,H="On a server powered by Intel GPUs, TGI can be launched with the following command:",k,d,W,h,A,u,N="Intel® Extension for PyTorch (IPEX) also provides further optimizations for Intel CPUs. The IPEX provides optimization operations such as flash attention, page attention, Add + LayerNorm, ROPE and more.",L,w,S="On a server powered by Intel CPU, TGI can be launched with the following command:",R,M,V,T,z='The launched TGI server can then be queried from clients, make sure to check out the <a href="./basic_tutorials/consuming_tgi">Consuming TGI</a> guide.',X,J,_,f,x;return r=new Q({props:{title:"Using TGI with Intel GPUs",local:"using-tgi-with-intel-gpus",headingTag:"h1"}}),d=new F({props:{code:"bW9kZWwlM0R0ZWtuaXVtJTJGT3Blbkhlcm1lcy0yLjUtTWlzdHJhbC03QiUwQXZvbHVtZSUzRCUyNFBXRCUyRmRhdGElMjAlMjMlMjBzaGFyZSUyMGElMjB2b2x1bWUlMjB3aXRoJTIwdGhlJTIwRG9ja2VyJTIwY29udGFpbmVyJTIwdG8lMjBhdm9pZCUyMGRvd25sb2FkaW5nJTIwd2VpZ2h0cyUyMGV2ZXJ5JTIwcnVuJTBBJTBBZG9ja2VyJTIwcnVuJTIwLS1ybSUyMC0tcHJpdmlsZWdlZCUyMC0tY2FwLWFkZCUzRHN5c19uaWNlJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1kZXZpY2UlM0QlMkZkZXYlMkZkcmklMjAlNUMlMEElMjAlMjAlMjAlMjAtLWlwYyUzRGhvc3QlMjAtLXNobS1zaXplJTIwMWclMjAtLW5ldCUyMGhvc3QlMjAtdiUyMCUyNHZvbHVtZSUzQSUyRmRhdGElMjAlNUMlMEElMjAlMjAlMjAlMjBnaGNyLmlvJTJGaHVnZ2luZ2ZhY2UlMkZ0ZXh0LWdlbmVyYXRpb24taW5mZXJlbmNlJTNBMi4yLjAtaW50ZWwteHB1JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1tb2RlbC1pZCUyMCUyNG1vZGVsJTIwLS1jdWRhLWdyYXBocyUyMDA=",highlighted:`model=teknium/OpenHermes-2.5-Mistral-7B
volume=<span class="hljs-variable">$PWD</span>/data <span class="hljs-comment"># share a volume with the Docker container to avoid downloading weights every run</span>

docker run --<span class="hljs-built_in">rm</span> --privileged --cap-add=sys_nice \\
    --device=/dev/dri \\
    --ipc=host --shm-size 1g --net host -v <span class="hljs-variable">$volume</span>:/data \\
    ghcr.io/huggingface/text-generation-inference:2.2.0-intel-xpu \\
    --model-id <span class="hljs-variable">$model</span> --cuda-graphs 0`,wrap:!1}}),h=new Q({props:{title:"Using TGI with Intel CPUs",local:"using-tgi-with-intel-cpus",headingTag:"h1"}}),M=new F({props:{code:"bW9kZWwlM0R0ZWtuaXVtJTJGT3Blbkhlcm1lcy0yLjUtTWlzdHJhbC03QiUwQXZvbHVtZSUzRCUyNFBXRCUyRmRhdGElMjAlMjMlMjBzaGFyZSUyMGElMjB2b2x1bWUlMjB3aXRoJTIwdGhlJTIwRG9ja2VyJTIwY29udGFpbmVyJTIwdG8lMjBhdm9pZCUyMGRvd25sb2FkaW5nJTIwd2VpZ2h0cyUyMGV2ZXJ5JTIwcnVuJTBBJTBBZG9ja2VyJTIwcnVuJTIwLS1ybSUyMC0tcHJpdmlsZWdlZCUyMC0tY2FwLWFkZCUzRHN5c19uaWNlJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1kZXZpY2UlM0QlMkZkZXYlMkZkcmklMjAlNUMlMEElMjAlMjAlMjAlMjAtLWlwYyUzRGhvc3QlMjAtLXNobS1zaXplJTIwMWclMjAtLW5ldCUyMGhvc3QlMjAtdiUyMCUyNHZvbHVtZSUzQSUyRmRhdGElMjAlNUMlMEElMjAlMjAlMjAlMjBnaGNyLmlvJTJGaHVnZ2luZ2ZhY2UlMkZ0ZXh0LWdlbmVyYXRpb24taW5mZXJlbmNlJTNBMi4yLjAtaW50ZWwtY3B1JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1tb2RlbC1pZCUyMCUyNG1vZGVsJTIwLS1jdWRhLWdyYXBocyUyMDA=",highlighted:`model=teknium/OpenHermes-2.5-Mistral-7B
volume=<span class="hljs-variable">$PWD</span>/data <span class="hljs-comment"># share a volume with the Docker container to avoid downloading weights every run</span>

docker run --<span class="hljs-built_in">rm</span> --privileged --cap-add=sys_nice \\
    --device=/dev/dri \\
    --ipc=host --shm-size 1g --net host -v <span class="hljs-variable">$volume</span>:/data \\
    ghcr.io/huggingface/text-generation-inference:2.2.0-intel-cpu \\
    --model-id <span class="hljs-variable">$model</span> --cuda-graphs 0`,wrap:!1}}),J=new ae({props:{source:"https://github.com/huggingface/text-generation-inference/blob/main/docs/source/installation_intel.md"}}),{c(){i=o("meta"),G=a(),y=o("p"),$=a(),U(r.$$.fragment),C=a(),p=o("p"),p.innerHTML=P,B=a(),c=o("p"),c.textContent=H,k=a(),U(d.$$.fragment),W=a(),U(h.$$.fragment),A=a(),u=o("p"),u.textContent=N,L=a(),w=o("p"),w.textContent=S,R=a(),U(M.$$.fragment),V=a(),T=o("p"),T.innerHTML=z,X=a(),U(J.$$.fragment),_=a(),f=o("p"),this.h()},l(e){const t=le("svelte-u9bgzb",document.head);i=m(t,"META",{name:!0,content:!0}),t.forEach(l),G=s(e),y=m(e,"P",{}),Y(y).forEach(l),$=s(e),j(r.$$.fragment,e),C=s(e),p=m(e,"P",{"data-svelte-h":!0}),v(p)!=="svelte-eqifmc"&&(p.innerHTML=P),B=s(e),c=m(e,"P",{"data-svelte-h":!0}),v(c)!=="svelte-hhnokj"&&(c.textContent=H),k=s(e),j(d.$$.fragment,e),W=s(e),j(h.$$.fragment,e),A=s(e),u=m(e,"P",{"data-svelte-h":!0}),v(u)!=="svelte-rvri8q"&&(u.textContent=N),L=s(e),w=m(e,"P",{"data-svelte-h":!0}),v(w)!=="svelte-vqse1y"&&(w.textContent=S),R=s(e),j(M.$$.fragment,e),V=s(e),T=m(e,"P",{"data-svelte-h":!0}),v(T)!=="svelte-uo4xf6"&&(T.innerHTML=z),X=s(e),j(J.$$.fragment,e),_=s(e),f=m(e,"P",{}),Y(f).forEach(l),this.h()},h(){D(i,"name","hf:doc:metadata"),D(i,"content",ie)},m(e,t){ne(document.head,i),n(e,G,t),n(e,y,t),n(e,$,t),I(r,e,t),n(e,C,t),n(e,p,t),n(e,B,t),n(e,c,t),n(e,k,t),I(d,e,t),n(e,W,t),I(h,e,t),n(e,A,t),n(e,u,t),n(e,L,t),n(e,w,t),n(e,R,t),I(M,e,t),n(e,V,t),n(e,T,t),n(e,X,t),I(J,e,t),n(e,_,t),n(e,f,t),x=!0},p:q,i(e){x||(g(r.$$.fragment,e),g(d.$$.fragment,e),g(h.$$.fragment,e),g(M.$$.fragment,e),g(J.$$.fragment,e),x=!0)},o(e){Z(r.$$.fragment,e),Z(d.$$.fragment,e),Z(h.$$.fragment,e),Z(M.$$.fragment,e),Z(J.$$.fragment,e),x=!1},d(e){e&&(l(G),l(y),l($),l(C),l(p),l(B),l(c),l(k),l(W),l(A),l(u),l(L),l(w),l(R),l(V),l(T),l(X),l(_),l(f)),l(i),b(r,e),b(d,e),b(h,e),b(M,e),b(J,e)}}}const ie='{"title":"Using TGI with Intel GPUs","local":"using-tgi-with-intel-gpus","sections":[],"depth":1}';function oe(E){return K(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class de extends ee{constructor(i){super(),te(this,i,oe,se,O,{})}}export{de as component};