bitsandbytes/pr_1503/en/_app/immutable/nodes/12.4fe7bc77.js

import{s as Ft,o as Bt,n as Ut}from"../chunks/scheduler.852ec091.js";import{S as Dt,i as Et,g as o,s as i,r as m,A as Gt,h as r,f as n,c as s,j as Ht,u as d,x as f,k as At,y as Nt,a as l,v as h,d as c,t as $,w as b}from"../chunks/index.28275fd3.js";import{T as It}from"../chunks/Tip.9f398c59.js";import{H as z,E as Rt}from"../chunks/index.f9a1508e.js";function St(O){let a,u="If you feel these docs need some additional info, please consider submitting a PR or respectfully request the missing info in one of the below mentioned Github discussion spaces.";return{c(){a=o("p"),a.textContent=u},l(p){a=r(p,"P",{"data-svelte-h":!0}),f(a)!=="svelte-1mb3sz"&&(a.textContent=u)},m(p,y){l(p,a,y)},p:Ut,d(p){p&&n(a)}}}function zt(O){let a,u='Apple Silicon support is planned for Q4 2024. We are actively seeking contributors to help implement this, develop a concrete plan, and create a detailed list of requirements. Due to limited resources, we rely on community contributions for this implementation effort. To discuss further, please spell out your thoughts and discuss in <a href="https://github.com/bitsandbytes-foundation/bitsandbytes/discussions/1340" rel="nofollow">this GitHub discussion</a> and tag <code>@Titus-von-Koeller</code> and <code>@matthewdouglas</code>. Thank you!';return{c(){a=o("p"),a.innerHTML=u},l(p){a=r(p,"P",{"data-svelte-h":!0}),f(a)!=="svelte-ztwrs6"&&(a.innerHTML=u)},m(p,y){l(p,a,y)},p:Ut,d(p){p&&n(a)}}}function Ot(O){let a,u,p,y,v,X,g,Y,x,ct="As part of a recent refactoring effort, we will soon offer official multi-backend support. Currently, this feature is available in a preview alpha release, allowing us to gather early feedback from users to improve the functionality and identify any bugs.",j,_,$t="At present, the Intel CPU and AMD ROCm backends are considered fully functional. The Intel XPU backend has limited functionality and is less mature.",K,w,bt='Please refer to the <a href="./installation#multi-backend">installation instructions</a> for details on installing the backend you intend to test (and hopefully provide feedback on).',Q,T,W,C,J,k,yt="As we are currently in the alpha testing phase, bugs are expected, and performance might not meet expectations. However, this is exactly what we want to discover from <strong>your</strong> perspective as the end user!",V,P,gt="Please share and discuss your feedback with us here:",Z,M,Tt='<li><a href="https://github.com/bitsandbytes-foundation/bitsandbytes/discussions/1339" rel="nofollow">Github Discussion: Multi-backend refactor: Alpha release ( AMD ROCm ONLY )</a></li> <li><a href="https://github.com/bitsandbytes-foundation/bitsandbytes/discussions/1338" rel="nofollow">Github Discussion: Multi-backend refactor: Alpha release ( Intel ONLY )</a></li>',tt,L,vt="Thank you for your support!",et,H,nt,A,lt,I,xt='The below performance data is collected from the Intel 4th Gen Xeon (SPR) platform. The tables show speed-up and memory compared with different data types of <a href="https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct" rel="nofollow">meta-llama/Llama-3.1-8B-Instruct</a>.',at,U,_t="You may run <code>benchmarking/generation_benchmark.py</code> to reproduce the below model memory and inference results. Please note that you need to bind cores if you are using the CPU to benchmark. For example, run <code>numactl -C 0-55 -m 0 python generation_benchmark.py --quant_type nf4</code> on Intel 4th Gen Xeon with single socket.",it,F,wt='The finetune results are selected from <a href="https://github.com/huggingface/peft/blob/main/examples/olora_finetuning/olora_finetuning.py" rel="nofollow">peft</a>.',st,B,ot,D,Ct="<thead><tr><th>Data Type</th> <th>BF16</th> <th>INT8</th> <th>NF4</th> <th>FP4</th></tr></thead> <tbody><tr><td>Memory (GB)</td> <td>15.0</td> <td>8.5</td> <td>5.2</td> <td>5.2</td></tr></tbody>",rt,E,pt,G,kt="<thead><tr><th>Data Type</th> <th>BF16</th> <th>INT8</th> <th>NF4</th> <th>FP4</th></tr></thead> <tbody><tr><td>Speed-Up (vs BF16)</td> <td>1.0x</td> <td>0.57x</td> <td>2.6x</td> <td>0.1x</td></tr></tbody>",ft,N,ut,R,Pt="<thead><tr><th>Data Type</th> <th>BF16</th> <th>INT8</th> <th>NF4</th> <th>FP4</th></tr></thead> <tbody><tr><td>Speed-Up (vs BF16)</td> <td>1.0x</td> <td>0.91x</td> <td>1.0x</td> <td>1.0x</td></tr></tbody>",mt,S,dt,q,ht;return v=new z({props:{title:"Multi-backend support (non-CUDA backends)",local:"multi-backend-support-non-cuda-backends",headingTag:"h1"}}),g=new It({props:{warning:!1,$$slots:{default:[St]},$$scope:{ctx:O}}}),T=new It({props:{warning:!1,$$slots:{default:[zt]},$$scope:{ctx:O}}}),C=new z({props:{title:"Alpha Release",local:"alpha-release",headingTag:"h2"}}),H=new z({props:{title:"Benchmarks",local:"benchmarks",headingTag:"h2"}}),A=new z({props:{title:"Intel",local:"intel",headingTag:"h3"}}),B=new z({props:{title:"Model memory (CPU)",local:"model-memory-cpu",headingTag:"h4"}}),E=new z({props:{title:"Inference (CPU)",local:"inference-cpu",headingTag:"h4"}}),N=new z({props:{title:"Fine-Tuning (CPU)",local:"fine-tuning-cpu",headingTag:"h4"}}),S=new Rt({props:{source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/docs/source/non_cuda_backends.mdx"}}),{c(){a=o("meta"),u=i(),p=o("p"),y=i(),m(v.$$.fragment),X=i(),m(g.$$.fragment),Y=i(),x=o("p"),x.textContent=ct,j=i(),_=o("p"),_.textContent=$t,K=i(),w=o("p"),w.innerHTML=bt,Q=i(),m(T.$$.fragment),W=i(),m(C.$$.fragment),J=i(),k=o("p"),k.innerHTML=yt,V=i(),P=o("p"),P.textContent=gt,Z=i(),M=o("ul"),M.innerHTML=Tt,tt=i(),L=o("p"),L.textContent=vt,et=i(),m(H.$$.fragment),nt=i(),m(A.$$.fragment),lt=i(),I=o("p"),I.innerHTML=xt,at=i(),U=o("p"),U.innerHTML=_t,it=i(),F=o("p"),F.innerHTML=wt,st=i(),m(B.$$.fragment),ot=i(),D=o("table"),D.innerHTML=Ct,rt=i(),m(E.$$.fragment),pt=i(),G=o("table"),G.innerHTML=kt,ft=i(),m(N.$$.fragment),ut=i(),R=o("table"),R.innerHTML=Pt,mt=i(),m(S.$$.fragment),dt=i(),q=o("p"),this.h()},l(t){const e=Gt("svelte-u9bgzb",document.head);a=r(e,"META",{name:!0,content:!0}),e.forEach(n),u=s(t),p=r(t,"P",{}),Ht(p).forEach(n),y=s(t),d(v.$$.fragment,t),X=s(t),d(g.$$.fragment,t),Y=s(t),x=r(t,"P",{"data-svelte-h":!0}),f(x)!=="svelte-vyk3k9"&&(x.textContent=ct),j=s(t),_=r(t,"P",{"data-svelte-h":!0}),f(_)!=="svelte-12ne44y"&&(_.textContent=$t),K=s(t),w=r(t,"P",{"data-svelte-h":!0}),f(w)!=="svelte-1wpsr8f"&&(w.innerHTML=bt),Q=s(t),d(T.$$.fragment,t),W=s(t),d(C.$$.fragment,t),J=s(t),k=r(t,"P",{"data-svelte-h":!0}),f(k)!=="svelte-njzoox"&&(k.innerHTML=yt),V=s(t),P=r(t,"P",{"data-svelte-h":!0}),f(P)!=="svelte-5kenpg"&&(P.textContent=gt),Z=s(t),M=r(t,"UL",{"data-svelte-h":!0}),f(M)!=="svelte-7ea0dg"&&(M.innerHTML=Tt),tt=s(t),L=r(t,"P",{"data-svelte-h":!0}),f(L)!=="svelte-16dyb3n"&&(L.textContent=vt),et=s(t),d(H.$$.fragment,t),nt=s(t),d(A.$$.fragment,t),lt=s(t),I=r(t,"P",{"data-svelte-h":!0}),f(I)!=="svelte-amshsc"&&(I.innerHTML=xt),at=s(t),U=r(t,"P",{"data-svelte-h":!0}),f(U)!=="svelte-14mmg4n"&&(U.innerHTML=_t),it=s(t),F=r(t,"P",{"data-svelte-h":!0}),f(F)!=="svelte-rgpm01"&&(F.innerHTML=wt),st=s(t),d(B.$$.fragment,t),ot=s(t),D=r(t,"TABLE",{"data-svelte-h":!0}),f(D)!=="svelte-19czuoc"&&(D.innerHTML=Ct),rt=s(t),d(E.$$.fragment,t),pt=s(t),G=r(t,"TABLE",{"data-svelte-h":!0}),f(G)!=="svelte-1hhredc"&&(G.innerHTML=kt),ft=s(t),d(N.$$.fragment,t),ut=s(t),R=r(t,"TABLE",{"data-svelte-h":!0}),f(R)!=="svelte-174tddt"&&(R.innerHTML=Pt),mt=s(t),d(S.$$.fragment,t),dt=s(t),q=r(t,"P",{}),Ht(q).forEach(n),this.h()},h(){At(a,"name","hf:doc:metadata"),At(a,"content",qt)},m(t,e){Nt(document.head,a),l(t,u,e),l(t,p,e),l(t,y,e),h(v,t,e),l(t,X,e),h(g,t,e),l(t,Y,e),l(t,x,e),l(t,j,e),l(t,_,e),l(t,K,e),l(t,w,e),l(t,Q,e),h(T,t,e),l(t,W,e),h(C,t,e),l(t,J,e),l(t,k,e),l(t,V,e),l(t,P,e),l(t,Z,e),l(t,M,e),l(t,tt,e),l(t,L,e),l(t,et,e),h(H,t,e),l(t,nt,e),h(A,t,e),l(t,lt,e),l(t,I,e),l(t,at,e),l(t,U,e),l(t,it,e),l(t,F,e),l(t,st,e),h(B,t,e),l(t,ot,e),l(t,D,e),l(t,rt,e),h(E,t,e),l(t,pt,e),l(t,G,e),l(t,ft,e),h(N,t,e),l(t,ut,e),l(t,R,e),l(t,mt,e),h(S,t,e),l(t,dt,e),l(t,q,e),ht=!0},p(t,[e]){const Mt={};e&2&&(Mt.$$scope={dirty:e,ctx:t}),g.$set(Mt);const Lt={};e&2&&(Lt.$$scope={dirty:e,ctx:t}),T.$set(Lt)},i(t){ht||(c(v.$$.fragment,t),c(g.$$.fragment,t),c(T.$$.fragment,t),c(C.$$.fragment,t),c(H.$$.fragment,t),c(A.$$.fragment,t),c(B.$$.fragment,t),c(E.$$.fragment,t),c(N.$$.fragment,t),c(S.$$.fragment,t),ht=!0)},o(t){$(v.$$.fragment,t),$(g.$$.fragment,t),$(T.$$.fragment,t),$(C.$$.fragment,t),$(H.$$.fragment,t),$(A.$$.fragment,t),$(B.$$.fragment,t),$(E.$$.fragment,t),$(N.$$.fragment,t),$(S.$$.fragment,t),ht=!1},d(t){t&&(n(u),n(p),n(y),n(X),n(Y),n(x),n(j),n(_),n(K),n(w),n(Q),n(W),n(J),n(k),n(V),n(P),n(Z),n(M),n(tt),n(L),n(et),n(nt),n(lt),n(I),n(at),n(U),n(it),n(F),n(st),n(ot),n(D),n(rt),n(pt),n(G),n(ft),n(ut),n(R),n(mt),n(dt),n(q)),n(a),b(v,t),b(g,t),b(T,t),b(C,t),b(H,t),b(A,t),b(B,t),b(E,t),b(N,t),b(S,t)}}}const qt='{"title":"Multi-backend support (non-CUDA backends)","local":"multi-backend-support-non-cuda-backends","sections":[{"title":"Alpha Release","local":"alpha-release","sections":[],"depth":2},{"title":"Benchmarks","local":"benchmarks","sections":[{"title":"Intel","local":"intel","sections":[{"title":"Model memory (CPU)","local":"model-memory-cpu","sections":[],"depth":4},{"title":"Inference (CPU)","local":"inference-cpu","sections":[],"depth":4},{"title":"Fine-Tuning (CPU)","local":"fine-tuning-cpu","sections":[],"depth":4}],"depth":3}],"depth":2}],"depth":1}';function Xt(O){return Bt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Wt extends Dt{constructor(a){super(),Et(this,a,Xt,Ot,Ft,{})}}export{Wt as component};