bitsandbytes/pr_1827/en/_app/immutable/nodes/16.35d2bd57.js

import{s as gt,o as vt,n as yt}from"../chunks/scheduler.8a2cc2fa.js";import{S as wt,i as Mt,e as m,s,c as h,h as Lt,a as d,d as n,b as l,f as D,g as $,j as P,k as Q,l as p,m as i,n as _,t as g,o as v,p as y}from"../chunks/index.7079e750.js";import{C as Tt,H as ut,E as jt}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.edc8d25f.js";import{D as st}from"../chunks/Docstring.5a99e2dc.js";import{C as xt}from"../chunks/CodeBlock.135fbe6b.js";import{E as Jt}from"../chunks/ExampleCodeBlock.0f1d2f50.js";function It(X){let r,x="Example:",w,b,c;return b=new xt({props:{code:"aW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjB0b3JjaC5ubiUyMGFzJTIwbm4lMEElMEFpbXBvcnQlMjBiaXRzYW5kYnl0ZXMlMjBhcyUyMGJuYiUwQWZyb20lMjBibmIubm4lMjBpbXBvcnQlMjBMaW5lYXI4Yml0THQlMEElMEFmcDE2X21vZGVsJTIwJTNEJTIwbm4uU2VxdWVudGlhbCglMEElMjAlMjAlMjAlMjBubi5MaW5lYXIoNjQlMkMlMjA2NCklMkMlMEElMjAlMjAlMjAlMjBubi5MaW5lYXIoNjQlMkMlMjA2NCklMEEpJTBBJTBBaW50OF9tb2RlbCUyMCUzRCUyMG5uLlNlcXVlbnRpYWwoJTBBJTIwJTIwJTIwJTIwTGluZWFyOGJpdEx0KDY0JTJDJTIwNjQlMkMlMjBoYXNfZnAxNl93ZWlnaHRzJTNERmFsc2UpJTJDJTBBJTIwJTIwJTIwJTIwTGluZWFyOGJpdEx0KDY0JTJDJTIwNjQlMkMlMjBoYXNfZnAxNl93ZWlnaHRzJTNERmFsc2UpJTBBKSUwQSUwQWludDhfbW9kZWwubG9hZF9zdGF0ZV9kaWN0KGZwMTZfbW9kZWwuc3RhdGVfZGljdCgpKSUwQWludDhfbW9kZWwlMjAlM0QlMjBpbnQ4X21vZGVsLnRvKDApJTIwJTIzJTIwUXVhbnRpemF0aW9uJTIwaGFwcGVucyUyMGhlcmU=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> torch.nn <span class="hljs-keyword">as</span> nn

<span class="hljs-keyword">import</span> bitsandbytes <span class="hljs-keyword">as</span> bnb
<span class="hljs-keyword">from</span> bnb.nn <span class="hljs-keyword">import</span> Linear8bitLt

fp16_model = nn.Sequential(
    nn.Linear(<span class="hljs-number">64</span>, <span class="hljs-number">64</span>),
    nn.Linear(<span class="hljs-number">64</span>, <span class="hljs-number">64</span>)
)

int8_model = nn.Sequential(
    Linear8bitLt(<span class="hljs-number">64</span>, <span class="hljs-number">64</span>, has_fp16_weights=<span class="hljs-literal">False</span>),
    Linear8bitLt(<span class="hljs-number">64</span>, <span class="hljs-number">64</span>, has_fp16_weights=<span class="hljs-literal">False</span>)
)

int8_model.load_state_dict(fp16_model.state_dict())
int8_model = int8_model.to(<span class="hljs-number">0</span>) <span class="hljs-comment"># Quantization happens here</span>`,wrap:!1}}),{c(){r=m("p"),r.textContent=x,w=s(),h(b.$$.fragment)},l(a){r=d(a,"P",{"data-svelte-h":!0}),P(r)!=="svelte-11lpom8"&&(r.textContent=x),w=l(a),$(b.$$.fragment,a)},m(a,f){i(a,r,f),i(a,w,f),_(b,a,f),c=!0},p:yt,i(a){c||(g(b.$$.fragment,a),c=!0)},o(a){v(b.$$.fragment,a),c=!1},d(a){a&&(n(r),n(w)),y(b,a)}}}function Et(X){let r,x,w,b,c,a,f,H,J,bt='<a href="https://hf.co/papers/2208.07339" rel="nofollow">LLM.int8()</a> is a quantization method that aims to make large language model inference more accessible without significant degradation. Unlike naive 8-bit quantization, which can result in loss of critical information and accuracy, LLM.int8() dynamically adapts to ensure sensitive components of the computation retain higher precision when needed. The key is to extract the outliers from the inputs and weights and multiply them in 16-bit. All other values are multiplied in 8-bit before being dequantized back to 16-bits. The outputs from the 16-bit and 8-bit multiplication are combined to produce the final output.',R,I,ct='<a href="../../explanations/resources#llm-int8">Further Resources</a>',V,E,Y,o,k,lt,Z,ft=`This class is the base module for the <a href="https://arxiv.org/abs/2208.07339" rel="nofollow">LLM.int8()</a> algorithm.
To read more about it, have a look at the paper.`,it,z,ht=`In order to quantize a linear layer one should first load the original fp16 / bf16 weights into
the Linear8bitLt module, then call <code>int8_module.to(&quot;cuda&quot;)</code> to quantize the fp16 weights.`,rt,L,ot,T,C,pt,F,$t="Initialize Linear8bitLt class.",q,B,S,M,N,mt,j,W,dt,U,_t="Initialize self.  See help(type(self)) for accurate signature.",O,G,K,A,tt;return c=new Tt({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),f=new ut({props:{title:"LLM.int8()",local:"llmint8",headingTag:"h1"}}),E=new ut({props:{title:"Linear8bitLt",local:"bitsandbytes.nn.Linear8bitLt",headingTag:"h2"}}),k=new st({props:{name:"class bitsandbytes.nn.Linear8bitLt",anchor:"bitsandbytes.nn.Linear8bitLt",parameters:[{name:"input_features",val:": int"},{name:"output_features",val:": int"},{name:"bias",val:" = True"},{name:"has_fp16_weights",val:" = True"},{name:"threshold",val:" = 0.0"},{name:"index",val:" = None"},{name:"device",val:" = None"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1827/bitsandbytes/nn/modules.py#L935"}}),L=new Jt({props:{anchor:"bitsandbytes.nn.Linear8bitLt.example",$$slots:{default:[It]},$$scope:{ctx:X}}}),C=new st({props:{name:"__init__",anchor:"bitsandbytes.nn.Linear8bitLt.__init__",parameters:[{name:"input_features",val:": int"},{name:"output_features",val:": int"},{name:"bias",val:" = True"},{name:"has_fp16_weights",val:" = True"},{name:"threshold",val:" = 0.0"},{name:"index",val:" = None"},{name:"device",val:" = None"}],parametersDescription:[{anchor:"bitsandbytes.nn.Linear8bitLt.__init__.input_features",description:`<strong>input_features</strong> (<code>int</code>) &#x2014;
Number of input features of the linear layer.`,name:"input_features"},{anchor:"bitsandbytes.nn.Linear8bitLt.__init__.output_features",description:`<strong>output_features</strong> (<code>int</code>) &#x2014;
Number of output features of the linear layer.`,name:"output_features"},{anchor:"bitsandbytes.nn.Linear8bitLt.__init__.bias",description:`<strong>bias</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether the linear class uses the bias term as well.`,name:"bias"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1827/bitsandbytes/nn/modules.py#L967"}}),B=new ut({props:{title:"Int8Params",local:"bitsandbytes.nn.Int8Params",headingTag:"h2"}}),N=new st({props:{name:"class bitsandbytes.nn.Int8Params",anchor:"bitsandbytes.nn.Int8Params",parameters:[{name:"data",val:": typing.Optional[torch.Tensor] = None"},{name:"requires_grad",val:" = True"},{name:"has_fp16_weights",val:" = False"},{name:"CB",val:": typing.Optional[torch.Tensor] = None"},{name:"SCB",val:": typing.Optional[torch.Tensor] = None"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1827/bitsandbytes/nn/modules.py#L637"}}),W=new st({props:{name:"__init__",anchor:"bitsandbytes.nn.Int8Params.__init__",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}]}}),G=new jt({props:{source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/docs/source/reference/nn/linear8bit.mdx"}}),{c(){r=m("meta"),x=s(),w=m("p"),b=s(),h(c.$$.fragment),a=s(),h(f.$$.fragment),H=s(),J=m("p"),J.innerHTML=bt,R=s(),I=m("p"),I.innerHTML=ct,V=s(),h(E.$$.fragment),Y=s(),o=m("div"),h(k.$$.fragment),lt=s(),Z=m("p"),Z.innerHTML=ft,it=s(),z=m("p"),z.innerHTML=ht,rt=s(),h(L.$$.fragment),ot=s(),T=m("div"),h(C.$$.fragment),pt=s(),F=m("p"),F.textContent=$t,q=s(),h(B.$$.fragment),S=s(),M=m("div"),h(N.$$.fragment),mt=s(),j=m("div"),h(W.$$.fragment),dt=s(),U=m("p"),U.textContent=_t,O=s(),h(G.$$.fragment),K=s(),A=m("p"),this.h()},l(t){const e=Lt("svelte-u9bgzb",document.head);r=d(e,"META",{name:!0,content:!0}),e.forEach(n),x=l(t),w=d(t,"P",{}),D(w).forEach(n),b=l(t),$(c.$$.fragment,t),a=l(t),$(f.$$.fragment,t),H=l(t),J=d(t,"P",{"data-svelte-h":!0}),P(J)!=="svelte-3b0d0i"&&(J.innerHTML=bt),R=l(t),I=d(t,"P",{"data-svelte-h":!0}),P(I)!=="svelte-1a4aty"&&(I.innerHTML=ct),V=l(t),$(E.$$.fragment,t),Y=l(t),o=d(t,"DIV",{class:!0});var u=D(o);$(k.$$.fragment,u),lt=l(u),Z=d(u,"P",{"data-svelte-h":!0}),P(Z)!=="svelte-xo5v9s"&&(Z.innerHTML=ft),it=l(u),z=d(u,"P",{"data-svelte-h":!0}),P(z)!=="svelte-13qed9e"&&(z.innerHTML=ht),rt=l(u),$(L.$$.fragment,u),ot=l(u),T=d(u,"DIV",{class:!0});var et=D(T);$(C.$$.fragment,et),pt=l(et),F=d(et,"P",{"data-svelte-h":!0}),P(F)!=="svelte-p49gdy"&&(F.textContent=$t),et.forEach(n),u.forEach(n),q=l(t),$(B.$$.fragment,t),S=l(t),M=d(t,"DIV",{class:!0});var nt=D(M);$(N.$$.fragment,nt),mt=l(nt),j=d(nt,"DIV",{class:!0});var at=D(j);$(W.$$.fragment,at),dt=l(at),U=d(at,"P",{"data-svelte-h":!0}),P(U)!=="svelte-gef1cn"&&(U.textContent=_t),at.forEach(n),nt.forEach(n),O=l(t),$(G.$$.fragment,t),K=l(t),A=d(t,"P",{}),D(A).forEach(n),this.h()},h(){Q(r,"name","hf:doc:metadata"),Q(r,"content",kt),Q(T,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),Q(o,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),Q(j,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),Q(M,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(t,e){p(document.head,r),i(t,x,e),i(t,w,e),i(t,b,e),_(c,t,e),i(t,a,e),_(f,t,e),i(t,H,e),i(t,J,e),i(t,R,e),i(t,I,e),i(t,V,e),_(E,t,e),i(t,Y,e),i(t,o,e),_(k,o,null),p(o,lt),p(o,Z),p(o,it),p(o,z),p(o,rt),_(L,o,null),p(o,ot),p(o,T),_(C,T,null),p(T,pt),p(T,F),i(t,q,e),_(B,t,e),i(t,S,e),i(t,M,e),_(N,M,null),p(M,mt),p(M,j),_(W,j,null),p(j,dt),p(j,U),i(t,O,e),_(G,t,e),i(t,K,e),i(t,A,e),tt=!0},p(t,[e]){const u={};e&2&&(u.$$scope={dirty:e,ctx:t}),L.$set(u)},i(t){tt||(g(c.$$.fragment,t),g(f.$$.fragment,t),g(E.$$.fragment,t),g(k.$$.fragment,t),g(L.$$.fragment,t),g(C.$$.fragment,t),g(B.$$.fragment,t),g(N.$$.fragment,t),g(W.$$.fragment,t),g(G.$$.fragment,t),tt=!0)},o(t){v(c.$$.fragment,t),v(f.$$.fragment,t),v(E.$$.fragment,t),v(k.$$.fragment,t),v(L.$$.fragment,t),v(C.$$.fragment,t),v(B.$$.fragment,t),v(N.$$.fragment,t),v(W.$$.fragment,t),v(G.$$.fragment,t),tt=!1},d(t){t&&(n(x),n(w),n(b),n(a),n(H),n(J),n(R),n(I),n(V),n(Y),n(o),n(q),n(S),n(M),n(O),n(K),n(A)),n(r),y(c,t),y(f,t),y(E,t),y(k),y(L),y(C),y(B,t),y(N),y(W),y(G,t)}}}const kt='{"title":"LLM.int8()","local":"llmint8","sections":[{"title":"Linear8bitLt","local":"bitsandbytes.nn.Linear8bitLt","sections":[],"depth":2},{"title":"Int8Params","local":"bitsandbytes.nn.Int8Params","sections":[],"depth":2}],"depth":1}';function Ct(X){return vt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class zt extends wt{constructor(r){super(),Mt(this,r,Ct,Et,gt,{})}}export{zt as component};