bitsandbytes/pr_1908/en/_app/immutable/nodes/15.8433bf3f.js

import{s as At,o as Kt,n as Ot}from"../chunks/scheduler.8a2cc2fa.js";import{S as te,i as ee,e as o,s as a,c as p,h as ne,a as l,d as e,b as s,f as N,g as b,j as x,k as F,l as r,m as i,n as c,t as f,o as _,p as h}from"../chunks/index.7079e750.js";import{C as ae,H as lt,E as se}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.2b7ce466.js";import{D as J}from"../chunks/Docstring.8c9a5003.js";import{C as re}from"../chunks/CodeBlock.a326412a.js";import{E as ie}from"../chunks/ExampleCodeBlock.7664d7e9.js";function oe(dt){let m,z="Example:",M,$,v;return $=new re({props:{code:"aW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjB0b3JjaC5ubiUyMGFzJTIwbm4lMEElMEFpbXBvcnQlMjBiaXRzYW5kYnl0ZXMlMjBhcyUyMGJuYiUwQWZyb20lMjBiaXRzYW5kYnl0ZXMubm4lMjBpbXBvcnQlMjBMaW5lYXI0Yml0JTBBJTBBZnAxNl9tb2RlbCUyMCUzRCUyMG5uLlNlcXVlbnRpYWwoJTBBJTIwJTIwJTIwJTIwbm4uTGluZWFyKDY0JTJDJTIwNjQpJTJDJTBBJTIwJTIwJTIwJTIwbm4uTGluZWFyKDY0JTJDJTIwNjQpJTBBKSUwQSUwQXF1YW50aXplZF9tb2RlbCUyMCUzRCUyMG5uLlNlcXVlbnRpYWwoJTBBJTIwJTIwJTIwJTIwTGluZWFyNGJpdCg2NCUyQyUyMDY0KSUyQyUwQSUyMCUyMCUyMCUyMExpbmVhcjRiaXQoNjQlMkMlMjA2NCklMEEpJTBBJTBBcXVhbnRpemVkX21vZGVsLmxvYWRfc3RhdGVfZGljdChmcDE2X21vZGVsLnN0YXRlX2RpY3QoKSklMEFxdWFudGl6ZWRfbW9kZWwlMjAlM0QlMjBxdWFudGl6ZWRfbW9kZWwudG8oMCklMjAlMjMlMjBRdWFudGl6YXRpb24lMjBoYXBwZW5zJTIwaGVyZQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> torch.nn <span class="hljs-keyword">as</span> nn

<span class="hljs-keyword">import</span> bitsandbytes <span class="hljs-keyword">as</span> bnb
<span class="hljs-keyword">from</span> bitsandbytes.nn <span class="hljs-keyword">import</span> Linear4bit

fp16_model = nn.Sequential(
    nn.Linear(<span class="hljs-number">64</span>, <span class="hljs-number">64</span>),
    nn.Linear(<span class="hljs-number">64</span>, <span class="hljs-number">64</span>)
)

quantized_model = nn.Sequential(
    Linear4bit(<span class="hljs-number">64</span>, <span class="hljs-number">64</span>),
    Linear4bit(<span class="hljs-number">64</span>, <span class="hljs-number">64</span>)
)

quantized_model.load_state_dict(fp16_model.state_dict())
quantized_model = quantized_model.to(<span class="hljs-number">0</span>) <span class="hljs-comment"># Quantization happens here</span>`,wrap:!1}}),{c(){m=o("p"),m.textContent=z,M=a(),p($.$$.fragment)},l(d){m=l(d,"P",{"data-svelte-h":!0}),x(m)!=="svelte-11lpom8"&&(m.textContent=z),M=s(d),b($.$$.fragment,d)},m(d,L){i(d,m,L),i(d,M,L),c($,d,L),v=!0},p:Ot,i(d){v||(f($.$$.fragment,d),v=!0)},o(d){_($.$$.fragment,d),v=!1},d(d){d&&(e(m),e(M)),h($,d)}}}function le(dt){let m,z,M,$,v,d,L,mt,I,Rt='<a href="https://hf.co/papers/2305.14314" rel="nofollow">QLoRA</a> is a finetuning method that quantizes a model to 4-bits and adds a set of low-rank adaptation (LoRA) weights to the model and tuning them through the quantized weights. This method also introduces a new data type, 4-bit NormalFloat (<code>LinearNF4</code>) in addition to the standard Float4 data type (<code>LinearFP4</code>). <code>LinearNF4</code> is a quantization data type for normally distributed data and can improve performance.',ut,B,pt,y,k,xt,S,Wt=`This class is the base module for the 4-bit quantization algorithm presented in <a href="https://arxiv.org/abs/2305.14314" rel="nofollow">QLoRA</a>.
QLoRA 4-bit linear layers uses blockwise k-bit quantization under the hood, with the possibility of selecting various
compute datatypes such as FP4 and NF4.`,Mt,H,Qt=`In order to quantize a linear layer one should first load the original fp16 / bf16 weights into
the Linear4bit module, then call <code>quantized_module.to(&quot;cuda&quot;)</code> to quantize the fp16 / bf16 weights.`,Nt,q,Ft,j,E,Ct,A,Dt="Initialize Linear4bit class.",bt,R,ct,T,W,qt,K,Ut="Implements the FP4 data type.",jt,O,Q,ft,D,_t,u,U,Pt,tt,Xt="Implements the NF4 data type.",Jt,et,Gt=`Constructs a quantization data type where each bin has equal area under a standard normal distribution N(0, 1) that
is normalized into the range [-1, 1].`,zt,nt,Yt='For more information read the paper: QLoRA: Efficient Finetuning of Quantized LLMs (<a href="https://arxiv.org/abs/2305.14314" rel="nofollow">https://arxiv.org/abs/2305.14314</a>)',It,at,Vt=`Implementation of the NF4 data type in bitsandbytes can be found in the <code>create_normal_map</code> function in
the <code>functional.py</code> file: <a href="https://github.com/TimDettmers/bitsandbytes/blob/main/bitsandbytes/functional.py#L236" rel="nofollow">https://github.com/TimDettmers/bitsandbytes/blob/main/bitsandbytes/functional.py#L236</a>.`,Bt,st,X,ht,G,yt,C,Y,kt,P,V,Et,rt,Zt="Initialize self.  See help(type(self)) for accurate signature.",gt,Z,$t,ot,vt;return v=new ae({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),L=new lt({props:{title:"4-bit quantization",local:"4-bit-quantization",headingTag:"h1"}}),B=new lt({props:{title:"Linear4bit",local:"bitsandbytes.nn.Linear4bit",headingTag:"h2"}}),k=new J({props:{name:"class bitsandbytes.nn.Linear4bit",anchor:"bitsandbytes.nn.Linear4bit",parameters:[{name:"input_features",val:""},{name:"output_features",val:""},{name:"bias",val:" = True"},{name:"compute_dtype",val:" = None"},{name:"compress_statistics",val:" = True"},{name:"quant_type",val:" = 'fp4'"},{name:"quant_storage",val:" = torch.uint8"},{name:"device",val:" = None"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/nn/modules.py#L461"}}),q=new ie({props:{anchor:"bitsandbytes.nn.Linear4bit.example",$$slots:{default:[oe]},$$scope:{ctx:dt}}}),E=new J({props:{name:"__init__",anchor:"bitsandbytes.nn.Linear4bit.__init__",parameters:[{name:"input_features",val:""},{name:"output_features",val:""},{name:"bias",val:" = True"},{name:"compute_dtype",val:" = None"},{name:"compress_statistics",val:" = True"},{name:"quant_type",val:" = 'fp4'"},{name:"quant_storage",val:" = torch.uint8"},{name:"device",val:" = None"}],parametersDescription:[{anchor:"bitsandbytes.nn.Linear4bit.__init__.input_features",description:`<strong>input_features</strong> (<code>str</code>) &#x2014;
Number of input features of the linear layer.`,name:"input_features"},{anchor:"bitsandbytes.nn.Linear4bit.__init__.output_features",description:`<strong>output_features</strong> (<code>str</code>) &#x2014;
Number of output features of the linear layer.`,name:"output_features"},{anchor:"bitsandbytes.nn.Linear4bit.__init__.bias",description:`<strong>bias</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether the linear class uses the bias term as well.`,name:"bias"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/nn/modules.py#L494"}}),R=new lt({props:{title:"LinearFP4",local:"bitsandbytes.nn.LinearFP4",headingTag:"h2"}}),W=new J({props:{name:"class bitsandbytes.nn.LinearFP4",anchor:"bitsandbytes.nn.LinearFP4",parameters:[{name:"input_features",val:""},{name:"output_features",val:""},{name:"bias",val:" = True"},{name:"compute_dtype",val:" = None"},{name:"compress_statistics",val:" = True"},{name:"quant_storage",val:" = torch.uint8"},{name:"device",val:" = None"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/nn/modules.py#L597"}}),Q=new J({props:{name:"__init__",anchor:"bitsandbytes.nn.LinearFP4.__init__",parameters:[{name:"input_features",val:""},{name:"output_features",val:""},{name:"bias",val:" = True"},{name:"compute_dtype",val:" = None"},{name:"compress_statistics",val:" = True"},{name:"quant_storage",val:" = torch.uint8"},{name:"device",val:" = None"}],parametersDescription:[{anchor:"bitsandbytes.nn.LinearFP4.__init__.input_features",description:`<strong>input_features</strong> (<code>str</code>) &#x2014;
Number of input features of the linear layer.`,name:"input_features"},{anchor:"bitsandbytes.nn.LinearFP4.__init__.output_features",description:`<strong>output_features</strong> (<code>str</code>) &#x2014;
Number of output features of the linear layer.`,name:"output_features"},{anchor:"bitsandbytes.nn.LinearFP4.__init__.bias",description:`<strong>bias</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether the linear class uses the bias term as well.`,name:"bias"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/nn/modules.py#L602"}}),D=new lt({props:{title:"LinearNF4",local:"bitsandbytes.nn.LinearNF4",headingTag:"h2"}}),U=new J({props:{name:"class bitsandbytes.nn.LinearNF4",anchor:"bitsandbytes.nn.LinearNF4",parameters:[{name:"input_features",val:""},{name:"output_features",val:""},{name:"bias",val:" = True"},{name:"compute_dtype",val:" = None"},{name:"compress_statistics",val:" = True"},{name:"quant_storage",val:" = torch.uint8"},{name:"device",val:" = None"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/nn/modules.py#L633"}}),X=new J({props:{name:"__init__",anchor:"bitsandbytes.nn.LinearNF4.__init__",parameters:[{name:"input_features",val:""},{name:"output_features",val:""},{name:"bias",val:" = True"},{name:"compute_dtype",val:" = None"},{name:"compress_statistics",val:" = True"},{name:"quant_storage",val:" = torch.uint8"},{name:"device",val:" = None"}],parametersDescription:[{anchor:"bitsandbytes.nn.LinearNF4.__init__.input_features",description:`<strong>input_features</strong> (<code>str</code>) &#x2014;
Number of input features of the linear layer.`,name:"input_features"},{anchor:"bitsandbytes.nn.LinearNF4.__init__.output_features",description:`<strong>output_features</strong> (<code>str</code>) &#x2014;
Number of output features of the linear layer.`,name:"output_features"},{anchor:"bitsandbytes.nn.LinearNF4.__init__.bias",description:`<strong>bias</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether the linear class uses the bias term as well.`,name:"bias"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/nn/modules.py#L645"}}),G=new lt({props:{title:"Params4bit",local:"bitsandbytes.nn.Params4bit",headingTag:"h2"}}),Y=new J({props:{name:"class bitsandbytes.nn.Params4bit",anchor:"bitsandbytes.nn.Params4bit",parameters:[{name:"data",val:": typing.Optional[torch.Tensor] = None"},{name:"requires_grad",val:" = False"},{name:"quant_state",val:": typing.Optional[bitsandbytes.functional.QuantState] = None"},{name:"blocksize",val:": typing.Optional[int] = None"},{name:"compress_statistics",val:": bool = True"},{name:"quant_type",val:": str = 'fp4'"},{name:"quant_storage",val:": dtype = torch.uint8"},{name:"module",val:": typing.Optional[ForwardRef('Linear4bit')] = None"},{name:"bnb_quantized",val:": bool = False"},{name:"**kwargs",val:""}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/nn/modules.py#L213"}}),V=new J({props:{name:"__init__",anchor:"bitsandbytes.nn.Params4bit.__init__",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}]}}),Z=new se({props:{source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/docs/source/reference/nn/linear4bit.mdx"}}),{c(){m=o("meta"),z=a(),M=o("p"),$=a(),p(v.$$.fragment),d=a(),p(L.$$.fragment),mt=a(),I=o("p"),I.innerHTML=Rt,ut=a(),p(B.$$.fragment),pt=a(),y=o("div"),p(k.$$.fragment),xt=a(),S=o("p"),S.innerHTML=Wt,Mt=a(),H=o("p"),H.innerHTML=Qt,Nt=a(),p(q.$$.fragment),Ft=a(),j=o("div"),p(E.$$.fragment),Ct=a(),A=o("p"),A.textContent=Dt,bt=a(),p(R.$$.fragment),ct=a(),T=o("div"),p(W.$$.fragment),qt=a(),K=o("p"),K.textContent=Ut,jt=a(),O=o("div"),p(Q.$$.fragment),ft=a(),p(D.$$.fragment),_t=a(),u=o("div"),p(U.$$.fragment),Pt=a(),tt=o("p"),tt.textContent=Xt,Jt=a(),et=o("p"),et.textContent=Gt,zt=a(),nt=o("p"),nt.innerHTML=Yt,It=a(),at=o("p"),at.innerHTML=Vt,Bt=a(),st=o("div"),p(X.$$.fragment),ht=a(),p(G.$$.fragment),yt=a(),C=o("div"),p(Y.$$.fragment),kt=a(),P=o("div"),p(V.$$.fragment),Et=a(),rt=o("p"),rt.textContent=Zt,gt=a(),p(Z.$$.fragment),$t=a(),ot=o("p"),this.h()},l(t){const n=ne("svelte-u9bgzb",document.head);m=l(n,"META",{name:!0,content:!0}),n.forEach(e),z=s(t),M=l(t,"P",{}),N(M).forEach(e),$=s(t),b(v.$$.fragment,t),d=s(t),b(L.$$.fragment,t),mt=s(t),I=l(t,"P",{"data-svelte-h":!0}),x(I)!=="svelte-j7lmsi"&&(I.innerHTML=Rt),ut=s(t),b(B.$$.fragment,t),pt=s(t),y=l(t,"DIV",{class:!0});var g=N(y);b(k.$$.fragment,g),xt=s(g),S=l(g,"P",{"data-svelte-h":!0}),x(S)!=="svelte-b56pxf"&&(S.innerHTML=Wt),Mt=s(g),H=l(g,"P",{"data-svelte-h":!0}),x(H)!=="svelte-yrth4q"&&(H.innerHTML=Qt),Nt=s(g),b(q.$$.fragment,g),Ft=s(g),j=l(g,"DIV",{class:!0});var wt=N(j);b(E.$$.fragment,wt),Ct=s(wt),A=l(wt,"P",{"data-svelte-h":!0}),x(A)!=="svelte-9bim4q"&&(A.textContent=Dt),wt.forEach(e),g.forEach(e),bt=s(t),b(R.$$.fragment,t),ct=s(t),T=l(t,"DIV",{class:!0});var it=N(T);b(W.$$.fragment,it),qt=s(it),K=l(it,"P",{"data-svelte-h":!0}),x(K)!=="svelte-1wlk05v"&&(K.textContent=Ut),jt=s(it),O=l(it,"DIV",{class:!0});var St=N(O);b(Q.$$.fragment,St),St.forEach(e),it.forEach(e),ft=s(t),b(D.$$.fragment,t),_t=s(t),u=l(t,"DIV",{class:!0});var w=N(u);b(U.$$.fragment,w),Pt=s(w),tt=l(w,"P",{"data-svelte-h":!0}),x(tt)!=="svelte-5d8bnh"&&(tt.textContent=Xt),Jt=s(w),et=l(w,"P",{"data-svelte-h":!0}),x(et)!=="svelte-4zan40"&&(et.textContent=Gt),zt=s(w),nt=l(w,"P",{"data-svelte-h":!0}),x(nt)!=="svelte-1ditbm8"&&(nt.innerHTML=Yt),It=s(w),at=l(w,"P",{"data-svelte-h":!0}),x(at)!=="svelte-weczdq"&&(at.innerHTML=Vt),Bt=s(w),st=l(w,"DIV",{class:!0});var Ht=N(st);b(X.$$.fragment,Ht),Ht.forEach(e),w.forEach(e),ht=s(t),b(G.$$.fragment,t),yt=s(t),C=l(t,"DIV",{class:!0});var Lt=N(C);b(Y.$$.fragment,Lt),kt=s(Lt),P=l(Lt,"DIV",{class:!0});var Tt=N(P);b(V.$$.fragment,Tt),Et=s(Tt),rt=l(Tt,"P",{"data-svelte-h":!0}),x(rt)!=="svelte-gef1cn"&&(rt.textContent=Zt),Tt.forEach(e),Lt.forEach(e),gt=s(t),b(Z.$$.fragment,t),$t=s(t),ot=l(t,"P",{}),N(ot).forEach(e),this.h()},h(){F(m,"name","hf:doc:metadata"),F(m,"content",de),F(j,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),F(y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),F(O,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),F(T,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),F(st,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),F(u,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),F(P,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),F(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(t,n){r(document.head,m),i(t,z,n),i(t,M,n),i(t,$,n),c(v,t,n),i(t,d,n),c(L,t,n),i(t,mt,n),i(t,I,n),i(t,ut,n),c(B,t,n),i(t,pt,n),i(t,y,n),c(k,y,null),r(y,xt),r(y,S),r(y,Mt),r(y,H),r(y,Nt),c(q,y,null),r(y,Ft),r(y,j),c(E,j,null),r(j,Ct),r(j,A),i(t,bt,n),c(R,t,n),i(t,ct,n),i(t,T,n),c(W,T,null),r(T,qt),r(T,K),r(T,jt),r(T,O),c(Q,O,null),i(t,ft,n),c(D,t,n),i(t,_t,n),i(t,u,n),c(U,u,null),r(u,Pt),r(u,tt),r(u,Jt),r(u,et),r(u,zt),r(u,nt),r(u,It),r(u,at),r(u,Bt),r(u,st),c(X,st,null),i(t,ht,n),c(G,t,n),i(t,yt,n),i(t,C,n),c(Y,C,null),r(C,kt),r(C,P),c(V,P,null),r(P,Et),r(P,rt),i(t,gt,n),c(Z,t,n),i(t,$t,n),i(t,ot,n),vt=!0},p(t,[n]){const g={};n&2&&(g.$$scope={dirty:n,ctx:t}),q.$set(g)},i(t){vt||(f(v.$$.fragment,t),f(L.$$.fragment,t),f(B.$$.fragment,t),f(k.$$.fragment,t),f(q.$$.fragment,t),f(E.$$.fragment,t),f(R.$$.fragment,t),f(W.$$.fragment,t),f(Q.$$.fragment,t),f(D.$$.fragment,t),f(U.$$.fragment,t),f(X.$$.fragment,t),f(G.$$.fragment,t),f(Y.$$.fragment,t),f(V.$$.fragment,t),f(Z.$$.fragment,t),vt=!0)},o(t){_(v.$$.fragment,t),_(L.$$.fragment,t),_(B.$$.fragment,t),_(k.$$.fragment,t),_(q.$$.fragment,t),_(E.$$.fragment,t),_(R.$$.fragment,t),_(W.$$.fragment,t),_(Q.$$.fragment,t),_(D.$$.fragment,t),_(U.$$.fragment,t),_(X.$$.fragment,t),_(G.$$.fragment,t),_(Y.$$.fragment,t),_(V.$$.fragment,t),_(Z.$$.fragment,t),vt=!1},d(t){t&&(e(z),e(M),e($),e(d),e(mt),e(I),e(ut),e(pt),e(y),e(bt),e(ct),e(T),e(ft),e(_t),e(u),e(ht),e(yt),e(C),e(gt),e($t),e(ot)),e(m),h(v,t),h(L,t),h(B,t),h(k),h(q),h(E),h(R,t),h(W),h(Q),h(D,t),h(U),h(X),h(G,t),h(Y),h(V),h(Z,t)}}}const de='{"title":"4-bit quantization","local":"4-bit-quantization","sections":[{"title":"Linear4bit","local":"bitsandbytes.nn.Linear4bit","sections":[],"depth":2},{"title":"LinearFP4","local":"bitsandbytes.nn.LinearFP4","sections":[],"depth":2},{"title":"LinearNF4","local":"bitsandbytes.nn.LinearNF4","sections":[],"depth":2},{"title":"Params4bit","local":"bitsandbytes.nn.Params4bit","sections":[],"depth":2}],"depth":1}';function me(dt){return Kt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class he extends te{constructor(m){super(),ee(this,m,me,le,At,{})}}export{he as component};