bitsandbytes/pr_1908/en/_app/immutable/nodes/22.a510274b.js

import{s as $t,n as xt,o as Lt}from"../chunks/scheduler.8a2cc2fa.js";import{S as At,i as St,e as s,s as a,c as d,h as Rt,a as r,d as e,b as i,f as x,g as l,j as mt,k as L,l as m,m as o,n as p,t as b,o as _,p as c}from"../chunks/index.7079e750.js";import{C as wt,H as dt,E as zt}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.2b7ce466.js";import{D as V}from"../chunks/Docstring.8c9a5003.js";function Tt(ut){let g,q,H,B,A,G,S,O,R,ft='<a href="https:/hf.co/papers/1708.03888" rel="nofollow">LARS (Layer-wise Adaptive Rate Scaling)</a> is an optimizer designed for training with large batch sizes to accelerate training. LARS uses a separate learning rate for each <em>layer</em> instead of each parameter. The learning rate is calculated from a <em>trust ratio</em> between the weight and gradient norm in a layer. This helps calibrate a stable update size.',U,w,J,u,z,lt,v,T,pt,k,ht="Base LARS optimizer.",K,C,Q,f,E,bt,y,N,_t,I,vt="8-bit LARS optimizer.",X,D,Y,h,P,ct,$,F,gt,M,yt="32-bit LARS optimizer.",Z,j,tt,W,et;return A=new wt({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),S=new dt({props:{title:"LARS",local:"lars",headingTag:"h1"}}),w=new dt({props:{title:"LARS",local:"api-class ][ bitsandbytes.optim.LARS",headingTag:"h2"}}),z=new V({props:{name:"class bitsandbytes.optim.LARS",anchor:"bitsandbytes.optim.LARS",parameters:[{name:"params",val:""},{name:"lr",val:""},{name:"momentum",val:" = 0"},{name:"dampening",val:" = 0"},{name:"weight_decay",val:" = 0"},{name:"nesterov",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 0.02"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/optim/lars.py#L11"}}),T=new V({props:{name:"__init__",anchor:"bitsandbytes.optim.LARS.__init__",parameters:[{name:"params",val:""},{name:"lr",val:""},{name:"momentum",val:" = 0"},{name:"dampening",val:" = 0"},{name:"weight_decay",val:" = 0"},{name:"nesterov",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 0.02"}],parametersDescription:[{anchor:"bitsandbytes.optim.LARS.__init__.params",description:`<strong>params</strong> (<code>torch.tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.LARS.__init__.lr",description:`<strong>lr</strong> (<code>float</code>) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.LARS.__init__.momentum",description:`<strong>momentum</strong> (<code>float</code>, defaults to 0) &#x2014;
The momentum value speeds up the optimizer by taking bigger steps.`,name:"momentum"},{anchor:"bitsandbytes.optim.LARS.__init__.dampening",description:`<strong>dampening</strong> (<code>float</code>, defaults to 0) &#x2014;
The dampening value reduces the momentum of the optimizer.`,name:"dampening"},{anchor:"bitsandbytes.optim.LARS.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.LARS.__init__.nesterov",description:`<strong>nesterov</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use Nesterov momentum.`,name:"nesterov"},{anchor:"bitsandbytes.optim.LARS.__init__.optim_bits",description:`<strong>optim_bits</strong> (<code>int</code>, defaults to 32) &#x2014;
The number of bits of the optimizer state.`,name:"optim_bits"},{anchor:"bitsandbytes.optim.LARS.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.LARS.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.LARS.__init__.max_unorm",description:`<strong>max_unorm</strong> (<code>float</code>, defaults to 0.02) &#x2014;
The maximum gradient norm.`,name:"max_unorm"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/optim/lars.py#L12"}}),C=new dt({props:{title:"LARS8bit",local:"bitsandbytes.optim.LARS8bit",headingTag:"h2"}}),E=new V({props:{name:"class bitsandbytes.optim.LARS8bit",anchor:"bitsandbytes.optim.LARS8bit",parameters:[{name:"params",val:""},{name:"lr",val:""},{name:"momentum",val:" = 0"},{name:"dampening",val:" = 0"},{name:"weight_decay",val:" = 0"},{name:"nesterov",val:" = False"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 0.02"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/optim/lars.py#L66"}}),N=new V({props:{name:"__init__",anchor:"bitsandbytes.optim.LARS8bit.__init__",parameters:[{name:"params",val:""},{name:"lr",val:""},{name:"momentum",val:" = 0"},{name:"dampening",val:" = 0"},{name:"weight_decay",val:" = 0"},{name:"nesterov",val:" = False"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 0.02"}],parametersDescription:[{anchor:"bitsandbytes.optim.LARS8bit.__init__.params",description:`<strong>params</strong> (<code>torch.tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.LARS8bit.__init__.lr",description:`<strong>lr</strong> (<code>float</code>) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.LARS8bit.__init__.momentum",description:`<strong>momentum</strong> (<code>float</code>, defaults to 0) &#x2014;
The momentum value speeds up the optimizer by taking bigger steps.`,name:"momentum"},{anchor:"bitsandbytes.optim.LARS8bit.__init__.dampening",description:`<strong>dampening</strong> (<code>float</code>, defaults to 0) &#x2014;
The dampening value reduces the momentum of the optimizer.`,name:"dampening"},{anchor:"bitsandbytes.optim.LARS8bit.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.LARS8bit.__init__.nesterov",description:`<strong>nesterov</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use Nesterov momentum.`,name:"nesterov"},{anchor:"bitsandbytes.optim.LARS8bit.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.LARS8bit.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.LARS8bit.__init__.max_unorm",description:`<strong>max_unorm</strong> (<code>float</code>, defaults to 0.02) &#x2014;
The maximum gradient norm.`,name:"max_unorm"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/optim/lars.py#L67"}}),D=new dt({props:{title:"LARS32bit",local:"bitsandbytes.optim.LARS32bit",headingTag:"h2"}}),P=new V({props:{name:"class bitsandbytes.optim.LARS32bit",anchor:"bitsandbytes.optim.LARS32bit",parameters:[{name:"params",val:""},{name:"lr",val:""},{name:"momentum",val:" = 0"},{name:"dampening",val:" = 0"},{name:"weight_decay",val:" = 0"},{name:"nesterov",val:" = False"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 0.02"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/optim/lars.py#L118"}}),F=new V({props:{name:"__init__",anchor:"bitsandbytes.optim.LARS32bit.__init__",parameters:[{name:"params",val:""},{name:"lr",val:""},{name:"momentum",val:" = 0"},{name:"dampening",val:" = 0"},{name:"weight_decay",val:" = 0"},{name:"nesterov",val:" = False"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 0.02"}],parametersDescription:[{anchor:"bitsandbytes.optim.LARS32bit.__init__.params",description:`<strong>params</strong> (<code>torch.tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.LARS32bit.__init__.lr",description:`<strong>lr</strong> (<code>float</code>) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.LARS32bit.__init__.momentum",description:`<strong>momentum</strong> (<code>float</code>, defaults to 0) &#x2014;
The momentum value speeds up the optimizer by taking bigger steps.`,name:"momentum"},{anchor:"bitsandbytes.optim.LARS32bit.__init__.dampening",description:`<strong>dampening</strong> (<code>float</code>, defaults to 0) &#x2014;
The dampening value reduces the momentum of the optimizer.`,name:"dampening"},{anchor:"bitsandbytes.optim.LARS32bit.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.LARS32bit.__init__.nesterov",description:`<strong>nesterov</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use Nesterov momentum.`,name:"nesterov"},{anchor:"bitsandbytes.optim.LARS32bit.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.LARS32bit.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.LARS32bit.__init__.max_unorm",description:`<strong>max_unorm</strong> (<code>float</code>, defaults to 0.02) &#x2014;
The maximum gradient norm.`,name:"max_unorm"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1908/bitsandbytes/optim/lars.py#L119"}}),j=new zt({props:{source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/docs/source/reference/optim/lars.mdx"}}),{c(){g=s("meta"),q=a(),H=s("p"),B=a(),d(A.$$.fragment),G=a(),d(S.$$.fragment),O=a(),R=s("p"),R.innerHTML=ft,U=a(),d(w.$$.fragment),J=a(),u=s("div"),d(z.$$.fragment),lt=a(),v=s("div"),d(T.$$.fragment),pt=a(),k=s("p"),k.textContent=ht,K=a(),d(C.$$.fragment),Q=a(),f=s("div"),d(E.$$.fragment),bt=a(),y=s("div"),d(N.$$.fragment),_t=a(),I=s("p"),I.textContent=vt,X=a(),d(D.$$.fragment),Y=a(),h=s("div"),d(P.$$.fragment),ct=a(),$=s("div"),d(F.$$.fragment),gt=a(),M=s("p"),M.textContent=yt,Z=a(),d(j.$$.fragment),tt=a(),W=s("p"),this.h()},l(t){const n=Rt("svelte-u9bgzb",document.head);g=r(n,"META",{name:!0,content:!0}),n.forEach(e),q=i(t),H=r(t,"P",{}),x(H).forEach(e),B=i(t),l(A.$$.fragment,t),G=i(t),l(S.$$.fragment,t),O=i(t),R=r(t,"P",{"data-svelte-h":!0}),mt(R)!=="svelte-18efkb3"&&(R.innerHTML=ft),U=i(t),l(w.$$.fragment,t),J=i(t),u=r(t,"DIV",{class:!0});var nt=x(u);l(z.$$.fragment,nt),lt=i(nt),v=r(nt,"DIV",{class:!0});var at=x(v);l(T.$$.fragment,at),pt=i(at),k=r(at,"P",{"data-svelte-h":!0}),mt(k)!=="svelte-1vko6mi"&&(k.textContent=ht),at.forEach(e),nt.forEach(e),K=i(t),l(C.$$.fragment,t),Q=i(t),f=r(t,"DIV",{class:!0});var it=x(f);l(E.$$.fragment,it),bt=i(it),y=r(it,"DIV",{class:!0});var ot=x(y);l(N.$$.fragment,ot),_t=i(ot),I=r(ot,"P",{"data-svelte-h":!0}),mt(I)!=="svelte-mgnzq1"&&(I.textContent=vt),ot.forEach(e),it.forEach(e),X=i(t),l(D.$$.fragment,t),Y=i(t),h=r(t,"DIV",{class:!0});var st=x(h);l(P.$$.fragment,st),ct=i(st),$=r(st,"DIV",{class:!0});var rt=x($);l(F.$$.fragment,rt),gt=i(rt),M=r(rt,"P",{"data-svelte-h":!0}),mt(M)!=="svelte-1l89br0"&&(M.textContent=yt),rt.forEach(e),st.forEach(e),Z=i(t),l(j.$$.fragment,t),tt=i(t),W=r(t,"P",{}),x(W).forEach(e),this.h()},h(){L(g,"name","hf:doc:metadata"),L(g,"content",Ct),L(v,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),L(u,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),L(y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),L(f,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),L($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),L(h,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(t,n){m(document.head,g),o(t,q,n),o(t,H,n),o(t,B,n),p(A,t,n),o(t,G,n),p(S,t,n),o(t,O,n),o(t,R,n),o(t,U,n),p(w,t,n),o(t,J,n),o(t,u,n),p(z,u,null),m(u,lt),m(u,v),p(T,v,null),m(v,pt),m(v,k),o(t,K,n),p(C,t,n),o(t,Q,n),o(t,f,n),p(E,f,null),m(f,bt),m(f,y),p(N,y,null),m(y,_t),m(y,I),o(t,X,n),p(D,t,n),o(t,Y,n),o(t,h,n),p(P,h,null),m(h,ct),m(h,$),p(F,$,null),m($,gt),m($,M),o(t,Z,n),p(j,t,n),o(t,tt,n),o(t,W,n),et=!0},p:xt,i(t){et||(b(A.$$.fragment,t),b(S.$$.fragment,t),b(w.$$.fragment,t),b(z.$$.fragment,t),b(T.$$.fragment,t),b(C.$$.fragment,t),b(E.$$.fragment,t),b(N.$$.fragment,t),b(D.$$.fragment,t),b(P.$$.fragment,t),b(F.$$.fragment,t),b(j.$$.fragment,t),et=!0)},o(t){_(A.$$.fragment,t),_(S.$$.fragment,t),_(w.$$.fragment,t),_(z.$$.fragment,t),_(T.$$.fragment,t),_(C.$$.fragment,t),_(E.$$.fragment,t),_(N.$$.fragment,t),_(D.$$.fragment,t),_(P.$$.fragment,t),_(F.$$.fragment,t),_(j.$$.fragment,t),et=!1},d(t){t&&(e(q),e(H),e(B),e(G),e(O),e(R),e(U),e(J),e(u),e(K),e(Q),e(f),e(X),e(Y),e(h),e(Z),e(tt),e(W)),e(g),c(A,t),c(S,t),c(w,t),c(z),c(T),c(C,t),c(E),c(N),c(D,t),c(P),c(F),c(j,t)}}}const Ct='{"title":"LARS","local":"lars","sections":[{"title":"LARS","local":"api-class ][ bitsandbytes.optim.LARS","sections":[],"depth":2},{"title":"LARS8bit","local":"bitsandbytes.optim.LARS8bit","sections":[],"depth":2},{"title":"LARS32bit","local":"bitsandbytes.optim.LARS32bit","sections":[],"depth":2}],"depth":1}';function Et(ut){return Lt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class jt extends At{constructor(g){super(),St(this,g,Et,Tt,$t,{})}}export{jt as component};