bitsandbytes/pr_1925/en/_app/immutable/nodes/21.480eba9d.js

import{s as At,n as Mt,o as wt}from"../chunks/scheduler.8a2cc2fa.js";import{S as Bt,i as Tt,e as s,s as o,c as d,h as zt,a as r,d as e,b as i,f as x,g as l,j as R,k as L,l as m,m as n,n as b,t as c,o as _,p}from"../chunks/index.7079e750.js";import{C as Ct,H as bt,E as Et}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.0d032bd2.js";import{D as I}from"../chunks/Docstring.a1d84405.js";function Wt(ut){let g,U,V,k,A,O,M,J,w,vt='<a href="https://hf.co/papers/1904.00962" rel="nofollow">LAMB (Layerwise adaptive large batch optimization)</a> is an adaptive optimizer designed for training with large batch sizes to accelerate training, combining ideas from <code>LARS</code> and <code>Adam</code> to automatically scale the learning rate for each layer:',K,B,yt="<li>calculates a <em>trust ratio</em> between the weight and gradient norm in a layer and clips the ratio to prevent overly large or small updates</li> <li>updates weights with the first and second-moments</li>",Q,T,X,h,z,ct,v,C,_t,j,$t="Base LAMB optimizer.",Y,E,Z,f,W,pt,y,D,gt,q,xt="8-bit LAMB optimizer.",tt,P,et,u,F,ht,$,N,ft,H,Lt="32-bit LAMB optimizer.",at,S,nt,G,ot;return A=new Ct({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),M=new bt({props:{title:"LAMB",local:"lamb",headingTag:"h1"}}),T=new bt({props:{title:"LAMB",local:"api-class ][ bitsandbytes.optim.LAMB",headingTag:"h2"}}),z=new I({props:{name:"class bitsandbytes.optim.LAMB",anchor:"bitsandbytes.optim.LAMB",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"bias_correction",val:" = True"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0"},{name:"amsgrad",val:" = False"},{name:"adam_w_mode",val:" = True"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 1.0"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1925/bitsandbytes/optim/lamb.py#L8"}}),C=new I({props:{name:"__init__",anchor:"bitsandbytes.optim.LAMB.__init__",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"bias_correction",val:" = True"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0"},{name:"amsgrad",val:" = False"},{name:"adam_w_mode",val:" = True"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 1.0"}],parametersDescription:[{anchor:"bitsandbytes.optim.LAMB.__init__.params",description:`<strong>params</strong> (<code>torch.tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.LAMB.__init__.lr",description:`<strong>lr</strong> (<code>float</code>, defaults to 1e-3) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.LAMB.__init__.bias_correction",description:`<strong>bias_correction</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to apply bias correction to the first and second-order moments.`,name:"bias_correction"},{anchor:"bitsandbytes.optim.LAMB.__init__.betas",description:`<strong>betas</strong> (<code>tuple(float, float)</code>, defaults to (0.9, 0.999)) &#x2014;
The beta values are the decay rates of the first and second-order moment of the optimizer.`,name:"betas"},{anchor:"bitsandbytes.optim.LAMB.__init__.eps",description:`<strong>eps</strong> (<code>float</code>, defaults to 1e-8) &#x2014;
The epsilon value prevents division by zero in the optimizer.`,name:"eps"},{anchor:"bitsandbytes.optim.LAMB.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.LAMB.__init__.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use the <a href="https://hf.co/papers/1904.09237" rel="nofollow">AMSGrad</a> variant of Adam that uses the maximum of past squared gradients instead.`,name:"amsgrad"},{anchor:"bitsandbytes.optim.LAMB.__init__.adam_w_mode",description:`<strong>adam_w_mode</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to use the AdamW variant.`,name:"adam_w_mode"},{anchor:"bitsandbytes.optim.LAMB.__init__.optim_bits",description:`<strong>optim_bits</strong> (<code>int</code>, defaults to 32) &#x2014;
The number of bits of the optimizer state.`,name:"optim_bits"},{anchor:"bitsandbytes.optim.LAMB.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.LAMB.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.LAMB.__init__.max_unorm",description:`<strong>max_unorm</strong> (<code>float</code>, defaults to 1.0) &#x2014;
The maximum gradient norm.`,name:"max_unorm"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1925/bitsandbytes/optim/lamb.py#L9"}}),E=new bt({props:{title:"LAMB8bit",local:"bitsandbytes.optim.LAMB8bit",headingTag:"h2"}}),W=new I({props:{name:"class bitsandbytes.optim.LAMB8bit",anchor:"bitsandbytes.optim.LAMB8bit",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"bias_correction",val:" = True"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0"},{name:"amsgrad",val:" = False"},{name:"adam_w_mode",val:" = True"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 1.0"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1925/bitsandbytes/optim/lamb.py#L67"}}),D=new I({props:{name:"__init__",anchor:"bitsandbytes.optim.LAMB8bit.__init__",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"bias_correction",val:" = True"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0"},{name:"amsgrad",val:" = False"},{name:"adam_w_mode",val:" = True"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 1.0"}],parametersDescription:[{anchor:"bitsandbytes.optim.LAMB8bit.__init__.params",description:`<strong>params</strong> (<code>torch.tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.lr",description:`<strong>lr</strong> (<code>float</code>, defaults to 1e-3) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.bias_correction",description:`<strong>bias_correction</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to apply bias correction to the first and second-order moments.`,name:"bias_correction"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.betas",description:`<strong>betas</strong> (<code>tuple(float, float)</code>, defaults to (0.9, 0.999)) &#x2014;
The beta values are the decay rates of the first and second-order moment of the optimizer.`,name:"betas"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.eps",description:`<strong>eps</strong> (<code>float</code>, defaults to 1e-8) &#x2014;
The epsilon value prevents division by zero in the optimizer.`,name:"eps"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use the <a href="https://hf.co/papers/1904.09237" rel="nofollow">AMSGrad</a> variant of Adam that uses the maximum of past squared gradients instead.`,name:"amsgrad"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.adam_w_mode",description:`<strong>adam_w_mode</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to use the AdamW variant.`,name:"adam_w_mode"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.max_unorm",description:`<strong>max_unorm</strong> (<code>float</code>, defaults to 1.0) &#x2014;
The maximum gradient norm.`,name:"max_unorm"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1925/bitsandbytes/optim/lamb.py#L68"}}),P=new bt({props:{title:"LAMB32bit",local:"bitsandbytes.optim.LAMB32bit",headingTag:"h2"}}),F=new I({props:{name:"class bitsandbytes.optim.LAMB32bit",anchor:"bitsandbytes.optim.LAMB32bit",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"bias_correction",val:" = True"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0"},{name:"amsgrad",val:" = False"},{name:"adam_w_mode",val:" = True"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 1.0"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1925/bitsandbytes/optim/lamb.py#L123"}}),N=new I({props:{name:"__init__",anchor:"bitsandbytes.optim.LAMB32bit.__init__",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"bias_correction",val:" = True"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0"},{name:"amsgrad",val:" = False"},{name:"adam_w_mode",val:" = True"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 1.0"}],parametersDescription:[{anchor:"bitsandbytes.optim.LAMB32bit.__init__.params",description:`<strong>params</strong> (<code>torch.tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.lr",description:`<strong>lr</strong> (<code>float</code>, defaults to 1e-3) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.bias_correction",description:`<strong>bias_correction</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to apply bias correction to the first and second-order moments.`,name:"bias_correction"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.betas",description:`<strong>betas</strong> (<code>tuple(float, float)</code>, defaults to (0.9, 0.999)) &#x2014;
The beta values are the decay rates of the first and second-order moment of the optimizer.`,name:"betas"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.eps",description:`<strong>eps</strong> (<code>float</code>, defaults to 1e-8) &#x2014;
The epsilon value prevents division by zero in the optimizer.`,name:"eps"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use the <a href="https://hf.co/papers/1904.09237" rel="nofollow">AMSGrad</a> variant of Adam that uses the maximum of past squared gradients instead.`,name:"amsgrad"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.adam_w_mode",description:`<strong>adam_w_mode</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to use the AdamW variant.`,name:"adam_w_mode"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.max_unorm",description:`<strong>max_unorm</strong> (<code>float</code>, defaults to 1.0) &#x2014;
The maximum gradient norm.`,name:"max_unorm"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1925/bitsandbytes/optim/lamb.py#L124"}}),S=new Et({props:{source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/docs/source/reference/optim/lamb.mdx"}}),{c(){g=s("meta"),U=o(),V=s("p"),k=o(),d(A.$$.fragment),O=o(),d(M.$$.fragment),J=o(),w=s("p"),w.innerHTML=vt,K=o(),B=s("ul"),B.innerHTML=yt,Q=o(),d(T.$$.fragment),X=o(),h=s("div"),d(z.$$.fragment),ct=o(),v=s("div"),d(C.$$.fragment),_t=o(),j=s("p"),j.textContent=$t,Y=o(),d(E.$$.fragment),Z=o(),f=s("div"),d(W.$$.fragment),pt=o(),y=s("div"),d(D.$$.fragment),gt=o(),q=s("p"),q.textContent=xt,tt=o(),d(P.$$.fragment),et=o(),u=s("div"),d(F.$$.fragment),ht=o(),$=s("div"),d(N.$$.fragment),ft=o(),H=s("p"),H.textContent=Lt,at=o(),d(S.$$.fragment),nt=o(),G=s("p"),this.h()},l(t){const a=zt("svelte-u9bgzb",document.head);g=r(a,"META",{name:!0,content:!0}),a.forEach(e),U=i(t),V=r(t,"P",{}),x(V).forEach(e),k=i(t),l(A.$$.fragment,t),O=i(t),l(M.$$.fragment,t),J=i(t),w=r(t,"P",{"data-svelte-h":!0}),R(w)!=="svelte-166pr93"&&(w.innerHTML=vt),K=i(t),B=r(t,"UL",{"data-svelte-h":!0}),R(B)!=="svelte-1v2rbtd"&&(B.innerHTML=yt),Q=i(t),l(T.$$.fragment,t),X=i(t),h=r(t,"DIV",{class:!0});var it=x(h);l(z.$$.fragment,it),ct=i(it),v=r(it,"DIV",{class:!0});var st=x(v);l(C.$$.fragment,st),_t=i(st),j=r(st,"P",{"data-svelte-h":!0}),R(j)!=="svelte-q646"&&(j.textContent=$t),st.forEach(e),it.forEach(e),Y=i(t),l(E.$$.fragment,t),Z=i(t),f=r(t,"DIV",{class:!0});var rt=x(f);l(W.$$.fragment,rt),pt=i(rt),y=r(rt,"DIV",{class:!0});var mt=x(y);l(D.$$.fragment,mt),gt=i(mt),q=r(mt,"P",{"data-svelte-h":!0}),R(q)!=="svelte-1glmsph"&&(q.textContent=xt),mt.forEach(e),rt.forEach(e),tt=i(t),l(P.$$.fragment,t),et=i(t),u=r(t,"DIV",{class:!0});var dt=x(u);l(F.$$.fragment,dt),ht=i(dt),$=r(dt,"DIV",{class:!0});var lt=x($);l(N.$$.fragment,lt),ft=i(lt),H=r(lt,"P",{"data-svelte-h":!0}),R(H)!=="svelte-1vurq1c"&&(H.textContent=Lt),lt.forEach(e),dt.forEach(e),at=i(t),l(S.$$.fragment,t),nt=i(t),G=r(t,"P",{}),x(G).forEach(e),this.h()},h(){L(g,"name","hf:doc:metadata"),L(g,"content",Dt),L(v,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),L(h,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),L(y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),L(f,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),L($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),L(u,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(t,a){m(document.head,g),n(t,U,a),n(t,V,a),n(t,k,a),b(A,t,a),n(t,O,a),b(M,t,a),n(t,J,a),n(t,w,a),n(t,K,a),n(t,B,a),n(t,Q,a),b(T,t,a),n(t,X,a),n(t,h,a),b(z,h,null),m(h,ct),m(h,v),b(C,v,null),m(v,_t),m(v,j),n(t,Y,a),b(E,t,a),n(t,Z,a),n(t,f,a),b(W,f,null),m(f,pt),m(f,y),b(D,y,null),m(y,gt),m(y,q),n(t,tt,a),b(P,t,a),n(t,et,a),n(t,u,a),b(F,u,null),m(u,ht),m(u,$),b(N,$,null),m($,ft),m($,H),n(t,at,a),b(S,t,a),n(t,nt,a),n(t,G,a),ot=!0},p:Mt,i(t){ot||(c(A.$$.fragment,t),c(M.$$.fragment,t),c(T.$$.fragment,t),c(z.$$.fragment,t),c(C.$$.fragment,t),c(E.$$.fragment,t),c(W.$$.fragment,t),c(D.$$.fragment,t),c(P.$$.fragment,t),c(F.$$.fragment,t),c(N.$$.fragment,t),c(S.$$.fragment,t),ot=!0)},o(t){_(A.$$.fragment,t),_(M.$$.fragment,t),_(T.$$.fragment,t),_(z.$$.fragment,t),_(C.$$.fragment,t),_(E.$$.fragment,t),_(W.$$.fragment,t),_(D.$$.fragment,t),_(P.$$.fragment,t),_(F.$$.fragment,t),_(N.$$.fragment,t),_(S.$$.fragment,t),ot=!1},d(t){t&&(e(U),e(V),e(k),e(O),e(J),e(w),e(K),e(B),e(Q),e(X),e(h),e(Y),e(Z),e(f),e(tt),e(et),e(u),e(at),e(nt),e(G)),e(g),p(A,t),p(M,t),p(T,t),p(z),p(C),p(E,t),p(W),p(D),p(P,t),p(F),p(N),p(S,t)}}}const Dt='{"title":"LAMB","local":"lamb","sections":[{"title":"LAMB","local":"api-class ][ bitsandbytes.optim.LAMB","sections":[],"depth":2},{"title":"LAMB8bit","local":"bitsandbytes.optim.LAMB8bit","sections":[],"depth":2},{"title":"LAMB32bit","local":"bitsandbytes.optim.LAMB32bit","sections":[],"depth":2}],"depth":1}';function Pt(ut){return wt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class qt extends Bt{constructor(g){super(),Tt(this,g,Pt,Wt,At,{})}}export{qt as component};