bitsandbytes/pr_1855/en/_app/immutable/nodes/21.8728efb3.js

import{s as Le,n as Me,o as we}from"../chunks/scheduler.8a2cc2fa.js";import{S as Be,i as Te,e as s,s as i,c as d,h as ze,a as r,d as t,b as o,f as x,g as l,j as G,k as A,l as m,m as n,n as c,t as p,o as b,p as _}from"../chunks/index.7079e750.js";import{C as ke,H as ce,E as Fe}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.5308db9a.js";import{D as H}from"../chunks/Docstring.1eacd8f6.js";function We(fe){let g,R,I,U,L,O,M,J,w,ve='<a href="https://hf.co/papers/1904.00962" rel="nofollow">LAMB (Layerwise adaptive large batch optimization)</a> is an adaptive optimizer designed for training with large batch sizes to accelerate training, combining ideas from <code>LARS</code> and <code>Adam</code> to automatically scale the learning rate for each layer:',K,B,ye="<li>calculates a <em>trust ratio</em> between the weight and gradient norm in a layer and clips the ratio to prevent overly large or small updates</li> <li>updates weights with the first and second-moments</li>",Q,T,X,h,z,pe,v,k,be,N,$e="Base LAMB optimizer.",Y,F,Z,u,W,_e,y,C,ge,S,xe="8-bit LAMB optimizer.",ee,E,te,f,D,he,$,P,ue,j,Ae="32-bit LAMB optimizer.",ae,q,ne,V,ie;return L=new ke({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),M=new ce({props:{title:"LAMB",local:"lamb",headingTag:"h1"}}),T=new ce({props:{title:"LAMB",local:"api-class ][ bitsandbytes.optim.LAMB",headingTag:"h2"}}),z=new H({props:{name:"class bitsandbytes.optim.LAMB",anchor:"bitsandbytes.optim.LAMB",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"bias_correction",val:" = True"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0"},{name:"amsgrad",val:" = False"},{name:"adam_w_mode",val:" = True"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"percentile_clipping",val:" = 100"},{name:"block_wise",val:" = False"},{name:"max_unorm",val:" = 1.0"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1855/bitsandbytes/optim/lamb.py#L8"}}),k=new H({props:{name:"__init__",anchor:"bitsandbytes.optim.LAMB.__init__",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"bias_correction",val:" = True"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0"},{name:"amsgrad",val:" = False"},{name:"adam_w_mode",val:" = True"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"percentile_clipping",val:" = 100"},{name:"block_wise",val:" = False"},{name:"max_unorm",val:" = 1.0"}],parametersDescription:[{anchor:"bitsandbytes.optim.LAMB.__init__.params",description:`<strong>params</strong> (<code>torch.tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.LAMB.__init__.lr",description:`<strong>lr</strong> (<code>float</code>, defaults to 1e-3) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.LAMB.__init__.bias_correction",description:`<strong>bias_correction</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to apply bias correction to the first and second-order moments.`,name:"bias_correction"},{anchor:"bitsandbytes.optim.LAMB.__init__.betas",description:`<strong>betas</strong> (<code>tuple(float, float)</code>, defaults to (0.9, 0.999)) &#x2014;
The beta values are the decay rates of the first and second-order moment of the optimizer.`,name:"betas"},{anchor:"bitsandbytes.optim.LAMB.__init__.eps",description:`<strong>eps</strong> (<code>float</code>, defaults to 1e-8) &#x2014;
The epsilon value prevents division by zero in the optimizer.`,name:"eps"},{anchor:"bitsandbytes.optim.LAMB.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.LAMB.__init__.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use the <a href="https://hf.co/papers/1904.09237" rel="nofollow">AMSGrad</a> variant of Adam that uses the maximum of past squared gradients instead.`,name:"amsgrad"},{anchor:"bitsandbytes.optim.LAMB.__init__.adam_w_mode",description:`<strong>adam_w_mode</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to use the AdamW variant.`,name:"adam_w_mode"},{anchor:"bitsandbytes.optim.LAMB.__init__.optim_bits",description:`<strong>optim_bits</strong> (<code>int</code>, defaults to 32) &#x2014;
The number of bits of the optimizer state.`,name:"optim_bits"},{anchor:"bitsandbytes.optim.LAMB.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.LAMB.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.LAMB.__init__.percentile_clipping",description:`<strong>percentile_clipping</strong> (<code>int</code>, defaults to 100) &#x2014;
Adapts clipping threshold automatically by tracking the last 100 gradient norms and clipping the gradient at a certain percentile to improve stability.`,name:"percentile_clipping"},{anchor:"bitsandbytes.optim.LAMB.__init__.block_wise",description:`<strong>block_wise</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to independently quantize each block of tensors to reduce outlier effects and improve stability.`,name:"block_wise"},{anchor:"bitsandbytes.optim.LAMB.__init__.max_unorm",description:`<strong>max_unorm</strong> (<code>float</code>, defaults to 1.0) &#x2014;
The maximum gradient norm.`,name:"max_unorm"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1855/bitsandbytes/optim/lamb.py#L9"}}),F=new ce({props:{title:"LAMB8bit",local:"bitsandbytes.optim.LAMB8bit",headingTag:"h2"}}),W=new H({props:{name:"class bitsandbytes.optim.LAMB8bit",anchor:"bitsandbytes.optim.LAMB8bit",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"bias_correction",val:" = True"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0"},{name:"amsgrad",val:" = False"},{name:"adam_w_mode",val:" = True"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"percentile_clipping",val:" = 100"},{name:"block_wise",val:" = False"},{name:"max_unorm",val:" = 1.0"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1855/bitsandbytes/optim/lamb.py#L75"}}),C=new H({props:{name:"__init__",anchor:"bitsandbytes.optim.LAMB8bit.__init__",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"bias_correction",val:" = True"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0"},{name:"amsgrad",val:" = False"},{name:"adam_w_mode",val:" = True"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"percentile_clipping",val:" = 100"},{name:"block_wise",val:" = False"},{name:"max_unorm",val:" = 1.0"}],parametersDescription:[{anchor:"bitsandbytes.optim.LAMB8bit.__init__.params",description:`<strong>params</strong> (<code>torch.tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.lr",description:`<strong>lr</strong> (<code>float</code>, defaults to 1e-3) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.bias_correction",description:`<strong>bias_correction</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to apply bias correction to the first and second-order moments.`,name:"bias_correction"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.betas",description:`<strong>betas</strong> (<code>tuple(float, float)</code>, defaults to (0.9, 0.999)) &#x2014;
The beta values are the decay rates of the first and second-order moment of the optimizer.`,name:"betas"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.eps",description:`<strong>eps</strong> (<code>float</code>, defaults to 1e-8) &#x2014;
The epsilon value prevents division by zero in the optimizer.`,name:"eps"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use the <a href="https://hf.co/papers/1904.09237" rel="nofollow">AMSGrad</a> variant of Adam that uses the maximum of past squared gradients instead.`,name:"amsgrad"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.adam_w_mode",description:`<strong>adam_w_mode</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to use the AdamW variant.`,name:"adam_w_mode"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.percentile_clipping",description:`<strong>percentile_clipping</strong> (<code>int</code>, defaults to 100) &#x2014;
Adapts clipping threshold automatically by tracking the last 100 gradient norms and clipping the gradient at a certain percentile to improve stability.`,name:"percentile_clipping"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.block_wise",description:`<strong>block_wise</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to independently quantize each block of tensors to reduce outlier effects and improve stability.`,name:"block_wise"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.max_unorm",description:`<strong>max_unorm</strong> (<code>float</code>, defaults to 1.0) &#x2014;
The maximum gradient norm.`,name:"max_unorm"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1855/bitsandbytes/optim/lamb.py#L76"}}),E=new ce({props:{title:"LAMB32bit",local:"bitsandbytes.optim.LAMB32bit",headingTag:"h2"}}),D=new H({props:{name:"class bitsandbytes.optim.LAMB32bit",anchor:"bitsandbytes.optim.LAMB32bit",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"bias_correction",val:" = True"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0"},{name:"amsgrad",val:" = False"},{name:"adam_w_mode",val:" = True"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"percentile_clipping",val:" = 100"},{name:"block_wise",val:" = False"},{name:"max_unorm",val:" = 1.0"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1855/bitsandbytes/optim/lamb.py#L139"}}),P=new H({props:{name:"__init__",anchor:"bitsandbytes.optim.LAMB32bit.__init__",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"bias_correction",val:" = True"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0"},{name:"amsgrad",val:" = False"},{name:"adam_w_mode",val:" = True"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"percentile_clipping",val:" = 100"},{name:"block_wise",val:" = False"},{name:"max_unorm",val:" = 1.0"}],parametersDescription:[{anchor:"bitsandbytes.optim.LAMB32bit.__init__.params",description:`<strong>params</strong> (<code>torch.tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.lr",description:`<strong>lr</strong> (<code>float</code>, defaults to 1e-3) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.bias_correction",description:`<strong>bias_correction</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to apply bias correction to the first and second-order moments.`,name:"bias_correction"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.betas",description:`<strong>betas</strong> (<code>tuple(float, float)</code>, defaults to (0.9, 0.999)) &#x2014;
The beta values are the decay rates of the first and second-order moment of the optimizer.`,name:"betas"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.eps",description:`<strong>eps</strong> (<code>float</code>, defaults to 1e-8) &#x2014;
The epsilon value prevents division by zero in the optimizer.`,name:"eps"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use the <a href="https://hf.co/papers/1904.09237" rel="nofollow">AMSGrad</a> variant of Adam that uses the maximum of past squared gradients instead.`,name:"amsgrad"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.adam_w_mode",description:`<strong>adam_w_mode</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to use the AdamW variant.`,name:"adam_w_mode"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.percentile_clipping",description:`<strong>percentile_clipping</strong> (<code>int</code>, defaults to 100) &#x2014;
Adapts clipping threshold automatically by tracking the last 100 gradient norms and clipping the gradient at a certain percentile to improve stability.`,name:"percentile_clipping"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.block_wise",description:`<strong>block_wise</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to independently quantize each block of tensors to reduce outlier effects and improve stability.`,name:"block_wise"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.max_unorm",description:`<strong>max_unorm</strong> (<code>float</code>, defaults to 1.0) &#x2014;
The maximum gradient norm.`,name:"max_unorm"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1855/bitsandbytes/optim/lamb.py#L140"}}),q=new Fe({props:{source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/docs/source/reference/optim/lamb.mdx"}}),{c(){g=s("meta"),R=i(),I=s("p"),U=i(),d(L.$$.fragment),O=i(),d(M.$$.fragment),J=i(),w=s("p"),w.innerHTML=ve,K=i(),B=s("ul"),B.innerHTML=ye,Q=i(),d(T.$$.fragment),X=i(),h=s("div"),d(z.$$.fragment),pe=i(),v=s("div"),d(k.$$.fragment),be=i(),N=s("p"),N.textContent=$e,Y=i(),d(F.$$.fragment),Z=i(),u=s("div"),d(W.$$.fragment),_e=i(),y=s("div"),d(C.$$.fragment),ge=i(),S=s("p"),S.textContent=xe,ee=i(),d(E.$$.fragment),te=i(),f=s("div"),d(D.$$.fragment),he=i(),$=s("div"),d(P.$$.fragment),ue=i(),j=s("p"),j.textContent=Ae,ae=i(),d(q.$$.fragment),ne=i(),V=s("p"),this.h()},l(e){const a=ze("svelte-u9bgzb",document.head);g=r(a,"META",{name:!0,content:!0}),a.forEach(t),R=o(e),I=r(e,"P",{}),x(I).forEach(t),U=o(e),l(L.$$.fragment,e),O=o(e),l(M.$$.fragment,e),J=o(e),w=r(e,"P",{"data-svelte-h":!0}),G(w)!=="svelte-166pr93"&&(w.innerHTML=ve),K=o(e),B=r(e,"UL",{"data-svelte-h":!0}),G(B)!=="svelte-1v2rbtd"&&(B.innerHTML=ye),Q=o(e),l(T.$$.fragment,e),X=o(e),h=r(e,"DIV",{class:!0});var oe=x(h);l(z.$$.fragment,oe),pe=o(oe),v=r(oe,"DIV",{class:!0});var se=x(v);l(k.$$.fragment,se),be=o(se),N=r(se,"P",{"data-svelte-h":!0}),G(N)!=="svelte-q646"&&(N.textContent=$e),se.forEach(t),oe.forEach(t),Y=o(e),l(F.$$.fragment,e),Z=o(e),u=r(e,"DIV",{class:!0});var re=x(u);l(W.$$.fragment,re),_e=o(re),y=r(re,"DIV",{class:!0});var me=x(y);l(C.$$.fragment,me),ge=o(me),S=r(me,"P",{"data-svelte-h":!0}),G(S)!=="svelte-1glmsph"&&(S.textContent=xe),me.forEach(t),re.forEach(t),ee=o(e),l(E.$$.fragment,e),te=o(e),f=r(e,"DIV",{class:!0});var de=x(f);l(D.$$.fragment,de),he=o(de),$=r(de,"DIV",{class:!0});var le=x($);l(P.$$.fragment,le),ue=o(le),j=r(le,"P",{"data-svelte-h":!0}),G(j)!=="svelte-1vurq1c"&&(j.textContent=Ae),le.forEach(t),de.forEach(t),ae=o(e),l(q.$$.fragment,e),ne=o(e),V=r(e,"P",{}),x(V).forEach(t),this.h()},h(){A(g,"name","hf:doc:metadata"),A(g,"content",Ce),A(v,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),A(h,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),A(y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),A(u,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),A($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),A(f,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,a){m(document.head,g),n(e,R,a),n(e,I,a),n(e,U,a),c(L,e,a),n(e,O,a),c(M,e,a),n(e,J,a),n(e,w,a),n(e,K,a),n(e,B,a),n(e,Q,a),c(T,e,a),n(e,X,a),n(e,h,a),c(z,h,null),m(h,pe),m(h,v),c(k,v,null),m(v,be),m(v,N),n(e,Y,a),c(F,e,a),n(e,Z,a),n(e,u,a),c(W,u,null),m(u,_e),m(u,y),c(C,y,null),m(y,ge),m(y,S),n(e,ee,a),c(E,e,a),n(e,te,a),n(e,f,a),c(D,f,null),m(f,he),m(f,$),c(P,$,null),m($,ue),m($,j),n(e,ae,a),c(q,e,a),n(e,ne,a),n(e,V,a),ie=!0},p:Me,i(e){ie||(p(L.$$.fragment,e),p(M.$$.fragment,e),p(T.$$.fragment,e),p(z.$$.fragment,e),p(k.$$.fragment,e),p(F.$$.fragment,e),p(W.$$.fragment,e),p(C.$$.fragment,e),p(E.$$.fragment,e),p(D.$$.fragment,e),p(P.$$.fragment,e),p(q.$$.fragment,e),ie=!0)},o(e){b(L.$$.fragment,e),b(M.$$.fragment,e),b(T.$$.fragment,e),b(z.$$.fragment,e),b(k.$$.fragment,e),b(F.$$.fragment,e),b(W.$$.fragment,e),b(C.$$.fragment,e),b(E.$$.fragment,e),b(D.$$.fragment,e),b(P.$$.fragment,e),b(q.$$.fragment,e),ie=!1},d(e){e&&(t(R),t(I),t(U),t(O),t(J),t(w),t(K),t(B),t(Q),t(X),t(h),t(Y),t(Z),t(u),t(ee),t(te),t(f),t(ae),t(ne),t(V)),t(g),_(L,e),_(M,e),_(T,e),_(z),_(k),_(F,e),_(W),_(C),_(E,e),_(D),_(P),_(q,e)}}}const Ce='{"title":"LAMB","local":"lamb","sections":[{"title":"LAMB","local":"api-class ][ bitsandbytes.optim.LAMB","sections":[],"depth":2},{"title":"LAMB8bit","local":"bitsandbytes.optim.LAMB8bit","sections":[],"depth":2},{"title":"LAMB32bit","local":"bitsandbytes.optim.LAMB32bit","sections":[],"depth":2}],"depth":1}';function Ee(fe){return we(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Se extends Be{constructor(g){super(),Te(this,g,Ee,We,Le,{})}}export{Se as component};