bitsandbytes/pr_2017/en/_app/immutable/nodes/10.Bn_ALQPG.js

import"../chunks/DsnmJJEf.js";import{i as F,h as j,C as S,H as i,D as e,E as D,s as G}from"../chunks/DT0OpeMJ.js";import{p as q,o as P,s as t,f as C,a as M,b as E,c as a,d as x,n,r as o}from"../chunks/Bb-LL0eD.js";const H='{"title":"LAMB","local":"lamb","sections":[{"title":"LAMB","local":"api-class ][ bitsandbytes.optim.LAMB","sections":[],"depth":2},{"title":"LAMB8bit","local":"bitsandbytes.optim.LAMB8bit","sections":[],"depth":2},{"title":"LAMB32bit","local":"bitsandbytes.optim.LAMB32bit","sections":[],"depth":2}],"depth":1}';var R=x('<meta name="hf:doc:metadata"/>'),k=x('<p></p> <!> <!> <p><a href="https://hf.co/papers/1904.00962" rel="nofollow">LAMB (Layerwise adaptive large batch optimization)</a> is an adaptive optimizer designed for training with large batch sizes to accelerate training, combining ideas from <code>LARS</code> and <code>Adam</code> to automatically scale the learning rate for each layer:</p> <ul><li>calculates a <em>trust ratio</em> between the weight and gradient norm in a layer and clips the ratio to prevent overly large or small updates</li> <li>updates weights with the first and second-moments</li></ul> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Base LAMB optimizer.</p></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>8-bit LAMB optimizer.</p></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>32-bit LAMB optimizer.</p></div></div> <!> <p></p>',1);function J(B,w){q(w,!1),P(()=>{new URLSearchParams(window.location.search).get("fw")}),F();var m=k();j("lccalj",A=>{var L=R();G(L,"content",H),M(A,L)});var b=t(C(m),2);S(b,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var c=t(b,2);i(c,{title:"LAMB",local:"lamb",headingTag:"h1"});var l=t(c,6);i(l,{title:"LAMB",local:"api-class ][ bitsandbytes.optim.LAMB",headingTag:"h2"});var s=t(l,2),_=a(s);e(_,{name:"class bitsandbytes.optim.LAMB",anchor:"bitsandbytes.optim.LAMB",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2017/bitsandbytes/optim/lamb.py#L8",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"bias_correction",val:" = True"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0"},{name:"amsgrad",val:" = False"},{name:"adam_w_mode",val:" = True"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 1.0"}]});var p=t(_,2),T=a(p);e(T,{name:"__init__",anchor:"bitsandbytes.optim.LAMB.__init__",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2017/bitsandbytes/optim/lamb.py#L9",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"bias_correction",val:" = True"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0"},{name:"amsgrad",val:" = False"},{name:"adam_w_mode",val:" = True"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 1.0"}],parametersDescription:[{anchor:"bitsandbytes.optim.LAMB.__init__.params",description:`<strong>params</strong> (<code>torch.tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.LAMB.__init__.lr",description:`<strong>lr</strong> (<code>float</code>, defaults to 1e-3) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.LAMB.__init__.bias_correction",description:`<strong>bias_correction</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to apply bias correction to the first and second-order moments.`,name:"bias_correction"},{anchor:"bitsandbytes.optim.LAMB.__init__.betas",description:`<strong>betas</strong> (<code>tuple(float, float)</code>, defaults to (0.9, 0.999)) &#x2014;
The beta values are the decay rates of the first and second-order moment of the optimizer.`,name:"betas"},{anchor:"bitsandbytes.optim.LAMB.__init__.eps",description:`<strong>eps</strong> (<code>float</code>, defaults to 1e-8) &#x2014;
The epsilon value prevents division by zero in the optimizer.`,name:"eps"},{anchor:"bitsandbytes.optim.LAMB.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.LAMB.__init__.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use the <a href="https://hf.co/papers/1904.09237" rel="nofollow">AMSGrad</a> variant of Adam that uses the maximum of past squared gradients instead.`,name:"amsgrad"},{anchor:"bitsandbytes.optim.LAMB.__init__.adam_w_mode",description:`<strong>adam_w_mode</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to use the AdamW variant.`,name:"adam_w_mode"},{anchor:"bitsandbytes.optim.LAMB.__init__.optim_bits",description:`<strong>optim_bits</strong> (<code>int</code>, defaults to 32) &#x2014;
The number of bits of the optimizer state.`,name:"optim_bits"},{anchor:"bitsandbytes.optim.LAMB.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.LAMB.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.LAMB.__init__.max_unorm",description:`<strong>max_unorm</strong> (<code>float</code>, defaults to 1.0) &#x2014;
The maximum gradient norm.`,name:"max_unorm"}]}),n(2),o(p),o(s);var h=t(s,2);i(h,{title:"LAMB8bit",local:"bitsandbytes.optim.LAMB8bit",headingTag:"h2"});var r=t(h,2),g=a(r);e(g,{name:"class bitsandbytes.optim.LAMB8bit",anchor:"bitsandbytes.optim.LAMB8bit",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2017/bitsandbytes/optim/lamb.py#L67",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"bias_correction",val:" = True"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0"},{name:"amsgrad",val:" = False"},{name:"adam_w_mode",val:" = True"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 1.0"}]});var u=t(g,2),z=a(u);e(z,{name:"__init__",anchor:"bitsandbytes.optim.LAMB8bit.__init__",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2017/bitsandbytes/optim/lamb.py#L68",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"bias_correction",val:" = True"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0"},{name:"amsgrad",val:" = False"},{name:"adam_w_mode",val:" = True"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 1.0"}],parametersDescription:[{anchor:"bitsandbytes.optim.LAMB8bit.__init__.params",description:`<strong>params</strong> (<code>torch.tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.lr",description:`<strong>lr</strong> (<code>float</code>, defaults to 1e-3) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.bias_correction",description:`<strong>bias_correction</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to apply bias correction to the first and second-order moments.`,name:"bias_correction"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.betas",description:`<strong>betas</strong> (<code>tuple(float, float)</code>, defaults to (0.9, 0.999)) &#x2014;
The beta values are the decay rates of the first and second-order moment of the optimizer.`,name:"betas"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.eps",description:`<strong>eps</strong> (<code>float</code>, defaults to 1e-8) &#x2014;
The epsilon value prevents division by zero in the optimizer.`,name:"eps"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use the <a href="https://hf.co/papers/1904.09237" rel="nofollow">AMSGrad</a> variant of Adam that uses the maximum of past squared gradients instead.
Note: This parameter is not supported in LAMB8bit and must be False.`,name:"amsgrad"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.adam_w_mode",description:`<strong>adam_w_mode</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to use the AdamW variant.`,name:"adam_w_mode"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.LAMB8bit.__init__.max_unorm",description:`<strong>max_unorm</strong> (<code>float</code>, defaults to 1.0) &#x2014;
The maximum update norm for trust-ratio clipping.
Note: This parameter is not supported in LAMB8bit and must be left at the
default 1.0. The 8-bit blockwise update does not implement update-norm
clipping; it is honored by the 32-bit LAMB / LAMB32bit optimizers.`,name:"max_unorm"}]}),n(2),o(u),o(r);var v=t(r,2);i(v,{title:"LAMB32bit",local:"bitsandbytes.optim.LAMB32bit",headingTag:"h2"});var d=t(v,2),f=a(d);e(f,{name:"class bitsandbytes.optim.LAMB32bit",anchor:"bitsandbytes.optim.LAMB32bit",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2017/bitsandbytes/optim/lamb.py#L137",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"bias_correction",val:" = True"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0"},{name:"amsgrad",val:" = False"},{name:"adam_w_mode",val:" = True"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 1.0"}]});var y=t(f,2),W=a(y);e(W,{name:"__init__",anchor:"bitsandbytes.optim.LAMB32bit.__init__",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2017/bitsandbytes/optim/lamb.py#L138",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"bias_correction",val:" = True"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0"},{name:"amsgrad",val:" = False"},{name:"adam_w_mode",val:" = True"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 1.0"}],parametersDescription:[{anchor:"bitsandbytes.optim.LAMB32bit.__init__.params",description:`<strong>params</strong> (<code>torch.tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.lr",description:`<strong>lr</strong> (<code>float</code>, defaults to 1e-3) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.bias_correction",description:`<strong>bias_correction</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to apply bias correction to the first and second-order moments.`,name:"bias_correction"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.betas",description:`<strong>betas</strong> (<code>tuple(float, float)</code>, defaults to (0.9, 0.999)) &#x2014;
The beta values are the decay rates of the first and second-order moment of the optimizer.`,name:"betas"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.eps",description:`<strong>eps</strong> (<code>float</code>, defaults to 1e-8) &#x2014;
The epsilon value prevents division by zero in the optimizer.`,name:"eps"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use the <a href="https://hf.co/papers/1904.09237" rel="nofollow">AMSGrad</a> variant of Adam that uses the maximum of past squared gradients instead.`,name:"amsgrad"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.adam_w_mode",description:`<strong>adam_w_mode</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to use the AdamW variant.`,name:"adam_w_mode"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.LAMB32bit.__init__.max_unorm",description:`<strong>max_unorm</strong> (<code>float</code>, defaults to 1.0) &#x2014;
The maximum gradient norm.`,name:"max_unorm"}]}),n(2),o(y),o(d);var N=t(d,2);D(N,{source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/docs/source/reference/optim/lamb.mdx"}),n(2),M(B,m),E()}export{J as component};