bitsandbytes/pr_2016/en/_app/immutable/nodes/12.W0ADcctq.js

import"../chunks/DsnmJJEf.js";import{i as j,h as D,C as k,H as i,D as e,E as M,s as P}from"../chunks/DdsaYIhL.js";import{p as W,o as C,s as t,f as E,a as x,b as H,c as a,d as S,n as o,r as n}from"../chunks/Cq6T2TQD.js";const q='{"title":"LARS","local":"lars","sections":[{"title":"LARS","local":"api-class ][ bitsandbytes.optim.LARS","sections":[],"depth":2},{"title":"LARS8bit","local":"bitsandbytes.optim.LARS8bit","sections":[],"depth":2},{"title":"LARS32bit","local":"bitsandbytes.optim.LARS32bit","sections":[],"depth":2}],"depth":1}';var B=S('<meta name="hf:doc:metadata"/>'),G=S('<p></p> <!> <!> <p><a href="https:/hf.co/papers/1708.03888" rel="nofollow">LARS (Layer-wise Adaptive Rate Scaling)</a> is an optimizer designed for training with large batch sizes to accelerate training. LARS uses a separate learning rate for each <em>layer</em> instead of each parameter. The learning rate is calculated from a <em>trust ratio</em> between the weight and gradient norm in a layer. This helps calibrate a stable update size.</p> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Base LARS optimizer.</p></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>8-bit LARS optimizer.</p></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>32-bit LARS optimizer.</p></div></div> <!> <p></p>',1);function J(R,z){W(z,!1),C(()=>{new URLSearchParams(window.location.search).get("fw")}),j();var d=G();D("1rv3qvb",L=>{var A=B();P(A,"content",q),x(L,A)});var b=t(E(d),2);k(b,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var l=t(b,2);i(l,{title:"LARS",local:"lars",headingTag:"h1"});var p=t(l,4);i(p,{title:"LARS",local:"api-class ][ bitsandbytes.optim.LARS",headingTag:"h2"});var s=t(p,2),c=a(s);e(c,{name:"class bitsandbytes.optim.LARS",anchor:"bitsandbytes.optim.LARS",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2016/bitsandbytes/optim/lars.py#L11",parameters:[{name:"params",val:""},{name:"lr",val:""},{name:"momentum",val:" = 0"},{name:"dampening",val:" = 0"},{name:"weight_decay",val:" = 0"},{name:"nesterov",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 0.02"}]});var _=t(c,2),w=a(_);e(w,{name:"__init__",anchor:"bitsandbytes.optim.LARS.__init__",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2016/bitsandbytes/optim/lars.py#L12",parameters:[{name:"params",val:""},{name:"lr",val:""},{name:"momentum",val:" = 0"},{name:"dampening",val:" = 0"},{name:"weight_decay",val:" = 0"},{name:"nesterov",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 0.02"}],parametersDescription:[{anchor:"bitsandbytes.optim.LARS.__init__.params",description:`<strong>params</strong> (<code>torch.tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.LARS.__init__.lr",description:`<strong>lr</strong> (<code>float</code>) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.LARS.__init__.momentum",description:`<strong>momentum</strong> (<code>float</code>, defaults to 0) &#x2014;
The momentum value speeds up the optimizer by taking bigger steps.`,name:"momentum"},{anchor:"bitsandbytes.optim.LARS.__init__.dampening",description:`<strong>dampening</strong> (<code>float</code>, defaults to 0) &#x2014;
The dampening value reduces the momentum of the optimizer.`,name:"dampening"},{anchor:"bitsandbytes.optim.LARS.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.LARS.__init__.nesterov",description:`<strong>nesterov</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use Nesterov momentum.`,name:"nesterov"},{anchor:"bitsandbytes.optim.LARS.__init__.optim_bits",description:`<strong>optim_bits</strong> (<code>int</code>, defaults to 32) &#x2014;
The number of bits of the optimizer state.`,name:"optim_bits"},{anchor:"bitsandbytes.optim.LARS.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.LARS.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.LARS.__init__.max_unorm",description:`<strong>max_unorm</strong> (<code>float</code>, defaults to 0.02) &#x2014;
The maximum gradient norm.`,name:"max_unorm"}]}),o(2),n(_),n(s);var g=t(s,2);i(g,{title:"LARS8bit",local:"bitsandbytes.optim.LARS8bit",headingTag:"h2"});var r=t(g,2),h=a(r);e(h,{name:"class bitsandbytes.optim.LARS8bit",anchor:"bitsandbytes.optim.LARS8bit",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2016/bitsandbytes/optim/lars.py#L66",parameters:[{name:"params",val:""},{name:"lr",val:""},{name:"momentum",val:" = 0"},{name:"dampening",val:" = 0"},{name:"weight_decay",val:" = 0"},{name:"nesterov",val:" = False"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 0.02"}]});var u=t(h,2),T=a(u);e(T,{name:"__init__",anchor:"bitsandbytes.optim.LARS8bit.__init__",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2016/bitsandbytes/optim/lars.py#L67",parameters:[{name:"params",val:""},{name:"lr",val:""},{name:"momentum",val:" = 0"},{name:"dampening",val:" = 0"},{name:"weight_decay",val:" = 0"},{name:"nesterov",val:" = False"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 0.02"}],parametersDescription:[{anchor:"bitsandbytes.optim.LARS8bit.__init__.params",description:`<strong>params</strong> (<code>torch.tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.LARS8bit.__init__.lr",description:`<strong>lr</strong> (<code>float</code>) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.LARS8bit.__init__.momentum",description:`<strong>momentum</strong> (<code>float</code>, defaults to 0) &#x2014;
The momentum value speeds up the optimizer by taking bigger steps.`,name:"momentum"},{anchor:"bitsandbytes.optim.LARS8bit.__init__.dampening",description:`<strong>dampening</strong> (<code>float</code>, defaults to 0) &#x2014;
The dampening value reduces the momentum of the optimizer.`,name:"dampening"},{anchor:"bitsandbytes.optim.LARS8bit.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.LARS8bit.__init__.nesterov",description:`<strong>nesterov</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use Nesterov momentum.`,name:"nesterov"},{anchor:"bitsandbytes.optim.LARS8bit.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.LARS8bit.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.LARS8bit.__init__.max_unorm",description:`<strong>max_unorm</strong> (<code>float</code>, defaults to 0.02) &#x2014;
The maximum gradient norm.`,name:"max_unorm"}]}),o(2),n(u),n(r);var v=t(r,2);i(v,{title:"LARS32bit",local:"bitsandbytes.optim.LARS32bit",headingTag:"h2"});var m=t(v,2),y=a(m);e(y,{name:"class bitsandbytes.optim.LARS32bit",anchor:"bitsandbytes.optim.LARS32bit",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2016/bitsandbytes/optim/lars.py#L118",parameters:[{name:"params",val:""},{name:"lr",val:""},{name:"momentum",val:" = 0"},{name:"dampening",val:" = 0"},{name:"weight_decay",val:" = 0"},{name:"nesterov",val:" = False"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 0.02"}]});var f=t(y,2),N=a(f);e(N,{name:"__init__",anchor:"bitsandbytes.optim.LARS32bit.__init__",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2016/bitsandbytes/optim/lars.py#L119",parameters:[{name:"params",val:""},{name:"lr",val:""},{name:"momentum",val:" = 0"},{name:"dampening",val:" = 0"},{name:"weight_decay",val:" = 0"},{name:"nesterov",val:" = False"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 0.02"}],parametersDescription:[{anchor:"bitsandbytes.optim.LARS32bit.__init__.params",description:`<strong>params</strong> (<code>torch.tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.LARS32bit.__init__.lr",description:`<strong>lr</strong> (<code>float</code>) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.LARS32bit.__init__.momentum",description:`<strong>momentum</strong> (<code>float</code>, defaults to 0) &#x2014;
The momentum value speeds up the optimizer by taking bigger steps.`,name:"momentum"},{anchor:"bitsandbytes.optim.LARS32bit.__init__.dampening",description:`<strong>dampening</strong> (<code>float</code>, defaults to 0) &#x2014;
The dampening value reduces the momentum of the optimizer.`,name:"dampening"},{anchor:"bitsandbytes.optim.LARS32bit.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.LARS32bit.__init__.nesterov",description:`<strong>nesterov</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use Nesterov momentum.`,name:"nesterov"},{anchor:"bitsandbytes.optim.LARS32bit.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.LARS32bit.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.LARS32bit.__init__.max_unorm",description:`<strong>max_unorm</strong> (<code>float</code>, defaults to 0.02) &#x2014;
The maximum gradient norm.`,name:"max_unorm"}]}),o(2),n(f),n(m);var F=t(m,2);M(F,{source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/docs/source/reference/optim/lars.mdx"}),o(2),x(R,d),H()}export{J as component};