bitsandbytes/pr_1997/en/_app/immutable/nodes/9.D2fMigvI.js

import"../chunks/DsnmJJEf.js";import{i as J,h as K,C as Q,H as o,D as e,E as V,s as X}from"../chunks/DdsaYIhL.js";import{p as Y,o as Z,s as t,f as $,a as S,b as tt,c as a,d as q,n as s,r as i}from"../chunks/Cq6T2TQD.js";const et='{"title":"AdamW","local":"adamw","sections":[{"title":"AdamW","local":"api-class ][ bitsandbytes.optim.AdamW","sections":[],"depth":2},{"title":"AdamW8bit","local":"bitsandbytes.optim.AdamW8bit","sections":[],"depth":2},{"title":"AdamW32bit","local":"bitsandbytes.optim.AdamW32bit","sections":[],"depth":2},{"title":"PagedAdamW","local":"bitsandbytes.optim.PagedAdamW","sections":[],"depth":2},{"title":"PagedAdamW8bit","local":"bitsandbytes.optim.PagedAdamW8bit","sections":[],"depth":2},{"title":"PagedAdamW32bit","local":"bitsandbytes.optim.PagedAdamW32bit","sections":[],"depth":2}],"depth":1}';var at=q('<meta name="hf:doc:metadata"/>'),it=q('<p></p> <!> <!> <p><a href="https://hf.co/papers/1711.05101" rel="nofollow">AdamW</a> is a variant of the <code>Adam</code> optimizer that separates weight decay from the gradient update based on the observation that the weight decay formulation is different when applied to <code>SGD</code> and <code>Adam</code>.</p> <p>bitsandbytes also supports paged optimizers which take advantage of CUDAs unified memory to transfer memory from the GPU to the CPU when GPU memory is exhausted.</p> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Base AdamW optimizer.</p></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>8-bit AdamW optimizer.</p></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>32-bit AdamW optimizer.</p></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Paged AdamW optimizer.</p></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Paged 8-bit AdamW optimizer.</p></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Paged 32-bit AdamW optimizer.</p></div></div> <!> <p></p>',1);function dt(U,C){Y(C,!1),Z(()=>{new URLSearchParams(window.location.search).get("fw")}),J();var l=it();K("1biwkk9",G=>{var M=at();X(M,"content",et),S(G,M)});var _=t($(l),2);Q(_,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var c=t(_,2);o(c,{title:"AdamW",local:"adamw",headingTag:"h1"});var g=t(c,6);o(g,{title:"AdamW",local:"api-class ][ bitsandbytes.optim.AdamW",headingTag:"h2"});var n=t(g,2),h=a(n);e(h,{name:"class bitsandbytes.optim.AdamW",anchor:"bitsandbytes.optim.AdamW",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1997/bitsandbytes/optim/adamw.py#L9",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.01"},{name:"amsgrad",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"is_paged",val:" = False"}]});var v=t(h,2),k=a(v);e(k,{name:"__init__",anchor:"bitsandbytes.optim.AdamW.__init__",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1997/bitsandbytes/optim/adamw.py#L10",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.01"},{name:"amsgrad",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"is_paged",val:" = False"}],parametersDescription:[{anchor:"bitsandbytes.optim.AdamW.__init__.params",description:`<strong>params</strong> (<code>torch.Tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.AdamW.__init__.lr",description:`<strong>lr</strong> (<code>float</code>, defaults to 1e-3) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.AdamW.__init__.betas",description:`<strong>betas</strong> (<code>tuple(float, float)</code>, defaults to (0.9, 0.999)) &#x2014;
The beta values are the decay rates of the first and second-order moment of the optimizer.`,name:"betas"},{anchor:"bitsandbytes.optim.AdamW.__init__.eps",description:`<strong>eps</strong> (<code>float</code>, defaults to 1e-8) &#x2014;
The epsilon value prevents division by zero in the optimizer.`,name:"eps"},{anchor:"bitsandbytes.optim.AdamW.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.AdamW.__init__.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use the <a href="https://hf.co/papers/1904.09237" rel="nofollow">AMSGrad</a> variant of Adam that uses the maximum of past squared gradients instead.`,name:"amsgrad"},{anchor:"bitsandbytes.optim.AdamW.__init__.optim_bits",description:`<strong>optim_bits</strong> (<code>int</code>, defaults to 32) &#x2014;
The number of bits of the optimizer state.`,name:"optim_bits"},{anchor:"bitsandbytes.optim.AdamW.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.AdamW.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.AdamW.__init__.is_paged",description:`<strong>is_paged</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether the optimizer is a paged optimizer or not.`,name:"is_paged"}]}),s(2),i(v),i(n);var y=t(n,2);o(y,{title:"AdamW8bit",local:"bitsandbytes.optim.AdamW8bit",headingTag:"h2"});var d=t(y,2),f=a(d);e(f,{name:"class bitsandbytes.optim.AdamW8bit",anchor:"bitsandbytes.optim.AdamW8bit",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1997/bitsandbytes/optim/adamw.py#L62",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.01"},{name:"amsgrad",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"is_paged",val:" = False"}]});var u=t(f,2),E=a(u);e(E,{name:"__init__",anchor:"bitsandbytes.optim.AdamW8bit.__init__",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1997/bitsandbytes/optim/adamw.py#L63",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.01"},{name:"amsgrad",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"is_paged",val:" = False"}],parametersDescription:[{anchor:"bitsandbytes.optim.AdamW8bit.__init__.params",description:`<strong>params</strong> (<code>torch.Tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.AdamW8bit.__init__.lr",description:`<strong>lr</strong> (<code>float</code>, defaults to 1e-3) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.AdamW8bit.__init__.betas",description:`<strong>betas</strong> (<code>tuple(float, float)</code>, defaults to (0.9, 0.999)) &#x2014;
The beta values are the decay rates of the first and second-order moment of the optimizer.`,name:"betas"},{anchor:"bitsandbytes.optim.AdamW8bit.__init__.eps",description:`<strong>eps</strong> (<code>float</code>, defaults to 1e-8) &#x2014;
The epsilon value prevents division by zero in the optimizer.`,name:"eps"},{anchor:"bitsandbytes.optim.AdamW8bit.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.AdamW8bit.__init__.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use the <a href="https://hf.co/papers/1904.09237" rel="nofollow">AMSGrad</a> variant of Adam that uses the maximum of past squared gradients instead.
Note: This parameter is not supported in AdamW8bit and must be False.`,name:"amsgrad"},{anchor:"bitsandbytes.optim.AdamW8bit.__init__.optim_bits",description:`<strong>optim_bits</strong> (<code>int</code>, defaults to 32) &#x2014;
The number of bits of the optimizer state.
Note: This parameter is not used in AdamW8bit as it always uses 8-bit optimization.`,name:"optim_bits"},{anchor:"bitsandbytes.optim.AdamW8bit.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.AdamW8bit.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.AdamW8bit.__init__.is_paged",description:`<strong>is_paged</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether the optimizer is a paged optimizer or not.`,name:"is_paged"}]}),s(2),i(u),i(d);var A=t(d,2);o(A,{title:"AdamW32bit",local:"bitsandbytes.optim.AdamW32bit",headingTag:"h2"});var r=t(A,2),W=a(r);e(W,{name:"class bitsandbytes.optim.AdamW32bit",anchor:"bitsandbytes.optim.AdamW32bit",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1997/bitsandbytes/optim/adamw.py#L126",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.01"},{name:"amsgrad",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"is_paged",val:" = False"}]});var z=t(W,2),H=a(z);e(H,{name:"__init__",anchor:"bitsandbytes.optim.AdamW32bit.__init__",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1997/bitsandbytes/optim/adamw.py#L127",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.01"},{name:"amsgrad",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"is_paged",val:" = False"}],parametersDescription:[{anchor:"bitsandbytes.optim.AdamW32bit.__init__.params",description:`<strong>params</strong> (<code>torch.Tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.AdamW32bit.__init__.lr",description:`<strong>lr</strong> (<code>float</code>, defaults to 1e-3) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.AdamW32bit.__init__.betas",description:`<strong>betas</strong> (<code>tuple(float, float)</code>, defaults to (0.9, 0.999)) &#x2014;
The beta values are the decay rates of the first and second-order moment of the optimizer.`,name:"betas"},{anchor:"bitsandbytes.optim.AdamW32bit.__init__.eps",description:`<strong>eps</strong> (<code>float</code>, defaults to 1e-8) &#x2014;
The epsilon value prevents division by zero in the optimizer.`,name:"eps"},{anchor:"bitsandbytes.optim.AdamW32bit.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.AdamW32bit.__init__.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use the <a href="https://hf.co/papers/1904.09237" rel="nofollow">AMSGrad</a> variant of Adam that uses the maximum of past squared gradients instead.`,name:"amsgrad"},{anchor:"bitsandbytes.optim.AdamW32bit.__init__.optim_bits",description:`<strong>optim_bits</strong> (<code>int</code>, defaults to 32) &#x2014;
The number of bits of the optimizer state.`,name:"optim_bits"},{anchor:"bitsandbytes.optim.AdamW32bit.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.AdamW32bit.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.AdamW32bit.__init__.is_paged",description:`<strong>is_paged</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether the optimizer is a paged optimizer or not.`,name:"is_paged"}]}),s(2),i(z),i(r);var x=t(r,2);o(x,{title:"PagedAdamW",local:"bitsandbytes.optim.PagedAdamW",headingTag:"h2"});var m=t(x,2),w=a(m);e(w,{name:"class bitsandbytes.optim.PagedAdamW",anchor:"bitsandbytes.optim.PagedAdamW",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1997/bitsandbytes/optim/adamw.py#L179",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.01"},{name:"amsgrad",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"}]});var T=t(w,2),B=a(T);e(B,{name:"__init__",anchor:"bitsandbytes.optim.PagedAdamW.__init__",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1997/bitsandbytes/optim/adamw.py#L180",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.01"},{name:"amsgrad",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"}],parametersDescription:[{anchor:"bitsandbytes.optim.PagedAdamW.__init__.params",description:`<strong>params</strong> (<code>torch.Tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.PagedAdamW.__init__.lr",description:`<strong>lr</strong> (<code>float</code>, defaults to 1e-3) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.PagedAdamW.__init__.betas",description:`<strong>betas</strong> (<code>tuple(float, float)</code>, defaults to (0.9, 0.999)) &#x2014;
The beta values are the decay rates of the first and second-order moment of the optimizer.`,name:"betas"},{anchor:"bitsandbytes.optim.PagedAdamW.__init__.eps",description:`<strong>eps</strong> (<code>float</code>, defaults to 1e-8) &#x2014;
The epsilon value prevents division by zero in the optimizer.`,name:"eps"},{anchor:"bitsandbytes.optim.PagedAdamW.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.PagedAdamW.__init__.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use the <a href="https://hf.co/papers/1904.09237" rel="nofollow">AMSGrad</a> variant of Adam that uses the maximum of past squared gradients instead.`,name:"amsgrad"},{anchor:"bitsandbytes.optim.PagedAdamW.__init__.optim_bits",description:`<strong>optim_bits</strong> (<code>int</code>, defaults to 32) &#x2014;
The number of bits of the optimizer state.`,name:"optim_bits"},{anchor:"bitsandbytes.optim.PagedAdamW.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.PagedAdamW.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"}]}),s(2),i(T),i(m);var P=t(m,2);o(P,{title:"PagedAdamW8bit",local:"bitsandbytes.optim.PagedAdamW8bit",headingTag:"h2"});var b=t(P,2),F=a(b);e(F,{name:"class bitsandbytes.optim.PagedAdamW8bit",anchor:"bitsandbytes.optim.PagedAdamW8bit",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1997/bitsandbytes/optim/adamw.py#L229",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.01"},{name:"amsgrad",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"}]});var N=t(F,2),O=a(N);e(O,{name:"__init__",anchor:"bitsandbytes.optim.PagedAdamW8bit.__init__",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1997/bitsandbytes/optim/adamw.py#L230",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.01"},{name:"amsgrad",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"}],parametersDescription:[{anchor:"bitsandbytes.optim.PagedAdamW8bit.__init__.params",description:`<strong>params</strong> (<code>torch.Tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.PagedAdamW8bit.__init__.lr",description:`<strong>lr</strong> (<code>float</code>, defaults to 1e-3) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.PagedAdamW8bit.__init__.betas",description:`<strong>betas</strong> (<code>tuple(float, float)</code>, defaults to (0.9, 0.999)) &#x2014;
The beta values are the decay rates of the first and second-order moment of the optimizer.`,name:"betas"},{anchor:"bitsandbytes.optim.PagedAdamW8bit.__init__.eps",description:`<strong>eps</strong> (<code>float</code>, defaults to 1e-8) &#x2014;
The epsilon value prevents division by zero in the optimizer.`,name:"eps"},{anchor:"bitsandbytes.optim.PagedAdamW8bit.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.PagedAdamW8bit.__init__.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use the <a href="https://hf.co/papers/1904.09237" rel="nofollow">AMSGrad</a> variant of Adam that uses the maximum of past squared gradients instead.
Note: This parameter is not supported in PagedAdamW8bit and must be False.`,name:"amsgrad"},{anchor:"bitsandbytes.optim.PagedAdamW8bit.__init__.optim_bits",description:`<strong>optim_bits</strong> (<code>int</code>, defaults to 32) &#x2014;
The number of bits of the optimizer state.
Note: This parameter is not used in PagedAdamW8bit as it always uses 8-bit optimization.`,name:"optim_bits"},{anchor:"bitsandbytes.optim.PagedAdamW8bit.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.PagedAdamW8bit.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"}]}),s(2),i(N),i(b);var L=t(b,2);o(L,{title:"PagedAdamW32bit",local:"bitsandbytes.optim.PagedAdamW32bit",headingTag:"h2"});var p=t(L,2),j=a(p);e(j,{name:"class bitsandbytes.optim.PagedAdamW32bit",anchor:"bitsandbytes.optim.PagedAdamW32bit",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1997/bitsandbytes/optim/adamw.py#L290",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.01"},{name:"amsgrad",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"}]});var D=t(j,2),R=a(D);e(R,{name:"__init__",anchor:"bitsandbytes.optim.PagedAdamW32bit.__init__",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1997/bitsandbytes/optim/adamw.py#L291",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.01"},{name:"amsgrad",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"}],parametersDescription:[{anchor:"bitsandbytes.optim.PagedAdamW32bit.__init__.params",description:`<strong>params</strong> (<code>torch.Tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.PagedAdamW32bit.__init__.lr",description:`<strong>lr</strong> (<code>float</code>, defaults to 1e-3) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.PagedAdamW32bit.__init__.betas",description:`<strong>betas</strong> (<code>tuple(float, float)</code>, defaults to (0.9, 0.999)) &#x2014;
The beta values are the decay rates of the first and second-order moment of the optimizer.`,name:"betas"},{anchor:"bitsandbytes.optim.PagedAdamW32bit.__init__.eps",description:`<strong>eps</strong> (<code>float</code>, defaults to 1e-8) &#x2014;
The epsilon value prevents division by zero in the optimizer.`,name:"eps"},{anchor:"bitsandbytes.optim.PagedAdamW32bit.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.PagedAdamW32bit.__init__.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use the <a href="https://hf.co/papers/1904.09237" rel="nofollow">AMSGrad</a> variant of Adam that uses the maximum of past squared gradients instead.`,name:"amsgrad"},{anchor:"bitsandbytes.optim.PagedAdamW32bit.__init__.optim_bits",description:`<strong>optim_bits</strong> (<code>int</code>, defaults to 32) &#x2014;
The number of bits of the optimizer state.`,name:"optim_bits"},{anchor:"bitsandbytes.optim.PagedAdamW32bit.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.PagedAdamW32bit.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"}]}),s(2),i(D),i(p);var I=t(p,2);V(I,{source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/docs/source/reference/optim/adamw.mdx"}),s(2),S(U,l),tt()}export{dt as component};