bitsandbytes/pr_1876/en/_app/immutable/nodes/19.f34b60e0.js

import{s as mt,n as lt,o as pt}from"../chunks/scheduler.8a2cc2fa.js";import{S as bt,i as ct,e as r,s as i,c as m,h as _t,a as d,d as t,b as n,f as g,g as l,j as C,k as h,l as s,m as o,n as p,t as b,o as c,p as _}from"../chunks/index.7079e750.js";import{C as gt,H as te,E as ht}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.15a81876.js";import{D as f}from"../chunks/Docstring.89e69224.js";function ft(et){let u,le,de,pe,D,be,N,ce,E,tt='<a href="https://hf.co/papers/1711.05101" rel="nofollow">AdamW</a> is a variant of the <code>Adam</code> optimizer that separates weight decay from the gradient update based on the observation that the weight decay formulation is different when applied to <code>SGD</code> and <code>Adam</code>.',_e,L,at="bitsandbytes also supports paged optimizers which take advantage of CUDAs unified memory to transfer memory from the GPU to the CPU when GPU memory is exhausted.",ge,q,he,v,j,Ge,x,I,Ue,ae,it="Base AdamW optimizer.",fe,M,ue,y,V,He,z,S,Be,ie,nt="8-bit AdamW optimizer.",ve,G,ye,$,U,Oe,T,H,Re,ne,ot="32-bit AdamW optimizer.",$e,B,Ae,A,O,Je,P,R,Ke,oe,st="Paged AdamW optimizer.",We,J,we,W,K,Qe,k,Q,Xe,se,rt="Paged 8-bit AdamW optimizer.",xe,X,ze,w,Y,Ye,F,Z,Ze,re,dt="Paged 32-bit AdamW optimizer.",Te,ee,Pe,me,ke;return D=new gt({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),N=new te({props:{title:"AdamW",local:"adamw",headingTag:"h1"}}),q=new te({props:{title:"AdamW",local:"api-class ][ bitsandbytes.optim.AdamW",headingTag:"h2"}}),j=new f({props:{name:"class bitsandbytes.optim.AdamW",anchor:"bitsandbytes.optim.AdamW",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.01"},{name:"amsgrad",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"percentile_clipping",val:" = 100"},{name:"block_wise",val:" = True"},{name:"is_paged",val:" = False"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1876/bitsandbytes/optim/adamw.py#L9"}}),I=new f({props:{name:"__init__",anchor:"bitsandbytes.optim.AdamW.__init__",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.01"},{name:"amsgrad",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"percentile_clipping",val:" = 100"},{name:"block_wise",val:" = True"},{name:"is_paged",val:" = False"}],parametersDescription:[{anchor:"bitsandbytes.optim.AdamW.__init__.params",description:`<strong>params</strong> (<code>torch.Tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.AdamW.__init__.lr",description:`<strong>lr</strong> (<code>float</code>, defaults to 1e-3) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.AdamW.__init__.betas",description:`<strong>betas</strong> (<code>tuple(float, float)</code>, defaults to (0.9, 0.999)) &#x2014;
The beta values are the decay rates of the first and second-order moment of the optimizer.`,name:"betas"},{anchor:"bitsandbytes.optim.AdamW.__init__.eps",description:`<strong>eps</strong> (<code>float</code>, defaults to 1e-8) &#x2014;
The epsilon value prevents division by zero in the optimizer.`,name:"eps"},{anchor:"bitsandbytes.optim.AdamW.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.AdamW.__init__.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use the <a href="https://hf.co/papers/1904.09237" rel="nofollow">AMSGrad</a> variant of Adam that uses the maximum of past squared gradients instead.`,name:"amsgrad"},{anchor:"bitsandbytes.optim.AdamW.__init__.optim_bits",description:`<strong>optim_bits</strong> (<code>int</code>, defaults to 32) &#x2014;
The number of bits of the optimizer state.`,name:"optim_bits"},{anchor:"bitsandbytes.optim.AdamW.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.AdamW.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.AdamW.__init__.percentile_clipping",description:`<strong>percentile_clipping</strong> (<code>int</code>, defaults to 100) &#x2014;
Adapts clipping threshold automatically by tracking the last 100 gradient norms and clipping the gradient at a certain percentile to improve stability.`,name:"percentile_clipping"},{anchor:"bitsandbytes.optim.AdamW.__init__.block_wise",description:`<strong>block_wise</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to independently quantize each block of tensors to reduce outlier effects and improve stability.`,name:"block_wise"},{anchor:"bitsandbytes.optim.AdamW.__init__.is_paged",description:`<strong>is_paged</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether the optimizer is a paged optimizer or not.`,name:"is_paged"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1876/bitsandbytes/optim/adamw.py#L10"}}),M=new te({props:{title:"AdamW8bit",local:"bitsandbytes.optim.AdamW8bit",headingTag:"h2"}}),V=new f({props:{name:"class bitsandbytes.optim.AdamW8bit",anchor:"bitsandbytes.optim.AdamW8bit",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.01"},{name:"amsgrad",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"percentile_clipping",val:" = 100"},{name:"block_wise",val:" = True"},{name:"is_paged",val:" = False"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1876/bitsandbytes/optim/adamw.py#L70"}}),S=new f({props:{name:"__init__",anchor:"bitsandbytes.optim.AdamW8bit.__init__",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.01"},{name:"amsgrad",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"percentile_clipping",val:" = 100"},{name:"block_wise",val:" = True"},{name:"is_paged",val:" = False"}],parametersDescription:[{anchor:"bitsandbytes.optim.AdamW8bit.__init__.params",description:`<strong>params</strong> (<code>torch.Tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.AdamW8bit.__init__.lr",description:`<strong>lr</strong> (<code>float</code>, defaults to 1e-3) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.AdamW8bit.__init__.betas",description:`<strong>betas</strong> (<code>tuple(float, float)</code>, defaults to (0.9, 0.999)) &#x2014;
The beta values are the decay rates of the first and second-order moment of the optimizer.`,name:"betas"},{anchor:"bitsandbytes.optim.AdamW8bit.__init__.eps",description:`<strong>eps</strong> (<code>float</code>, defaults to 1e-8) &#x2014;
The epsilon value prevents division by zero in the optimizer.`,name:"eps"},{anchor:"bitsandbytes.optim.AdamW8bit.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.AdamW8bit.__init__.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use the <a href="https://hf.co/papers/1904.09237" rel="nofollow">AMSGrad</a> variant of Adam that uses the maximum of past squared gradients instead.
Note: This parameter is not supported in AdamW8bit and must be False.`,name:"amsgrad"},{anchor:"bitsandbytes.optim.AdamW8bit.__init__.optim_bits",description:`<strong>optim_bits</strong> (<code>int</code>, defaults to 32) &#x2014;
The number of bits of the optimizer state.
Note: This parameter is not used in AdamW8bit as it always uses 8-bit optimization.`,name:"optim_bits"},{anchor:"bitsandbytes.optim.AdamW8bit.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.AdamW8bit.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.AdamW8bit.__init__.percentile_clipping",description:`<strong>percentile_clipping</strong> (<code>int</code>, defaults to 100) &#x2014;
Adapts clipping threshold automatically by tracking the last 100 gradient norms and clipping the gradient at a certain percentile to improve stability.`,name:"percentile_clipping"},{anchor:"bitsandbytes.optim.AdamW8bit.__init__.block_wise",description:`<strong>block_wise</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to independently quantize each block of tensors to reduce outlier effects and improve stability.`,name:"block_wise"},{anchor:"bitsandbytes.optim.AdamW8bit.__init__.is_paged",description:`<strong>is_paged</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether the optimizer is a paged optimizer or not.`,name:"is_paged"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1876/bitsandbytes/optim/adamw.py#L71"}}),G=new te({props:{title:"AdamW32bit",local:"bitsandbytes.optim.AdamW32bit",headingTag:"h2"}}),U=new f({props:{name:"class bitsandbytes.optim.AdamW32bit",anchor:"bitsandbytes.optim.AdamW32bit",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.01"},{name:"amsgrad",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"percentile_clipping",val:" = 100"},{name:"block_wise",val:" = True"},{name:"is_paged",val:" = False"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1876/bitsandbytes/optim/adamw.py#L142"}}),H=new f({props:{name:"__init__",anchor:"bitsandbytes.optim.AdamW32bit.__init__",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.01"},{name:"amsgrad",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"percentile_clipping",val:" = 100"},{name:"block_wise",val:" = True"},{name:"is_paged",val:" = False"}],parametersDescription:[{anchor:"bitsandbytes.optim.AdamW32bit.__init__.params",description:`<strong>params</strong> (<code>torch.Tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.AdamW32bit.__init__.lr",description:`<strong>lr</strong> (<code>float</code>, defaults to 1e-3) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.AdamW32bit.__init__.betas",description:`<strong>betas</strong> (<code>tuple(float, float)</code>, defaults to (0.9, 0.999)) &#x2014;
The beta values are the decay rates of the first and second-order moment of the optimizer.`,name:"betas"},{anchor:"bitsandbytes.optim.AdamW32bit.__init__.eps",description:`<strong>eps</strong> (<code>float</code>, defaults to 1e-8) &#x2014;
The epsilon value prevents division by zero in the optimizer.`,name:"eps"},{anchor:"bitsandbytes.optim.AdamW32bit.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.AdamW32bit.__init__.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use the <a href="https://hf.co/papers/1904.09237" rel="nofollow">AMSGrad</a> variant of Adam that uses the maximum of past squared gradients instead.`,name:"amsgrad"},{anchor:"bitsandbytes.optim.AdamW32bit.__init__.optim_bits",description:`<strong>optim_bits</strong> (<code>int</code>, defaults to 32) &#x2014;
The number of bits of the optimizer state.`,name:"optim_bits"},{anchor:"bitsandbytes.optim.AdamW32bit.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.AdamW32bit.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.AdamW32bit.__init__.percentile_clipping",description:`<strong>percentile_clipping</strong> (<code>int</code>, defaults to 100) &#x2014;
Adapts clipping threshold automatically by tracking the last 100 gradient norms and clipping the gradient at a certain percentile to improve stability.`,name:"percentile_clipping"},{anchor:"bitsandbytes.optim.AdamW32bit.__init__.block_wise",description:`<strong>block_wise</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to independently quantize each block of tensors to reduce outlier effects and improve stability.`,name:"block_wise"},{anchor:"bitsandbytes.optim.AdamW32bit.__init__.is_paged",description:`<strong>is_paged</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether the optimizer is a paged optimizer or not.`,name:"is_paged"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1876/bitsandbytes/optim/adamw.py#L143"}}),B=new te({props:{title:"PagedAdamW",local:"bitsandbytes.optim.PagedAdamW",headingTag:"h2"}}),O=new f({props:{name:"class bitsandbytes.optim.PagedAdamW",anchor:"bitsandbytes.optim.PagedAdamW",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.01"},{name:"amsgrad",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"percentile_clipping",val:" = 100"},{name:"block_wise",val:" = True"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1876/bitsandbytes/optim/adamw.py#L203"}}),R=new f({props:{name:"__init__",anchor:"bitsandbytes.optim.PagedAdamW.__init__",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.01"},{name:"amsgrad",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"percentile_clipping",val:" = 100"},{name:"block_wise",val:" = True"}],parametersDescription:[{anchor:"bitsandbytes.optim.PagedAdamW.__init__.params",description:`<strong>params</strong> (<code>torch.Tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.PagedAdamW.__init__.lr",description:`<strong>lr</strong> (<code>float</code>, defaults to 1e-3) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.PagedAdamW.__init__.betas",description:`<strong>betas</strong> (<code>tuple(float, float)</code>, defaults to (0.9, 0.999)) &#x2014;
The beta values are the decay rates of the first and second-order moment of the optimizer.`,name:"betas"},{anchor:"bitsandbytes.optim.PagedAdamW.__init__.eps",description:`<strong>eps</strong> (<code>float</code>, defaults to 1e-8) &#x2014;
The epsilon value prevents division by zero in the optimizer.`,name:"eps"},{anchor:"bitsandbytes.optim.PagedAdamW.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.PagedAdamW.__init__.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use the <a href="https://hf.co/papers/1904.09237" rel="nofollow">AMSGrad</a> variant of Adam that uses the maximum of past squared gradients instead.`,name:"amsgrad"},{anchor:"bitsandbytes.optim.PagedAdamW.__init__.optim_bits",description:`<strong>optim_bits</strong> (<code>int</code>, defaults to 32) &#x2014;
The number of bits of the optimizer state.`,name:"optim_bits"},{anchor:"bitsandbytes.optim.PagedAdamW.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.PagedAdamW.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.PagedAdamW.__init__.percentile_clipping",description:`<strong>percentile_clipping</strong> (<code>int</code>, defaults to 100) &#x2014;
Adapts clipping threshold automatically by tracking the last 100 gradient norms and clipping the gradient at a certain percentile to improve stability.`,name:"percentile_clipping"},{anchor:"bitsandbytes.optim.PagedAdamW.__init__.block_wise",description:`<strong>block_wise</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to independently quantize each block of tensors to reduce outlier effects and improve stability.`,name:"block_wise"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1876/bitsandbytes/optim/adamw.py#L204"}}),J=new te({props:{title:"PagedAdamW8bit",local:"bitsandbytes.optim.PagedAdamW8bit",headingTag:"h2"}}),K=new f({props:{name:"class bitsandbytes.optim.PagedAdamW8bit",anchor:"bitsandbytes.optim.PagedAdamW8bit",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.01"},{name:"amsgrad",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"percentile_clipping",val:" = 100"},{name:"block_wise",val:" = True"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1876/bitsandbytes/optim/adamw.py#L261"}}),Q=new f({props:{name:"__init__",anchor:"bitsandbytes.optim.PagedAdamW8bit.__init__",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.01"},{name:"amsgrad",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"percentile_clipping",val:" = 100"},{name:"block_wise",val:" = True"}],parametersDescription:[{anchor:"bitsandbytes.optim.PagedAdamW8bit.__init__.params",description:`<strong>params</strong> (<code>torch.Tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.PagedAdamW8bit.__init__.lr",description:`<strong>lr</strong> (<code>float</code>, defaults to 1e-3) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.PagedAdamW8bit.__init__.betas",description:`<strong>betas</strong> (<code>tuple(float, float)</code>, defaults to (0.9, 0.999)) &#x2014;
The beta values are the decay rates of the first and second-order moment of the optimizer.`,name:"betas"},{anchor:"bitsandbytes.optim.PagedAdamW8bit.__init__.eps",description:`<strong>eps</strong> (<code>float</code>, defaults to 1e-8) &#x2014;
The epsilon value prevents division by zero in the optimizer.`,name:"eps"},{anchor:"bitsandbytes.optim.PagedAdamW8bit.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.PagedAdamW8bit.__init__.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use the <a href="https://hf.co/papers/1904.09237" rel="nofollow">AMSGrad</a> variant of Adam that uses the maximum of past squared gradients instead.
Note: This parameter is not supported in PagedAdamW8bit and must be False.`,name:"amsgrad"},{anchor:"bitsandbytes.optim.PagedAdamW8bit.__init__.optim_bits",description:`<strong>optim_bits</strong> (<code>int</code>, defaults to 32) &#x2014;
The number of bits of the optimizer state.
Note: This parameter is not used in PagedAdamW8bit as it always uses 8-bit optimization.`,name:"optim_bits"},{anchor:"bitsandbytes.optim.PagedAdamW8bit.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.PagedAdamW8bit.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.PagedAdamW8bit.__init__.percentile_clipping",description:`<strong>percentile_clipping</strong> (<code>int</code>, defaults to 100) &#x2014;
Adapts clipping threshold automatically by tracking the last 100 gradient norms and clipping the gradient at a certain percentile to improve stability.`,name:"percentile_clipping"},{anchor:"bitsandbytes.optim.PagedAdamW8bit.__init__.block_wise",description:`<strong>block_wise</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to independently quantize each block of tensors to reduce outlier effects and improve stability.`,name:"block_wise"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1876/bitsandbytes/optim/adamw.py#L262"}}),X=new te({props:{title:"PagedAdamW32bit",local:"bitsandbytes.optim.PagedAdamW32bit",headingTag:"h2"}}),Y=new f({props:{name:"class bitsandbytes.optim.PagedAdamW32bit",anchor:"bitsandbytes.optim.PagedAdamW32bit",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.01"},{name:"amsgrad",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"percentile_clipping",val:" = 100"},{name:"block_wise",val:" = True"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1876/bitsandbytes/optim/adamw.py#L330"}}),Z=new f({props:{name:"__init__",anchor:"bitsandbytes.optim.PagedAdamW32bit.__init__",parameters:[{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.01"},{name:"amsgrad",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"percentile_clipping",val:" = 100"},{name:"block_wise",val:" = True"}],parametersDescription:[{anchor:"bitsandbytes.optim.PagedAdamW32bit.__init__.params",description:`<strong>params</strong> (<code>torch.Tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.PagedAdamW32bit.__init__.lr",description:`<strong>lr</strong> (<code>float</code>, defaults to 1e-3) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.PagedAdamW32bit.__init__.betas",description:`<strong>betas</strong> (<code>tuple(float, float)</code>, defaults to (0.9, 0.999)) &#x2014;
The beta values are the decay rates of the first and second-order moment of the optimizer.`,name:"betas"},{anchor:"bitsandbytes.optim.PagedAdamW32bit.__init__.eps",description:`<strong>eps</strong> (<code>float</code>, defaults to 1e-8) &#x2014;
The epsilon value prevents division by zero in the optimizer.`,name:"eps"},{anchor:"bitsandbytes.optim.PagedAdamW32bit.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.PagedAdamW32bit.__init__.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use the <a href="https://hf.co/papers/1904.09237" rel="nofollow">AMSGrad</a> variant of Adam that uses the maximum of past squared gradients instead.`,name:"amsgrad"},{anchor:"bitsandbytes.optim.PagedAdamW32bit.__init__.optim_bits",description:`<strong>optim_bits</strong> (<code>int</code>, defaults to 32) &#x2014;
The number of bits of the optimizer state.`,name:"optim_bits"},{anchor:"bitsandbytes.optim.PagedAdamW32bit.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.PagedAdamW32bit.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.PagedAdamW32bit.__init__.percentile_clipping",description:`<strong>percentile_clipping</strong> (<code>int</code>, defaults to 100) &#x2014;
Adapts clipping threshold automatically by tracking the last 100 gradient norms and clipping the gradient at a certain percentile to improve stability.`,name:"percentile_clipping"},{anchor:"bitsandbytes.optim.PagedAdamW32bit.__init__.block_wise",description:`<strong>block_wise</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether to independently quantize each block of tensors to reduce outlier effects and improve stability.`,name:"block_wise"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1876/bitsandbytes/optim/adamw.py#L331"}}),ee=new ht({props:{source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/docs/source/reference/optim/adamw.mdx"}}),{c(){u=r("meta"),le=i(),de=r("p"),pe=i(),m(D.$$.fragment),be=i(),m(N.$$.fragment),ce=i(),E=r("p"),E.innerHTML=tt,_e=i(),L=r("p"),L.textContent=at,ge=i(),m(q.$$.fragment),he=i(),v=r("div"),m(j.$$.fragment),Ge=i(),x=r("div"),m(I.$$.fragment),Ue=i(),ae=r("p"),ae.textContent=it,fe=i(),m(M.$$.fragment),ue=i(),y=r("div"),m(V.$$.fragment),He=i(),z=r("div"),m(S.$$.fragment),Be=i(),ie=r("p"),ie.textContent=nt,ve=i(),m(G.$$.fragment),ye=i(),$=r("div"),m(U.$$.fragment),Oe=i(),T=r("div"),m(H.$$.fragment),Re=i(),ne=r("p"),ne.textContent=ot,$e=i(),m(B.$$.fragment),Ae=i(),A=r("div"),m(O.$$.fragment),Je=i(),P=r("div"),m(R.$$.fragment),Ke=i(),oe=r("p"),oe.textContent=st,We=i(),m(J.$$.fragment),we=i(),W=r("div"),m(K.$$.fragment),Qe=i(),k=r("div"),m(Q.$$.fragment),Xe=i(),se=r("p"),se.textContent=rt,xe=i(),m(X.$$.fragment),ze=i(),w=r("div"),m(Y.$$.fragment),Ye=i(),F=r("div"),m(Z.$$.fragment),Ze=i(),re=r("p"),re.textContent=dt,Te=i(),m(ee.$$.fragment),Pe=i(),me=r("p"),this.h()},l(e){const a=_t("svelte-u9bgzb",document.head);u=d(a,"META",{name:!0,content:!0}),a.forEach(t),le=n(e),de=d(e,"P",{}),g(de).forEach(t),pe=n(e),l(D.$$.fragment,e),be=n(e),l(N.$$.fragment,e),ce=n(e),E=d(e,"P",{"data-svelte-h":!0}),C(E)!=="svelte-zypq46"&&(E.innerHTML=tt),_e=n(e),L=d(e,"P",{"data-svelte-h":!0}),C(L)!=="svelte-qpasov"&&(L.textContent=at),ge=n(e),l(q.$$.fragment,e),he=n(e),v=d(e,"DIV",{class:!0});var Fe=g(v);l(j.$$.fragment,Fe),Ge=n(Fe),x=d(Fe,"DIV",{class:!0});var Ce=g(x);l(I.$$.fragment,Ce),Ue=n(Ce),ae=d(Ce,"P",{"data-svelte-h":!0}),C(ae)!=="svelte-cz2sic"&&(ae.textContent=it),Ce.forEach(t),Fe.forEach(t),fe=n(e),l(M.$$.fragment,e),ue=n(e),y=d(e,"DIV",{class:!0});var De=g(y);l(V.$$.fragment,De),He=n(De),z=d(De,"DIV",{class:!0});var Ne=g(z);l(S.$$.fragment,Ne),Be=n(Ne),ie=d(Ne,"P",{"data-svelte-h":!0}),C(ie)!=="svelte-1jiw1x7"&&(ie.textContent=nt),Ne.forEach(t),De.forEach(t),ve=n(e),l(G.$$.fragment,e),ye=n(e),$=d(e,"DIV",{class:!0});var Ee=g($);l(U.$$.fragment,Ee),Oe=n(Ee),T=d(Ee,"DIV",{class:!0});var Le=g(T);l(H.$$.fragment,Le),Re=n(Le),ne=d(Le,"P",{"data-svelte-h":!0}),C(ne)!=="svelte-gm8nxe"&&(ne.textContent=ot),Le.forEach(t),Ee.forEach(t),$e=n(e),l(B.$$.fragment,e),Ae=n(e),A=d(e,"DIV",{class:!0});var qe=g(A);l(O.$$.fragment,qe),Je=n(qe),P=d(qe,"DIV",{class:!0});var je=g(P);l(R.$$.fragment,je),Ke=n(je),oe=d(je,"P",{"data-svelte-h":!0}),C(oe)!=="svelte-we884g"&&(oe.textContent=st),je.forEach(t),qe.forEach(t),We=n(e),l(J.$$.fragment,e),we=n(e),W=d(e,"DIV",{class:!0});var Ie=g(W);l(K.$$.fragment,Ie),Qe=n(Ie),k=d(Ie,"DIV",{class:!0});var Me=g(k);l(Q.$$.fragment,Me),Xe=n(Me),se=d(Me,"P",{"data-svelte-h":!0}),C(se)!=="svelte-y5nwi"&&(se.textContent=rt),Me.forEach(t),Ie.forEach(t),xe=n(e),l(X.$$.fragment,e),ze=n(e),w=d(e,"DIV",{class:!0});var Ve=g(w);l(Y.$$.fragment,Ve),Ye=n(Ve),F=d(Ve,"DIV",{class:!0});var Se=g(F);l(Z.$$.fragment,Se),Ze=n(Se),re=d(Se,"P",{"data-svelte-h":!0}),C(re)!=="svelte-10llmb"&&(re.textContent=dt),Se.forEach(t),Ve.forEach(t),Te=n(e),l(ee.$$.fragment,e),Pe=n(e),me=d(e,"P",{}),g(me).forEach(t),this.h()},h(){h(u,"name","hf:doc:metadata"),h(u,"content",ut),h(x,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(v,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(T,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(P,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(A,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(k,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(F,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(w,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,a){s(document.head,u),o(e,le,a),o(e,de,a),o(e,pe,a),p(D,e,a),o(e,be,a),p(N,e,a),o(e,ce,a),o(e,E,a),o(e,_e,a),o(e,L,a),o(e,ge,a),p(q,e,a),o(e,he,a),o(e,v,a),p(j,v,null),s(v,Ge),s(v,x),p(I,x,null),s(x,Ue),s(x,ae),o(e,fe,a),p(M,e,a),o(e,ue,a),o(e,y,a),p(V,y,null),s(y,He),s(y,z),p(S,z,null),s(z,Be),s(z,ie),o(e,ve,a),p(G,e,a),o(e,ye,a),o(e,$,a),p(U,$,null),s($,Oe),s($,T),p(H,T,null),s(T,Re),s(T,ne),o(e,$e,a),p(B,e,a),o(e,Ae,a),o(e,A,a),p(O,A,null),s(A,Je),s(A,P),p(R,P,null),s(P,Ke),s(P,oe),o(e,We,a),p(J,e,a),o(e,we,a),o(e,W,a),p(K,W,null),s(W,Qe),s(W,k),p(Q,k,null),s(k,Xe),s(k,se),o(e,xe,a),p(X,e,a),o(e,ze,a),o(e,w,a),p(Y,w,null),s(w,Ye),s(w,F),p(Z,F,null),s(F,Ze),s(F,re),o(e,Te,a),p(ee,e,a),o(e,Pe,a),o(e,me,a),ke=!0},p:lt,i(e){ke||(b(D.$$.fragment,e),b(N.$$.fragment,e),b(q.$$.fragment,e),b(j.$$.fragment,e),b(I.$$.fragment,e),b(M.$$.fragment,e),b(V.$$.fragment,e),b(S.$$.fragment,e),b(G.$$.fragment,e),b(U.$$.fragment,e),b(H.$$.fragment,e),b(B.$$.fragment,e),b(O.$$.fragment,e),b(R.$$.fragment,e),b(J.$$.fragment,e),b(K.$$.fragment,e),b(Q.$$.fragment,e),b(X.$$.fragment,e),b(Y.$$.fragment,e),b(Z.$$.fragment,e),b(ee.$$.fragment,e),ke=!0)},o(e){c(D.$$.fragment,e),c(N.$$.fragment,e),c(q.$$.fragment,e),c(j.$$.fragment,e),c(I.$$.fragment,e),c(M.$$.fragment,e),c(V.$$.fragment,e),c(S.$$.fragment,e),c(G.$$.fragment,e),c(U.$$.fragment,e),c(H.$$.fragment,e),c(B.$$.fragment,e),c(O.$$.fragment,e),c(R.$$.fragment,e),c(J.$$.fragment,e),c(K.$$.fragment,e),c(Q.$$.fragment,e),c(X.$$.fragment,e),c(Y.$$.fragment,e),c(Z.$$.fragment,e),c(ee.$$.fragment,e),ke=!1},d(e){e&&(t(le),t(de),t(pe),t(be),t(ce),t(E),t(_e),t(L),t(ge),t(he),t(v),t(fe),t(ue),t(y),t(ve),t(ye),t($),t($e),t(Ae),t(A),t(We),t(we),t(W),t(xe),t(ze),t(w),t(Te),t(Pe),t(me)),t(u),_(D,e),_(N,e),_(q,e),_(j),_(I),_(M,e),_(V),_(S),_(G,e),_(U),_(H),_(B,e),_(O),_(R),_(J,e),_(K),_(Q),_(X,e),_(Y),_(Z),_(ee,e)}}}const ut='{"title":"AdamW","local":"adamw","sections":[{"title":"AdamW","local":"api-class ][ bitsandbytes.optim.AdamW","sections":[],"depth":2},{"title":"AdamW8bit","local":"bitsandbytes.optim.AdamW8bit","sections":[],"depth":2},{"title":"AdamW32bit","local":"bitsandbytes.optim.AdamW32bit","sections":[],"depth":2},{"title":"PagedAdamW","local":"bitsandbytes.optim.PagedAdamW","sections":[],"depth":2},{"title":"PagedAdamW8bit","local":"bitsandbytes.optim.PagedAdamW8bit","sections":[],"depth":2},{"title":"PagedAdamW32bit","local":"bitsandbytes.optim.PagedAdamW32bit","sections":[],"depth":2}],"depth":1}';function vt(et){return pt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class wt extends bt{constructor(u){super(),ct(this,u,vt,ft,mt,{})}}export{wt as component};