bitsandbytes/pr_2002/en/_app/immutable/nodes/13.2EdWHKHh.js

import"../chunks/DsnmJJEf.js";import{i as R,h as A,C,H as o,D as t,E as I,s as L,a as Y}from"../chunks/DT0OpeMJ.js";import{p as V,o as H,s as e,f as j,a as b,b as Q,c as i,d as c,n,r as a}from"../chunks/Bb-LL0eD.js";import{E as D}from"../chunks/-HcYSbRW.js";const K='{"title":"Overview","local":"overview","sections":[{"title":"Optimizer8bit","local":"bitsandbytes.optim.optimizer.Optimizer8bit","sections":[],"depth":2},{"title":"Optimizer2State","local":"bitsandbytes.optim.optimizer.Optimizer2State","sections":[],"depth":2},{"title":"Optimizer1State","local":"bitsandbytes.optim.optimizer.Optimizer1State","sections":[],"depth":2},{"title":"Utilities","local":"bitsandbytes.optim.GlobalOptimManager","sections":[],"depth":2}],"depth":1}';var P=c('<meta name="hf:doc:metadata"/>'),$=c("<p>Example:</p> <!>",1),q=c(`<p></p> <!> <!> <p><a href="https://hf.co/papers/2110.02861" rel="nofollow">8-bit optimizers</a> reduce the memory footprint of 32-bit optimizers without any performance degradation which means you can train large models with many parameters faster. At the core of 8-bit optimizers is block-wise quantization which enables quantization accuracy, computational efficiency, and stability.</p> <p>bitsandbytes provides 8-bit optimizers through the base <code>Optimizer8bit</code> class, and additionally provides <code>Optimizer2State</code> and <code>Optimizer1State</code> for 2-state (for example, <code>Adam</code>) and 1-state (for example, <code>Adagrad</code>) optimizers respectively. To provide custom optimizer hyperparameters, use the <code>GlobalOptimManager</code> class to configure the optimizer.</p> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Base 8-bit optimizer class.</p></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Base 2-state update optimizer class.</p></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Base 1-state update optimizer class.</p></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>A global optimizer manager for enabling custom optimizer configs.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Override initial optimizer config with specific hyperparameters.</p> <p>The key-values of the optimizer config for the input parameters are overridden
This can be both, optimizer parameters like <code>betas</code> or <code>lr</code>, or it can be
8-bit specific parameters like <code>optim_bits</code>.</p> <!></div></div> <!> <p></p>`,1);function oe(W,U){V(U,!1),H(()=>{new URLSearchParams(window.location.search).get("fw")}),R();var _=q();A("14gbhh6",r=>{var l=P();L(l,"content",K),b(r,l)});var h=e(j(_),2);C(h,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var g=e(h,2);o(g,{title:"Overview",local:"overview",headingTag:"h1"});var z=e(g,6);o(z,{title:"Optimizer8bit",local:"bitsandbytes.optim.optimizer.Optimizer8bit",headingTag:"h2"});var s=e(z,2),v=i(s);t(v,{name:"class bitsandbytes.optim.optimizer.Optimizer8bit",anchor:"bitsandbytes.optim.optimizer.Optimizer8bit",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2002/bitsandbytes/optim/optimizer.py#L117",parameters:[{name:"params",val:""},{name:"defaults",val:""},{name:"optim_bits",val:" = 32"},{name:"is_paged",val:" = False"}]});var y=e(v,2),Z=i(y);t(Z,{name:"__init__",anchor:"bitsandbytes.optim.optimizer.Optimizer8bit.__init__",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2002/bitsandbytes/optim/optimizer.py#L120",parameters:[{name:"params",val:""},{name:"defaults",val:""},{name:"optim_bits",val:" = 32"},{name:"is_paged",val:" = False"}],parametersDescription:[{anchor:"bitsandbytes.optim.optimizer.Optimizer8bit.__init__.params",description:`<strong>params</strong> (<code>torch.Tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.optimizer.Optimizer8bit.__init__.optim_bits",description:`<strong>optim_bits</strong> (<code>int</code>, defaults to 32) &#x2014;
The number of bits of the optimizer state.`,name:"optim_bits"},{anchor:"bitsandbytes.optim.optimizer.Optimizer8bit.__init__.is_paged",description:`<strong>is_paged</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether the optimizer is a paged optimizer or not.`,name:"is_paged"}]}),n(2),a(y),a(s);var u=e(s,2);o(u,{title:"Optimizer2State",local:"bitsandbytes.optim.optimizer.Optimizer2State",headingTag:"h2"});var m=e(u,2),f=i(m);t(f,{name:"class bitsandbytes.optim.optimizer.Optimizer2State",anchor:"bitsandbytes.optim.optimizer.Optimizer2State",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2002/bitsandbytes/optim/optimizer.py#L403",parameters:[{name:"optimizer_name",val:""},{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.0"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 0.0"},{name:"skip_zeros",val:" = False"},{name:"is_paged",val:" = False"},{name:"alpha",val:" = 0.0"},{name:"t_alpha",val:": typing.Optional[int] = None"},{name:"t_beta3",val:": typing.Optional[int] = None"}]});var O=e(f,2),B=i(O);t(B,{name:"__init__",anchor:"bitsandbytes.optim.optimizer.Optimizer2State.__init__",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2002/bitsandbytes/optim/optimizer.py#L404",parameters:[{name:"optimizer_name",val:""},{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.999)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.0"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 0.0"},{name:"skip_zeros",val:" = False"},{name:"is_paged",val:" = False"},{name:"alpha",val:" = 0.0"},{name:"t_alpha",val:": typing.Optional[int] = None"},{name:"t_beta3",val:": typing.Optional[int] = None"}],parametersDescription:[{anchor:"bitsandbytes.optim.optimizer.Optimizer2State.__init__.optimizer_name",description:`<strong>optimizer_name</strong> (<code>str</code>) &#x2014;
The name of the optimizer.`,name:"optimizer_name"},{anchor:"bitsandbytes.optim.optimizer.Optimizer2State.__init__.params",description:`<strong>params</strong> (<code>torch.Tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.optimizer.Optimizer2State.__init__.lr",description:`<strong>lr</strong> (<code>float</code>, defaults to 1e-3) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.optimizer.Optimizer2State.__init__.betas",description:`<strong>betas</strong> (<code>tuple</code>, defaults to (0.9, 0.999)) &#x2014;
The beta values for the optimizer.`,name:"betas"},{anchor:"bitsandbytes.optim.optimizer.Optimizer2State.__init__.eps",description:`<strong>eps</strong> (<code>float</code>, defaults to 1e-8) &#x2014;
The epsilon value for the optimizer.`,name:"eps"},{anchor:"bitsandbytes.optim.optimizer.Optimizer2State.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 0.0) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.optimizer.Optimizer2State.__init__.optim_bits",description:`<strong>optim_bits</strong> (<code>int</code>, defaults to 32) &#x2014;
The number of bits of the optimizer state.`,name:"optim_bits"},{anchor:"bitsandbytes.optim.optimizer.Optimizer2State.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.optimizer.Optimizer2State.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.optimizer.Optimizer2State.__init__.max_unorm",description:`<strong>max_unorm</strong> (<code>float</code>, defaults to 0.0) &#x2014;
The maximum value to normalize each block with.`,name:"max_unorm"},{anchor:"bitsandbytes.optim.optimizer.Optimizer2State.__init__.skip_zeros",description:`<strong>skip_zeros</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to skip zero values for sparse gradients and models to ensure correct updates.`,name:"skip_zeros"},{anchor:"bitsandbytes.optim.optimizer.Optimizer2State.__init__.is_paged",description:`<strong>is_paged</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether the optimizer is a paged optimizer or not.`,name:"is_paged"},{anchor:"bitsandbytes.optim.optimizer.Optimizer2State.__init__.alpha",description:`<strong>alpha</strong> (<code>float</code>, defaults to 0.0) &#x2014;
The alpha value for the AdEMAMix optimizer.`,name:"alpha"},{anchor:"bitsandbytes.optim.optimizer.Optimizer2State.__init__.t_alpha",description:`<strong>t_alpha</strong> (<code>Optional[int]</code>, defaults to <code>None</code>) &#x2014;
Number of iterations for alpha scheduling with AdEMAMix.`,name:"t_alpha"},{anchor:"bitsandbytes.optim.optimizer.Optimizer2State.__init__.t_beta3",description:`<strong>t_beta3</strong> (<code>Optional[int]</code>, defaults to <code>None</code>) &#x2014;
Number of iterations for beta scheduling with AdEMAMix.`,name:"t_beta3"}]}),n(2),a(O),a(m);var x=e(m,2);o(x,{title:"Optimizer1State",local:"bitsandbytes.optim.optimizer.Optimizer1State",headingTag:"h2"});var p=e(x,2),M=i(p);t(M,{name:"class bitsandbytes.optim.optimizer.Optimizer1State",anchor:"bitsandbytes.optim.optimizer.Optimizer1State",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2002/bitsandbytes/optim/optimizer.py#L593",parameters:[{name:"optimizer_name",val:""},{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.0)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.0"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 0.0"},{name:"skip_zeros",val:" = False"},{name:"is_paged",val:" = False"}]});var w=e(M,2),F=i(w);t(F,{name:"__init__",anchor:"bitsandbytes.optim.optimizer.Optimizer1State.__init__",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2002/bitsandbytes/optim/optimizer.py#L594",parameters:[{name:"optimizer_name",val:""},{name:"params",val:""},{name:"lr",val:" = 0.001"},{name:"betas",val:" = (0.9, 0.0)"},{name:"eps",val:" = 1e-08"},{name:"weight_decay",val:" = 0.0"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"max_unorm",val:" = 0.0"},{name:"skip_zeros",val:" = False"},{name:"is_paged",val:" = False"}],parametersDescription:[{anchor:"bitsandbytes.optim.optimizer.Optimizer1State.__init__.optimizer_name",description:`<strong>optimizer_name</strong> (<code>str</code>) &#x2014;
The name of the optimizer.`,name:"optimizer_name"},{anchor:"bitsandbytes.optim.optimizer.Optimizer1State.__init__.params",description:`<strong>params</strong> (<code>torch.Tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.optimizer.Optimizer1State.__init__.lr",description:`<strong>lr</strong> (<code>float</code>, defaults to 1e-3) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.optimizer.Optimizer1State.__init__.betas",description:`<strong>betas</strong> (<code>tuple</code>, defaults to (0.9, 0.0)) &#x2014;
The beta values for the optimizer.`,name:"betas"},{anchor:"bitsandbytes.optim.optimizer.Optimizer1State.__init__.eps",description:`<strong>eps</strong> (<code>float</code>, defaults to 1e-8) &#x2014;
The epsilon value for the optimizer.`,name:"eps"},{anchor:"bitsandbytes.optim.optimizer.Optimizer1State.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 0.0) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.optimizer.Optimizer1State.__init__.optim_bits",description:`<strong>optim_bits</strong> (<code>int</code>, defaults to 32) &#x2014;
The number of bits of the optimizer state.`,name:"optim_bits"},{anchor:"bitsandbytes.optim.optimizer.Optimizer1State.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.optimizer.Optimizer1State.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.optimizer.Optimizer1State.__init__.max_unorm",description:`<strong>max_unorm</strong> (<code>float</code>, defaults to 0.0) &#x2014;
The maximum value to normalize each block with.`,name:"max_unorm"},{anchor:"bitsandbytes.optim.optimizer.Optimizer1State.__init__.skip_zeros",description:`<strong>skip_zeros</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to skip zero values for sparse gradients and models to ensure correct updates.`,name:"skip_zeros"},{anchor:"bitsandbytes.optim.optimizer.Optimizer1State.__init__.is_paged",description:`<strong>is_paged</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether the optimizer is a paged optimizer or not.`,name:"is_paged"}]}),n(2),a(w),a(p);var T=e(p,2);o(T,{title:"Utilities",local:"bitsandbytes.optim.GlobalOptimManager",headingTag:"h2"});var d=e(T,2),S=i(d);t(S,{name:"class bitsandbytes.optim.GlobalOptimManager",anchor:"bitsandbytes.optim.GlobalOptimManager",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2002/bitsandbytes/optim/optimizer.py#L26",parameters:[]});var k=e(S,4),G=i(k);t(G,{name:"override_config",anchor:"bitsandbytes.optim.GlobalOptimManager.override_config",source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_2002/bitsandbytes/optim/optimizer.py#L60",parameters:[{name:"parameters",val:""},{name:"key",val:" = None"},{name:"value",val:" = None"},{name:"key_value_dict",val:" = None"}],parametersDescription:[{anchor:"bitsandbytes.optim.GlobalOptimManager.override_config.parameters",description:`<strong>parameters</strong> (<code>torch.Tensor</code> or <code>list(torch.Tensors)</code>) &#x2014;
The input parameters.`,name:"parameters"},{anchor:"bitsandbytes.optim.GlobalOptimManager.override_config.key",description:`<strong>key</strong> (<code>str</code>) &#x2014;
The hyperparameter to override.`,name:"key"},{anchor:"bitsandbytes.optim.GlobalOptimManager.override_config.value",description:`<strong>value</strong> &#x2014;
The hyperparameter value.`,name:"value"},{anchor:"bitsandbytes.optim.GlobalOptimManager.override_config.key_value_dict",description:`<strong>key_value_dict</strong> (<code>dict</code>) &#x2014;
A dictionary with multiple key-values to override.`,name:"key_value_dict"}]});var N=e(G,6);D(N,{anchor:"bitsandbytes.optim.GlobalOptimManager.override_config.example",children:(r,l)=>{var J=$(),X=e(j(J),2);Y(X,{code:"aW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjBiaXRzYW5kYnl0ZXMlMjBhcyUyMGJuYiUwQSUwQW1uZyUyMCUzRCUyMGJuYi5vcHRpbS5HbG9iYWxPcHRpbU1hbmFnZXIuZ2V0X2luc3RhbmNlKCklMEElMEFtb2RlbCUyMCUzRCUyME15TW9kZWwoKSUwQW1uZy5yZWdpc3Rlcl9wYXJhbWV0ZXJzKG1vZGVsLnBhcmFtZXRlcnMoKSklMjAlMjMlMjAxLiUyMHJlZ2lzdGVyJTIwcGFyYW1ldGVycyUyMHdoaWxlJTIwc3RpbGwlMjBvbiUyMENQVSUwQSUwQW1vZGVsJTIwJTNEJTIwbW9kZWwuY3VkYSgpJTBBJTIzJTIwdXNlJTIwOC1iaXQlMjBvcHRpbWl6ZXIlMjBzdGF0ZXMlMjBmb3IlMjBhbGwlMjBwYXJhbWV0ZXJzJTBBYWRhbSUyMCUzRCUyMGJuYi5vcHRpbS5BZGFtKG1vZGVsLnBhcmFtZXRlcnMoKSUyQyUyMGxyJTNEMC4wMDElMkMlMjBvcHRpbV9iaXRzJTNEOCklMEElMEElMjMlMjAyLiUyMG92ZXJyaWRlJTNBJTIwdGhlJTIwcGFyYW1ldGVyJTIwbW9kZWwuZmMxLndlaWdodCUyMG5vdyUyMHVzZXMlMjAzMi1iaXQlMjBBZGFtJTBBbW5nLm92ZXJyaWRlX2NvbmZpZyhtb2RlbC5mYzEud2VpZ2h0JTJDJTIwJ29wdGltX2JpdHMnJTJDJTIwMzIp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> bitsandbytes <span class="hljs-keyword">as</span> bnb

mng = bnb.optim.GlobalOptimManager.get_instance()

model = MyModel()
mng.register_parameters(model.parameters()) <span class="hljs-comment"># 1. register parameters while still on CPU</span>

model = model.cuda()
<span class="hljs-comment"># use 8-bit optimizer states for all parameters</span>
adam = bnb.optim.Adam(model.parameters(), lr=<span class="hljs-number">0.001</span>, optim_bits=<span class="hljs-number">8</span>)

<span class="hljs-comment"># 2. override: the parameter model.fc1.weight now uses 32-bit Adam</span>
mng.override_config(model.fc1.weight, <span class="hljs-string">&#x27;optim_bits&#x27;</span>, <span class="hljs-number">32</span>)`,lang:"py",wrap:!1}),b(r,J)},$$slots:{default:!0}}),a(k),a(d);var E=e(d,2);I(E,{source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/docs/source/reference/optim/optim_overview.mdx"}),n(2),b(W,_),Q()}export{oe as component};