bitsandbytes/pr_2032/en/_app/immutable/nodes/11.9Su_AFKJ.js

import"../chunks/DsnmJJEf.js";import{i as T,h as U,C as f,H as s,a,E as G,s as Z}from"../chunks/CHailA3B.js";import{p as W,o as z,s as e,f as k,a as u,b as I,c as w,n as v}from"../chunks/CMkz9frW.js";const _='{"title":"8-bit optimizers","local":"8-bit-optimizers","sections":[{"title":"Optimize unstable parameters","local":"optimize-unstable-parameters","sections":[],"depth":2},{"title":"Next steps","local":"next-steps","sections":[],"depth":2}],"depth":1}';var A=w('<meta name="hf:doc:metadata"/>'),B=w('<p></p> <!> <!> <p>With 8-bit optimizers, large models can be finetuned with 75% less GPU memory without losing any accuracy compared to training with standard 32-bit optimizers. The reduced memory requirements means 8-bit optimizers are 4x faster than a standard optimizer, and no hyperparameter tuning is required.</p> <p>This guide will show you how to use 8-bit optimizers.</p> <blockquote class="warning"><p>8-bit optimizers reduce memory usage and accelerate optimization on a wide range of tasks. However, since 8-bit optimizers only reduce memory proportional to the number of parameters, models that use large amounts of activation memory, such as convolutional networks, don’t really benefit from 8-bit optimizers. 8-bit optimizers are most beneficial for training or finetuning models with many parameters on highly memory-constrained GPUs.</p></blockquote> <p>8-bit optimizers are a drop-in replacement for regular optimizers which means they also accept the same arguments as a regular optimizer. For NLP models, it is recommended to use the <a href="/docs/bitsandbytes/pr_2032/en/reference/nn/embeddings#bitsandbytes.nn.StableEmbedding">StableEmbedding</a> class to improve stability and results.</p> <!> <p>By default, all parameter tensors with less than 4096 elements are kept at 32-bits even if you initialize those parameters with 8-bit optimizers. This is done because small tensors do not save much memory and often contain highly variable parameters (biases) or parameters that require high precision (batch norm, layer norm).</p> <p>You can change this value with the <code>min_8bit_size</code> parameter. For example, if you want to optimize parameters to 8-bits only if the minimum size is 16384 values (it is recommended to use multiples of 4096):</p> <!> <p>Other parameters you can configure include the learning rate (<code>lr</code>), the decay rates (<code>betas</code>), and the number of bits of the optimizer state (<code>optim_bits</code>). For example, to initialize a 32-bit <a href="/docs/bitsandbytes/pr_2032/en/reference/optim/adam#bitsandbytes.optim.Adam">Adam</a> optimizer:</p> <!> <!> <p>To optimize some unstable parameters with 32-bit Adam and others with 8-bit Adam, use the <a href="/docs/bitsandbytes/pr_2032/en/reference/optim/optim_overview#bitsandbytes.optim.GlobalOptimManager">GlobalOptimManager</a> class to override the specific hyperparameters for a particular layer. You’ll need to:</p> <ol><li>Register the parameters while they’re on the CPU.</li></ol> <!> <ol start="2"><li>Override the config with the new desired hyperparameters. For example, let’s override the <code>model.fc1.weight</code> layer to use 32-bit Adam.</li></ol> <blockquote class="tip"><p>Check the optimizer API documentation for more information about other hyperparameters you can override.</p></blockquote> <!> <p>You can also override multiple layers at once by passing them as a list and the new hyperparameters as a dictionary. For example, let’s override the <code>model.special.weight</code> and <code>model.also_special.weight</code> layers to use sparse optimization and a lower learning and decay rate.</p> <!> <p>For a specific layer, we recommend overriding locally in each module. Pass the module, the parameter, and its attribute name to the <a href="/docs/bitsandbytes/pr_2032/en/reference/optim/optim_overview#bitsandbytes.optim.GlobalOptimManager">GlobalOptimManager</a>:</p> <!> <!> <p>For more conceptual details and explanation about 8-bit optimizers, take a look at the <a href="./explanations/optimizers">8-bit optimizers</a> guide.</p> <!> <p></p>',1);function C(j,J){W(J,!1),z(()=>{new URLSearchParams(window.location.search).get("fw")}),T();var l=B();U("p4xky5",y=>{var M=A();Z(M,"content",_),u(y,M)});var t=e(k(l),2);f(t,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var i=e(t,2);s(i,{title:"8-bit optimizers",local:"8-bit-optimizers",headingTag:"h1"});var n=e(i,10);a(n,{code:"aW1wb3J0JTIwYml0c2FuZGJ5dGVzJTIwYXMlMjBibmIlMEElMEEtJTIwYWRhbSUyMCUzRCUyMHRvcmNoLm9wdGltLkFkYW0oLi4uKSUwQSUyQiUyMGFkYW0lMjAlM0QlMjBibmIub3B0aW0uQWRhbThiaXQoLi4uKSUwQSUwQSUyMyUyMHJlY29tbWVuZGVkJTIwZm9yJTIwTkxQJTIwbW9kZWxzJTBBLSUyMGJlZm9yZSUzQSUyMHRvcmNoLm5uLkVtYmVkZGluZyguLi4pJTBBJTJCJTIwYm5iLm5uLlN0YWJsZUVtYmVkZGluZyguLi4p",highlighted:`import bitsandbytes as bnb

<span class="hljs-deletion">- adam = torch.optim.Adam(...)</span>
<span class="hljs-addition">+ adam = bnb.optim.Adam8bit(...)</span>

# recommended for NLP models
<span class="hljs-deletion">- before: torch.nn.Embedding(...)</span>
<span class="hljs-addition">+ bnb.nn.StableEmbedding(...)</span>`,lang:"diff",wrap:!1});var o=e(n,6);a(o,{code:"aW1wb3J0JTIwYml0c2FuZGJ5dGVzJTIwYXMlMjBibmIlMEElMEFhZGFtJTIwJTNEJTIwYm5iLm9wdGltLkFkYW04Yml0KG1vZGVsLnBhcmFtZXRlcnMoKSUyQyUyMG1pbl84Yml0X3NpemUlM0QxNjM4NCk=",highlighted:`<span class="hljs-keyword">import</span> bitsandbytes <span class="hljs-keyword">as</span> bnb

adam = bnb.optim.Adam8bit(model.parameters(), min_8bit_size=<span class="hljs-number">16384</span>)`,lang:"py",wrap:!1});var r=e(o,4);a(r,{code:"aW1wb3J0JTIwYml0c2FuZGJ5dGVzJTIwYXMlMjBibmIlMEElMEFhZGFtJTIwJTNEJTIwYm5iLm9wdGltLkFkYW0obW9kZWwucGFyYW1ldGVycygpJTJDJTIwbHIlM0QwLjAwMSUyQyUyMGJldGFzJTNEKDAuOSUyQyUyMDAuOTk1KSUyQyUyMG9wdGltX2JpdHMlM0QzMik=",highlighted:`<span class="hljs-keyword">import</span> bitsandbytes <span class="hljs-keyword">as</span> bnb

adam = bnb.optim.Adam(model.parameters(), lr=<span class="hljs-number">0.001</span>, betas=(<span class="hljs-number">0.9</span>, <span class="hljs-number">0.995</span>), optim_bits=<span class="hljs-number">32</span>)`,lang:"py",wrap:!1});var p=e(r,2);s(p,{title:"Optimize unstable parameters",local:"optimize-unstable-parameters",headingTag:"h2"});var m=e(p,6);a(m,{code:"aW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjBiaXRzYW5kYnl0ZXMlMjBhcyUyMGJuYiUwQSUwQW1uZyUyMCUzRCUyMGJuYi5vcHRpbS5HbG9iYWxPcHRpbU1hbmFnZXIuZ2V0X2luc3RhbmNlKCklMEElMEFtb2RlbCUyMCUzRCUyME15TW9kZWwoKSUwQW1uZy5yZWdpc3Rlcl9wYXJhbWV0ZXJzKG1vZGVsLnBhcmFtZXRlcnMoKSk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> bitsandbytes <span class="hljs-keyword">as</span> bnb

mng = bnb.optim.GlobalOptimManager.get_instance()

model = MyModel()
mng.register_parameters(model.parameters())`,lang:"py",wrap:!1});var d=e(m,6);a(d,{code:"bW9kZWwlMjAlM0QlMjBtb2RlbC5jdWRhKCklMEElMjMlMjB1c2UlMjA4LWJpdCUyMG9wdGltaXplciUyMHN0YXRlcyUyMGZvciUyMGFsbCUyMHBhcmFtZXRlcnMlMEFhZGFtJTIwJTNEJTIwYm5iLm9wdGltLkFkYW0obW9kZWwucGFyYW1ldGVycygpJTJDJTIwbHIlM0QwLjAwMSUyQyUyMG9wdGltX2JpdHMlM0Q4KSUwQSUwQSUyMyUyMG92ZXJyaWRlJTIwdGhlJTIwcGFyYW1ldGVyJTIwbW9kZWwuZmMxLndlaWdodCUyMG5vdyUyMHVzZXMlMjAzMi1iaXQlMjBBZGFtJTBBbW5nLm92ZXJyaWRlX2NvbmZpZyhtb2RlbC5mYzEud2VpZ2h0JTJDJTIwJTIyb3B0aW1fYml0cyUyMiUyQyUyMDMyKQ==",highlighted:`model = model.cuda()
<span class="hljs-comment"># use 8-bit optimizer states for all parameters</span>
adam = bnb.optim.Adam(model.parameters(), lr=<span class="hljs-number">0.001</span>, optim_bits=<span class="hljs-number">8</span>)

<span class="hljs-comment"># override the parameter model.fc1.weight now uses 32-bit Adam</span>
mng.override_config(model.fc1.weight, <span class="hljs-string">&quot;optim_bits&quot;</span>, <span class="hljs-number">32</span>)`,lang:"py",wrap:!1});var c=e(d,4);a(c,{code:"bW5nLm92ZXJyaWRlX2NvbmZpZyglNUJtb2RlbC5zcGVjaWFsLndlaWdodCUyQyUyMG1vZGVsLmFsc29fc3BlY2lhbC53ZWlnaHQlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBrZXlfdmFsdWVfZGljdCUyMCUzRCU3Qidpc19zcGFyc2UnJTNBJTIwVHJ1ZSUyQyUyMCdsciclM0ElMjAxZS01JTJDJTIwJ2JldGFzJyUzRCgwLjklMkMlMjAwLjk4KSU3RCk=",highlighted:`mng.override_config([model.special.weight, model.also_special.weight],
                    key_value_dict ={<span class="hljs-string">&#x27;is_sparse&#x27;</span>: <span class="hljs-literal">True</span>, <span class="hljs-string">&#x27;lr&#x27;</span>: <span class="hljs-number">1e-5</span>, <span class="hljs-string">&#x27;betas&#x27;</span>=(<span class="hljs-number">0.9</span>, <span class="hljs-number">0.98</span>)})`,lang:"py",wrap:!1});var b=e(c,4);a(b,{code:"Y2xhc3MlMjBNeU1vZHVsZSh0b3JjaC5ubi5Nb2R1bGUpJTNBJTBBJTIwJTIwZGVmJTIwX19pbml0X18oZF9pbiUyQyUyMGRfb3V0KSUzQSUwQSUyMCUyMCUyMCUyMHN1cGVyKE15TW9kdWxlJTJDJTIwc2VsZikuX19pbml0X18oKSUwQSUyMCUyMCUyMCUyMHNlbGYubGluZWFyJTIwJTNEJTIwdG9yY2gubm4uTGluZWFyKGRfaW4lMkMlMjBkX291dCklMEElMjAlMjAlMjAlMjAlMjMlMjBvcHRpbWl6YXRpb24lMjB3aWxsJTIwaGFwcGVuJTIwaW4lMjAzMi1iaXQlMjBhbmQlMEElMjAlMjAlMjAlMjAlMjMlMjBsZWFybmluZyUyMHJhdGUlMjB3aWxsJTIwYmUlMjBzZXQlMjB0byUyMDAuMDAwMSUyMGluZGVwZW5kZW50JTIwb2YlMjB0aGUlMjBtYWluJTIwbGVhcm5pbmclMjByYXRlJTBBJTIwJTIwJTIwJTIwY29uZmlnJTIwJTNEJTIwJTdCJ29wdGltX2JpdHMnJTNBJTIwMzIlMkMlMjAnbHInJTIwJTNBJTIwMC4wMDAxJTdEJTBBJTIwJTIwJTIwJTIwR2xvYmFsT3B0aW1NYW5hZ2VyLmdldF9pbnN0YW5jZSgpLnJlZ2lzdGVyX21vZHVsZV9vdmVycmlkZShzZWxmJTJDJTIwJ3dlaWdodCclMkMlMjBjb25maWcpJTBB",highlighted:`<span class="hljs-keyword">class</span> <span class="hljs-title class_">MyModule</span>(torch.nn.Module):
  <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">d_in, d_out</span>):
    <span class="hljs-built_in">super</span>(MyModule, <span class="hljs-variable language_">self</span>).__init__()
    <span class="hljs-variable language_">self</span>.linear = torch.nn.Linear(d_in, d_out)
    <span class="hljs-comment"># optimization will happen in 32-bit and</span>
    <span class="hljs-comment"># learning rate will be set to 0.0001 independent of the main learning rate</span>
    config = {<span class="hljs-string">&#x27;optim_bits&#x27;</span>: <span class="hljs-number">32</span>, <span class="hljs-string">&#x27;lr&#x27;</span> : <span class="hljs-number">0.0001</span>}
    GlobalOptimManager.get_instance().register_module_override(<span class="hljs-variable language_">self</span>, <span class="hljs-string">&#x27;weight&#x27;</span>, config)
`,lang:"py",wrap:!1});var h=e(b,2);s(h,{title:"Next steps",local:"next-steps",headingTag:"h2"});var g=e(h,4);G(g,{source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/docs/source/optimizers.mdx"}),v(2),u(j,l),I()}export{C as component};