bitsandbytes/pr_1894/en/_app/immutable/nodes/11.9dfa9df8.js

import{s as Ee,n as Fe,o as Qe}from"../chunks/scheduler.8a2cc2fa.js";import{S as Se,i as Xe,e as n,s as a,c as p,h as Re,a as m,d as s,b as i,f as He,g as o,j as r,k as S,l as Ve,m as l,n as d,t as b,o as M,p as c}from"../chunks/index.7079e750.js";import{C as Ne,H as $e,E as Pe}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.1c92186b.js";import{C as Q}from"../chunks/CodeBlock.e12988bd.js";function Ke(ge){let u,V,X,N,T,P,J,K,j,Ue="With 8-bit optimizers, large models can be finetuned with 75% less GPU memory without losing any accuracy compared to training with standard 32-bit optimizers. The reduced memory requirements means 8-bit optimizers are 4x faster than a standard optimizer, and no hyperparameter tuning is required.",O,$,Ge="This guide will show you how to use 8-bit optimizers.",q,y,ve="<p>8-bit optimizers reduce memory usage and accelerate optimization on a wide range of tasks. However, since 8-bit optimizers only reduce memory proportional to the number of parameters, models that use large amounts of activation memory, such as convolutional networks, don’t really benefit from 8-bit optimizers. 8-bit optimizers are most beneficial for training or finetuning models with many parameters on highly memory-constrained GPUs.</p>",D,g,ke='8-bit optimizers are designed as drop-in replacements for regular optimizers, so most arguments are supported. For NLP models, it is recommended to use the <a href="/docs/bitsandbytes/pr_1894/en/reference/nn/embeddings#bitsandbytes.nn.StableEmbedding">StableEmbedding</a> class to improve stability and results.',ee,w,xe="<p><code>Adam8bit</code>/<code>PagedAdam8bit</code> always use 8-bit optimizer state, and <code>amsgrad</code> is not supported (must be <code>False</code>).</p>",te,U,se,G,ze="By default, all parameter tensors with less than 4096 elements are kept at 32-bits even if you initialize those parameters with 8-bit optimizers. This is done because small tensors do not save much memory and often contain highly variable parameters (biases) or parameters that require high precision (batch norm, layer norm).",le,v,_e="You can change this value with the <code>min_8bit_size</code> parameter. For example, if you want to optimize parameters to 8-bits only if the minimum size is 16384 values (it is recommended to use multiples of 4096):",ae,k,ie,x,Ze='Other parameters you can configure include the learning rate (<code>lr</code>) and the decay rates (<code>betas</code>). The <code>optim_bits</code> argument applies to the base optimizers (for example <code>Adam</code>), while <code>Adam8bit</code>/<code>PagedAdam8bit</code> always use 8-bit optimizer state. For example, to initialize a 32-bit <a href="/docs/bitsandbytes/pr_1894/en/reference/optim/adam#bitsandbytes.optim.Adam">Adam</a> optimizer:',ne,z,me,_,re,Z,We='To optimize some unstable parameters with 32-bit Adam and others with 8-bit Adam, use the <a href="/docs/bitsandbytes/pr_1894/en/reference/optim/optim_overview#bitsandbytes.optim.GlobalOptimManager">GlobalOptimManager</a> class to override the specific hyperparameters for a particular layer. You’ll need to:',pe,W,Ce="<li>Register the parameters while they’re on the CPU.</li>",oe,C,de,h,Le="<li>Override the config with the new desired hyperparameters. For example, let’s override the <code>model.fc1.weight</code> layer to use 32-bit Adam.</li>",be,f,Ie="<p>Check the optimizer API documentation for more information about other hyperparameters you can override.</p>",Me,L,ce,I,Ae="You can also override multiple layers at once by passing them as a list and the new hyperparameters as a dictionary. For example, let’s override the <code>model.special.weight</code> and <code>model.also_special.weight</code> layers to use sparse optimization and a lower learning and decay rate.",ue,A,ye,B,Be='For a specific layer, we recommend overriding locally in each module. Pass the module, the parameter, and its attribute name to the <a href="/docs/bitsandbytes/pr_1894/en/reference/optim/optim_overview#bitsandbytes.optim.GlobalOptimManager">GlobalOptimManager</a>:',we,Y,he,H,fe,E,Ye='For more conceptual details and explanation about 8-bit optimizers, take a look at the <a href="./explanations/optimizers">8-bit optimizers</a> guide.',Te,F,Je,R,je;return T=new Ne({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),J=new $e({props:{title:"8-bit optimizers",local:"8-bit-optimizers",headingTag:"h1"}}),U=new Q({props:{code:"aW1wb3J0JTIwYml0c2FuZGJ5dGVzJTIwYXMlMjBibmIlMEElMEEtJTIwYWRhbSUyMCUzRCUyMHRvcmNoLm9wdGltLkFkYW0oLi4uKSUwQSUyQiUyMGFkYW0lMjAlM0QlMjBibmIub3B0aW0uQWRhbThiaXQoLi4uKSUwQSUwQSUyMyUyMHJlY29tbWVuZGVkJTIwZm9yJTIwTkxQJTIwbW9kZWxzJTBBLSUyMGJlZm9yZSUzQSUyMHRvcmNoLm5uLkVtYmVkZGluZyguLi4pJTBBJTJCJTIwYm5iLm5uLlN0YWJsZUVtYmVkZGluZyguLi4p",highlighted:`import bitsandbytes as bnb

<span class="hljs-deletion">- adam = torch.optim.Adam(...)</span>
<span class="hljs-addition">+ adam = bnb.optim.Adam8bit(...)</span>

# recommended for NLP models
<span class="hljs-deletion">- before: torch.nn.Embedding(...)</span>
<span class="hljs-addition">+ bnb.nn.StableEmbedding(...)</span>`,wrap:!1}}),k=new Q({props:{code:"aW1wb3J0JTIwYml0c2FuZGJ5dGVzJTIwYXMlMjBibmIlMEElMEFhZGFtJTIwJTNEJTIwYm5iLm9wdGltLkFkYW04Yml0KG1vZGVsLnBhcmFtZXRlcnMoKSUyQyUyMG1pbl84Yml0X3NpemUlM0QxNjM4NCk=",highlighted:`<span class="hljs-keyword">import</span> bitsandbytes <span class="hljs-keyword">as</span> bnb

adam = bnb.optim.Adam8bit(model.parameters(), min_8bit_size=<span class="hljs-number">16384</span>)`,wrap:!1}}),z=new Q({props:{code:"aW1wb3J0JTIwYml0c2FuZGJ5dGVzJTIwYXMlMjBibmIlMEElMEFhZGFtJTIwJTNEJTIwYm5iLm9wdGltLkFkYW0obW9kZWwucGFyYW1ldGVycygpJTJDJTIwbHIlM0QwLjAwMSUyQyUyMGJldGFzJTNEKDAuOSUyQyUyMDAuOTk1KSUyQyUyMG9wdGltX2JpdHMlM0QzMik=",highlighted:`<span class="hljs-keyword">import</span> bitsandbytes <span class="hljs-keyword">as</span> bnb

adam = bnb.optim.Adam(model.parameters(), lr=<span class="hljs-number">0.001</span>, betas=(<span class="hljs-number">0.9</span>, <span class="hljs-number">0.995</span>), optim_bits=<span class="hljs-number">32</span>)`,wrap:!1}}),_=new $e({props:{title:"Optimize unstable parameters",local:"optimize-unstable-parameters",headingTag:"h2"}}),C=new Q({props:{code:"aW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjBiaXRzYW5kYnl0ZXMlMjBhcyUyMGJuYiUwQSUwQW1uZyUyMCUzRCUyMGJuYi5vcHRpbS5HbG9iYWxPcHRpbU1hbmFnZXIuZ2V0X2luc3RhbmNlKCklMEElMEFtb2RlbCUyMCUzRCUyME15TW9kZWwoKSUwQW1uZy5yZWdpc3Rlcl9wYXJhbWV0ZXJzKG1vZGVsLnBhcmFtZXRlcnMoKSk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> bitsandbytes <span class="hljs-keyword">as</span> bnb

mng = bnb.optim.GlobalOptimManager.get_instance()

model = MyModel()
mng.register_parameters(model.parameters())`,wrap:!1}}),L=new Q({props:{code:"bW9kZWwlMjAlM0QlMjBtb2RlbC5jdWRhKCklMEElMjMlMjB1c2UlMjA4LWJpdCUyMG9wdGltaXplciUyMHN0YXRlcyUyMGZvciUyMGFsbCUyMHBhcmFtZXRlcnMlMEFhZGFtJTIwJTNEJTIwYm5iLm9wdGltLkFkYW0obW9kZWwucGFyYW1ldGVycygpJTJDJTIwbHIlM0QwLjAwMSUyQyUyMG9wdGltX2JpdHMlM0Q4KSUwQSUwQSUyMyUyMG92ZXJyaWRlJTIwdGhlJTIwcGFyYW1ldGVyJTIwbW9kZWwuZmMxLndlaWdodCUyMG5vdyUyMHVzZXMlMjAzMi1iaXQlMjBBZGFtJTBBbW5nLm92ZXJyaWRlX2NvbmZpZyhtb2RlbC5mYzEud2VpZ2h0JTJDJTIwJTIyb3B0aW1fYml0cyUyMiUyQyUyMDMyKQ==",highlighted:`model = model.cuda()
<span class="hljs-comment"># use 8-bit optimizer states for all parameters</span>
adam = bnb.optim.Adam(model.parameters(), lr=<span class="hljs-number">0.001</span>, optim_bits=<span class="hljs-number">8</span>)

<span class="hljs-comment"># override the parameter model.fc1.weight now uses 32-bit Adam</span>
mng.override_config(model.fc1.weight, <span class="hljs-string">&quot;optim_bits&quot;</span>, <span class="hljs-number">32</span>)`,wrap:!1}}),A=new Q({props:{code:"bW5nLm92ZXJyaWRlX2NvbmZpZyglNUJtb2RlbC5zcGVjaWFsLndlaWdodCUyQyUyMG1vZGVsLmFsc29fc3BlY2lhbC53ZWlnaHQlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBrZXlfdmFsdWVfZGljdCUyMCUzRCU3Qidpc19zcGFyc2UnJTNBJTIwVHJ1ZSUyQyUyMCdsciclM0ElMjAxZS01JTJDJTIwJ2JldGFzJyUzRCgwLjklMkMlMjAwLjk4KSU3RCk=",highlighted:`mng.override_config([model.special.weight, model.also_special.weight],
                    key_value_dict ={<span class="hljs-string">&#x27;is_sparse&#x27;</span>: <span class="hljs-literal">True</span>, <span class="hljs-string">&#x27;lr&#x27;</span>: <span class="hljs-number">1e-5</span>, <span class="hljs-string">&#x27;betas&#x27;</span>=(<span class="hljs-number">0.9</span>, <span class="hljs-number">0.98</span>)})`,wrap:!1}}),Y=new Q({props:{code:"Y2xhc3MlMjBNeU1vZHVsZSh0b3JjaC5ubi5Nb2R1bGUpJTNBJTBBJTIwJTIwZGVmJTIwX19pbml0X18oZF9pbiUyQyUyMGRfb3V0KSUzQSUwQSUyMCUyMCUyMCUyMHN1cGVyKE15TW9kdWxlJTJDJTIwc2VsZikuX19pbml0X18oKSUwQSUyMCUyMCUyMCUyMHNlbGYubGluZWFyJTIwJTNEJTIwdG9yY2gubm4uTGluZWFyKGRfaW4lMkMlMjBkX291dCklMEElMjAlMjAlMjAlMjAlMjMlMjBvcHRpbWl6YXRpb24lMjB3aWxsJTIwaGFwcGVuJTIwaW4lMjAzMi1iaXQlMjBhbmQlMEElMjAlMjAlMjAlMjAlMjMlMjBsZWFybmluZyUyMHJhdGUlMjB3aWxsJTIwYmUlMjBzZXQlMjB0byUyMDAuMDAwMSUyMGluZGVwZW5kZW50JTIwb2YlMjB0aGUlMjBtYWluJTIwbGVhcm5pbmclMjByYXRlJTBBJTIwJTIwJTIwJTIwY29uZmlnJTIwJTNEJTIwJTdCJ29wdGltX2JpdHMnJTNBJTIwMzIlMkMlMjAnbHInJTIwJTNBJTIwMC4wMDAxJTdEJTBBJTIwJTIwJTIwJTIwR2xvYmFsT3B0aW1NYW5hZ2VyLmdldF9pbnN0YW5jZSgpLnJlZ2lzdGVyX21vZHVsZV9vdmVycmlkZShzZWxmJTJDJTIwJ3dlaWdodCclMkMlMjBjb25maWcpJTBB",highlighted:`<span class="hljs-keyword">class</span> <span class="hljs-title class_">MyModule</span>(torch.nn.Module):
  <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">d_in, d_out</span>):
    <span class="hljs-built_in">super</span>(MyModule, self).__init__()
    self.linear = torch.nn.Linear(d_in, d_out)
    <span class="hljs-comment"># optimization will happen in 32-bit and</span>
    <span class="hljs-comment"># learning rate will be set to 0.0001 independent of the main learning rate</span>
    config = {<span class="hljs-string">&#x27;optim_bits&#x27;</span>: <span class="hljs-number">32</span>, <span class="hljs-string">&#x27;lr&#x27;</span> : <span class="hljs-number">0.0001</span>}
    GlobalOptimManager.get_instance().register_module_override(self, <span class="hljs-string">&#x27;weight&#x27;</span>, config)
`,wrap:!1}}),H=new $e({props:{title:"Next steps",local:"next-steps",headingTag:"h2"}}),F=new Pe({props:{source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/docs/source/optimizers.mdx"}}),{c(){u=n("meta"),V=a(),X=n("p"),N=a(),p(T.$$.fragment),P=a(),p(J.$$.fragment),K=a(),j=n("p"),j.textContent=Ue,O=a(),$=n("p"),$.textContent=Ge,q=a(),y=n("blockquote"),y.innerHTML=ve,D=a(),g=n("p"),g.innerHTML=ke,ee=a(),w=n("blockquote"),w.innerHTML=xe,te=a(),p(U.$$.fragment),se=a(),G=n("p"),G.textContent=ze,le=a(),v=n("p"),v.innerHTML=_e,ae=a(),p(k.$$.fragment),ie=a(),x=n("p"),x.innerHTML=Ze,ne=a(),p(z.$$.fragment),me=a(),p(_.$$.fragment),re=a(),Z=n("p"),Z.innerHTML=We,pe=a(),W=n("ol"),W.innerHTML=Ce,oe=a(),p(C.$$.fragment),de=a(),h=n("ol"),h.innerHTML=Le,be=a(),f=n("blockquote"),f.innerHTML=Ie,Me=a(),p(L.$$.fragment),ce=a(),I=n("p"),I.innerHTML=Ae,ue=a(),p(A.$$.fragment),ye=a(),B=n("p"),B.innerHTML=Be,we=a(),p(Y.$$.fragment),he=a(),p(H.$$.fragment),fe=a(),E=n("p"),E.innerHTML=Ye,Te=a(),p(F.$$.fragment),Je=a(),R=n("p"),this.h()},l(e){const t=Re("svelte-u9bgzb",document.head);u=m(t,"META",{name:!0,content:!0}),t.forEach(s),V=i(e),X=m(e,"P",{}),He(X).forEach(s),N=i(e),o(T.$$.fragment,e),P=i(e),o(J.$$.fragment,e),K=i(e),j=m(e,"P",{"data-svelte-h":!0}),r(j)!=="svelte-7xifi0"&&(j.textContent=Ue),O=i(e),$=m(e,"P",{"data-svelte-h":!0}),r($)!=="svelte-anq6iu"&&($.textContent=Ge),q=i(e),y=m(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),r(y)!=="svelte-1n1g6xz"&&(y.innerHTML=ve),D=i(e),g=m(e,"P",{"data-svelte-h":!0}),r(g)!=="svelte-17l9ew3"&&(g.innerHTML=ke),ee=i(e),w=m(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),r(w)!=="svelte-4kncj5"&&(w.innerHTML=xe),te=i(e),o(U.$$.fragment,e),se=i(e),G=m(e,"P",{"data-svelte-h":!0}),r(G)!=="svelte-1swxts3"&&(G.textContent=ze),le=i(e),v=m(e,"P",{"data-svelte-h":!0}),r(v)!=="svelte-swi3zx"&&(v.innerHTML=_e),ae=i(e),o(k.$$.fragment,e),ie=i(e),x=m(e,"P",{"data-svelte-h":!0}),r(x)!=="svelte-185mgu4"&&(x.innerHTML=Ze),ne=i(e),o(z.$$.fragment,e),me=i(e),o(_.$$.fragment,e),re=i(e),Z=m(e,"P",{"data-svelte-h":!0}),r(Z)!=="svelte-yazl54"&&(Z.innerHTML=We),pe=i(e),W=m(e,"OL",{"data-svelte-h":!0}),r(W)!=="svelte-1revex8"&&(W.innerHTML=Ce),oe=i(e),o(C.$$.fragment,e),de=i(e),h=m(e,"OL",{start:!0,"data-svelte-h":!0}),r(h)!=="svelte-127zgse"&&(h.innerHTML=Le),be=i(e),f=m(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),r(f)!=="svelte-pqrln3"&&(f.innerHTML=Ie),Me=i(e),o(L.$$.fragment,e),ce=i(e),I=m(e,"P",{"data-svelte-h":!0}),r(I)!=="svelte-kxfpon"&&(I.innerHTML=Ae),ue=i(e),o(A.$$.fragment,e),ye=i(e),B=m(e,"P",{"data-svelte-h":!0}),r(B)!=="svelte-izxtyg"&&(B.innerHTML=Be),we=i(e),o(Y.$$.fragment,e),he=i(e),o(H.$$.fragment,e),fe=i(e),E=m(e,"P",{"data-svelte-h":!0}),r(E)!=="svelte-1er8qov"&&(E.innerHTML=Ye),Te=i(e),o(F.$$.fragment,e),Je=i(e),R=m(e,"P",{}),He(R).forEach(s),this.h()},h(){S(u,"name","hf:doc:metadata"),S(u,"content",Oe),S(y,"class","warning"),S(w,"class","note"),S(h,"start","2"),S(f,"class","tip")},m(e,t){Ve(document.head,u),l(e,V,t),l(e,X,t),l(e,N,t),d(T,e,t),l(e,P,t),d(J,e,t),l(e,K,t),l(e,j,t),l(e,O,t),l(e,$,t),l(e,q,t),l(e,y,t),l(e,D,t),l(e,g,t),l(e,ee,t),l(e,w,t),l(e,te,t),d(U,e,t),l(e,se,t),l(e,G,t),l(e,le,t),l(e,v,t),l(e,ae,t),d(k,e,t),l(e,ie,t),l(e,x,t),l(e,ne,t),d(z,e,t),l(e,me,t),d(_,e,t),l(e,re,t),l(e,Z,t),l(e,pe,t),l(e,W,t),l(e,oe,t),d(C,e,t),l(e,de,t),l(e,h,t),l(e,be,t),l(e,f,t),l(e,Me,t),d(L,e,t),l(e,ce,t),l(e,I,t),l(e,ue,t),d(A,e,t),l(e,ye,t),l(e,B,t),l(e,we,t),d(Y,e,t),l(e,he,t),d(H,e,t),l(e,fe,t),l(e,E,t),l(e,Te,t),d(F,e,t),l(e,Je,t),l(e,R,t),je=!0},p:Fe,i(e){je||(b(T.$$.fragment,e),b(J.$$.fragment,e),b(U.$$.fragment,e),b(k.$$.fragment,e),b(z.$$.fragment,e),b(_.$$.fragment,e),b(C.$$.fragment,e),b(L.$$.fragment,e),b(A.$$.fragment,e),b(Y.$$.fragment,e),b(H.$$.fragment,e),b(F.$$.fragment,e),je=!0)},o(e){M(T.$$.fragment,e),M(J.$$.fragment,e),M(U.$$.fragment,e),M(k.$$.fragment,e),M(z.$$.fragment,e),M(_.$$.fragment,e),M(C.$$.fragment,e),M(L.$$.fragment,e),M(A.$$.fragment,e),M(Y.$$.fragment,e),M(H.$$.fragment,e),M(F.$$.fragment,e),je=!1},d(e){e&&(s(V),s(X),s(N),s(P),s(K),s(j),s(O),s($),s(q),s(y),s(D),s(g),s(ee),s(w),s(te),s(se),s(G),s(le),s(v),s(ae),s(ie),s(x),s(ne),s(me),s(re),s(Z),s(pe),s(W),s(oe),s(de),s(h),s(be),s(f),s(Me),s(ce),s(I),s(ue),s(ye),s(B),s(we),s(he),s(fe),s(E),s(Te),s(Je),s(R)),s(u),c(T,e),c(J,e),c(U,e),c(k,e),c(z,e),c(_,e),c(C,e),c(L,e),c(A,e),c(Y,e),c(H,e),c(F,e)}}}const Oe='{"title":"8-bit optimizers","local":"8-bit-optimizers","sections":[{"title":"Optimize unstable parameters","local":"optimize-unstable-parameters","sections":[],"depth":2},{"title":"Next steps","local":"next-steps","sections":[],"depth":2}],"depth":1}';function qe(ge){return Qe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class lt extends Se{constructor(u){super(),Xe(this,u,qe,Ke,Ee,{})}}export{lt as component};