accelerate/pr_4076/en/_app/immutable/nodes/56.e9dec2dd.js

import{s as zt,n as xt,o as Rt}from"../chunks/scheduler.b9285784.js";import{S as qt,i as Wt,e as s,s as a,c as m,h as Bt,a as o,d as l,b as i,f as kt,g as d,j as p,k as Xt,l as Ht,m as n,n as r,t as u,o as f,p as b}from"../chunks/index.26bc89a1.js";import{C as Yt,H as g,E as Ft}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.fae74550.js";import{C as c}from"../chunks/CodeBlock.4108d4bd.js";function It(it){let h,re,me,ue,w,fe,y,be,M,ce,$,st="Accelerate brings <code>bitsandbytes</code> quantization to your model. You can now load any pytorch model in 8-bit or 4-bit with a few lines of code.",he,_,ot='If you want to use Transformers models with <code>bitsandbytes</code>, you should follow this <a href="https://huggingface.co/docs/transformers/main_classes/quantization" rel="nofollow">documentation</a>.',ge,T,pt='To learn more about how the <code>bitsandbytes</code> quantization works, check out the blog posts on <a href="https://huggingface.co/blog/hf-bitsandbytes-integration" rel="nofollow">8-bit quantization</a> and <a href="https://huggingface.co/blog/4bit-transformers-bitsandbytes" rel="nofollow">4-bit quantization</a>.',we,J,ye,v,mt="You will need to install the following requirements:",Me,j,dt="<li>Install <code>bitsandbytes</code> library</li>",$e,G,_e,U,rt='For non-cuda devices, you can refer to the bitsandbytes installation guide <a href="https://huggingface.co/docs/bitsandbytes/main/en/installation#multi-backend" rel="nofollow">here</a>.',Te,Z,ut="<li>Install latest <code>accelerate</code> from source</li>",Je,C,ve,k,ft="<li>Install <code>minGPT</code> and <code>huggingface_hub</code> to run examples</li>",je,X,Ge,z,Ue,x,bt='First, we need to initialize our model. To save memory, we can initialize an empty model using the context manager <a href="/docs/accelerate/pr_4076/en/package_reference/big_modeling#accelerate.init_empty_weights">init_empty_weights()</a>.',Ze,R,ct="Let’s take the GPT2 model from minGPT library.",Ce,q,ke,W,ht="Then, we need to get the path to the weights of your model. The path can be the state_dict file (e.g. “pytorch_model.bin”) or a folder containing the sharded checkpoints.",Xe,B,ze,H,gt='Finally, you need to set your quantization configuration with <a href="/docs/accelerate/pr_4076/en/package_reference/utilities#accelerate.utils.BnbQuantizationConfig">BnbQuantizationConfig</a>.',xe,Y,wt="Here’s an example for 8-bit quantization:",Re,F,qe,I,yt="Here’s an example for 4-bit quantization:",We,L,Be,P,Mt='To quantize your empty model with the selected configuration, you need to use <a href="/docs/accelerate/pr_4076/en/package_reference/utilities#accelerate.utils.load_and_quantize_model">load_and_quantize_model()</a>.',He,V,Ye,Q,Fe,N,$t='You can save your 8-bit model with accelerate using <a href="/docs/accelerate/pr_4076/en/package_reference/accelerator#accelerate.Accelerator.save_model">save_model()</a>.',Ie,A,Le,E,_t="Note that 4-bit model serialization is currently not supported.",Pe,S,Ve,D,Tt=`You can offload some modules to cpu/disk if you don’t have enough space on the GPU to store the entire model on your GPUs.
This uses big model inference under the hood. Check this <a href="https://huggingface.co/docs/accelerate/usage_guides/big_modeling" rel="nofollow">documentation</a> for more details.`,Qe,K,Jt="For 8-bit quantization, the selected modules will be converted to 8-bit precision.",Ne,O,vt="For 4-bit quantization, the selected modules will be kept in <code>torch_dtype</code> that the user passed in <code>BnbQuantizationConfig</code>.  We will add support to convert these offloaded modules in 4-bit when 4-bit serialization will be possible.",Ae,ee,jt="You just need to pass a custom <code>device_map</code> in order to offload modules on cpu/disk. The offload modules will be dispatched on the GPU when needed. Here’s an example :",Ee,te,Se,le,De,ne,Gt='It is not possible to perform pure 8bit or 4bit training on these models. However, you can train these models by leveraging parameter efficient fine tuning methods (PEFT) and train for example adapters on top of them. Please have a look at <a href="https://github.com/huggingface/peft" rel="nofollow">peft</a> library for more details.',Ke,ae,Ut='Currently, you can’t add adapters on top of any quantized model. However, with the official support of adapters with Transformers models, you can fine-tune quantized models. If you want to fine-tune a Transformers model , follow this <a href="https://huggingface.co/docs/transformers/main_classes/quantization" rel="nofollow">documentation</a> instead. Check out this <a href="https://colab.research.google.com/drive/1VoYNfYDKcKRQRor98Zbf2-9VQTtGJ24k?usp=sharing" rel="nofollow">demo</a> on how to fine-tune a 4-bit Transformers model.',Oe,ie,Zt="Note that you don’t need to pass <code>device_map</code> when loading the model for training. It will automatically load your model on your GPU. Please note that <code>device_map=auto</code> should be used for inference only.",et,se,tt,oe,Ct='Check out the Google Colab <a href="https://colab.research.google.com/drive/1T1pOgewAWVpR9gKpaEWw4orOrzPFb3yM?usp=sharing" rel="nofollow">demo</a> for running quantized models on a GPT2 model. The GPT2-1.5B model checkpoint is in FP32 which uses 6GB of memory. After quantization, it uses 1.6GB with 8-bit modules and 1.2GB with 4-bit modules.',lt,pe,nt,de,at;return w=new Yt({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),y=new g({props:{title:"Model quantization",local:"model-quantization",headingTag:"h1"}}),M=new g({props:{title:"bitsandbytes Integration",local:"bitsandbytes-integration",headingTag:"h2"}}),J=new g({props:{title:"Pre-Requisites",local:"pre-requisites",headingTag:"h3"}}),G=new c({props:{code:"cGlwJTIwaW5zdGFsbCUyMGJpdHNhbmRieXRlcw==",highlighted:"pip install bitsandbytes",lang:"bash",wrap:!1}}),C=new c({props:{code:"cGlwJTIwaW5zdGFsbCUyMGdpdCUyQmh0dHBzJTNBJTJGJTJGZ2l0aHViLmNvbSUyRmh1Z2dpbmdmYWNlJTJGYWNjZWxlcmF0ZS5naXQ=",highlighted:"pip install git+https://github.com/huggingface/accelerate.git",lang:"bash",wrap:!1}}),X=new c({props:{code:"Z2l0JTIwY2xvbmUlMjBodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZrYXJwYXRoeSUyRm1pbkdQVC5naXQlMEFwaXAlMjBpbnN0YWxsJTIwbWluR1BUJTJGJTBBcGlwJTIwaW5zdGFsbCUyMGh1Z2dpbmdmYWNlX2h1Yg==",highlighted:`git <span class="hljs-built_in">clone</span> https://github.com/karpathy/minGPT.git
pip install minGPT/
pip install huggingface_hub`,lang:"bash",wrap:!1}}),z=new g({props:{title:"How it works",local:"how-it-works",headingTag:"h3"}}),q=new c({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBpbml0X2VtcHR5X3dlaWdodHMlMEFmcm9tJTIwbWluZ3B0Lm1vZGVsJTIwaW1wb3J0JTIwR1BUJTBBJTBBbW9kZWxfY29uZmlnJTIwJTNEJTIwR1BULmdldF9kZWZhdWx0X2NvbmZpZygpJTBBbW9kZWxfY29uZmlnLm1vZGVsX3R5cGUlMjAlM0QlMjAnZ3B0Mi14bCclMEFtb2RlbF9jb25maWcudm9jYWJfc2l6ZSUyMCUzRCUyMDUwMjU3JTBBbW9kZWxfY29uZmlnLmJsb2NrX3NpemUlMjAlM0QlMjAxMDI0JTBBJTBBd2l0aCUyMGluaXRfZW1wdHlfd2VpZ2h0cygpJTNBJTBBJTIwJTIwJTIwJTIwZW1wdHlfbW9kZWwlMjAlM0QlMjBHUFQobW9kZWxfY29uZmlnKQ==",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> init_empty_weights
<span class="hljs-keyword">from</span> mingpt.model <span class="hljs-keyword">import</span> GPT

model_config = GPT.get_default_config()
model_config.model_type = <span class="hljs-string">&#x27;gpt2-xl&#x27;</span>
model_config.vocab_size = <span class="hljs-number">50257</span>
model_config.block_size = <span class="hljs-number">1024</span>

<span class="hljs-keyword">with</span> init_empty_weights():
    empty_model = GPT(model_config)`,lang:"py",wrap:!1}}),B=new c({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMHNuYXBzaG90X2Rvd25sb2FkJTBBd2VpZ2h0c19sb2NhdGlvbiUyMCUzRCUyMHNuYXBzaG90X2Rvd25sb2FkKHJlcG9faWQlM0QlMjJtYXJjc3VuMTMlMkZncHQyLXhsLWxpbmVhci1zaGFyZGVkJTIyKQ==",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> snapshot_download
weights_location = snapshot_download(repo_id=<span class="hljs-string">&quot;marcsun13/gpt2-xl-linear-sharded&quot;</span>)`,lang:"py",wrap:!1}}),F=new c({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUudXRpbHMlMjBpbXBvcnQlMjBCbmJRdWFudGl6YXRpb25Db25maWclMEFibmJfcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJuYlF1YW50aXphdGlvbkNvbmZpZyhsb2FkX2luXzhiaXQlM0RUcnVlJTJDJTIwbGxtX2ludDhfdGhyZXNob2xkJTIwJTNEJTIwNik=",highlighted:`<span class="hljs-keyword">from</span> accelerate.utils <span class="hljs-keyword">import</span> BnbQuantizationConfig
bnb_quantization_config = BnbQuantizationConfig(load_in_8bit=<span class="hljs-literal">True</span>, llm_int8_threshold = <span class="hljs-number">6</span>)`,lang:"py",wrap:!1}}),L=new c({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUudXRpbHMlMjBpbXBvcnQlMjBCbmJRdWFudGl6YXRpb25Db25maWclMEFibmJfcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJuYlF1YW50aXphdGlvbkNvbmZpZyhsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTIwYm5iXzRiaXRfY29tcHV0ZV9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTJDJTIwYm5iXzRiaXRfdXNlX2RvdWJsZV9xdWFudCUzRFRydWUlMkMlMjBibmJfNGJpdF9xdWFudF90eXBlJTNEJTIybmY0JTIyKQ==",highlighted:`<span class="hljs-keyword">from</span> accelerate.utils <span class="hljs-keyword">import</span> BnbQuantizationConfig
bnb_quantization_config = BnbQuantizationConfig(load_in_4bit=<span class="hljs-literal">True</span>, bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>, bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>)`,lang:"py",wrap:!1}}),V=new c({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUudXRpbHMlMjBpbXBvcnQlMjBsb2FkX2FuZF9xdWFudGl6ZV9tb2RlbCUwQXF1YW50aXplZF9tb2RlbCUyMCUzRCUyMGxvYWRfYW5kX3F1YW50aXplX21vZGVsKGVtcHR5X21vZGVsJTJDJTIwd2VpZ2h0c19sb2NhdGlvbiUzRHdlaWdodHNfbG9jYXRpb24lMkMlMjBibmJfcXVhbnRpemF0aW9uX2NvbmZpZyUzRGJuYl9xdWFudGl6YXRpb25fY29uZmlnKQ==",highlighted:`<span class="hljs-keyword">from</span> accelerate.utils <span class="hljs-keyword">import</span> load_and_quantize_model
quantized_model = load_and_quantize_model(empty_model, weights_location=weights_location, bnb_quantization_config=bnb_quantization_config)`,lang:"py",wrap:!1}}),Q=new g({props:{title:"Saving and loading 8-bit model",local:"saving-and-loading-8-bit-model",headingTag:"h3"}}),A=new c({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQWFjY2VsZXJhdGUlMjAlM0QlMjBBY2NlbGVyYXRvcigpJTBBbmV3X3dlaWdodHNfbG9jYXRpb24lMjAlM0QlMjAlMjJwYXRoJTJGdG8lMkZzYXZlX2RpcmVjdG9yeSUyMiUwQWFjY2VsZXJhdGUuc2F2ZV9tb2RlbChxdWFudGl6ZWRfbW9kZWwlMkMlMjBuZXdfd2VpZ2h0c19sb2NhdGlvbiklMEElMEFxdWFudGl6ZWRfbW9kZWxfZnJvbV9zYXZlZCUyMCUzRCUyMGxvYWRfYW5kX3F1YW50aXplX21vZGVsKGVtcHR5X21vZGVsJTJDJTIwd2VpZ2h0c19sb2NhdGlvbiUzRG5ld193ZWlnaHRzX2xvY2F0aW9uJTJDJTIwYm5iX3F1YW50aXphdGlvbl9jb25maWclM0RibmJfcXVhbnRpemF0aW9uX2NvbmZpZyUyQyUyMGRldmljZV9tYXAlMjAlM0QlMjAlMjJhdXRvJTIyKQ==",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator
accelerate = Accelerator()
new_weights_location = <span class="hljs-string">&quot;path/to/save_directory&quot;</span>
accelerate.save_model(quantized_model, new_weights_location)

quantized_model_from_saved = load_and_quantize_model(empty_model, weights_location=new_weights_location, bnb_quantization_config=bnb_quantization_config, device_map = <span class="hljs-string">&quot;auto&quot;</span>)`,lang:"py",wrap:!1}}),S=new g({props:{title:"Offload modules to cpu and disk",local:"offload-modules-to-cpu-and-disk",headingTag:"h3"}}),te=new c({props:{code:"ZGV2aWNlX21hcCUyMCUzRCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMnRyYW5zZm9ybWVyLnd0ZSUyMiUzQSUyMDAlMkMlMEElMjAlMjAlMjAlMjAlMjJ0cmFuc2Zvcm1lci53cGUlMjIlM0ElMjAwJTJDJTBBJTIwJTIwJTIwJTIwJTIydHJhbnNmb3JtZXIuZHJvcCUyMiUzQSUyMDAlMkMlMEElMjAlMjAlMjAlMjAlMjJ0cmFuc2Zvcm1lci5oJTIyJTNBJTIwJTIyY3B1JTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIydHJhbnNmb3JtZXIubG5fZiUyMiUzQSUyMCUyMmRpc2slMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJsbV9oZWFkJTIyJTNBJTIwJTIyZGlzayUyMiUyQyUwQSU3RA==",highlighted:`device_map = {
    <span class="hljs-string">&quot;transformer.wte&quot;</span>: <span class="hljs-number">0</span>,
    <span class="hljs-string">&quot;transformer.wpe&quot;</span>: <span class="hljs-number">0</span>,
    <span class="hljs-string">&quot;transformer.drop&quot;</span>: <span class="hljs-number">0</span>,
    <span class="hljs-string">&quot;transformer.h&quot;</span>: <span class="hljs-string">&quot;cpu&quot;</span>,
    <span class="hljs-string">&quot;transformer.ln_f&quot;</span>: <span class="hljs-string">&quot;disk&quot;</span>,
    <span class="hljs-string">&quot;lm_head&quot;</span>: <span class="hljs-string">&quot;disk&quot;</span>,
}`,lang:"py",wrap:!1}}),le=new g({props:{title:"Fine-tune a quantized model",local:"fine-tune-a-quantized-model",headingTag:"h3"}}),se=new g({props:{title:"Example demo - running GPT2 1.5b on a Google Colab",local:"example-demo---running-gpt2-15b-on-a-google-colab",headingTag:"h3"}}),pe=new Ft({props:{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/usage_guides/quantization.md"}}),{c(){h=s("meta"),re=a(),me=s("p"),ue=a(),m(w.$$.fragment),fe=a(),m(y.$$.fragment),be=a(),m(M.$$.fragment),ce=a(),$=s("p"),$.innerHTML=st,he=a(),_=s("p"),_.innerHTML=ot,ge=a(),T=s("p"),T.innerHTML=pt,we=a(),m(J.$$.fragment),ye=a(),v=s("p"),v.textContent=mt,Me=a(),j=s("ul"),j.innerHTML=dt,$e=a(),m(G.$$.fragment),_e=a(),U=s("p"),U.innerHTML=rt,Te=a(),Z=s("ul"),Z.innerHTML=ut,Je=a(),m(C.$$.fragment),ve=a(),k=s("ul"),k.innerHTML=ft,je=a(),m(X.$$.fragment),Ge=a(),m(z.$$.fragment),Ue=a(),x=s("p"),x.innerHTML=bt,Ze=a(),R=s("p"),R.textContent=ct,Ce=a(),m(q.$$.fragment),ke=a(),W=s("p"),W.textContent=ht,Xe=a(),m(B.$$.fragment),ze=a(),H=s("p"),H.innerHTML=gt,xe=a(),Y=s("p"),Y.textContent=wt,Re=a(),m(F.$$.fragment),qe=a(),I=s("p"),I.textContent=yt,We=a(),m(L.$$.fragment),Be=a(),P=s("p"),P.innerHTML=Mt,He=a(),m(V.$$.fragment),Ye=a(),m(Q.$$.fragment),Fe=a(),N=s("p"),N.innerHTML=$t,Ie=a(),m(A.$$.fragment),Le=a(),E=s("p"),E.textContent=_t,Pe=a(),m(S.$$.fragment),Ve=a(),D=s("p"),D.innerHTML=Tt,Qe=a(),K=s("p"),K.textContent=Jt,Ne=a(),O=s("p"),O.innerHTML=vt,Ae=a(),ee=s("p"),ee.innerHTML=jt,Ee=a(),m(te.$$.fragment),Se=a(),m(le.$$.fragment),De=a(),ne=s("p"),ne.innerHTML=Gt,Ke=a(),ae=s("p"),ae.innerHTML=Ut,Oe=a(),ie=s("p"),ie.innerHTML=Zt,et=a(),m(se.$$.fragment),tt=a(),oe=s("p"),oe.innerHTML=Ct,lt=a(),m(pe.$$.fragment),nt=a(),de=s("p"),this.h()},l(e){const t=Bt("svelte-u9bgzb",document.head);h=o(t,"META",{name:!0,content:!0}),t.forEach(l),re=i(e),me=o(e,"P",{}),kt(me).forEach(l),ue=i(e),d(w.$$.fragment,e),fe=i(e),d(y.$$.fragment,e),be=i(e),d(M.$$.fragment,e),ce=i(e),$=o(e,"P",{"data-svelte-h":!0}),p($)!=="svelte-1uk3wou"&&($.innerHTML=st),he=i(e),_=o(e,"P",{"data-svelte-h":!0}),p(_)!=="svelte-1p54xze"&&(_.innerHTML=ot),ge=i(e),T=o(e,"P",{"data-svelte-h":!0}),p(T)!=="svelte-hrid2v"&&(T.innerHTML=pt),we=i(e),d(J.$$.fragment,e),ye=i(e),v=o(e,"P",{"data-svelte-h":!0}),p(v)!=="svelte-50fa4n"&&(v.textContent=mt),Me=i(e),j=o(e,"UL",{"data-svelte-h":!0}),p(j)!=="svelte-t2geh8"&&(j.innerHTML=dt),$e=i(e),d(G.$$.fragment,e),_e=i(e),U=o(e,"P",{"data-svelte-h":!0}),p(U)!=="svelte-s1psu3"&&(U.innerHTML=rt),Te=i(e),Z=o(e,"UL",{"data-svelte-h":!0}),p(Z)!=="svelte-l0cm28"&&(Z.innerHTML=ut),Je=i(e),d(C.$$.fragment,e),ve=i(e),k=o(e,"UL",{"data-svelte-h":!0}),p(k)!=="svelte-1c2t6dx"&&(k.innerHTML=ft),je=i(e),d(X.$$.fragment,e),Ge=i(e),d(z.$$.fragment,e),Ue=i(e),x=o(e,"P",{"data-svelte-h":!0}),p(x)!=="svelte-1u2qp9s"&&(x.innerHTML=bt),Ze=i(e),R=o(e,"P",{"data-svelte-h":!0}),p(R)!=="svelte-1fwkss5"&&(R.textContent=ct),Ce=i(e),d(q.$$.fragment,e),ke=i(e),W=o(e,"P",{"data-svelte-h":!0}),p(W)!=="svelte-bbyjp5"&&(W.textContent=ht),Xe=i(e),d(B.$$.fragment,e),ze=i(e),H=o(e,"P",{"data-svelte-h":!0}),p(H)!=="svelte-1icf2em"&&(H.innerHTML=gt),xe=i(e),Y=o(e,"P",{"data-svelte-h":!0}),p(Y)!=="svelte-4hyyr7"&&(Y.textContent=wt),Re=i(e),d(F.$$.fragment,e),qe=i(e),I=o(e,"P",{"data-svelte-h":!0}),p(I)!=="svelte-1qtcdkv"&&(I.textContent=yt),We=i(e),d(L.$$.fragment,e),Be=i(e),P=o(e,"P",{"data-svelte-h":!0}),p(P)!=="svelte-1ah5e6c"&&(P.innerHTML=Mt),He=i(e),d(V.$$.fragment,e),Ye=i(e),d(Q.$$.fragment,e),Fe=i(e),N=o(e,"P",{"data-svelte-h":!0}),p(N)!=="svelte-19h5r7s"&&(N.innerHTML=$t),Ie=i(e),d(A.$$.fragment,e),Le=i(e),E=o(e,"P",{"data-svelte-h":!0}),p(E)!=="svelte-1x0s82b"&&(E.textContent=_t),Pe=i(e),d(S.$$.fragment,e),Ve=i(e),D=o(e,"P",{"data-svelte-h":!0}),p(D)!=="svelte-1vy35js"&&(D.innerHTML=Tt),Qe=i(e),K=o(e,"P",{"data-svelte-h":!0}),p(K)!=="svelte-11qxtld"&&(K.textContent=Jt),Ne=i(e),O=o(e,"P",{"data-svelte-h":!0}),p(O)!=="svelte-qaw7ck"&&(O.innerHTML=vt),Ae=i(e),ee=o(e,"P",{"data-svelte-h":!0}),p(ee)!=="svelte-gyjts5"&&(ee.innerHTML=jt),Ee=i(e),d(te.$$.fragment,e),Se=i(e),d(le.$$.fragment,e),De=i(e),ne=o(e,"P",{"data-svelte-h":!0}),p(ne)!=="svelte-rtwtrf"&&(ne.innerHTML=Gt),Ke=i(e),ae=o(e,"P",{"data-svelte-h":!0}),p(ae)!=="svelte-1c6wgo7"&&(ae.innerHTML=Ut),Oe=i(e),ie=o(e,"P",{"data-svelte-h":!0}),p(ie)!=="svelte-1n8dj9v"&&(ie.innerHTML=Zt),et=i(e),d(se.$$.fragment,e),tt=i(e),oe=o(e,"P",{"data-svelte-h":!0}),p(oe)!=="svelte-19uggsw"&&(oe.innerHTML=Ct),lt=i(e),d(pe.$$.fragment,e),nt=i(e),de=o(e,"P",{}),kt(de).forEach(l),this.h()},h(){Xt(h,"name","hf:doc:metadata"),Xt(h,"content",Lt)},m(e,t){Ht(document.head,h),n(e,re,t),n(e,me,t),n(e,ue,t),r(w,e,t),n(e,fe,t),r(y,e,t),n(e,be,t),r(M,e,t),n(e,ce,t),n(e,$,t),n(e,he,t),n(e,_,t),n(e,ge,t),n(e,T,t),n(e,we,t),r(J,e,t),n(e,ye,t),n(e,v,t),n(e,Me,t),n(e,j,t),n(e,$e,t),r(G,e,t),n(e,_e,t),n(e,U,t),n(e,Te,t),n(e,Z,t),n(e,Je,t),r(C,e,t),n(e,ve,t),n(e,k,t),n(e,je,t),r(X,e,t),n(e,Ge,t),r(z,e,t),n(e,Ue,t),n(e,x,t),n(e,Ze,t),n(e,R,t),n(e,Ce,t),r(q,e,t),n(e,ke,t),n(e,W,t),n(e,Xe,t),r(B,e,t),n(e,ze,t),n(e,H,t),n(e,xe,t),n(e,Y,t),n(e,Re,t),r(F,e,t),n(e,qe,t),n(e,I,t),n(e,We,t),r(L,e,t),n(e,Be,t),n(e,P,t),n(e,He,t),r(V,e,t),n(e,Ye,t),r(Q,e,t),n(e,Fe,t),n(e,N,t),n(e,Ie,t),r(A,e,t),n(e,Le,t),n(e,E,t),n(e,Pe,t),r(S,e,t),n(e,Ve,t),n(e,D,t),n(e,Qe,t),n(e,K,t),n(e,Ne,t),n(e,O,t),n(e,Ae,t),n(e,ee,t),n(e,Ee,t),r(te,e,t),n(e,Se,t),r(le,e,t),n(e,De,t),n(e,ne,t),n(e,Ke,t),n(e,ae,t),n(e,Oe,t),n(e,ie,t),n(e,et,t),r(se,e,t),n(e,tt,t),n(e,oe,t),n(e,lt,t),r(pe,e,t),n(e,nt,t),n(e,de,t),at=!0},p:xt,i(e){at||(u(w.$$.fragment,e),u(y.$$.fragment,e),u(M.$$.fragment,e),u(J.$$.fragment,e),u(G.$$.fragment,e),u(C.$$.fragment,e),u(X.$$.fragment,e),u(z.$$.fragment,e),u(q.$$.fragment,e),u(B.$$.fragment,e),u(F.$$.fragment,e),u(L.$$.fragment,e),u(V.$$.fragment,e),u(Q.$$.fragment,e),u(A.$$.fragment,e),u(S.$$.fragment,e),u(te.$$.fragment,e),u(le.$$.fragment,e),u(se.$$.fragment,e),u(pe.$$.fragment,e),at=!0)},o(e){f(w.$$.fragment,e),f(y.$$.fragment,e),f(M.$$.fragment,e),f(J.$$.fragment,e),f(G.$$.fragment,e),f(C.$$.fragment,e),f(X.$$.fragment,e),f(z.$$.fragment,e),f(q.$$.fragment,e),f(B.$$.fragment,e),f(F.$$.fragment,e),f(L.$$.fragment,e),f(V.$$.fragment,e),f(Q.$$.fragment,e),f(A.$$.fragment,e),f(S.$$.fragment,e),f(te.$$.fragment,e),f(le.$$.fragment,e),f(se.$$.fragment,e),f(pe.$$.fragment,e),at=!1},d(e){e&&(l(re),l(me),l(ue),l(fe),l(be),l(ce),l($),l(he),l(_),l(ge),l(T),l(we),l(ye),l(v),l(Me),l(j),l($e),l(_e),l(U),l(Te),l(Z),l(Je),l(ve),l(k),l(je),l(Ge),l(Ue),l(x),l(Ze),l(R),l(Ce),l(ke),l(W),l(Xe),l(ze),l(H),l(xe),l(Y),l(Re),l(qe),l(I),l(We),l(Be),l(P),l(He),l(Ye),l(Fe),l(N),l(Ie),l(Le),l(E),l(Pe),l(Ve),l(D),l(Qe),l(K),l(Ne),l(O),l(Ae),l(ee),l(Ee),l(Se),l(De),l(ne),l(Ke),l(ae),l(Oe),l(ie),l(et),l(tt),l(oe),l(lt),l(nt),l(de)),l(h),b(w,e),b(y,e),b(M,e),b(J,e),b(G,e),b(C,e),b(X,e),b(z,e),b(q,e),b(B,e),b(F,e),b(L,e),b(V,e),b(Q,e),b(A,e),b(S,e),b(te,e),b(le,e),b(se,e),b(pe,e)}}}const Lt='{"title":"Model quantization","local":"model-quantization","sections":[{"title":"bitsandbytes Integration","local":"bitsandbytes-integration","sections":[{"title":"Pre-Requisites","local":"pre-requisites","sections":[],"depth":3},{"title":"How it works","local":"how-it-works","sections":[],"depth":3},{"title":"Saving and loading 8-bit model","local":"saving-and-loading-8-bit-model","sections":[],"depth":3},{"title":"Offload modules to cpu and disk","local":"offload-modules-to-cpu-and-disk","sections":[],"depth":3},{"title":"Fine-tune a quantized model","local":"fine-tune-a-quantized-model","sections":[],"depth":3},{"title":"Example demo - running GPT2 1.5b on a Google Colab","local":"example-demo---running-gpt2-15b-on-a-google-colab","sections":[],"depth":3}],"depth":2}],"depth":1}';function Pt(it){return Rt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Et extends qt{constructor(h){super(),Wt(this,h,Pt,It,zt,{})}}export{Et as component};