bitsandbytes/pr_2036/en/_app/immutable/nodes/12.JwDAYpyI.js

import"../chunks/DsnmJJEf.js";import{i as k,h as Z,C as Q,H as s,a as l,E as W,s as z}from"../chunks/CHailA3B.js";import{p as X,o as E,s as a,f as _,a as j,b as F,c as I,n as R}from"../chunks/CMkz9frW.js";const v='{"title":"Quickstart","local":"quickstart","sections":[{"title":"Installation","local":"installation","sections":[],"depth":2},{"title":"What is bitsandbytes?","local":"what-is-bitsandbytes","sections":[],"depth":2},{"title":"Quick Examples","local":"quick-examples","sections":[{"title":"8-bit Inference","local":"8-bit-inference","sections":[],"depth":3},{"title":"4-bit Quantization","local":"4-bit-quantization","sections":[],"depth":3},{"title":"QLoRA Fine-tuning","local":"qlora-fine-tuning","sections":[],"depth":3},{"title":"8-bit Optimizers","local":"8-bit-optimizers","sections":[],"depth":3},{"title":"Custom Quantized Layers","local":"custom-quantized-layers","sections":[],"depth":3}],"depth":2},{"title":"Next Steps","local":"next-steps","sections":[],"depth":2},{"title":"Getting Help","local":"getting-help","sections":[],"depth":2}],"depth":1}';var V=I('<meta name="hf:doc:metadata"/>'),Y=I('<p></p> <!> <!> <p>Welcome to bitsandbytes! This library enables accessible large language models via k-bit quantization for PyTorch, dramatically reducing memory consumption for inference and training.</p> <!> <!> <p><strong>Requirements:</strong> Python 3.10+, PyTorch 2.4+</p> <p>For detailed installation instructions, see the <a href="./installation">Installation Guide</a>.</p> <!> <p>bitsandbytes provides three main features:</p> <ul><li><strong>LLM.int8()</strong>: 8-bit quantization for inference (50% memory reduction)</li> <li><strong>QLoRA</strong>: 4-bit quantization for training (75% memory reduction)</li> <li><strong>8-bit Optimizers</strong>: Memory-efficient optimizers for training</li></ul> <!> <!> <p>Load and run a model using 8-bit quantization:</p> <!> <blockquote><p><strong>Learn more:</strong> See the <a href="./integrations">Integrations guide</a> for more details on using bitsandbytes with Transformers.</p></blockquote> <!> <p>For even greater memory savings:</p> <!> <!> <p>Combine 4-bit quantization with LoRA for efficient training:</p> <!> <blockquote><p><strong>Learn more:</strong> See the <a href="./fsdp_qlora">FSDP-QLoRA guide</a> for advanced training techniques and the <a href="./integrations">Integrations guide</a> for using with PEFT.</p></blockquote> <!> <p>Use 8-bit optimizers to reduce training memory by 75%:</p> <!> <blockquote><p><strong>Learn more:</strong> See the <a href="./optimizers">8-bit Optimizers guide</a> for detailed usage and configuration options.</p></blockquote> <!> <p>Use quantized linear layers directly in your models:</p> <!> <!> <ul><li><a href="./optimizers">8-bit Optimizers Guide</a> - Detailed optimizer usage</li> <li><a href="./fsdp_qlora">FSDP-QLoRA</a> - Train 70B+ models on consumer GPUs</li> <li><a href="./integrations">Integrations</a> - Use with Transformers, PEFT, Accelerate</li> <li><a href="./faqs">FAQs</a> - Common questions and troubleshooting</li></ul> <!> <ul><li>Check the <a href="./faqs">FAQs</a> and <a href="./errors">Common Errors</a></li> <li>Visit <a href="https://huggingface.co/docs/bitsandbytes" rel="nofollow">official documentation</a></li> <li>Open an issue on <a href="https://github.com/bitsandbytes-foundation/bitsandbytes/issues" rel="nofollow">GitHub</a></li></ul> <!> <p></p>',1);function S(B,G){X(G,!1),E(()=>{new URLSearchParams(window.location.search).get("fw")}),k();var t=Y();Z("11na0ra",U=>{var f=V();z(f,"content",v),j(U,f)});var n=a(_(t),2);Q(n,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var e=a(n,2);s(e,{title:"Quickstart",local:"quickstart",headingTag:"h1"});var i=a(e,4);s(i,{title:"Installation",local:"installation",headingTag:"h2"});var o=a(i,2);l(o,{code:"cGlwJTIwaW5zdGFsbCUyMGJpdHNhbmRieXRlcw==",highlighted:"pip install bitsandbytes",lang:"bash",wrap:!1});var r=a(o,6);s(r,{title:"What is bitsandbytes?",local:"what-is-bitsandbytes",headingTag:"h2"});var p=a(r,6);s(p,{title:"Quick Examples",local:"quick-examples",headingTag:"h2"});var d=a(p,2);s(d,{title:"8-bit Inference",local:"8-bit-inference",headingTag:"h3"});var c=a(d,4);l(c,{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMm1ldGEtbGxhbWElMkZMbGFtYS0yLTdiLWhmJTIyJTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fOGJpdCUzRFRydWUpJTJDJTBBKSUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMm1ldGEtbGxhbWElMkZMbGFtYS0yLTdiLWhmJTIyKSUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplciglMjJIZWxsbyUyQyUyMG15JTIwbmFtZSUyMGlzJTIyJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8oJTIyY3VkYSUyMiklMEFvdXRwdXRzJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDIwKSUwQXByaW50KHRva2VuaXplci5kZWNvZGUob3V0cHV0cyU1QjAlNUQpKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;meta-llama/Llama-2-7b-hf&quot;</span>,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
    quantization_config=BitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>),
)

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-2-7b-hf&quot;</span>)
inputs = tokenizer(<span class="hljs-string">&quot;Hello, my name is&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)
outputs = model.generate(**inputs, max_new_tokens=<span class="hljs-number">20</span>)
<span class="hljs-built_in">print</span>(tokenizer.decode(outputs[<span class="hljs-number">0</span>]))`,lang:"py",wrap:!1});var m=a(c,4);s(m,{title:"4-bit Quantization",local:"4-bit-quantization",headingTag:"h3"});var b=a(m,4);l(b,{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBCaXRzQW5kQnl0ZXNDb25maWclMEElMEFibmJfY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKCUwQSUyMCUyMCUyMCUyMGxvYWRfaW5fNGJpdCUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBibmJfNGJpdF9jb21wdXRlX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYlMkMlMEElMjAlMjAlMjAlMjBibmJfNGJpdF9xdWFudF90eXBlJTNEJTIybmY0JTIyJTJDJTBBKSUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMm1ldGEtbGxhbWElMkZMbGFtYS0yLTdiLWhmJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRGJuYl9jb25maWclMkMlMEElMjAlMjAlMjAlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUwQSk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig

bnb_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
)

model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;meta-llama/Llama-2-7b-hf&quot;</span>,
    quantization_config=bnb_config,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
)`,lang:"py",wrap:!1});var y=a(b,2);s(y,{title:"QLoRA Fine-tuning",local:"qlora-fine-tuning",headingTag:"h3"});var M=a(y,4);l(M,{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBZnJvbSUyMHBlZnQlMjBpbXBvcnQlMjBMb3JhQ29uZmlnJTJDJTIwZ2V0X3BlZnRfbW9kZWwlMkMlMjBwcmVwYXJlX21vZGVsX2Zvcl9rYml0X3RyYWluaW5nJTBBJTBBJTIzJTIwTG9hZCUyMDQtYml0JTIwbW9kZWwlMEFibmJfY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fNGJpdCUzRFRydWUpJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIybWV0YS1sbGFtYSUyRkxsYW1hLTItN2ItaGYlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEYm5iX2NvbmZpZyUyQyUwQSklMEElMEElMjMlMjBQcmVwYXJlJTIwZm9yJTIwdHJhaW5pbmclMEFtb2RlbCUyMCUzRCUyMHByZXBhcmVfbW9kZWxfZm9yX2tiaXRfdHJhaW5pbmcobW9kZWwpJTBBJTBBJTIzJTIwQWRkJTIwTG9SQSUyMGFkYXB0ZXJzJTBBbG9yYV9jb25maWclMjAlM0QlMjBMb3JhQ29uZmlnKCUwQSUyMCUyMCUyMCUyMHIlM0QxNiUyQyUwQSUyMCUyMCUyMCUyMGxvcmFfYWxwaGElM0QzMiUyQyUwQSUyMCUyMCUyMCUyMHRhcmdldF9tb2R1bGVzJTNEJTVCJTIycV9wcm9qJTIyJTJDJTIwJTIydl9wcm9qJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwdGFza190eXBlJTNEJTIyQ0FVU0FMX0xNJTIyJTJDJTBBKSUwQW1vZGVsJTIwJTNEJTIwZ2V0X3BlZnRfbW9kZWwobW9kZWwlMkMlMjBsb3JhX2NvbmZpZyklMEElMEElMjMlMjBOb3clMjB0cmFpbiUyMHdpdGglMjB5b3VyJTIwcHJlZmVycmVkJTIwdHJhaW5lcg==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig
<span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> LoraConfig, get_peft_model, prepare_model_for_kbit_training

<span class="hljs-comment"># Load 4-bit model</span>
bnb_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>)
model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;meta-llama/Llama-2-7b-hf&quot;</span>,
    quantization_config=bnb_config,
)

<span class="hljs-comment"># Prepare for training</span>
model = prepare_model_for_kbit_training(model)

<span class="hljs-comment"># Add LoRA adapters</span>
lora_config = LoraConfig(
    r=<span class="hljs-number">16</span>,
    lora_alpha=<span class="hljs-number">32</span>,
    target_modules=[<span class="hljs-string">&quot;q_proj&quot;</span>, <span class="hljs-string">&quot;v_proj&quot;</span>],
    task_type=<span class="hljs-string">&quot;CAUSAL_LM&quot;</span>,
)
model = get_peft_model(model, lora_config)

<span class="hljs-comment"># Now train with your preferred trainer</span>`,lang:"py",wrap:!1});var h=a(M,4);s(h,{title:"8-bit Optimizers",local:"8-bit-optimizers",headingTag:"h3"});var u=a(h,4);l(u,{code:"aW1wb3J0JTIwYml0c2FuZGJ5dGVzJTIwYXMlMjBibmIlMEElMEFtb2RlbCUyMCUzRCUyMFlvdXJNb2RlbCgpJTBBJTBBJTIzJTIwUmVwbGFjZSUyMHN0YW5kYXJkJTIwb3B0aW1pemVyJTIwd2l0aCUyMDgtYml0JTIwdmVyc2lvbiUwQW9wdGltaXplciUyMCUzRCUyMGJuYi5vcHRpbS5BZGFtOGJpdChtb2RlbC5wYXJhbWV0ZXJzKCklMkMlMjBsciUzRDFlLTMpJTBBJTBBJTIzJTIwVXNlJTIwaW4lMjB0cmFpbmluZyUyMGxvb3AlMjBhcyUyMG5vcm1hbCUwQWZvciUyMGJhdGNoJTIwaW4lMjBkYXRhbG9hZGVyJTNBJTBBJTIwJTIwJTIwJTIwbG9zcyUyMCUzRCUyMG1vZGVsKGJhdGNoKSUwQSUyMCUyMCUyMCUyMGxvc3MuYmFja3dhcmQoKSUwQSUyMCUyMCUyMCUyMG9wdGltaXplci5zdGVwKCklMEElMjAlMjAlMjAlMjBvcHRpbWl6ZXIuemVyb19ncmFkKCk=",highlighted:`<span class="hljs-keyword">import</span> bitsandbytes <span class="hljs-keyword">as</span> bnb

model = YourModel()

<span class="hljs-comment"># Replace standard optimizer with 8-bit version</span>
optimizer = bnb.optim.Adam8bit(model.parameters(), lr=<span class="hljs-number">1e-3</span>)

<span class="hljs-comment"># Use in training loop as normal</span>
<span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> dataloader:
    loss = model(batch)
    loss.backward()
    optimizer.step()
    optimizer.zero_grad()`,lang:"py",wrap:!1});var J=a(u,4);s(J,{title:"Custom Quantized Layers",local:"custom-quantized-layers",headingTag:"h3"});var T=a(J,4);l(T,{code:"aW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjBiaXRzYW5kYnl0ZXMlMjBhcyUyMGJuYiUwQSUwQSUyMyUyMDgtYml0JTIwbGluZWFyJTIwbGF5ZXIlMEFsaW5lYXJfOGJpdCUyMCUzRCUyMGJuYi5ubi5MaW5lYXI4Yml0THQoMTAyNCUyQyUyMDEwMjQlMkMlMjBoYXNfZnAxNl93ZWlnaHRzJTNERmFsc2UpJTBBJTBBJTIzJTIwNC1iaXQlMjBsaW5lYXIlMjBsYXllciUwQWxpbmVhcl80Yml0JTIwJTNEJTIwYm5iLm5uLkxpbmVhcjRiaXQoMTAyNCUyQyUyMDEwMjQlMkMlMjBjb21wdXRlX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> bitsandbytes <span class="hljs-keyword">as</span> bnb

<span class="hljs-comment"># 8-bit linear layer</span>
linear_8bit = bnb.nn.Linear8bitLt(<span class="hljs-number">1024</span>, <span class="hljs-number">1024</span>, has_fp16_weights=<span class="hljs-literal">False</span>)

<span class="hljs-comment"># 4-bit linear layer</span>
linear_4bit = bnb.nn.Linear4bit(<span class="hljs-number">1024</span>, <span class="hljs-number">1024</span>, compute_dtype=torch.bfloat16)`,lang:"py",wrap:!1});var w=a(T,2);s(w,{title:"Next Steps",local:"next-steps",headingTag:"h2"});var g=a(w,4);s(g,{title:"Getting Help",local:"getting-help",headingTag:"h2"});var C=a(g,4);W(C,{source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/docs/source/quickstart.mdx"}),R(2),j(B,t),F()}export{S as component};