bitsandbytes/pr_1925/en/_app/immutable/nodes/7.bbf3339f.js

import{s as ze,n as Ne,o as We}from"../chunks/scheduler.8a2cc2fa.js";import{S as De,i as Ve,e as i,s as l,c as p,h as Oe,a as o,d as a,b as s,f as Ye,g as f,j as r,k as N,l as Ke,m as n,n as d,t as m,o as c,p as u}from"../chunks/index.7079e750.js";import{C as et,H as Be,E as tt}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.0d032bd2.js";import{C as X}from"../chunks/CodeBlock.507aecff.js";function at(je){let b,W,Y,D,g,V,w,O,T,ve='FSDP-QLoRA combines data parallelism (FSDP enables sharding model parameters, optimizer states, and gradients across GPUs), 4-bit quantization, and LoRA to train LLMs up to 70B parameters on a dual 24GB GPU system. This technique was released by <a href="https://www.answer.ai/posts/2024-03-06-fsdp-qlora" rel="nofollow">Answer.AI</a> in collaboration with bitsandbytes to make training LLMs more efficient and accessible for everyone.',K,J,Le="This guide provides a brief guide on how bitsandbytes supports storing quantized weights to enable FSDP-QLoRA, and how to run training with the Hugging Face libraries.",ee,h,Fe='<p>Other changes required for bitsandbytes to support FSDP-QLoRA, such as reconstructing the weights from the quantization metadata and preventing quantizing already quantized weights when they’re moved from a CPU to GPU, are documented in this <a href="https://github.com/bitsandbytes-foundation/bitsandbytes/pull/970" rel="nofollow">Pull Request</a> and described in the <a href="https://www.answer.ai/posts/2024-03-14-fsdp-qlora-deep-dive" rel="nofollow">Enabling 70B Finetuning on Consumer GPUs</a> blog post. We highly recommend reading these resources for a better understanding of FSDP-QLoRA!</p>',te,U,ae,_,Ae='FSDP only supports sharding float data types which can be problematic because quantized weights are typically stored as integer data types (uint8). bitsandbytes doesn’t have this problem because it uses <code>StoreChar</code> to read and write quantized weights regardless of the data type storage. This makes it simple to add a <code>quant_storage</code> parameter to the <a href="/docs/bitsandbytes/pr_1925/en/reference/nn/linear4bit#bitsandbytes.nn.Linear4bit">Linear4bit</a> and <a href="/docs/bitsandbytes/pr_1925/en/reference/nn/linear4bit#bitsandbytes.nn.Params4bit">Params4bit</a> classes and set it to <code>torch.uint8</code> to maintain backward compatibility with the codebase. With the <code>quant_storage</code> parameter, you can select any of the FSDP supported data types to shard <a href="/docs/bitsandbytes/pr_1925/en/reference/nn/linear4bit#bitsandbytes.nn.Linear4bit">Linear4bit</a> with such as bfloat16, float16 or float32.',ne,C,Re='You’ll typically access and configure this option from <a href="https://huggingface.co/docs/transformers/main/en/main_classes/quantization#transformers.BitsAndBytesConfig" rel="nofollow">transformers.BitsAndBytesConfig</a> by setting the <code>bnb_4bit_quant_storage</code> parameter. It is very <strong>important</strong> the <code>quant_storage</code> data type matches the data types used throughout the model because FSDP can only wrap layers and modules that have the <em>same floating data type</em>. Making sure the data types are aligned will ensure the model is correctly sharded.',le,y,qe="<p>The <code>compute_dtype</code> is the data type used for computation inside the CUDA kernel, where the 4-bit quantized weights are unpacked from the data type in <code>quant_storage</code> and dequantized to <code>compute_dtype</code>. We recommend using torch.bfloat16 (if available on your hardware) for better numerical stability.</p>",se,$,ie,B,Ze='Check out this <a href="https://hf.co/docs/peft/main/en/accelerate/fsdp#use-peft-qlora-and-fsdp-for-finetuning-large-models-on-multiple-gpus" rel="nofollow">section</a> of the PEFT documentation for the config file and training code to run FSDP-QLoRA training.',oe,j,re,M,Qe='<p>FSDP is a distributed training framework that needs to be launched as a distributed training job with a library like <a href="https://hf.co/docs/accelerate/index" rel="nofollow">Accelerate</a> or <a href="https://pytorch.org/docs/stable/elastic/run.html" rel="nofollow">torchrun</a>. The launch command provided in this section uses Accelerate to launch the training script.</p>',pe,v,ke='bitsandbytes is deeply integrated with the Hugging Face ecosystem, making it easy to use with libraries like <a href="https://hf.co/docs/transformers" rel="nofollow">Transformers</a>, <a href="https://hf.co/docs/peft" rel="nofollow">PEFT</a>, and <a href="https://hf.co/docs/trl" rel="nofollow">TRL</a>.',fe,L,Ee='PEFT provides a configuration file (<a href="https://github.com/huggingface/peft/blob/main/examples/sft/configs/fsdp_config_qlora.yaml" rel="nofollow">fsdp_config_qlora.yaml</a>), launch command (<a href="https://github.com/huggingface/peft/blob/main/examples/sft/run_peft_qlora_fsdp.sh" rel="nofollow">run_peft_qlora_fsdp.sh</a>), and training script (<a href="https://github.com/huggingface/peft/blob/main/examples/sft/train.py" rel="nofollow">train.py</a>) for running FSDP-QLoRA. To learn more, check out the <a href="https://huggingface.co/docs/peft/main/en/accelerate/fsdp#use-peft-qlora-and-fsdp-for-finetuning-large-models-on-multiple-gpus" rel="nofollow">Use PEFT QLoRA and FSDP for finetuning large models on multiple GPUs</a> documentation. This section briefly covers the steps to run FSDP-QLoRA training.',de,F,xe="Before you begin, make sure you have the latest libraries installed.",me,A,ce,R,Ge='The important change that enables FSDP-QLoRA training is the <code>bnb_4bit_quant_storage</code> parameter in the <a href="https://huggingface.co/docs/transformers/main/en/main_classes/quantization#transformers.BitsAndBytesConfig" rel="nofollow">BitsAndBytesConfig</a> class. This allows you to set the storage data type of the quantized weights to a float data type.',ue,q,be,Z,He='Pass the <a href="https://huggingface.co/docs/transformers/main/en/main_classes/quantization#transformers.BitsAndBytesConfig" rel="nofollow">BitsAndBytesConfig</a> to a model to set it up for FSDP-QLoRA. You should set the <code>torch_dtype</code> parameter to match <code>bnb_4bit_quant_storage</code> so that the <a href="/docs/bitsandbytes/pr_1925/en/reference/nn/linear4bit#bitsandbytes.nn.Linear4bit">Linear4bit</a> layers are wrapped identically to the <code>Linear</code> layers. If the storage types do not match, then each <a href="/docs/bitsandbytes/pr_1925/en/reference/nn/linear4bit#bitsandbytes.nn.Linear4bit">Linear4bit</a> layer is wrapped individually.',he,Q,ye,k,Se='Configure the <a href="https://huggingface.co/docs/peft/main/en/package_reference/lora#peft.LoraConfig" rel="nofollow">LoraConfig</a> class for QLoRA training by setting <code>target_modules=&quot;all-linear&quot;</code>.',Me,E,ge,x,Ie='Now you can pass everything to the <a href="https://huggingface.co/docs/trl/main/en/sft_trainer#trl.SFTTrainer" rel="nofollow">SFTTrainer</a> for training.',we,G,Te,H,Je,S,Pe="To learn more about FSDP and QLoRA, check out the following resources:",Ue,I,Xe='<li>The <a href="https://github.com/AnswerDotAI/fsdp_qlora" rel="nofollow">AnswerDotAI/fsdp_qlora</a> repository.</li> <li>The introductory <a href="https://www.answer.ai/posts/2024-03-06-fsdp-qlora.html" rel="nofollow">You can now train a 70b language model at home</a> blog post by Answer.AI.</li> <li>For an introduction to FSDP, read the <a href="https://pytorch.org/blog/introducing-pytorch-fully-sharded-data-parallel-api" rel="nofollow">Introducing PyTorch Fully Sharded Data Parallel (FSDP) API</a> blog post.</li> <li>For more details about QLoRA, take a look at the <a href="https://huggingface.co/blog/4bit-transformers-bitsandbytes" rel="nofollow">Making LLMs even more accessible with bitsandbytes, 4-bit quantization and QLoRA</a> blog post.</li>',_e,P,Ce,z,$e;return g=new et({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),w=new Be({props:{title:"FSDP-QLoRA",local:"fsdp-qlora",headingTag:"h1"}}),U=new Be({props:{title:"Quantized data storage",local:"quantized-data-storage",headingTag:"h2"}}),$=new X({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBYm5iX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyglMEElMjAlMjAlMjAlMjBsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwYm5iXzRiaXRfcXVhbnRfdHlwZSUzRCUyMm5mNCUyMiUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X2NvbXB1dGVfZHR5cGUlM0R0b3JjaC5iZmxvYXQxNiUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3N0b3JhZ2UlM0R0b3JjaC5iZmxvYXQxNiUyQyUwQSklMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJtZXRhLWxsYW1hJTJGTGxhbWEtMi03MGIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEYm5iX2NvbmZpZyUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYlMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig, AutoModelForCausalLM

bnb_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_quant_storage=torch.bfloat16,
)

model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;meta-llama/Llama-2-70b&quot;</span>,
    quantization_config=bnb_config,
    torch_dtype=torch.bfloat16,
)`,wrap:!1}}),j=new Be({props:{title:"Training",local:"training",headingTag:"h2"}}),A=new X({props:{code:"cGlwJTIwaW5zdGFsbCUyMC1VJTIwYml0c2FuZGJ5dGVzJTIwYWNjZWxlcmF0ZSUyMHRyYW5zZm9ybWVycyUyMHBlZnQlMjB0cmw=",highlighted:"pip install -U bitsandbytes accelerate transformers peft trl",wrap:!1}}),q=new X({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQWJuYl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3R5cGUlM0QlMjJuZjQlMjIlMkMlMEElMjAlMjAlMjAlMjBibmJfNGJpdF9jb21wdXRlX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYlMkMlMEElMjAlMjAlMjAlMjBibmJfNGJpdF91c2VfZG91YmxlX3F1YW50JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3N0b3JhZ2UlM0R0b3JjaC5iZmxvYXQxNiUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig

bnb_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_storage=torch.bfloat16,
)`,wrap:!1}}),Q=new X({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIybWV0YS1sbGFtYSUyRkxsYW1hLTItNzBiJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRGJuYl9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;meta-llama/Llama-2-70b&quot;</span>,
    quantization_config=bnb_config,
    torch_dtype=torch.bfloat16,
)`,wrap:!1}}),E=new X({props:{code:"ZnJvbSUyMHBlZnQlMjBpbXBvcnQlMjBMb3JhQ29uZmlnJTBBJTBBcGVmdF9jb25maWclMjAlM0QlMjBMb3JhQ29uZmlnKCUwQSUyMCUyMCUyMCUyMGxvcmFfYWxwaGElM0QxNiUyQyUwQSUyMCUyMCUyMCUyMGxvcmFfZHJvcG91dCUzRDAuMSUyQyUwQSUyMCUyMCUyMCUyMHIlM0Q2NCUyQyUwQSUyMCUyMCUyMCUyMGJpYXMlM0QlMjJub25lJTIyJTJDJTBBJTIwJTIwJTIwJTIwdGFza190eXBlJTNEJTIyQ0FVU0FMX0xNJTIyJTJDJTBBJTIwJTIwJTIwJTIwdGFyZ2V0X21vZHVsZXMlM0QlMjJhbGwtbGluZWFyJTIyJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> LoraConfig

peft_config = LoraConfig(
    lora_alpha=<span class="hljs-number">16</span>,
    lora_dropout=<span class="hljs-number">0.1</span>,
    r=<span class="hljs-number">64</span>,
    bias=<span class="hljs-string">&quot;none&quot;</span>,
    task_type=<span class="hljs-string">&quot;CAUSAL_LM&quot;</span>,
    target_modules=<span class="hljs-string">&quot;all-linear&quot;</span>,
)`,wrap:!1}}),G=new X({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMFNGVFRyYWluZXIlMEElMEF0cmFpbmVyJTIwJTNEJTIwU0ZUVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRG1vZGVsJTJDJTBBJTIwJTIwJTIwJTIwdHJhaW5fZGF0YXNldCUzRGRhdGFzZXQlMkMlMEElMjAlMjAlMjAlMjBwZWZ0X2NvbmZpZyUzRHBlZnRfY29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwcHJvY2Vzc2luZ19jbGFzcyUzRHRva2VuaXplciUyQyUwQSUyMCUyMCUyMCUyMGFyZ3MlM0R0cmFpbmluZ19hcmd1bWVudHMlMkMlMEEpJTBBdHJhaW5lci50cmFpbigp",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> SFTTrainer

trainer = SFTTrainer(
    model=model,
    train_dataset=dataset,
    peft_config=peft_config,
    processing_class=tokenizer,
    args=training_arguments,
)
trainer.train()`,wrap:!1}}),H=new Be({props:{title:"Resources",local:"resources",headingTag:"h2"}}),P=new tt({props:{source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/docs/source/fsdp_qlora.md"}}),{c(){b=i("meta"),W=l(),Y=i("p"),D=l(),p(g.$$.fragment),V=l(),p(w.$$.fragment),O=l(),T=i("p"),T.innerHTML=ve,K=l(),J=i("p"),J.textContent=Le,ee=l(),h=i("blockquote"),h.innerHTML=Fe,te=l(),p(U.$$.fragment),ae=l(),_=i("p"),_.innerHTML=Ae,ne=l(),C=i("p"),C.innerHTML=Re,le=l(),y=i("blockquote"),y.innerHTML=qe,se=l(),p($.$$.fragment),ie=l(),B=i("p"),B.innerHTML=Ze,oe=l(),p(j.$$.fragment),re=l(),M=i("blockquote"),M.innerHTML=Qe,pe=l(),v=i("p"),v.innerHTML=ke,fe=l(),L=i("p"),L.innerHTML=Ee,de=l(),F=i("p"),F.textContent=xe,me=l(),p(A.$$.fragment),ce=l(),R=i("p"),R.innerHTML=Ge,ue=l(),p(q.$$.fragment),be=l(),Z=i("p"),Z.innerHTML=He,he=l(),p(Q.$$.fragment),ye=l(),k=i("p"),k.innerHTML=Se,Me=l(),p(E.$$.fragment),ge=l(),x=i("p"),x.innerHTML=Ie,we=l(),p(G.$$.fragment),Te=l(),p(H.$$.fragment),Je=l(),S=i("p"),S.textContent=Pe,Ue=l(),I=i("ul"),I.innerHTML=Xe,_e=l(),p(P.$$.fragment),Ce=l(),z=i("p"),this.h()},l(e){const t=Oe("svelte-u9bgzb",document.head);b=o(t,"META",{name:!0,content:!0}),t.forEach(a),W=s(e),Y=o(e,"P",{}),Ye(Y).forEach(a),D=s(e),f(g.$$.fragment,e),V=s(e),f(w.$$.fragment,e),O=s(e),T=o(e,"P",{"data-svelte-h":!0}),r(T)!=="svelte-5l5q4j"&&(T.innerHTML=ve),K=s(e),J=o(e,"P",{"data-svelte-h":!0}),r(J)!=="svelte-1naqdkg"&&(J.textContent=Le),ee=s(e),h=o(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),r(h)!=="svelte-1bablpl"&&(h.innerHTML=Fe),te=s(e),f(U.$$.fragment,e),ae=s(e),_=o(e,"P",{"data-svelte-h":!0}),r(_)!=="svelte-1a8v7yk"&&(_.innerHTML=Ae),ne=s(e),C=o(e,"P",{"data-svelte-h":!0}),r(C)!=="svelte-2kgr0m"&&(C.innerHTML=Re),le=s(e),y=o(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),r(y)!=="svelte-5h53j4"&&(y.innerHTML=qe),se=s(e),f($.$$.fragment,e),ie=s(e),B=o(e,"P",{"data-svelte-h":!0}),r(B)!=="svelte-1mcsrsf"&&(B.innerHTML=Ze),oe=s(e),f(j.$$.fragment,e),re=s(e),M=o(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),r(M)!=="svelte-1nwuwp3"&&(M.innerHTML=Qe),pe=s(e),v=o(e,"P",{"data-svelte-h":!0}),r(v)!=="svelte-11niy1x"&&(v.innerHTML=ke),fe=s(e),L=o(e,"P",{"data-svelte-h":!0}),r(L)!=="svelte-eapz72"&&(L.innerHTML=Ee),de=s(e),F=o(e,"P",{"data-svelte-h":!0}),r(F)!=="svelte-1x35k4g"&&(F.textContent=xe),me=s(e),f(A.$$.fragment,e),ce=s(e),R=o(e,"P",{"data-svelte-h":!0}),r(R)!=="svelte-2os12x"&&(R.innerHTML=Ge),ue=s(e),f(q.$$.fragment,e),be=s(e),Z=o(e,"P",{"data-svelte-h":!0}),r(Z)!=="svelte-ndx4t6"&&(Z.innerHTML=He),he=s(e),f(Q.$$.fragment,e),ye=s(e),k=o(e,"P",{"data-svelte-h":!0}),r(k)!=="svelte-qa6nm8"&&(k.innerHTML=Se),Me=s(e),f(E.$$.fragment,e),ge=s(e),x=o(e,"P",{"data-svelte-h":!0}),r(x)!=="svelte-1yk5yum"&&(x.innerHTML=Ie),we=s(e),f(G.$$.fragment,e),Te=s(e),f(H.$$.fragment,e),Je=s(e),S=o(e,"P",{"data-svelte-h":!0}),r(S)!=="svelte-ijbsql"&&(S.textContent=Pe),Ue=s(e),I=o(e,"UL",{"data-svelte-h":!0}),r(I)!=="svelte-47gauf"&&(I.innerHTML=Xe),_e=s(e),f(P.$$.fragment,e),Ce=s(e),z=o(e,"P",{}),Ye(z).forEach(a),this.h()},h(){N(b,"name","hf:doc:metadata"),N(b,"content",nt),N(h,"class","tip"),N(y,"class","tip"),N(M,"class","tip")},m(e,t){Ke(document.head,b),n(e,W,t),n(e,Y,t),n(e,D,t),d(g,e,t),n(e,V,t),d(w,e,t),n(e,O,t),n(e,T,t),n(e,K,t),n(e,J,t),n(e,ee,t),n(e,h,t),n(e,te,t),d(U,e,t),n(e,ae,t),n(e,_,t),n(e,ne,t),n(e,C,t),n(e,le,t),n(e,y,t),n(e,se,t),d($,e,t),n(e,ie,t),n(e,B,t),n(e,oe,t),d(j,e,t),n(e,re,t),n(e,M,t),n(e,pe,t),n(e,v,t),n(e,fe,t),n(e,L,t),n(e,de,t),n(e,F,t),n(e,me,t),d(A,e,t),n(e,ce,t),n(e,R,t),n(e,ue,t),d(q,e,t),n(e,be,t),n(e,Z,t),n(e,he,t),d(Q,e,t),n(e,ye,t),n(e,k,t),n(e,Me,t),d(E,e,t),n(e,ge,t),n(e,x,t),n(e,we,t),d(G,e,t),n(e,Te,t),d(H,e,t),n(e,Je,t),n(e,S,t),n(e,Ue,t),n(e,I,t),n(e,_e,t),d(P,e,t),n(e,Ce,t),n(e,z,t),$e=!0},p:Ne,i(e){$e||(m(g.$$.fragment,e),m(w.$$.fragment,e),m(U.$$.fragment,e),m($.$$.fragment,e),m(j.$$.fragment,e),m(A.$$.fragment,e),m(q.$$.fragment,e),m(Q.$$.fragment,e),m(E.$$.fragment,e),m(G.$$.fragment,e),m(H.$$.fragment,e),m(P.$$.fragment,e),$e=!0)},o(e){c(g.$$.fragment,e),c(w.$$.fragment,e),c(U.$$.fragment,e),c($.$$.fragment,e),c(j.$$.fragment,e),c(A.$$.fragment,e),c(q.$$.fragment,e),c(Q.$$.fragment,e),c(E.$$.fragment,e),c(G.$$.fragment,e),c(H.$$.fragment,e),c(P.$$.fragment,e),$e=!1},d(e){e&&(a(W),a(Y),a(D),a(V),a(O),a(T),a(K),a(J),a(ee),a(h),a(te),a(ae),a(_),a(ne),a(C),a(le),a(y),a(se),a(ie),a(B),a(oe),a(re),a(M),a(pe),a(v),a(fe),a(L),a(de),a(F),a(me),a(ce),a(R),a(ue),a(be),a(Z),a(he),a(ye),a(k),a(Me),a(ge),a(x),a(we),a(Te),a(Je),a(S),a(Ue),a(I),a(_e),a(Ce),a(z)),a(b),u(g,e),u(w,e),u(U,e),u($,e),u(j,e),u(A,e),u(q,e),u(Q,e),u(E,e),u(G,e),u(H,e),u(P,e)}}}const nt='{"title":"FSDP-QLoRA","local":"fsdp-qlora","sections":[{"title":"Quantized data storage","local":"quantized-data-storage","sections":[],"depth":2},{"title":"Training","local":"training","sections":[],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2}],"depth":1}';function lt(je){return We(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class pt extends De{constructor(b){super(),Ve(this,b,lt,at,ze,{})}}export{pt as component};