bitsandbytes/pr_1521/en/_app/immutable/nodes/8.4a64124c.js

import{s as ze,o as Ne,n as Be}from"../chunks/scheduler.852ec091.js";import{S as We,i as De,g as r,s as l,r as d,A as Ve,h as p,f as a,c as i,j as Xe,u as m,x as f,k as Ye,y as Ke,a as n,v as c,d as u,t as b,w as h}from"../chunks/index.28275fd3.js";import{T as Ce}from"../chunks/Tip.9f398c59.js";import{C as N}from"../chunks/CodeBlock.c3366071.js";import{H as Ue,E as Oe}from"../chunks/EditOnGithub.582011f0.js";function et(g){let s,y='Other changes required for bitsandbytes to support FSDP-QLoRA, such as reconstructing the weights from the quantization metadata and preventing quantizing already quantized weights when they’re moved from a CPU to GPU, are documented in this <a href="https://github.com/TimDettmers/bitsandbytes/pull/970" rel="nofollow">Pull Request</a> and described in the <a href="https://www.answer.ai/posts/2024-03-14-fsdp-qlora-deep-dive" rel="nofollow">Enabling 70B Finetuning on Consumer GPUs</a> blog post. We highly recommend reading these resources for a better understanding of FSDP-QLoRA!';return{c(){s=r("p"),s.innerHTML=y},l(o){s=p(o,"P",{"data-svelte-h":!0}),f(s)!=="svelte-10tshw4"&&(s.innerHTML=y)},m(o,M){n(o,s,M)},p:Be,d(o){o&&a(s)}}}function tt(g){let s,y="The <code>compute_dtype</code> is the data type used for computation inside the CUDA kernel, where the 4-bit quantized weights are unpacked from the data type in <code>quant_storage</code> and dequantized to <code>compute_dtype</code>. We recommend using torch.bfloat16 (if available on your hardware) for better numerical stability.";return{c(){s=r("p"),s.innerHTML=y},l(o){s=p(o,"P",{"data-svelte-h":!0}),f(s)!=="svelte-1xdoh7b"&&(s.innerHTML=y)},m(o,M){n(o,s,M)},p:Be,d(o){o&&a(s)}}}function at(g){let s,y='FSDP is a distributed training framework that needs to be launched as a distributed training job with a library like <a href="https://hf.co/docs/accelerate/index" rel="nofollow">Accelerate</a> or <a href="https://pytorch.org/docs/stable/elastic/run.html" rel="nofollow">torchrun</a>. The launch command provided in this section uses Accelerate to launch the training script.';return{c(){s=r("p"),s.innerHTML=y},l(o){s=p(o,"P",{"data-svelte-h":!0}),f(s)!=="svelte-1k9ae9o"&&(s.innerHTML=y)},m(o,M){n(o,s,M)},p:Be,d(o){o&&a(s)}}}function nt(g){let s,y,o,M,J,D,_,je='FSDP-QLoRA combines data parallelism (FSDP enables sharding model parameters, optimizer states, and gradients across GPUs), 4-bit quantization, and LoRA to train LLMs up to 70B parameters on a dual 24GB GPU system. This technique was released by <a href="https://www.answer.ai/posts/2024-03-06-fsdp-qlora" rel="nofollow">Answer.AI</a> in collaboration with bitsandbytes to make training LLMs more efficient and accessible for everyone.',V,U,ve="This guide provides a brief guide on how bitsandbytes supports storing quantized weights to enable FSDP-QLoRA, and how to run training with the Hugging Face libraries.",K,w,O,C,ee,B,Le='FSDP only supports sharding float data types which can be problematic because quantized weights are typically stored as integer data types (uint8). bitsandbytes doesn’t have this problem because it uses <code>StoreChar</code> to read and write quantized weights regardless of the data type storage. This makes it simple to add a <code>quant_storage</code> parameter to the <a href="/docs/bitsandbytes/pr_1521/en/reference/nn/linear4bit#bitsandbytes.nn.Linear4bit">Linear4bit</a> and <a href="/docs/bitsandbytes/pr_1521/en/reference/nn/linear4bit#bitsandbytes.nn.Params4bit">Params4bit</a> classes and set it to <code>torch.uint8</code> to maintain backward compatibility with the codebase. With the <code>quant_storage</code> parameter, you can select any of the FSDP supported data types to shard <a href="/docs/bitsandbytes/pr_1521/en/reference/nn/linear4bit#bitsandbytes.nn.Linear4bit">Linear4bit</a> with such as bfloat16, float16 or float32.',te,j,Fe='You’ll typically access and configure this option from <a href="https://huggingface.co/docs/transformers/main/en/main_classes/quantization#transformers.BitsAndBytesConfig" rel="nofollow">transformers.BitsAndBytesConfig</a> by setting the <code>bnb_4bit_quant_storage</code> parameter. It is very <strong>important</strong> the <code>quant_storage</code> data type matches the data types used throughout the model because FSDP can only wrap layers and modules that have the <em>same floating data type</em>. Making sure the data types are aligned will ensure the model is correctly sharded.',ae,T,ne,v,se,L,Ae='Check out this <a href="https://hf.co/docs/peft/main/en/accelerate/fsdp#use-peft-qlora-and-fsdp-for-finetuning-large-models-on-multiple-gpus" rel="nofollow">section</a> of the PEFT documentation for the config file and training code to run FSDP-QLoRA training.',le,F,ie,$,oe,A,Re='bitsandbytes is deeply integrated with the Hugging Face ecosystem, making it easy to use with libraries like <a href="https://hf.co/docs/transformers" rel="nofollow">Transformers</a>, <a href="https://hf.co/docs/peft" rel="nofollow">PEFT</a>, and <a href="https://hf.co/docs/trl" rel="nofollow">TRL</a>.',re,R,Ze='PEFT provides a configuration file (<a href="https://github.com/huggingface/peft/blob/main/examples/sft/configs/fsdp_config_qlora.yaml" rel="nofollow">fsdp_config_qlora.yaml</a>), launch command (<a href="https://github.com/huggingface/peft/blob/main/examples/sft/run_peft_qlora_fsdp.sh" rel="nofollow">run_peft_qlora_fsdp.sh</a>), and training script (<a href="https://github.com/huggingface/peft/blob/main/examples/sft/train.py" rel="nofollow">train.py</a>) for running FSDP-QLoRA. To learn more, check out the <a href="https://huggingface.co/docs/peft/main/en/accelerate/fsdp#use-peft-qlora-and-fsdp-for-finetuning-large-models-on-multiple-gpus" rel="nofollow">Use PEFT QLoRA and FSDP for finetuning large models on multiple GPUs</a> documentation. This section briefly covers the steps to run FSDP-QLoRA training.',pe,Z,Qe="Before you begin, make sure you have the latest libraries installed.",fe,Q,de,q,qe='The important change that enables FSDP-QLoRA training is the <code>bnb_4bit_quant_storage</code> parameter in the <a href="https://huggingface.co/docs/transformers/main/en/main_classes/quantization#transformers.BitsAndBytesConfig" rel="nofollow">BitsAndBytesConfig</a> class. This allows you to set the storage data type of the quantized weights to a float data type.',me,k,ce,G,ke='Pass the <a href="https://huggingface.co/docs/transformers/main/en/main_classes/quantization#transformers.BitsAndBytesConfig" rel="nofollow">BitsAndBytesConfig</a> to a model to set it up for FSDP-QLoRA. You should set the <code>torch_dtype</code> parameter to match <code>bnb_4bit_quant_storage</code> so that the <a href="/docs/bitsandbytes/pr_1521/en/reference/nn/linear4bit#bitsandbytes.nn.Linear4bit">Linear4bit</a> layers are wrapped identically to the <code>Linear</code> layers. If the storage types do not match, then each <a href="/docs/bitsandbytes/pr_1521/en/reference/nn/linear4bit#bitsandbytes.nn.Linear4bit">Linear4bit</a> layer is wrapped individually.',ue,H,be,E,Ge="Configure the <code>~peft.LoraConfig</code> class for QLoRA training by setting <code>target_modules=&quot;all-linear&quot;</code>.",he,P,ye,I,He='Now you can pass everything to the <a href="https://huggingface.co/docs/trl/main/en/sft_trainer#trl.SFTTrainer" rel="nofollow">SFTTrainer</a> for training.',Me,S,ge,x,we,X,Ee="To learn more about FSDP and QLoRA, check out the following resources:",Te,Y,Pe='<li>The <a href="https://github.com/AnswerDotAI/fsdp_qlora" rel="nofollow">AnswerDotAI/fsdp_qlora</a> repository.</li> <li>The introductory <a href="https://www.answer.ai/posts/2024-03-06-fsdp-qlora.html" rel="nofollow">You can now train a 70b language model at home</a> blog post by Answer.AI.</li> <li>For an introduction to FSDP, read the <a href="https://pytorch.org/blog/introducing-pytorch-fully-sharded-data-parallel-api" rel="nofollow">Introducing PyTorch Fully Sharded Data Parallel (FSDP) API</a> blog post.</li> <li>For more details about QLoRA, take a look at the <a href="https://huggingface.co/blog/4bit-transformers-bitsandbytes" rel="nofollow">Making LLMs even more accessible with bitsandbytes, 4-bit quantization and QLoRA</a> blog post.</li>',$e,z,Je,W,_e;return J=new Ue({props:{title:"FSDP-QLoRA",local:"fsdp-qlora",headingTag:"h1"}}),w=new Ce({props:{warning:!1,$$slots:{default:[et]},$$scope:{ctx:g}}}),C=new Ue({props:{title:"Quantized data storage",local:"quantized-data-storage",headingTag:"h2"}}),T=new Ce({props:{warning:!1,$$slots:{default:[tt]},$$scope:{ctx:g}}}),v=new N({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBYm5iX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyglMEElMjAlMjAlMjAlMjBsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwYm5iXzRiaXRfcXVhbnRfdHlwZSUzRCUyMm5mNCUyMiUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X2NvbXB1dGVfZHR5cGUlM0R0b3JjaC5iZmxvYXQxNiUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3N0b3JhZ2UlM0R0b3JjaC5iZmxvYXQxNiUyQyUwQSklMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJtZXRhLWxsYW1hJTJGTGxhbWEtMi03MGIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEYm5iX2NvbmZpZyUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYlMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig, AutoModelForCausalLM

bnb_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_quant_storage=torch.bfloat16,
)

model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;meta-llama/Llama-2-70b&quot;</span>,
    quantization_config=bnb_config,
    torch_dtype=torch.bfloat16,
)`,wrap:!1}}),F=new Ue({props:{title:"Training",local:"training",headingTag:"h2"}}),$=new Ce({props:{warning:!1,$$slots:{default:[at]},$$scope:{ctx:g}}}),Q=new N({props:{code:"cGlwJTIwaW5zdGFsbCUyMC1VJTIwYml0c2FuZGJ5dGVzJTIwYWNjZWxlcmF0ZSUyMHRyYW5zZm9ybWVycyUyMHBlZnQlMjB0cmw=",highlighted:"pip install -U bitsandbytes accelerate transformers peft trl",wrap:!1}}),k=new N({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQWJuYl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3R5cGUlM0QlMjJuZjQlMjIlMkMlMEElMjAlMjAlMjAlMjBibmJfNGJpdF9jb21wdXRlX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYlMkMlMEElMjAlMjAlMjAlMjBibmJfNGJpdF91c2VfZG91YmxlX3F1YW50JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3N0b3JhZ2UlM0R0b3JjaC5iZmxvYXQxNiUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig

bnb_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_storage=torch.bfloat16,
)`,wrap:!1}}),H=new N({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIybWV0YS1sbGFtYSUyRkxsYW1hLTItNzBiJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRGJuYl9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;meta-llama/Llama-2-70b&quot;</span>,
    quantization_config=bnb_config,
    torch_dtype=torch.bfloat16,
)`,wrap:!1}}),P=new N({props:{code:"ZnJvbSUyMHBlZnQlMjBpbXBvcnQlMjBMb3JhQ29uZmlnJTBBJTBBcGVmdF9jb25maWclMjAlM0QlMjBMb3JhQ29uZmlnKCUwQSUyMCUyMCUyMCUyMGxvcmFfYWxwaGElM0QxNiUyQyUwQSUyMCUyMCUyMCUyMGxvcmFfZHJvcG91dCUzRDAuMSUyQyUwQSUyMCUyMCUyMCUyMHIlM0Q2NCUyQyUwQSUyMCUyMCUyMCUyMGJpYXMlM0QlMjJub25lJTIyJTJDJTBBJTIwJTIwJTIwJTIwdGFza190eXBlJTNEJTIyQ0FVU0FMX0xNJTIyJTJDJTBBJTIwJTIwJTIwJTIwdGFyZ2V0X21vZHVsZXMlM0QlMjJhbGwtbGluZWFyJTIyJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> LoraConfig

peft_config = LoraConfig(
    lora_alpha=<span class="hljs-number">16</span>,
    lora_dropout=<span class="hljs-number">0.1</span>,
    r=<span class="hljs-number">64</span>,
    bias=<span class="hljs-string">&quot;none&quot;</span>,
    task_type=<span class="hljs-string">&quot;CAUSAL_LM&quot;</span>,
    target_modules=<span class="hljs-string">&quot;all-linear&quot;</span>,
)`,wrap:!1}}),S=new N({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMFNGVFRyYWluZXIlMEElMEF0cmFpbmVyJTIwJTNEJTIwU0ZUVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRG1vZGVsJTJDJTBBJTIwJTIwJTIwJTIwdHJhaW5fZGF0YXNldCUzRGRhdGFzZXQlMkMlMEElMjAlMjAlMjAlMjBwZWZ0X2NvbmZpZyUzRHBlZnRfY29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwcHJvY2Vzc2luZ19jbGFzcyUzRHRva2VuaXplciUyQyUwQSUyMCUyMCUyMCUyMGFyZ3MlM0R0cmFpbmluZ19hcmd1bWVudHMlMkMlMEEpJTBBdHJhaW5lci50cmFpbigp",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> SFTTrainer

trainer = SFTTrainer(
    model=model,
    train_dataset=dataset,
    peft_config=peft_config,
    processing_class=tokenizer,
    args=training_arguments,
)
trainer.train()`,wrap:!1}}),x=new Ue({props:{title:"Resources",local:"resources",headingTag:"h2"}}),z=new Oe({props:{source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/docs/source/fsdp_qlora.md"}}),{c(){s=r("meta"),y=l(),o=r("p"),M=l(),d(J.$$.fragment),D=l(),_=r("p"),_.innerHTML=je,V=l(),U=r("p"),U.textContent=ve,K=l(),d(w.$$.fragment),O=l(),d(C.$$.fragment),ee=l(),B=r("p"),B.innerHTML=Le,te=l(),j=r("p"),j.innerHTML=Fe,ae=l(),d(T.$$.fragment),ne=l(),d(v.$$.fragment),se=l(),L=r("p"),L.innerHTML=Ae,le=l(),d(F.$$.fragment),ie=l(),d($.$$.fragment),oe=l(),A=r("p"),A.innerHTML=Re,re=l(),R=r("p"),R.innerHTML=Ze,pe=l(),Z=r("p"),Z.textContent=Qe,fe=l(),d(Q.$$.fragment),de=l(),q=r("p"),q.innerHTML=qe,me=l(),d(k.$$.fragment),ce=l(),G=r("p"),G.innerHTML=ke,ue=l(),d(H.$$.fragment),be=l(),E=r("p"),E.innerHTML=Ge,he=l(),d(P.$$.fragment),ye=l(),I=r("p"),I.innerHTML=He,Me=l(),d(S.$$.fragment),ge=l(),d(x.$$.fragment),we=l(),X=r("p"),X.textContent=Ee,Te=l(),Y=r("ul"),Y.innerHTML=Pe,$e=l(),d(z.$$.fragment),Je=l(),W=r("p"),this.h()},l(e){const t=Ve("svelte-u9bgzb",document.head);s=p(t,"META",{name:!0,content:!0}),t.forEach(a),y=i(e),o=p(e,"P",{}),Xe(o).forEach(a),M=i(e),m(J.$$.fragment,e),D=i(e),_=p(e,"P",{"data-svelte-h":!0}),f(_)!=="svelte-5l5q4j"&&(_.innerHTML=je),V=i(e),U=p(e,"P",{"data-svelte-h":!0}),f(U)!=="svelte-1naqdkg"&&(U.textContent=ve),K=i(e),m(w.$$.fragment,e),O=i(e),m(C.$$.fragment,e),ee=i(e),B=p(e,"P",{"data-svelte-h":!0}),f(B)!=="svelte-1imkoks"&&(B.innerHTML=Le),te=i(e),j=p(e,"P",{"data-svelte-h":!0}),f(j)!=="svelte-2kgr0m"&&(j.innerHTML=Fe),ae=i(e),m(T.$$.fragment,e),ne=i(e),m(v.$$.fragment,e),se=i(e),L=p(e,"P",{"data-svelte-h":!0}),f(L)!=="svelte-1mcsrsf"&&(L.innerHTML=Ae),le=i(e),m(F.$$.fragment,e),ie=i(e),m($.$$.fragment,e),oe=i(e),A=p(e,"P",{"data-svelte-h":!0}),f(A)!=="svelte-11niy1x"&&(A.innerHTML=Re),re=i(e),R=p(e,"P",{"data-svelte-h":!0}),f(R)!=="svelte-eapz72"&&(R.innerHTML=Ze),pe=i(e),Z=p(e,"P",{"data-svelte-h":!0}),f(Z)!=="svelte-1x35k4g"&&(Z.textContent=Qe),fe=i(e),m(Q.$$.fragment,e),de=i(e),q=p(e,"P",{"data-svelte-h":!0}),f(q)!=="svelte-2os12x"&&(q.innerHTML=qe),me=i(e),m(k.$$.fragment,e),ce=i(e),G=p(e,"P",{"data-svelte-h":!0}),f(G)!=="svelte-1f3gfxm"&&(G.innerHTML=ke),ue=i(e),m(H.$$.fragment,e),be=i(e),E=p(e,"P",{"data-svelte-h":!0}),f(E)!=="svelte-1c6aae"&&(E.innerHTML=Ge),he=i(e),m(P.$$.fragment,e),ye=i(e),I=p(e,"P",{"data-svelte-h":!0}),f(I)!=="svelte-1yk5yum"&&(I.innerHTML=He),Me=i(e),m(S.$$.fragment,e),ge=i(e),m(x.$$.fragment,e),we=i(e),X=p(e,"P",{"data-svelte-h":!0}),f(X)!=="svelte-ijbsql"&&(X.textContent=Ee),Te=i(e),Y=p(e,"UL",{"data-svelte-h":!0}),f(Y)!=="svelte-47gauf"&&(Y.innerHTML=Pe),$e=i(e),m(z.$$.fragment,e),Je=i(e),W=p(e,"P",{}),Xe(W).forEach(a),this.h()},h(){Ye(s,"name","hf:doc:metadata"),Ye(s,"content",st)},m(e,t){Ke(document.head,s),n(e,y,t),n(e,o,t),n(e,M,t),c(J,e,t),n(e,D,t),n(e,_,t),n(e,V,t),n(e,U,t),n(e,K,t),c(w,e,t),n(e,O,t),c(C,e,t),n(e,ee,t),n(e,B,t),n(e,te,t),n(e,j,t),n(e,ae,t),c(T,e,t),n(e,ne,t),c(v,e,t),n(e,se,t),n(e,L,t),n(e,le,t),c(F,e,t),n(e,ie,t),c($,e,t),n(e,oe,t),n(e,A,t),n(e,re,t),n(e,R,t),n(e,pe,t),n(e,Z,t),n(e,fe,t),c(Q,e,t),n(e,de,t),n(e,q,t),n(e,me,t),c(k,e,t),n(e,ce,t),n(e,G,t),n(e,ue,t),c(H,e,t),n(e,be,t),n(e,E,t),n(e,he,t),c(P,e,t),n(e,ye,t),n(e,I,t),n(e,Me,t),c(S,e,t),n(e,ge,t),c(x,e,t),n(e,we,t),n(e,X,t),n(e,Te,t),n(e,Y,t),n(e,$e,t),c(z,e,t),n(e,Je,t),n(e,W,t),_e=!0},p(e,[t]){const Ie={};t&2&&(Ie.$$scope={dirty:t,ctx:e}),w.$set(Ie);const Se={};t&2&&(Se.$$scope={dirty:t,ctx:e}),T.$set(Se);const xe={};t&2&&(xe.$$scope={dirty:t,ctx:e}),$.$set(xe)},i(e){_e||(u(J.$$.fragment,e),u(w.$$.fragment,e),u(C.$$.fragment,e),u(T.$$.fragment,e),u(v.$$.fragment,e),u(F.$$.fragment,e),u($.$$.fragment,e),u(Q.$$.fragment,e),u(k.$$.fragment,e),u(H.$$.fragment,e),u(P.$$.fragment,e),u(S.$$.fragment,e),u(x.$$.fragment,e),u(z.$$.fragment,e),_e=!0)},o(e){b(J.$$.fragment,e),b(w.$$.fragment,e),b(C.$$.fragment,e),b(T.$$.fragment,e),b(v.$$.fragment,e),b(F.$$.fragment,e),b($.$$.fragment,e),b(Q.$$.fragment,e),b(k.$$.fragment,e),b(H.$$.fragment,e),b(P.$$.fragment,e),b(S.$$.fragment,e),b(x.$$.fragment,e),b(z.$$.fragment,e),_e=!1},d(e){e&&(a(y),a(o),a(M),a(D),a(_),a(V),a(U),a(K),a(O),a(ee),a(B),a(te),a(j),a(ae),a(ne),a(se),a(L),a(le),a(ie),a(oe),a(A),a(re),a(R),a(pe),a(Z),a(fe),a(de),a(q),a(me),a(ce),a(G),a(ue),a(be),a(E),a(he),a(ye),a(I),a(Me),a(ge),a(we),a(X),a(Te),a(Y),a($e),a(Je),a(W)),a(s),h(J,e),h(w,e),h(C,e),h(T,e),h(v,e),h(F,e),h($,e),h(Q,e),h(k,e),h(H,e),h(P,e),h(S,e),h(x,e),h(z,e)}}}const st='{"title":"FSDP-QLoRA","local":"fsdp-qlora","sections":[{"title":"Quantized data storage","local":"quantized-data-storage","sections":[],"depth":2},{"title":"Training","local":"training","sections":[],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2}],"depth":1}';function lt(g){return Ne(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class dt extends We{constructor(s){super(),De(this,s,lt,nt,ze,{})}}export{dt as component};