bitsandbytes/pr_1512/en/_app/immutable/nodes/7.de259aee.js

import{s as ze,o as Ne,n as ve}from"../chunks/scheduler.852ec091.js";import{S as We,i as De,g as r,s as l,r as d,A as Ve,h as p,f as a,c as i,j as Xe,u as m,x as f,k as Ye,y as Ke,a as n,v as c,d as u,t as b,w as h}from"../chunks/index.28275fd3.js";import{T as Ce}from"../chunks/Tip.9f398c59.js";import{C as N}from"../chunks/CodeBlock.c866bcf2.js";import{H as Ue,E as Oe}from"../chunks/getInferenceSnippets.f859b5ca.js";function et(g){let s,y='Other changes required for bitsandbytes to support FSDP-QLoRA, such as reconstructing the weights from the quantization metadata and preventing quantizing already quantized weights when they’re moved from a CPU to GPU, are documented in this <a href="https://github.com/bitsandbytes-foundation/bitsandbytes/pull/970" rel="nofollow">Pull Request</a> and described in the <a href="https://www.answer.ai/posts/2024-03-14-fsdp-qlora-deep-dive" rel="nofollow">Enabling 70B Finetuning on Consumer GPUs</a> blog post. We highly recommend reading these resources for a better understanding of FSDP-QLoRA!';return{c(){s=r("p"),s.innerHTML=y},l(o){s=p(o,"P",{"data-svelte-h":!0}),f(s)!=="svelte-1qu8ni6"&&(s.innerHTML=y)},m(o,M){n(o,s,M)},p:ve,d(o){o&&a(s)}}}function tt(g){let s,y="The <code>compute_dtype</code> is the data type used for computation inside the CUDA kernel, where the 4-bit quantized weights are unpacked from the data type in <code>quant_storage</code> and dequantized to <code>compute_dtype</code>. We recommend using torch.bfloat16 (if available on your hardware) for better numerical stability.";return{c(){s=r("p"),s.innerHTML=y},l(o){s=p(o,"P",{"data-svelte-h":!0}),f(s)!=="svelte-1xdoh7b"&&(s.innerHTML=y)},m(o,M){n(o,s,M)},p:ve,d(o){o&&a(s)}}}function at(g){let s,y='FSDP is a distributed training framework that needs to be launched as a distributed training job with a library like <a href="https://hf.co/docs/accelerate/index" rel="nofollow">Accelerate</a> or <a href="https://pytorch.org/docs/stable/elastic/run.html" rel="nofollow">torchrun</a>. The launch command provided in this section uses Accelerate to launch the training script.';return{c(){s=r("p"),s.innerHTML=y},l(o){s=p(o,"P",{"data-svelte-h":!0}),f(s)!=="svelte-1k9ae9o"&&(s.innerHTML=y)},m(o,M){n(o,s,M)},p:ve,d(o){o&&a(s)}}}function nt(g){let s,y,o,M,J,D,_,Be='FSDP-QLoRA combines data parallelism (FSDP enables sharding model parameters, optimizer states, and gradients across GPUs), 4-bit quantization, and LoRA to train LLMs up to 70B parameters on a dual 24GB GPU system. This technique was released by <a href="https://www.answer.ai/posts/2024-03-06-fsdp-qlora" rel="nofollow">Answer.AI</a> in collaboration with bitsandbytes to make training LLMs more efficient and accessible for everyone.',V,U,je="This guide provides a brief guide on how bitsandbytes supports storing quantized weights to enable FSDP-QLoRA, and how to run training with the Hugging Face libraries.",K,w,O,C,ee,v,Le='FSDP only supports sharding float data types which can be problematic because quantized weights are typically stored as integer data types (uint8). bitsandbytes doesn’t have this problem because it uses <code>StoreChar</code> to read and write quantized weights regardless of the data type storage. This makes it simple to add a <code>quant_storage</code> parameter to the <a href="/docs/bitsandbytes/pr_1512/en/reference/nn/linear4bit#bitsandbytes.nn.Linear4bit">Linear4bit</a> and <a href="/docs/bitsandbytes/pr_1512/en/reference/nn/linear4bit#bitsandbytes.nn.Params4bit">Params4bit</a> classes and set it to <code>torch.uint8</code> to maintain backward compatibility with the codebase. With the <code>quant_storage</code> parameter, you can select any of the FSDP supported data types to shard <a href="/docs/bitsandbytes/pr_1512/en/reference/nn/linear4bit#bitsandbytes.nn.Linear4bit">Linear4bit</a> with such as bfloat16, float16 or float32.',te,B,Fe='You’ll typically access and configure this option from <a href="https://huggingface.co/docs/transformers/main/en/main_classes/quantization#transformers.BitsAndBytesConfig" rel="nofollow">transformers.BitsAndBytesConfig</a> by setting the <code>bnb_4bit_quant_storage</code> parameter. It is very <strong>important</strong> the <code>quant_storage</code> data type matches the data types used throughout the model because FSDP can only wrap layers and modules that have the <em>same floating data type</em>. Making sure the data types are aligned will ensure the model is correctly sharded.',ae,T,ne,j,se,L,Ae='Check out this <a href="https://hf.co/docs/peft/main/en/accelerate/fsdp#use-peft-qlora-and-fsdp-for-finetuning-large-models-on-multiple-gpus" rel="nofollow">section</a> of the PEFT documentation for the config file and training code to run FSDP-QLoRA training.',le,F,ie,$,oe,A,Re='bitsandbytes is deeply integrated with the Hugging Face ecosystem, making it easy to use with libraries like <a href="https://hf.co/docs/transformers" rel="nofollow">Transformers</a>, <a href="https://hf.co/docs/peft" rel="nofollow">PEFT</a>, and <a href="https://hf.co/docs/trl" rel="nofollow">TRL</a>.',re,R,Ze='PEFT provides a configuration file (<a href="https://github.com/huggingface/peft/blob/main/examples/sft/configs/fsdp_config_qlora.yaml" rel="nofollow">fsdp_config_qlora.yaml</a>), launch command (<a href="https://github.com/huggingface/peft/blob/main/examples/sft/run_peft_qlora_fsdp.sh" rel="nofollow">run_peft_qlora_fsdp.sh</a>), and training script (<a href="https://github.com/huggingface/peft/blob/main/examples/sft/train.py" rel="nofollow">train.py</a>) for running FSDP-QLoRA. To learn more, check out the <a href="https://huggingface.co/docs/peft/main/en/accelerate/fsdp#use-peft-qlora-and-fsdp-for-finetuning-large-models-on-multiple-gpus" rel="nofollow">Use PEFT QLoRA and FSDP for finetuning large models on multiple GPUs</a> documentation. This section briefly covers the steps to run FSDP-QLoRA training.',pe,Z,qe="Before you begin, make sure you have the latest libraries installed.",fe,q,de,Q,Qe='The important change that enables FSDP-QLoRA training is the <code>bnb_4bit_quant_storage</code> parameter in the <a href="https://huggingface.co/docs/transformers/main/en/main_classes/quantization#transformers.BitsAndBytesConfig" rel="nofollow">BitsAndBytesConfig</a> class. This allows you to set the storage data type of the quantized weights to a float data type.',me,G,ce,H,Ge='Pass the <a href="https://huggingface.co/docs/transformers/main/en/main_classes/quantization#transformers.BitsAndBytesConfig" rel="nofollow">BitsAndBytesConfig</a> to a model to set it up for FSDP-QLoRA. You should set the <code>torch_dtype</code> parameter to match <code>bnb_4bit_quant_storage</code> so that the <a href="/docs/bitsandbytes/pr_1512/en/reference/nn/linear4bit#bitsandbytes.nn.Linear4bit">Linear4bit</a> layers are wrapped identically to the <code>Linear</code> layers. If the storage types do not match, then each <a href="/docs/bitsandbytes/pr_1512/en/reference/nn/linear4bit#bitsandbytes.nn.Linear4bit">Linear4bit</a> layer is wrapped individually.',ue,k,be,E,He="Configure the <code>~peft.LoraConfig</code> class for QLoRA training by setting <code>target_modules=&quot;all-linear&quot;</code>.",he,P,ye,I,ke='Now you can pass everything to the <a href="https://huggingface.co/docs/trl/main/en/sft_trainer#trl.SFTTrainer" rel="nofollow">SFTTrainer</a> for training.',Me,S,ge,x,we,X,Ee="To learn more about FSDP and QLoRA, check out the following resources:",Te,Y,Pe='<li>The <a href="https://github.com/AnswerDotAI/fsdp_qlora" rel="nofollow">AnswerDotAI/fsdp_qlora</a> repository.</li> <li>The introductory <a href="https://www.answer.ai/posts/2024-03-06-fsdp-qlora.html" rel="nofollow">You can now train a 70b language model at home</a> blog post by Answer.AI.</li> <li>For an introduction to FSDP, read the <a href="https://pytorch.org/blog/introducing-pytorch-fully-sharded-data-parallel-api" rel="nofollow">Introducing PyTorch Fully Sharded Data Parallel (FSDP) API</a> blog post.</li> <li>For more details about QLoRA, take a look at the <a href="https://huggingface.co/blog/4bit-transformers-bitsandbytes" rel="nofollow">Making LLMs even more accessible with bitsandbytes, 4-bit quantization and QLoRA</a> blog post.</li>',$e,z,Je,W,_e;return J=new Ue({props:{title:"FSDP-QLoRA",local:"fsdp-qlora",headingTag:"h1"}}),w=new Ce({props:{warning:!1,$$slots:{default:[et]},$$scope:{ctx:g}}}),C=new Ue({props:{title:"Quantized data storage",local:"quantized-data-storage",headingTag:"h2"}}),T=new Ce({props:{warning:!1,$$slots:{default:[tt]},$$scope:{ctx:g}}}),j=new N({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBYm5iX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyglMEElMjAlMjAlMjAlMjBsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwYm5iXzRiaXRfcXVhbnRfdHlwZSUzRCUyMm5mNCUyMiUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X2NvbXB1dGVfZHR5cGUlM0R0b3JjaC5iZmxvYXQxNiUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3N0b3JhZ2UlM0R0b3JjaC5iZmxvYXQxNiUyQyUwQSklMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJtZXRhLWxsYW1hJTJGTGxhbWEtMi03MGIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEYm5iX2NvbmZpZyUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYlMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig, AutoModelForCausalLM

bnb_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_quant_storage=torch.bfloat16,
)

model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;meta-llama/Llama-2-70b&quot;</span>,
    quantization_config=bnb_config,
    torch_dtype=torch.bfloat16,
)`,wrap:!1}}),F=new Ue({props:{title:"Training",local:"training",headingTag:"h2"}}),$=new Ce({props:{warning:!1,$$slots:{default:[at]},$$scope:{ctx:g}}}),q=new N({props:{code:"cGlwJTIwaW5zdGFsbCUyMC1VJTIwYml0c2FuZGJ5dGVzJTIwYWNjZWxlcmF0ZSUyMHRyYW5zZm9ybWVycyUyMHBlZnQlMjB0cmw=",highlighted:"pip install -U bitsandbytes accelerate transformers peft trl",wrap:!1}}),G=new N({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQWJuYl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3R5cGUlM0QlMjJuZjQlMjIlMkMlMEElMjAlMjAlMjAlMjBibmJfNGJpdF9jb21wdXRlX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYlMkMlMEElMjAlMjAlMjAlMjBibmJfNGJpdF91c2VfZG91YmxlX3F1YW50JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3N0b3JhZ2UlM0R0b3JjaC5iZmxvYXQxNiUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig

bnb_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_storage=torch.bfloat16,
)`,wrap:!1}}),k=new N({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIybWV0YS1sbGFtYSUyRkxsYW1hLTItNzBiJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRGJuYl9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;meta-llama/Llama-2-70b&quot;</span>,
    quantization_config=bnb_config,
    torch_dtype=torch.bfloat16,
)`,wrap:!1}}),P=new N({props:{code:"ZnJvbSUyMHBlZnQlMjBpbXBvcnQlMjBMb3JhQ29uZmlnJTBBJTBBcGVmdF9jb25maWclMjAlM0QlMjBMb3JhQ29uZmlnKCUwQSUyMCUyMCUyMCUyMGxvcmFfYWxwaGElM0QxNiUyQyUwQSUyMCUyMCUyMCUyMGxvcmFfZHJvcG91dCUzRDAuMSUyQyUwQSUyMCUyMCUyMCUyMHIlM0Q2NCUyQyUwQSUyMCUyMCUyMCUyMGJpYXMlM0QlMjJub25lJTIyJTJDJTBBJTIwJTIwJTIwJTIwdGFza190eXBlJTNEJTIyQ0FVU0FMX0xNJTIyJTJDJTBBJTIwJTIwJTIwJTIwdGFyZ2V0X21vZHVsZXMlM0QlMjJhbGwtbGluZWFyJTIyJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> LoraConfig

peft_config = LoraConfig(
    lora_alpha=<span class="hljs-number">16</span>,
    lora_dropout=<span class="hljs-number">0.1</span>,
    r=<span class="hljs-number">64</span>,
    bias=<span class="hljs-string">&quot;none&quot;</span>,
    task_type=<span class="hljs-string">&quot;CAUSAL_LM&quot;</span>,
    target_modules=<span class="hljs-string">&quot;all-linear&quot;</span>,
)`,wrap:!1}}),S=new N({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMFNGVFRyYWluZXIlMEElMEF0cmFpbmVyJTIwJTNEJTIwU0ZUVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRG1vZGVsJTJDJTBBJTIwJTIwJTIwJTIwdHJhaW5fZGF0YXNldCUzRGRhdGFzZXQlMkMlMEElMjAlMjAlMjAlMjBwZWZ0X2NvbmZpZyUzRHBlZnRfY29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwcHJvY2Vzc2luZ19jbGFzcyUzRHRva2VuaXplciUyQyUwQSUyMCUyMCUyMCUyMGFyZ3MlM0R0cmFpbmluZ19hcmd1bWVudHMlMkMlMEEpJTBBdHJhaW5lci50cmFpbigp",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> SFTTrainer

trainer = SFTTrainer(
    model=model,
    train_dataset=dataset,
    peft_config=peft_config,
    processing_class=tokenizer,
    args=training_arguments,
)
trainer.train()`,wrap:!1}}),x=new Ue({props:{title:"Resources",local:"resources",headingTag:"h2"}}),z=new Oe({props:{source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/docs/source/fsdp_qlora.md"}}),{c(){s=r("meta"),y=l(),o=r("p"),M=l(),d(J.$$.fragment),D=l(),_=r("p"),_.innerHTML=Be,V=l(),U=r("p"),U.textContent=je,K=l(),d(w.$$.fragment),O=l(),d(C.$$.fragment),ee=l(),v=r("p"),v.innerHTML=Le,te=l(),B=r("p"),B.innerHTML=Fe,ae=l(),d(T.$$.fragment),ne=l(),d(j.$$.fragment),se=l(),L=r("p"),L.innerHTML=Ae,le=l(),d(F.$$.fragment),ie=l(),d($.$$.fragment),oe=l(),A=r("p"),A.innerHTML=Re,re=l(),R=r("p"),R.innerHTML=Ze,pe=l(),Z=r("p"),Z.textContent=qe,fe=l(),d(q.$$.fragment),de=l(),Q=r("p"),Q.innerHTML=Qe,me=l(),d(G.$$.fragment),ce=l(),H=r("p"),H.innerHTML=Ge,ue=l(),d(k.$$.fragment),be=l(),E=r("p"),E.innerHTML=He,he=l(),d(P.$$.fragment),ye=l(),I=r("p"),I.innerHTML=ke,Me=l(),d(S.$$.fragment),ge=l(),d(x.$$.fragment),we=l(),X=r("p"),X.textContent=Ee,Te=l(),Y=r("ul"),Y.innerHTML=Pe,$e=l(),d(z.$$.fragment),Je=l(),W=r("p"),this.h()},l(e){const t=Ve("svelte-u9bgzb",document.head);s=p(t,"META",{name:!0,content:!0}),t.forEach(a),y=i(e),o=p(e,"P",{}),Xe(o).forEach(a),M=i(e),m(J.$$.fragment,e),D=i(e),_=p(e,"P",{"data-svelte-h":!0}),f(_)!=="svelte-5l5q4j"&&(_.innerHTML=Be),V=i(e),U=p(e,"P",{"data-svelte-h":!0}),f(U)!=="svelte-1naqdkg"&&(U.textContent=je),K=i(e),m(w.$$.fragment,e),O=i(e),m(C.$$.fragment,e),ee=i(e),v=p(e,"P",{"data-svelte-h":!0}),f(v)!=="svelte-1v8mfzi"&&(v.innerHTML=Le),te=i(e),B=p(e,"P",{"data-svelte-h":!0}),f(B)!=="svelte-2kgr0m"&&(B.innerHTML=Fe),ae=i(e),m(T.$$.fragment,e),ne=i(e),m(j.$$.fragment,e),se=i(e),L=p(e,"P",{"data-svelte-h":!0}),f(L)!=="svelte-1mcsrsf"&&(L.innerHTML=Ae),le=i(e),m(F.$$.fragment,e),ie=i(e),m($.$$.fragment,e),oe=i(e),A=p(e,"P",{"data-svelte-h":!0}),f(A)!=="svelte-11niy1x"&&(A.innerHTML=Re),re=i(e),R=p(e,"P",{"data-svelte-h":!0}),f(R)!=="svelte-eapz72"&&(R.innerHTML=Ze),pe=i(e),Z=p(e,"P",{"data-svelte-h":!0}),f(Z)!=="svelte-1x35k4g"&&(Z.textContent=qe),fe=i(e),m(q.$$.fragment,e),de=i(e),Q=p(e,"P",{"data-svelte-h":!0}),f(Q)!=="svelte-2os12x"&&(Q.innerHTML=Qe),me=i(e),m(G.$$.fragment,e),ce=i(e),H=p(e,"P",{"data-svelte-h":!0}),f(H)!=="svelte-pav6vq"&&(H.innerHTML=Ge),ue=i(e),m(k.$$.fragment,e),be=i(e),E=p(e,"P",{"data-svelte-h":!0}),f(E)!=="svelte-1c6aae"&&(E.innerHTML=He),he=i(e),m(P.$$.fragment,e),ye=i(e),I=p(e,"P",{"data-svelte-h":!0}),f(I)!=="svelte-1yk5yum"&&(I.innerHTML=ke),Me=i(e),m(S.$$.fragment,e),ge=i(e),m(x.$$.fragment,e),we=i(e),X=p(e,"P",{"data-svelte-h":!0}),f(X)!=="svelte-ijbsql"&&(X.textContent=Ee),Te=i(e),Y=p(e,"UL",{"data-svelte-h":!0}),f(Y)!=="svelte-47gauf"&&(Y.innerHTML=Pe),$e=i(e),m(z.$$.fragment,e),Je=i(e),W=p(e,"P",{}),Xe(W).forEach(a),this.h()},h(){Ye(s,"name","hf:doc:metadata"),Ye(s,"content",st)},m(e,t){Ke(document.head,s),n(e,y,t),n(e,o,t),n(e,M,t),c(J,e,t),n(e,D,t),n(e,_,t),n(e,V,t),n(e,U,t),n(e,K,t),c(w,e,t),n(e,O,t),c(C,e,t),n(e,ee,t),n(e,v,t),n(e,te,t),n(e,B,t),n(e,ae,t),c(T,e,t),n(e,ne,t),c(j,e,t),n(e,se,t),n(e,L,t),n(e,le,t),c(F,e,t),n(e,ie,t),c($,e,t),n(e,oe,t),n(e,A,t),n(e,re,t),n(e,R,t),n(e,pe,t),n(e,Z,t),n(e,fe,t),c(q,e,t),n(e,de,t),n(e,Q,t),n(e,me,t),c(G,e,t),n(e,ce,t),n(e,H,t),n(e,ue,t),c(k,e,t),n(e,be,t),n(e,E,t),n(e,he,t),c(P,e,t),n(e,ye,t),n(e,I,t),n(e,Me,t),c(S,e,t),n(e,ge,t),c(x,e,t),n(e,we,t),n(e,X,t),n(e,Te,t),n(e,Y,t),n(e,$e,t),c(z,e,t),n(e,Je,t),n(e,W,t),_e=!0},p(e,[t]){const Ie={};t&2&&(Ie.$$scope={dirty:t,ctx:e}),w.$set(Ie);const Se={};t&2&&(Se.$$scope={dirty:t,ctx:e}),T.$set(Se);const xe={};t&2&&(xe.$$scope={dirty:t,ctx:e}),$.$set(xe)},i(e){_e||(u(J.$$.fragment,e),u(w.$$.fragment,e),u(C.$$.fragment,e),u(T.$$.fragment,e),u(j.$$.fragment,e),u(F.$$.fragment,e),u($.$$.fragment,e),u(q.$$.fragment,e),u(G.$$.fragment,e),u(k.$$.fragment,e),u(P.$$.fragment,e),u(S.$$.fragment,e),u(x.$$.fragment,e),u(z.$$.fragment,e),_e=!0)},o(e){b(J.$$.fragment,e),b(w.$$.fragment,e),b(C.$$.fragment,e),b(T.$$.fragment,e),b(j.$$.fragment,e),b(F.$$.fragment,e),b($.$$.fragment,e),b(q.$$.fragment,e),b(G.$$.fragment,e),b(k.$$.fragment,e),b(P.$$.fragment,e),b(S.$$.fragment,e),b(x.$$.fragment,e),b(z.$$.fragment,e),_e=!1},d(e){e&&(a(y),a(o),a(M),a(D),a(_),a(V),a(U),a(K),a(O),a(ee),a(v),a(te),a(B),a(ae),a(ne),a(se),a(L),a(le),a(ie),a(oe),a(A),a(re),a(R),a(pe),a(Z),a(fe),a(de),a(Q),a(me),a(ce),a(H),a(ue),a(be),a(E),a(he),a(ye),a(I),a(Me),a(ge),a(we),a(X),a(Te),a(Y),a($e),a(Je),a(W)),a(s),h(J,e),h(w,e),h(C,e),h(T,e),h(j,e),h(F,e),h($,e),h(q,e),h(G,e),h(k,e),h(P,e),h(S,e),h(x,e),h(z,e)}}}const st='{"title":"FSDP-QLoRA","local":"fsdp-qlora","sections":[{"title":"Quantized data storage","local":"quantized-data-storage","sections":[],"depth":2},{"title":"Training","local":"training","sections":[],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2}],"depth":1}';function lt(g){return Ne(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class dt extends We{constructor(s){super(),De(this,s,lt,nt,ze,{})}}export{dt as component};