bitsandbytes/pr_1419/en/_app/immutable/nodes/8.ca04fda2.js

import{s as Ye,o as We,n as je}from"../chunks/scheduler.852ec091.js";import{S as ze,i as De,g as r,s,r as d,A as Ve,h as p,f as a,c as i,j as Se,u as m,x as f,k as Ne,y as Ke,a as n,v as u,d as c,t as h,w as b}from"../chunks/index.28275fd3.js";import{T as Ce}from"../chunks/Tip.9f398c59.js";import{C as W}from"../chunks/CodeBlock.c3366071.js";import{H as Ue,E as Oe}from"../chunks/EditOnGithub.582011f0.js";function et(g){let l,y='Other changes required for bitsandbytes to support FSDP-QLoRA, such as reconstructing the weights from the quantization metadata and preventing quantizing already quantized weights when they’re moved from a CPU to GPU, are documented in this <a href="https://github.com/TimDettmers/bitsandbytes/pull/970" rel="nofollow">Pull Request</a> and described in the <a href="https://www.answer.ai/posts/2024-03-14-fsdp-qlora-deep-dive" rel="nofollow">Enabling 70B Finetuning on Consumer GPUs</a> blog post. We highly recommend reading these resources for a better understanding of FSDP-QLoRA!';return{c(){l=r("p"),l.innerHTML=y},l(o){l=p(o,"P",{"data-svelte-h":!0}),f(l)!=="svelte-10tshw4"&&(l.innerHTML=y)},m(o,M){n(o,l,M)},p:je,d(o){o&&a(l)}}}function tt(g){let l,y="The <code>compute_dtype</code> is the data type used for computation inside the CUDA kernel, where the 4-bit quantized weights are unpacked from the data type in <code>quant_storage</code> and dequantized to <code>compute_dtype</code>. We recommend using torch.bfloat16 (if available on your hardware) for better numerical stability.";return{c(){l=r("p"),l.innerHTML=y},l(o){l=p(o,"P",{"data-svelte-h":!0}),f(l)!=="svelte-1xdoh7b"&&(l.innerHTML=y)},m(o,M){n(o,l,M)},p:je,d(o){o&&a(l)}}}function at(g){let l,y='FSDP is a distributed training framework that needs to be launched as a distributed training job with a library like <a href="https://hf.co/docs/accelerate/index" rel="nofollow">Accelerate</a> or <a href="https://pytorch.org/docs/stable/elastic/run.html" rel="nofollow">torchrun</a>. The launch command provided in this section uses Accelerate to launch the training script.';return{c(){l=r("p"),l.innerHTML=y},l(o){l=p(o,"P",{"data-svelte-h":!0}),f(l)!=="svelte-1k9ae9o"&&(l.innerHTML=y)},m(o,M){n(o,l,M)},p:je,d(o){o&&a(l)}}}function nt(g){let l,y,o,M,$,D,_,Be='FSDP-QLoRA combines data parallelism (FSDP enables sharding model parameters, optimizer states, and gradients across GPUs), 4-bit quantization, and LoRA to train LLMs up to 70B parameters on a dual 24GB GPU system. This technique was released by <a href="https://www.answer.ai/posts/2024-03-06-fsdp-qlora" rel="nofollow">Answer.AI</a> in collaboration with bitsandbytes to make training LLMs more efficient and accessible for everyone.',V,U,ve="This guide provides a brief guide on how bitsandbytes supports storing quantized weights to enable FSDP-QLoRA, and how to run training with the Hugging Face libraries.",K,w,O,C,ee,j,Ae='FSDP only supports sharding float data types which can be problematic because quantized weights are typically stored as integer data types (uint8). bitsandbytes doesn’t have this problem because it uses <code>StoreChar</code> to read and write quantized weights regardless of the data type storage. This makes it simple to add a <code>quant_storage</code> parameter to the <a href="/docs/bitsandbytes/pr_1419/en/reference/nn/linear4bit#bitsandbytes.nn.Linear4bit">Linear4bit</a> and <a href="/docs/bitsandbytes/pr_1419/en/reference/nn/linear4bit#bitsandbytes.nn.Params4bit">Params4bit</a> classes and set it to <code>torch.uint8</code> to maintain backward compatibility with the codebase. With the <code>quant_storage</code> parameter, you can select any of the FSDP supported data types to shard <a href="/docs/bitsandbytes/pr_1419/en/reference/nn/linear4bit#bitsandbytes.nn.Linear4bit">Linear4bit</a> with such as bfloat16, float16 or float32.',te,B,Ze='You’ll typically access and configure this option from <a href="https://huggingface.co/docs/transformers/main/en/main_classes/quantization#transformers.BitsAndBytesConfig" rel="nofollow">transformers.BitsAndBytesConfig</a> by setting the <code>bnb_4bit_quant_storage</code> parameter. It is very <strong>important</strong> the <code>quant_storage</code> data type matches the data types used throughout the model because FSDP can only wrap layers and modules that have the <em>same floating data type</em>. Making sure the data types are aligned will ensure the model is correctly sharded.',ae,T,ne,v,le,A,Fe='Check out this <a href="https://hf.co/docs/peft/main/en/accelerate/fsdp#use-peft-qlora-and-fsdp-for-finetuning-large-models-on-multiple-gpus" rel="nofollow">section</a> of the PEFT documentation for the config file and training code to run FSDP-QLoRA training.',se,Z,ie,J,oe,F,Le='bitsandbytes is deeply integrated with the Hugging Face ecosystem, making it easy to use with libraries like <a href="https://hf.co/docs/transformers" rel="nofollow">Transformers</a>, <a href="https://hf.co/docs/peft" rel="nofollow">PEFT</a>, and <a href="https://hf.co/docs/trl" rel="nofollow">TRL</a>.',re,L,Re='PEFT provides a configuration file (<a href="https://github.com/huggingface/peft/blob/main/examples/sft/configs/fsdp_config_qlora.yaml" rel="nofollow">fsdp_config_qlora.yaml</a>), launch command (<a href="https://github.com/huggingface/peft/blob/main/examples/sft/run_peft_qlora_fsdp.sh" rel="nofollow">run_peft_qlora_fsdp.sh</a>), and training script (<a href="https://github.com/huggingface/peft/blob/main/examples/sft/train.py" rel="nofollow">train.py</a>) for running FSDP-QLoRA. To learn more, check out the <a href="https://huggingface.co/docs/peft/main/en/accelerate/fsdp#use-peft-qlora-and-fsdp-for-finetuning-large-models-on-multiple-gpus" rel="nofollow">Use PEFT QLoRA and FSDP for finetuning large models on multiple GPUs</a> documentation. This section briefly covers the steps to run FSDP-QLoRA training.',pe,R,qe="Before you begin, make sure you have the latest libraries installed.",fe,q,de,Q,Qe='The important change that enables FSDP-QLoRA training is the <code>bnb_4bit_quant_storage</code> parameter in the <a href="https://huggingface.co/docs/transformers/main/en/main_classes/quantization#transformers.BitsAndBytesConfig" rel="nofollow">BitsAndBytesConfig</a> class. This allows you to set the storage data type of the quantized weights to a float data type.',me,E,ue,G,Ee='Pass the <a href="https://huggingface.co/docs/transformers/main/en/main_classes/quantization#transformers.BitsAndBytesConfig" rel="nofollow">BitsAndBytesConfig</a> to a model to set it up for FSDP-QLoRA. You should set the <code>torch_dtype</code> parameter to match <code>bnb_4bit_quant_storage</code> so that the <a href="/docs/bitsandbytes/pr_1419/en/reference/nn/linear4bit#bitsandbytes.nn.Linear4bit">Linear4bit</a> layers are wrapped identically to the <code>Linear</code> layers. If the storage types do not match, then each <a href="/docs/bitsandbytes/pr_1419/en/reference/nn/linear4bit#bitsandbytes.nn.Linear4bit">Linear4bit</a> layer is wrapped individually.',ce,k,he,x,Ge="Configure the <code>~peft.LoraConfig</code> class for QLoRA training by setting <code>target_modules=&quot;all-linear&quot;</code>.",be,I,ye,X,ke='Now you can pass everything to the <a href="https://huggingface.co/docs/trl/main/en/sft_trainer#trl.SFTTrainer" rel="nofollow">SFTTrainer</a> for training.',Me,H,ge,P,we,S,xe="To learn more about FSDP and QLoRA, check out the following resources:",Te,N,Ie='<li>The <a href="https://github.com/AnswerDotAI/fsdp_qlora" rel="nofollow">AnswerDotAI/fsdp_qlora</a> repository.</li> <li>The introductory <a href="https://www.answer.ai/posts/2024-03-06-fsdp-qlora.html" rel="nofollow">You can now train a 70b language model at home</a> blog post by Answer.AI.</li> <li>For an introduction to FSDP, read the <a href="https://pytorch.org/blog/introducing-pytorch-fully-sharded-data-parallel-api" rel="nofollow">Introducing PyTorch Fully Sharded Data Parallel (FSDP) API</a> blog post.</li> <li>For more details about QLoRA, take a look at the <a href="https://huggingface.co/blog/4bit-transformers-bitsandbytes" rel="nofollow">Making LLMs even more accessible with bitsandbytes, 4-bit quantization and QLoRA</a> blog post.</li>',Je,Y,$e,z,_e;return $=new Ue({props:{title:"FSDP-QLoRA",local:"fsdp-qlora",headingTag:"h1"}}),w=new Ce({props:{warning:!1,$$slots:{default:[et]},$$scope:{ctx:g}}}),C=new Ue({props:{title:"Quantized data storage",local:"quantized-data-storage",headingTag:"h2"}}),T=new Ce({props:{warning:!1,$$slots:{default:[tt]},$$scope:{ctx:g}}}),v=new W({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBYm5iX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyglMEElMjAlMjAlMjAlMjBsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwYm5iXzRiaXRfcXVhbnRfdHlwZSUzRCUyMm5mNCUyMiUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X2NvbXB1dGVfZHR5cGUlM0R0b3JjaC5iZmxvYXQxNiUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3N0b3JhZ2UlM0R0b3JjaC5iZmxvYXQxNiUyQyUwQSklMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJtZXRhLWxsYW1hJTJGTGxhbWEtMi03MGIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEYm5iX2NvbmZpZyUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYlMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig, AutoModelForCausalLM

bnb_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_quant_storage=torch.bfloat16,
)

model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;meta-llama/Llama-2-70b&quot;</span>,
    quantization_config=bnb_config,
    torch_dtype=torch.bfloat16,
)`,wrap:!1}}),Z=new Ue({props:{title:"Training",local:"training",headingTag:"h2"}}),J=new Ce({props:{warning:!1,$$slots:{default:[at]},$$scope:{ctx:g}}}),q=new W({props:{code:"cGlwJTIwaW5zdGFsbCUyMC1VJTIwYml0c2FuZGJ5dGVzJTIwYWNjZWxlcmF0ZSUyMHRyYW5zZm9ybWVycyUyMHBlZnQlMjB0cmw=",highlighted:"pip install -U bitsandbytes accelerate transformers peft trl",wrap:!1}}),E=new W({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQWJuYl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3R5cGUlM0QlMjJuZjQlMjIlMkMlMEElMjAlMjAlMjAlMjBibmJfNGJpdF9jb21wdXRlX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYlMkMlMEElMjAlMjAlMjAlMjBibmJfNGJpdF91c2VfZG91YmxlX3F1YW50JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3N0b3JhZ2UlM0R0b3JjaC5iZmxvYXQxNiUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig

bnb_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_storage=torch.bfloat16,
)`,wrap:!1}}),k=new W({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIybWV0YS1sbGFtYSUyRkxsYW1hLTItNzBiJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRGJuYl9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;meta-llama/Llama-2-70b&quot;</span>,
    quantization_config=bnb_config,
    torch_dtype=torch.bfloat16,
)`,wrap:!1}}),I=new W({props:{code:"ZnJvbSUyMHBlZnQlMjBpbXBvcnQlMjBMb3JhQ29uZmlnJTBBJTBBcGVmdF9jb25maWclMjAlM0QlMjBMb3JhQ29uZmlnKCUwQSUyMCUyMCUyMCUyMGxvcmFfYWxwaGElM0QxNiUyQyUwQSUyMCUyMCUyMCUyMGxvcmFfZHJvcG91dCUzRDAuMSUyQyUwQSUyMCUyMCUyMCUyMHIlM0Q2NCUyQyUwQSUyMCUyMCUyMCUyMGJpYXMlM0QlMjJub25lJTIyJTJDJTBBJTIwJTIwJTIwJTIwdGFza190eXBlJTNEJTIyQ0FVU0FMX0xNJTIyJTJDJTBBJTIwJTIwJTIwJTIwdGFyZ2V0X21vZHVsZXMlM0QlMjJhbGwtbGluZWFyJTIyJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> LoraConfig

peft_config = LoraConfig(
    lora_alpha=<span class="hljs-number">16</span>,
    lora_dropout=<span class="hljs-number">0.1</span>,
    r=<span class="hljs-number">64</span>,
    bias=<span class="hljs-string">&quot;none&quot;</span>,
    task_type=<span class="hljs-string">&quot;CAUSAL_LM&quot;</span>,
    target_modules=<span class="hljs-string">&quot;all-linear&quot;</span>,
)`,wrap:!1}}),H=new W({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMFNGVFRyYWluZXIlMEElMEF0cmFpbmVyJTIwJTNEJTIwU0ZUVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRG1vZGVsJTJDJTBBJTIwJTIwJTIwJTIwdHJhaW5fZGF0YXNldCUzRGRhdGFzZXQlMkMlMEElMjAlMjAlMjAlMjBwZWZ0X2NvbmZpZyUzRHBlZnRfY29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwZGF0YXNldF90ZXh0X2ZpZWxkJTNEJTIydGV4dCUyMiUyQyUwQSUyMCUyMCUyMCUyMG1heF9zZXFfbGVuZ3RoJTNEbWF4X3NlcV9sZW5ndGglMkMlMEElMjAlMjAlMjAlMjB0b2tlbml6ZXIlM0R0b2tlbml6ZXIlMkMlMEElMjAlMjAlMjAlMjBhcmdzJTNEdHJhaW5pbmdfYXJndW1lbnRzJTJDJTBBKSUwQXRyYWluZXIudHJhaW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> SFTTrainer

trainer = SFTTrainer(
    model=model,
    train_dataset=dataset,
    peft_config=peft_config,
    dataset_text_field=<span class="hljs-string">&quot;text&quot;</span>,
    max_seq_length=max_seq_length,
    tokenizer=tokenizer,
    args=training_arguments,
)
trainer.train()`,wrap:!1}}),P=new Ue({props:{title:"Resources",local:"resources",headingTag:"h2"}}),Y=new Oe({props:{source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/docs/source/fsdp_qlora.md"}}),{c(){l=r("meta"),y=s(),o=r("p"),M=s(),d($.$$.fragment),D=s(),_=r("p"),_.innerHTML=Be,V=s(),U=r("p"),U.textContent=ve,K=s(),d(w.$$.fragment),O=s(),d(C.$$.fragment),ee=s(),j=r("p"),j.innerHTML=Ae,te=s(),B=r("p"),B.innerHTML=Ze,ae=s(),d(T.$$.fragment),ne=s(),d(v.$$.fragment),le=s(),A=r("p"),A.innerHTML=Fe,se=s(),d(Z.$$.fragment),ie=s(),d(J.$$.fragment),oe=s(),F=r("p"),F.innerHTML=Le,re=s(),L=r("p"),L.innerHTML=Re,pe=s(),R=r("p"),R.textContent=qe,fe=s(),d(q.$$.fragment),de=s(),Q=r("p"),Q.innerHTML=Qe,me=s(),d(E.$$.fragment),ue=s(),G=r("p"),G.innerHTML=Ee,ce=s(),d(k.$$.fragment),he=s(),x=r("p"),x.innerHTML=Ge,be=s(),d(I.$$.fragment),ye=s(),X=r("p"),X.innerHTML=ke,Me=s(),d(H.$$.fragment),ge=s(),d(P.$$.fragment),we=s(),S=r("p"),S.textContent=xe,Te=s(),N=r("ul"),N.innerHTML=Ie,Je=s(),d(Y.$$.fragment),$e=s(),z=r("p"),this.h()},l(e){const t=Ve("svelte-u9bgzb",document.head);l=p(t,"META",{name:!0,content:!0}),t.forEach(a),y=i(e),o=p(e,"P",{}),Se(o).forEach(a),M=i(e),m($.$$.fragment,e),D=i(e),_=p(e,"P",{"data-svelte-h":!0}),f(_)!=="svelte-5l5q4j"&&(_.innerHTML=Be),V=i(e),U=p(e,"P",{"data-svelte-h":!0}),f(U)!=="svelte-1naqdkg"&&(U.textContent=ve),K=i(e),m(w.$$.fragment,e),O=i(e),m(C.$$.fragment,e),ee=i(e),j=p(e,"P",{"data-svelte-h":!0}),f(j)!=="svelte-uxlegc"&&(j.innerHTML=Ae),te=i(e),B=p(e,"P",{"data-svelte-h":!0}),f(B)!=="svelte-2kgr0m"&&(B.innerHTML=Ze),ae=i(e),m(T.$$.fragment,e),ne=i(e),m(v.$$.fragment,e),le=i(e),A=p(e,"P",{"data-svelte-h":!0}),f(A)!=="svelte-1mcsrsf"&&(A.innerHTML=Fe),se=i(e),m(Z.$$.fragment,e),ie=i(e),m(J.$$.fragment,e),oe=i(e),F=p(e,"P",{"data-svelte-h":!0}),f(F)!=="svelte-11niy1x"&&(F.innerHTML=Le),re=i(e),L=p(e,"P",{"data-svelte-h":!0}),f(L)!=="svelte-eapz72"&&(L.innerHTML=Re),pe=i(e),R=p(e,"P",{"data-svelte-h":!0}),f(R)!=="svelte-1x35k4g"&&(R.textContent=qe),fe=i(e),m(q.$$.fragment,e),de=i(e),Q=p(e,"P",{"data-svelte-h":!0}),f(Q)!=="svelte-2os12x"&&(Q.innerHTML=Qe),me=i(e),m(E.$$.fragment,e),ue=i(e),G=p(e,"P",{"data-svelte-h":!0}),f(G)!=="svelte-i4orp2"&&(G.innerHTML=Ee),ce=i(e),m(k.$$.fragment,e),he=i(e),x=p(e,"P",{"data-svelte-h":!0}),f(x)!=="svelte-1c6aae"&&(x.innerHTML=Ge),be=i(e),m(I.$$.fragment,e),ye=i(e),X=p(e,"P",{"data-svelte-h":!0}),f(X)!=="svelte-1yk5yum"&&(X.innerHTML=ke),Me=i(e),m(H.$$.fragment,e),ge=i(e),m(P.$$.fragment,e),we=i(e),S=p(e,"P",{"data-svelte-h":!0}),f(S)!=="svelte-ijbsql"&&(S.textContent=xe),Te=i(e),N=p(e,"UL",{"data-svelte-h":!0}),f(N)!=="svelte-47gauf"&&(N.innerHTML=Ie),Je=i(e),m(Y.$$.fragment,e),$e=i(e),z=p(e,"P",{}),Se(z).forEach(a),this.h()},h(){Ne(l,"name","hf:doc:metadata"),Ne(l,"content",lt)},m(e,t){Ke(document.head,l),n(e,y,t),n(e,o,t),n(e,M,t),u($,e,t),n(e,D,t),n(e,_,t),n(e,V,t),n(e,U,t),n(e,K,t),u(w,e,t),n(e,O,t),u(C,e,t),n(e,ee,t),n(e,j,t),n(e,te,t),n(e,B,t),n(e,ae,t),u(T,e,t),n(e,ne,t),u(v,e,t),n(e,le,t),n(e,A,t),n(e,se,t),u(Z,e,t),n(e,ie,t),u(J,e,t),n(e,oe,t),n(e,F,t),n(e,re,t),n(e,L,t),n(e,pe,t),n(e,R,t),n(e,fe,t),u(q,e,t),n(e,de,t),n(e,Q,t),n(e,me,t),u(E,e,t),n(e,ue,t),n(e,G,t),n(e,ce,t),u(k,e,t),n(e,he,t),n(e,x,t),n(e,be,t),u(I,e,t),n(e,ye,t),n(e,X,t),n(e,Me,t),u(H,e,t),n(e,ge,t),u(P,e,t),n(e,we,t),n(e,S,t),n(e,Te,t),n(e,N,t),n(e,Je,t),u(Y,e,t),n(e,$e,t),n(e,z,t),_e=!0},p(e,[t]){const Xe={};t&2&&(Xe.$$scope={dirty:t,ctx:e}),w.$set(Xe);const He={};t&2&&(He.$$scope={dirty:t,ctx:e}),T.$set(He);const Pe={};t&2&&(Pe.$$scope={dirty:t,ctx:e}),J.$set(Pe)},i(e){_e||(c($.$$.fragment,e),c(w.$$.fragment,e),c(C.$$.fragment,e),c(T.$$.fragment,e),c(v.$$.fragment,e),c(Z.$$.fragment,e),c(J.$$.fragment,e),c(q.$$.fragment,e),c(E.$$.fragment,e),c(k.$$.fragment,e),c(I.$$.fragment,e),c(H.$$.fragment,e),c(P.$$.fragment,e),c(Y.$$.fragment,e),_e=!0)},o(e){h($.$$.fragment,e),h(w.$$.fragment,e),h(C.$$.fragment,e),h(T.$$.fragment,e),h(v.$$.fragment,e),h(Z.$$.fragment,e),h(J.$$.fragment,e),h(q.$$.fragment,e),h(E.$$.fragment,e),h(k.$$.fragment,e),h(I.$$.fragment,e),h(H.$$.fragment,e),h(P.$$.fragment,e),h(Y.$$.fragment,e),_e=!1},d(e){e&&(a(y),a(o),a(M),a(D),a(_),a(V),a(U),a(K),a(O),a(ee),a(j),a(te),a(B),a(ae),a(ne),a(le),a(A),a(se),a(ie),a(oe),a(F),a(re),a(L),a(pe),a(R),a(fe),a(de),a(Q),a(me),a(ue),a(G),a(ce),a(he),a(x),a(be),a(ye),a(X),a(Me),a(ge),a(we),a(S),a(Te),a(N),a(Je),a($e),a(z)),a(l),b($,e),b(w,e),b(C,e),b(T,e),b(v,e),b(Z,e),b(J,e),b(q,e),b(E,e),b(k,e),b(I,e),b(H,e),b(P,e),b(Y,e)}}}const lt='{"title":"FSDP-QLoRA","local":"fsdp-qlora","sections":[{"title":"Quantized data storage","local":"quantized-data-storage","sections":[],"depth":2},{"title":"Training","local":"training","sections":[],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2}],"depth":1}';function st(g){return We(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class dt extends ze{constructor(l){super(),De(this,l,st,nt,Ye,{})}}export{dt as component};