Hugging Face's logo

Buckets:
hf-doc-build
/
doc-dev

Files
xet
hf-doc-build/doc-dev / peft /pr_3117 /en /_app /immutable /nodes /2.577a4e33.js
HuggingFaceDocBuilder's picture
HuggingFaceDocBuilder
download
raw
70.3 kB
import{s as ua,n as fa,o as ba}from"../chunks/scheduler.78382b47.js";import{S as wa,i as ga,e as i,s,c as r,h as Za,a as p,d as t,b as n,f as _t,g as o,j as M,k as zl,l as At,m as a,n as c,t as d,o as J,p as m}from"../chunks/index.6dd35eb6.js";import{C as _a,H as h,E as Aa}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.8caa866a.js";import{C as j}from"../chunks/CodeBlock.32c14a25.js";function Ba(Bt){let y,ql,Hl,Pl,b,Dl,w,Ol,g,Wt='<a href="https://www.deepspeed.ai/" rel="nofollow">DeepSpeed</a> is a library designed for speed and scale for distributed training of large models with billions of parameters. At its core is the Zero Redundancy Optimizer (ZeRO) that shards optimizer states (ZeRO-1), gradients (ZeRO-2), and parameters (ZeRO-3) across data parallel processes. This drastically reduces memory usage, allowing you to scale your training to billion parameter models. To unlock even more memory efficiency, ZeRO-Offload reduces GPU compute and memory by leveraging CPU resources during optimization.',Kl,Z,Gt="Both of these features are supported in 🤗 Accelerate, and you can use them with 🤗 PEFT.",le,_,ee,A,It='Below is a table that summarizes the compatibility between PEFT’s LoRA, <a href="https://github.com/TimDettmers/bitsandbytes" rel="nofollow"><code>bitsandbytes</code></a> library and DeepSpeed Zero stages with respect to fine-tuning. DeepSpeed Zero-1 and 2 will have no effect at inference as stage 1 shards the optimizer states and stage 2 shards the optimizer states and gradients:',te,B,Ct="<thead><tr><th>DeepSpeed stage</th> <th>Is compatible?</th></tr></thead> <tbody><tr><td>Zero-1</td> <td>🟢</td></tr> <tr><td>Zero-2</td> <td>🟢</td></tr> <tr><td>Zero-3</td> <td>🟢</td></tr></tbody>",ae,W,kt='For DeepSpeed Stage 3 + QLoRA, please refer to the section <a href="#use-peft-qlora-and-deepspeed-with-zero3-for-finetuning-large-models-on-multiple-gpus">Use PEFT QLoRA and DeepSpeed with ZeRO3 for finetuning large models on multiple GPUs</a> below.',se,G,Nt='For confirming these observations, we ran the SFT (Supervised Fine-tuning) <a href="https://github.com/huggingface/trl/tree/main/examples" rel="nofollow">offical example scripts</a> of the <a href="https://github.com/huggingface/trl" rel="nofollow">Transformers Reinforcement Learning (TRL) library</a> using QLoRA + PEFT and the accelerate configs available <a href="https://github.com/huggingface/trl/tree/main/examples/accelerate_configs" rel="nofollow">here</a>. We ran these experiments on a 2x NVIDIA T4 GPU.',ne,I,ie,C,Ft='This section of guide will help you learn how to use our DeepSpeed <a href="https://github.com/huggingface/peft/blob/main/examples/sft/train.py" rel="nofollow">training script</a> for performing SFT. You’ll configure the script to do SFT (supervised fine-tuning) of Llama-70B model with LoRA and ZeRO-3 on 8xH100 80GB GPUs on a single machine. You can configure it to scale to multiple machines by changing the accelerate config.',pe,k,Me,N,vt='Start by running the following command to <a href="https://huggingface.co/docs/accelerate/quicktour#launching-your-distributed-script" rel="nofollow">create a DeepSpeed configuration file</a> with 🤗 Accelerate. The <code>--config_file</code> flag allows you to save the configuration file to a specific location, otherwise it is saved as a <code>default_config.yaml</code> file in the 🤗 Accelerate cache.',re,F,Xt="The configuration file is used to set the default options when you launch the training script.",oe,v,ce,X,Vt="You’ll be asked a few questions about your setup, and configure the following arguments. In this example, you’ll use ZeRO-3 so make sure you pick those options.",de,V,Je,Y,Yt='Once this is done, the corresponding config should look like below and you can find it in config folder at <a href="https://github.com/huggingface/peft/blob/main/examples/sft/configs/deepspeed_config.yaml" rel="nofollow">deepspeed_config.yaml</a>:',me,E,je,x,he,R,Et='The launch command is available at <a href="https://github.com/huggingface/peft/blob/main/examples/sft/run_peft_deepspeed.sh" rel="nofollow">run_peft_deepspeed.sh</a> and it is also shown below:',ye,$,Te,Q,xt="Notice that we are using LoRA with rank=8, alpha=16 and targeting all linear layers. We are passing the deepspeed config file and finetuning 70B Llama model on a subset of the ultrachat dataset.",Ue,S,ue,z,Rt="Let’s dive a little deeper into the script so you can see what’s going on, and understand how it works.",fe,H,$t='The first thing to know is that the script uses DeepSpeed for distributed training as the DeepSpeed config has been passed. The <a href="https://huggingface.co/docs/trl/main/en/sft_trainer#trl.SFTTrainer" rel="nofollow">SFTTrainer</a> class handles all the heavy lifting of creating the PEFT model using the peft config that is passed. After that, when you call <code>trainer.train()</code>, <a href="https://huggingface.co/docs/trl/main/en/sft_trainer#trl.SFTTrainer" rel="nofollow">SFTTrainer</a> internally uses 🤗 Accelerate to prepare the model, optimizer and trainer using the DeepSpeed config to create DeepSpeed engine which is then trained. The main code snippet is below:',be,L,we,q,ge,P,Qt="In the above example, the memory consumed per GPU is 64 GB (80%) as seen in the screenshot below:",Ze,U,St='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/peft_deepspeed_mem_usage.png"/>',_e,D,zt="GPU memory usage for the training run",Ae,O,Be,K,Ht='You can also refer this blog post <a href="https://medium.com/@sourabmangrulkar/falcon-180b-finetuning-using-peft-and-deepspeed-b92643091d99" rel="nofollow">Falcon 180B Finetuning using 🤗 PEFT and DeepSpeed</a> on how to finetune 180B Falcon model on 16 A100 GPUs on 2 machines.',We,ll,Ge,el,Lt=`In this section, we will look at how to use QLoRA and DeepSpeed Stage-3 for finetuning 70B llama model on 2X40GB GPUs.
For this, we first need <code>bitsandbytes&gt;=0.43.3</code>, <code>accelerate&gt;=1.0.1</code>, <code>transformers&gt;4.44.2</code>, <code>trl&gt;0.11.4</code> and <code>peft&gt;0.13.0</code>. We need to set <code>zero3_init_flag</code> to true when using Accelerate config. Below is the config which can be found at <a href="https://github.com/huggingface/peft/blob/main/examples/sft/configs/deepspeed_config_z3_qlora.yaml" rel="nofollow">deepspeed_config_z3_qlora.yaml</a>:`,Ie,tl,Ce,al,qt='Launch command is given below which is available at <a href="https://github.com/huggingface/peft/blob/main/examples/sft/run_peft_qlora_deepspeed_stage3.sh" rel="nofollow">run_peft_qlora_deepspeed_stage3.sh</a>:',ke,sl,Ne,nl,Pt="Notice the new argument being passed <code>bnb_4bit_quant_storage_dtype</code> which denotes the data type for packing the 4-bit parameters. For example, when it is set to <code>bfloat16</code>, <strong>32/4 = 8</strong> 4-bit params are packed together post quantization.",Fe,il,Dt="In terms of training code, the important code changes are:",ve,pl,Xe,Ml,Ot="Notice that <code>dtype</code> for <code>AutoModelForCausalLM</code> is same as the <code>bnb_4bit_quant_storage</code> data type. That’s it. Everything else is handled by Trainer and TRL.",Ve,rl,Ye,ol,Kt="In the above example, the memory consumed per GPU is <strong>36.6 GB</strong>. Therefore, what took 8X80GB GPUs with DeepSpeed Stage 3+LoRA and a couple of 80GB GPUs with DDP+QLoRA now requires 2X40GB GPUs. This makes finetuning of large models more accessible.",Ee,cl,xe,dl,la='This section of guide will help you learn how to use our DeepSpeed <a href="https://github.com/huggingface/peft/blob/main/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py" rel="nofollow">training script</a>. You’ll configure the script to train a large model for conditional generation with ZeRO-3 and CPU Offload.',Re,u,ea='<p>💡 To help you get started, check out our example training scripts for <a href="https://github.com/huggingface/peft/blob/main/examples/causal_language_modeling/peft_lora_clm_accelerate_ds_zero3_offload.py" rel="nofollow">causal language modeling</a> and <a href="https://github.com/huggingface/peft/blob/main/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py" rel="nofollow">conditional generation</a>. You can adapt these scripts for your own applications or even use them out of the box if your task is similar to the one in the scripts.</p>',$e,Jl,Qe,ml,ta='Start by running the following command to <a href="https://huggingface.co/docs/accelerate/quicktour#launching-your-distributed-script" rel="nofollow">create a DeepSpeed configuration file</a> with 🤗 Accelerate. The <code>--config_file</code> flag allows you to save the configuration file to a specific location, otherwise it is saved as a <code>default_config.yaml</code> file in the 🤗 Accelerate cache.',Se,jl,aa="The configuration file is used to set the default options when you launch the training script.",ze,hl,He,yl,sa="You’ll be asked a few questions about your setup, and configure the following arguments. In this example, you’ll use ZeRO-3 along with CPU-Offload so make sure you pick those options.",Le,Tl,qe,Ul,na='An example <a href="https://github.com/huggingface/peft/blob/main/examples/conditional_generation/accelerate_ds_zero3_cpu_offload_config.yaml" rel="nofollow">configuration file</a> might look like the following. The most important thing to notice is that <code>zero_stage</code> is set to <code>3</code>, and <code>offload_optimizer_device</code> and <code>offload_param_device</code> are set to the <code>cpu</code>.',Pe,ul,De,fl,Oe,bl,ia="Let’s dive a little deeper into the script so you can see what’s going on, and understand how it works.",Ke,wl,pa='Within the <a href="https://github.com/huggingface/peft/blob/2822398fbe896f25d4dac5e468624dc5fd65a51b/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py#L103" rel="nofollow"><code>main</code></a> function, the script creates an <a href="https://huggingface.co/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator" rel="nofollow">Accelerator</a> class to initialize all the necessary requirements for distributed training.',lt,f,Ma="<p>💡 Feel free to change the model and dataset inside the <code>main</code> function. If your dataset format is different from the one in the script, you may also need to write your own preprocessing function.</p>",et,gl,ra='The script also creates a configuration for the 🤗 PEFT method you’re using, which in this case, is LoRA. The <a href="/docs/peft/pr_3117/en/package_reference/lora#peft.LoraConfig">LoraConfig</a> specifies the task type and important parameters such as the dimension of the low-rank matrices, the matrices scaling factor, and the dropout probability of the LoRA layers. If you want to use a different 🤗 PEFT method, make sure you replace <code>LoraConfig</code> with the appropriate <a href="../package_reference/tuners">class</a>.',tt,Zl,at,_l,oa='Throughout the script, you’ll see the <a href="https://huggingface.co/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator.main_process_first" rel="nofollow">main_process_first</a> and <a href="https://huggingface.co/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator.wait_for_everyone" rel="nofollow">wait_for_everyone</a> functions which help control and synchronize when processes are executed.',st,Al,ca='The <a href="/docs/peft/pr_3117/en/package_reference/peft_model#peft.get_peft_model">get_peft_model()</a> function takes a base model and the <code>peft_config</code> you prepared earlier to create a <a href="/docs/peft/pr_3117/en/package_reference/peft_model#peft.PeftModel">PeftModel</a>:',nt,Bl,it,Wl,da='Pass all the relevant training objects to 🤗 Accelerate’s <a href="https://huggingface.co/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator.prepare" rel="nofollow">prepare</a> which makes sure everything is ready for training:',pt,Gl,Mt,Il,Ja="The next bit of code checks whether the DeepSpeed plugin is used in the <code>Accelerator</code>, and if the plugin exists, then we check if we are using ZeRO-3. This conditional flag is used when calling <code>generate</code> function call during inference for syncing GPUs when the model parameters are sharded:",rt,Cl,ot,kl,ma='Inside the training loop, the usual <code>loss.backward()</code> is replaced by 🤗 Accelerate’s <a href="https://huggingface.co/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator.backward" rel="nofollow">backward</a> which uses the correct <code>backward()</code> method based on your configuration:',ct,Nl,dt,Fl,ja="That is all! The rest of the script handles the training loop, evaluation, and even pushes it to the Hub for you.",Jt,vl,mt,Xl,ha="Run the following command to launch the training script. Earlier, you saved the configuration file to <code>ds_zero3_cpu.yaml</code>, so you’ll need to pass the path to the launcher with the <code>--config_file</code> argument like this:",jt,Vl,ht,Yl,ya="You’ll see some output logs that track memory usage during training, and once it’s completed, the script returns the accuracy and compares the predictions to the labels:",yt,El,Tt,xl,Ut,Rl,Ta="<li>Merging when using PEFT and DeepSpeed is currently unsupported and will raise error.</li> <li>When using CPU offloading, the major gains from using PEFT to shrink the optimizer states and gradients to that of the adapter weights would be realized on CPU RAM and there won’t be savings with respect to GPU memory.</li> <li>DeepSpeed Stage 3 and qlora when used with CPU offloading leads to more GPU memory usage when compared to disabling CPU offloading.</li>",ut,T,Sl,Ua="💡 When you have code that requires merging (and unmerging) of weights, try to manually collect the parameters with DeepSpeed Zero-3 beforehand:",Zt,$l,ft,Ql,bt,Ll,wt;return b=new _a({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),w=new h({props:{title:"DeepSpeed",local:"deepspeed",headingTag:"h1"}}),_=new h({props:{title:"Compatibility with bitsandbytes quantization + LoRA",local:"compatibility-with-bitsandbytes-quantization--lora",headingTag:"h2"}}),I=new h({props:{title:"Use PEFT and DeepSpeed with ZeRO3 for finetuning large models on multiple devices and multiple nodes",local:"use-peft-and-deepspeed-with-zero3-for-finetuning-large-models-on-multiple-devices-and-multiple-nodes",headingTag:"h1"}}),k=new h({props:{title:"Configuration",local:"configuration",headingTag:"h2"}}),v=new j({props:{code:"YWNjZWxlcmF0ZSUyMGNvbmZpZyUyMC0tY29uZmlnX2ZpbGUlMjBkZWVwc3BlZWRfY29uZmlnLnlhbWw=",highlighted:"accelerate config --config_file deepspeed_config.yaml",lang:"bash",wrap:!1}}),V=new j({props:{code:"JTYwemVyb19zdGFnZSU2MCUzQSUyMCU1QjAlNUQlMjBEaXNhYmxlZCUyQyUyMCU1QjElNUQlMjBvcHRpbWl6ZXIlMjBzdGF0ZSUyMHBhcnRpdGlvbmluZyUyQyUyMCU1QjIlNUQlMjBvcHRpbWl6ZXIlMkJncmFkaWVudCUyMHN0YXRlJTIwcGFydGl0aW9uaW5nJTIwYW5kJTIwJTVCMyU1RCUyMG9wdGltaXplciUyQmdyYWRpZW50JTJCcGFyYW1ldGVyJTIwcGFydGl0aW9uaW5nJTBBJTYwZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTYwJTNBJTIwTnVtYmVyJTIwb2YlMjB0cmFpbmluZyUyMHN0ZXBzJTIwdG8lMjBhY2N1bXVsYXRlJTIwZ3JhZGllbnRzJTIwYmVmb3JlJTIwYXZlcmFnaW5nJTIwYW5kJTIwYXBwbHlpbmclMjB0aGVtLiUyMFBhc3MlMjB0aGUlMjBzYW1lJTIwdmFsdWUlMjBhcyUyMHlvdSUyMHdvdWxkJTIwcGFzcyUyMHZpYSUyMGNtZCUyMGFyZ3VtZW50JTIwZWxzZSUyMHlvdSUyMHdpbGwlMjBlbmNvdW50ZXIlMjBtaXNtYXRjaCUyMGVycm9yLiUwQSU2MGdyYWRpZW50X2NsaXBwaW5nJTYwJTNBJTIwRW5hYmxlJTIwZ3JhZGllbnQlMjBjbGlwcGluZyUyMHdpdGglMjB2YWx1ZS4lMjBEb24ndCUyMHNldCUyMHRoaXMlMjBhcyUyMHlvdSUyMHdpbGwlMjBiZSUyMHBhc3NpbmclMjBpdCUyMHZpYSUyMGNtZCUyMGFyZ3VtZW50cy4lMEElNjBvZmZsb2FkX29wdGltaXplcl9kZXZpY2UlNjAlM0ElMjAlNUJub25lJTVEJTIwRGlzYWJsZSUyMG9wdGltaXplciUyMG9mZmxvYWRpbmclMkMlMjAlNUJjcHUlNUQlMjBvZmZsb2FkJTIwb3B0aW1pemVyJTIwdG8lMjBDUFUlMkMlMjAlNUJudm1lJTVEJTIwb2ZmbG9hZCUyMG9wdGltaXplciUyMHRvJTIwTlZNZSUyMFNTRC4lMjBPbmx5JTIwYXBwbGljYWJsZSUyMHdpdGglMjBaZVJPJTIwJTNFJTNEJTIwU3RhZ2UtMi4lMjBTZXQlMjB0aGlzJTIwYXMlMjAlNjBub25lJTYwJTIwYXMlMjBkb24ndCUyMHdhbnQlMjB0byUyMGVuYWJsZSUyMG9mZmxvYWRpbmcuJTBBJTYwb2ZmbG9hZF9wYXJhbV9kZXZpY2UlNjAlM0ElMjAlNUJub25lJTVEJTIwRGlzYWJsZSUyMHBhcmFtZXRlciUyMG9mZmxvYWRpbmclMkMlMjAlNUJjcHUlNUQlMjBvZmZsb2FkJTIwcGFyYW1ldGVycyUyMHRvJTIwQ1BVJTJDJTIwJTVCbnZtZSU1RCUyMG9mZmxvYWQlMjBwYXJhbWV0ZXJzJTIwdG8lMjBOVk1lJTIwU1NELiUyME9ubHklMjBhcHBsaWNhYmxlJTIwd2l0aCUyMFplUk8lMjBTdGFnZS0zLiUyMFNldCUyMHRoaXMlMjBhcyUyMCU2MG5vbmUlNjAlMjBhcyUyMGRvbid0JTIwd2FudCUyMHRvJTIwZW5hYmxlJTIwb2ZmbG9hZGluZy4lMEElNjB6ZXJvM19pbml0X2ZsYWclNjAlM0ElMjBEZWNpZGVzJTIwd2hldGhlciUyMHRvJTIwZW5hYmxlJTIwJTYwZGVlcHNwZWVkLnplcm8uSW5pdCU2MCUyMGZvciUyMGNvbnN0cnVjdGluZyUyMG1hc3NpdmUlMjBtb2RlbHMuJTIwT25seSUyMGFwcGxpY2FibGUlMjB3aXRoJTIwWmVSTyUyMFN0YWdlLTMuJTIwU2V0JTIwdGhpcyUyMHRvJTIwJTYwVHJ1ZSU2MC4lMEElNjB6ZXJvM19zYXZlXzE2Yml0X21vZGVsJTYwJTNBJTIwRGVjaWRlcyUyMHdoZXRoZXIlMjB0byUyMHNhdmUlMjAxNi1iaXQlMjBtb2RlbCUyMHdlaWdodHMlMjB3aGVuJTIwdXNpbmclMjBaZVJPJTIwU3RhZ2UtMy4lMjBTZXQlMjB0aGlzJTIwdG8lMjAlNjBUcnVlJTYwLiUwQSU2MG1peGVkX3ByZWNpc2lvbiU2MCUzQSUyMCU2MG5vJTYwJTIwZm9yJTIwRlAzMiUyMHRyYWluaW5nJTJDJTIwJTYwZnAxNiU2MCUyMGZvciUyMEZQMTYlMjBtaXhlZC1wcmVjaXNpb24lMjB0cmFpbmluZyUyMGFuZCUyMCU2MGJmMTYlNjAlMjBmb3IlMjBCRjE2JTIwbWl4ZWQtcHJlY2lzaW9uJTIwdHJhaW5pbmcuJTIwU2V0JTIwdGhpcyUyMHRvJTIwJTYwVHJ1ZSU2MC4=",highlighted:'`zero_stage`: [0] Disabled, [1] optimizer state partitioning, [2] optimizer+gradient state partitioning and [3] optimizer+gradient+parameter partitioning\n`gradient_accumulation_steps`: Number of training steps to accumulate gradients before averaging and applying them. Pass the same value as you would pass via cmd argument <span class="hljs-keyword">else</span> you will encounter mismatch error.\n`gradient_clipping`: Enable gradient clipping with value. Don<span class="hljs-string">&#x27;t set this as you will be passing it via cmd arguments.\n`offload_optimizer_device`: [none] Disable optimizer offloading, [cpu] offload optimizer to CPU, [nvme] offload optimizer to NVMe SSD. Only applicable with ZeRO &gt;= Stage-2. Set this as `none` as don&#x27;</span>t want to <span class="hljs-built_in">enable</span> offloading.\n`offload_param_device`: [none] Disable parameter offloading, [cpu] offload parameters to CPU, [nvme] offload parameters to NVMe SSD. Only applicable with ZeRO Stage-3. Set this as `none` as don<span class="hljs-string">&#x27;t want to enable offloading.\n`zero3_init_flag`: Decides whether to enable `deepspeed.zero.Init` for constructing massive models. Only applicable with ZeRO Stage-3. Set this to `True`.\n`zero3_save_16bit_model`: Decides whether to save 16-bit model weights when using ZeRO Stage-3. Set this to `True`.\n`mixed_precision`: `no` for FP32 training, `fp16` for FP16 mixed-precision training and `bf16` for BF16 mixed-precision training. Set this to `True`.</span>',lang:"bash",wrap:!1}}),E=new j({props:{code:"Y29tcHV0ZV9lbnZpcm9ubWVudCUzQSUyMExPQ0FMX01BQ0hJTkUlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMEFkZWJ1ZyUzQSUyMGZhbHNlJTBBZGVlcHNwZWVkX2NvbmZpZyUzQSUwQSUyMCUyMGRlZXBzcGVlZF9tdWx0aW5vZGVfbGF1bmNoZXIlM0ElMjBzdGFuZGFyZCUwQSUyMCUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUzQSUyMDQlMEElMjAlMjBvZmZsb2FkX29wdGltaXplcl9kZXZpY2UlM0ElMjBub25lJTBBJTIwJTIwb2ZmbG9hZF9wYXJhbV9kZXZpY2UlM0ElMjBub25lJTBBJTIwJTIwemVybzNfaW5pdF9mbGFnJTNBJTIwdHJ1ZSUwQSUyMCUyMHplcm8zX3NhdmVfMTZiaXRfbW9kZWwlM0ElMjB0cnVlJTBBJTIwJTIwemVyb19zdGFnZSUzQSUyMDMlMEFkaXN0cmlidXRlZF90eXBlJTNBJTIwREVFUFNQRUVEJTBBZG93bmNhc3RfYmYxNiUzQSUyMCdubyclMEFtYWNoaW5lX3JhbmslM0ElMjAwJTBBbWFpbl90cmFpbmluZ19mdW5jdGlvbiUzQSUyMG1haW4lMEFtaXhlZF9wcmVjaXNpb24lM0ElMjBiZjE2JTBBbnVtX21hY2hpbmVzJTNBJTIwMSUwQW51bV9wcm9jZXNzZXMlM0ElMjA4JTBBcmR6dl9iYWNrZW5kJTNBJTIwc3RhdGljJTBBc2FtZV9uZXR3b3JrJTNBJTIwdHJ1ZSUwQXRwdV9lbnYlM0ElMjAlNUIlNUQlMEF0cHVfdXNlX2NsdXN0ZXIlM0ElMjBmYWxzZSUwQXRwdV91c2Vfc3VkbyUzQSUyMGZhbHNlJTBBdXNlX2NwdSUzQSUyMGZhbHNl",highlighted:`<span class="hljs-attr">compute_environment:</span> <span class="hljs-string">LOCAL_MACHINE</span>
<span class="hljs-attr">debug:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">deepspeed_config:</span>
 <span class="hljs-attr">deepspeed_multinode_launcher:</span> <span class="hljs-string">standard</span>
 <span class="hljs-attr">gradient_accumulation_steps:</span> <span class="hljs-number">4</span>
 <span class="hljs-attr">offload_optimizer_device:</span> <span class="hljs-string">none</span>
 <span class="hljs-attr">offload_param_device:</span> <span class="hljs-string">none</span>
 <span class="hljs-attr">zero3_init_flag:</span> <span class="hljs-literal">true</span>
 <span class="hljs-attr">zero3_save_16bit_model:</span> <span class="hljs-literal">true</span>
 <span class="hljs-attr">zero_stage:</span> <span class="hljs-number">3</span>
<span class="hljs-attr">distributed_type:</span> <span class="hljs-string">DEEPSPEED</span>
<span class="hljs-attr">downcast_bf16:</span> <span class="hljs-string">&#x27;no&#x27;</span>
<span class="hljs-attr">machine_rank:</span> <span class="hljs-number">0</span>
<span class="hljs-attr">main_training_function:</span> <span class="hljs-string">main</span>
<span class="hljs-attr">mixed_precision:</span> <span class="hljs-string">bf16</span>
<span class="hljs-attr">num_machines:</span> <span class="hljs-number">1</span>
<span class="hljs-attr">num_processes:</span> <span class="hljs-number">8</span>
<span class="hljs-attr">rdzv_backend:</span> <span class="hljs-string">static</span>
<span class="hljs-attr">same_network:</span> <span class="hljs-literal">true</span>
<span class="hljs-attr">tpu_env:</span> []
<span class="hljs-attr">tpu_use_cluster:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">tpu_use_sudo:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">use_cpu:</span> <span class="hljs-literal">false</span>`,lang:"yml",wrap:!1}}),x=new h({props:{title:"Launch command",local:"launch-command",headingTag:"h2"}}),$=new j({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMC0tY29uZmlnX2ZpbGUlMjAlMjJjb25maWdzJTJGZGVlcHNwZWVkX2NvbmZpZy55YW1sJTIyJTIwJTIwdHJhaW4ucHklMjAlNUMlMEEtLXNlZWQlMjAxMDAlMjAlNUMlMEEtLW1vZGVsX25hbWVfb3JfcGF0aCUyMCUyMm1ldGEtbGxhbWElMkZMbGFtYS0yLTcwYi1oZiUyMiUyMCU1QyUwQS0tZGF0YXNldF9uYW1lJTIwJTIyc21hbmdydWwlMkZ1bHRyYWNoYXQtMTBrLWNoYXRtbCUyMiUyMCU1QyUwQS0tY2hhdF90ZW1wbGF0ZV9mb3JtYXQlMjAlMjJjaGF0bWwlMjIlMjAlNUMlMEEtLWFkZF9zcGVjaWFsX3Rva2VucyUyMEZhbHNlJTIwJTVDJTBBLS1hcHBlbmRfY29uY2F0X3Rva2VuJTIwRmFsc2UlMjAlNUMlMEEtLXNwbGl0cyUyMCUyMnRyYWluJTJDdGVzdCUyMiUyMCU1QyUwQS0tbWF4X3NlcV9sZW4lMjAyMDQ4JTIwJTVDJTBBLS1udW1fdHJhaW5fZXBvY2hzJTIwMSUyMCU1QyUwQS0tbG9nZ2luZ19zdGVwcyUyMDUlMjAlNUMlMEEtLWxvZ19sZXZlbCUyMCUyMmluZm8lMjIlMjAlNUMlMEEtLWxvZ2dpbmdfc3RyYXRlZ3klMjAlMjJzdGVwcyUyMiUyMCU1QyUwQS0tZXZhbF9zdHJhdGVneSUyMCUyMmVwb2NoJTIyJTIwJTVDJTBBLS1zYXZlX3N0cmF0ZWd5JTIwJTIyZXBvY2glMjIlMjAlNUMlMEEtLXB1c2hfdG9faHViJTIwJTVDJTBBLS1odWJfcHJpdmF0ZV9yZXBvJTIwVHJ1ZSUyMCU1QyUwQS0taHViX3N0cmF0ZWd5JTIwJTIyZXZlcnlfc2F2ZSUyMiUyMCU1QyUwQS0tYmYxNiUyMFRydWUlMjAlNUMlMEEtLXBhY2tpbmclMjBUcnVlJTIwJTVDJTBBLS1sZWFybmluZ19yYXRlJTIwMWUtNCUyMCU1QyUwQS0tbHJfc2NoZWR1bGVyX3R5cGUlMjAlMjJjb3NpbmUlMjIlMjAlNUMlMEEtLXdlaWdodF9kZWNheSUyMDFlLTQlMjAlNUMlMEEtLXdhcm11cF9zdGVwcyUyMDAlMjAlNUMlMEEtLW1heF9ncmFkX25vcm0lMjAxLjAlMjAlNUMlMEEtLW91dHB1dF9kaXIlMjAlMjJsbGFtYS1zZnQtbG9yYS1kZWVwc3BlZWQlMjIlMjAlNUMlMEEtLXBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUyMDglMjAlNUMlMEEtLXBlcl9kZXZpY2VfZXZhbF9iYXRjaF9zaXplJTIwOCUyMCU1QyUwQS0tZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTIwNCUyMCU1QyUwQS0tZ3JhZGllbnRfY2hlY2twb2ludGluZyUyMFRydWUlMjAlNUMlMEEtLXVzZV9yZWVudHJhbnQlMjBGYWxzZSUyMCU1QyUwQS0tZGF0YXNldF90ZXh0X2ZpZWxkJTIwJTIyY29udGVudCUyMiUyMCU1QyUwQS0tdXNlX2ZsYXNoX2F0dG4lMjBUcnVlJTIwJTVDJTBBLS11c2VfcGVmdF9sb3JhJTIwVHJ1ZSUyMCU1QyUwQS0tbG9yYV9yJTIwOCUyMCU1QyUwQS0tbG9yYV9hbHBoYSUyMDE2JTIwJTVDJTBBLS1sb3JhX2Ryb3BvdXQlMjAwLjElMjAlNUMlMEEtLWxvcmFfdGFyZ2V0X21vZHVsZXMlMjAlMjJhbGwtbGluZWFyJTIyJTIwJTVDJTBBLS11c2VfNGJpdF9xdWFudGl6YXRpb24lMjBGYWxzZQ==",highlighted:`accelerate launch --config_file <span class="hljs-string">&quot;configs/deepspeed_config.yaml&quot;</span> train.py \\
--seed 100 \\
--model_name_or_path <span class="hljs-string">&quot;meta-llama/Llama-2-70b-hf&quot;</span> \\
--dataset_name <span class="hljs-string">&quot;smangrul/ultrachat-10k-chatml&quot;</span> \\
--chat_template_format <span class="hljs-string">&quot;chatml&quot;</span> \\
--add_special_tokens False \\
--append_concat_token False \\
--splits <span class="hljs-string">&quot;train,test&quot;</span> \\
--max_seq_len 2048 \\
--num_train_epochs 1 \\
--logging_steps 5 \\
--log_level <span class="hljs-string">&quot;info&quot;</span> \\
--logging_strategy <span class="hljs-string">&quot;steps&quot;</span> \\
--eval_strategy <span class="hljs-string">&quot;epoch&quot;</span> \\
--save_strategy <span class="hljs-string">&quot;epoch&quot;</span> \\
--push_to_hub \\
--hub_private_repo True \\
--hub_strategy <span class="hljs-string">&quot;every_save&quot;</span> \\
--bf16 True \\
--packing True \\
--learning_rate 1e-4 \\
--lr_scheduler_type <span class="hljs-string">&quot;cosine&quot;</span> \\
--weight_decay 1e-4 \\
--warmup_steps 0 \\
--max_grad_norm 1.0 \\
--output_dir <span class="hljs-string">&quot;llama-sft-lora-deepspeed&quot;</span> \\
--per_device_train_batch_size 8 \\
--per_device_eval_batch_size 8 \\
--gradient_accumulation_steps 4 \\
--gradient_checkpointing True \\
--use_reentrant False \\
--dataset_text_field <span class="hljs-string">&quot;content&quot;</span> \\
--use_flash_attn True \\
--use_peft_lora True \\
--lora_r 8 \\
--lora_alpha 16 \\
--lora_dropout 0.1 \\
--lora_target_modules <span class="hljs-string">&quot;all-linear&quot;</span> \\
--use_4bit_quantization False`,lang:"bash",wrap:!1}}),S=new h({props:{title:"The important parts",local:"the-important-parts",headingTag:"h2"}}),L=new j({props:{code:"JTIzJTIwdHJhaW5lciUwQXRyYWluZXIlMjAlM0QlMjBTRlRUcmFpbmVyKCUwQSUyMCUyMCUyMCUyMG1vZGVsJTNEbW9kZWwlMkMlMEElMjAlMjAlMjAlMjBwcm9jZXNzaW5nX2NsYXNzJTNEdG9rZW5pemVyJTJDJTBBJTIwJTIwJTIwJTIwYXJncyUzRHRyYWluaW5nX2FyZ3MlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEdHJhaW5fZGF0YXNldCUyQyUwQSUyMCUyMCUyMCUyMGV2YWxfZGF0YXNldCUzRGV2YWxfZGF0YXNldCUyQyUwQSUyMCUyMCUyMCUyMHBlZnRfY29uZmlnJTNEcGVmdF9jb25maWclMkMlMEEpJTBBdHJhaW5lci5hY2NlbGVyYXRvci5wcmludChmJTIyJTdCdHJhaW5lci5tb2RlbCU3RCUyMiklMEElMEElMjMlMjB0cmFpbiUwQWNoZWNrcG9pbnQlMjAlM0QlMjBOb25lJTBBaWYlMjB0cmFpbmluZ19hcmdzLnJlc3VtZV9mcm9tX2NoZWNrcG9pbnQlMjBpcyUyMG5vdCUyME5vbmUlM0ElMEElMjAlMjAlMjAlMjBjaGVja3BvaW50JTIwJTNEJTIwdHJhaW5pbmdfYXJncy5yZXN1bWVfZnJvbV9jaGVja3BvaW50JTBBdHJhaW5lci50cmFpbihyZXN1bWVfZnJvbV9jaGVja3BvaW50JTNEY2hlY2twb2ludCklMEElMEElMjMlMjBzYXZpbmclMjBmaW5hbCUyMG1vZGVsJTBBdHJhaW5lci5zYXZlX21vZGVsKCk=",highlighted:`<span class="hljs-comment"># trainer</span>
trainer = SFTTrainer(
 model=model,
 processing_class=tokenizer,
 args=training_args,
 train_dataset=train_dataset,
 eval_dataset=eval_dataset,
 peft_config=peft_config,
)
trainer.accelerator.<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{trainer.model}</span>&quot;</span>)
<span class="hljs-comment"># train</span>
checkpoint = <span class="hljs-literal">None</span>
<span class="hljs-keyword">if</span> training_args.resume_from_checkpoint <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>:
 checkpoint = training_args.resume_from_checkpoint
trainer.train(resume_from_checkpoint=checkpoint)
<span class="hljs-comment"># saving final model</span>
trainer.save_model()`,lang:"python",wrap:!1}}),q=new h({props:{title:"Memory usage",local:"memory-usage",headingTag:"h2"}}),O=new h({props:{title:"More resources",local:"more-resources",headingTag:"h2"}}),ll=new h({props:{title:"Use PEFT QLoRA and DeepSpeed with ZeRO3 for finetuning large models on multiple GPUs",local:"use-peft-qlora-and-deepspeed-with-zero3-for-finetuning-large-models-on-multiple-gpus",headingTag:"h1"}}),tl=new j({props:{code:"Y29tcHV0ZV9lbnZpcm9ubWVudCUzQSUyMExPQ0FMX01BQ0hJTkUlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMEFkZWJ1ZyUzQSUyMGZhbHNlJTBBZGVlcHNwZWVkX2NvbmZpZyUzQSUwQSUyMCUyMGRlZXBzcGVlZF9tdWx0aW5vZGVfbGF1bmNoZXIlM0ElMjBzdGFuZGFyZCUwQSUyMCUyMG9mZmxvYWRfb3B0aW1pemVyX2RldmljZSUzQSUyMG5vbmUlMEElMjAlMjBvZmZsb2FkX3BhcmFtX2RldmljZSUzQSUyMG5vbmUlMEElMjAlMjB6ZXJvM19pbml0X2ZsYWclM0ElMjB0cnVlJTBBJTIwJTIwemVybzNfc2F2ZV8xNmJpdF9tb2RlbCUzQSUyMHRydWUlMEElMjAlMjB6ZXJvX3N0YWdlJTNBJTIwMyUwQWRpc3RyaWJ1dGVkX3R5cGUlM0ElMjBERUVQU1BFRUQlMEFkb3duY2FzdF9iZjE2JTNBJTIwJ25vJyUwQW1hY2hpbmVfcmFuayUzQSUyMDAlMEFtYWluX3RyYWluaW5nX2Z1bmN0aW9uJTNBJTIwbWFpbiUwQW1peGVkX3ByZWNpc2lvbiUzQSUyMGJmMTYlMEFudW1fbWFjaGluZXMlM0ElMjAxJTBBbnVtX3Byb2Nlc3NlcyUzQSUyMDIlMEFyZHp2X2JhY2tlbmQlM0ElMjBzdGF0aWMlMEFzYW1lX25ldHdvcmslM0ElMjB0cnVlJTBBdHB1X2VudiUzQSUyMCU1QiU1RCUwQXRwdV91c2VfY2x1c3RlciUzQSUyMGZhbHNlJTBBdHB1X3VzZV9zdWRvJTNBJTIwZmFsc2UlMEF1c2VfY3B1JTNBJTIwZmFsc2U=",highlighted:`<span class="hljs-attr">compute_environment:</span> <span class="hljs-string">LOCAL_MACHINE</span>
<span class="hljs-attr">debug:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">deepspeed_config:</span>
 <span class="hljs-attr">deepspeed_multinode_launcher:</span> <span class="hljs-string">standard</span>
 <span class="hljs-attr">offload_optimizer_device:</span> <span class="hljs-string">none</span>
 <span class="hljs-attr">offload_param_device:</span> <span class="hljs-string">none</span>
 <span class="hljs-attr">zero3_init_flag:</span> <span class="hljs-literal">true</span>
 <span class="hljs-attr">zero3_save_16bit_model:</span> <span class="hljs-literal">true</span>
 <span class="hljs-attr">zero_stage:</span> <span class="hljs-number">3</span>
<span class="hljs-attr">distributed_type:</span> <span class="hljs-string">DEEPSPEED</span>
<span class="hljs-attr">downcast_bf16:</span> <span class="hljs-string">&#x27;no&#x27;</span>
<span class="hljs-attr">machine_rank:</span> <span class="hljs-number">0</span>
<span class="hljs-attr">main_training_function:</span> <span class="hljs-string">main</span>
<span class="hljs-attr">mixed_precision:</span> <span class="hljs-string">bf16</span>
<span class="hljs-attr">num_machines:</span> <span class="hljs-number">1</span>
<span class="hljs-attr">num_processes:</span> <span class="hljs-number">2</span>
<span class="hljs-attr">rdzv_backend:</span> <span class="hljs-string">static</span>
<span class="hljs-attr">same_network:</span> <span class="hljs-literal">true</span>
<span class="hljs-attr">tpu_env:</span> []
<span class="hljs-attr">tpu_use_cluster:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">tpu_use_sudo:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">use_cpu:</span> <span class="hljs-literal">false</span>`,lang:"yml",wrap:!1}}),sl=new j({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMC0tY29uZmlnX2ZpbGUlMjAlMjJjb25maWdzJTJGZGVlcHNwZWVkX2NvbmZpZ196M19xbG9yYS55YW1sJTIyJTIwJTIwdHJhaW4ucHklMjAlNUMlMEEtLXNlZWQlMjAxMDAlMjAlNUMlMEEtLW1vZGVsX25hbWVfb3JfcGF0aCUyMCUyMm1ldGEtbGxhbWElMkZMbGFtYS0yLTcwYi1oZiUyMiUyMCU1QyUwQS0tZGF0YXNldF9uYW1lJTIwJTIyc21hbmdydWwlMkZ1bHRyYWNoYXQtMTBrLWNoYXRtbCUyMiUyMCU1QyUwQS0tY2hhdF90ZW1wbGF0ZV9mb3JtYXQlMjAlMjJjaGF0bWwlMjIlMjAlNUMlMEEtLWFkZF9zcGVjaWFsX3Rva2VucyUyMEZhbHNlJTIwJTVDJTBBLS1hcHBlbmRfY29uY2F0X3Rva2VuJTIwRmFsc2UlMjAlNUMlMEEtLXNwbGl0cyUyMCUyMnRyYWluJTJDdGVzdCUyMiUyMCU1QyUwQS0tbWF4X3NlcV9sZW4lMjAyMDQ4JTIwJTVDJTBBLS1udW1fdHJhaW5fZXBvY2hzJTIwMSUyMCU1QyUwQS0tbG9nZ2luZ19zdGVwcyUyMDUlMjAlNUMlMEEtLWxvZ19sZXZlbCUyMCUyMmluZm8lMjIlMjAlNUMlMEEtLWxvZ2dpbmdfc3RyYXRlZ3klMjAlMjJzdGVwcyUyMiUyMCU1QyUwQS0tZXZhbF9zdHJhdGVneSUyMCUyMmVwb2NoJTIyJTIwJTVDJTBBLS1zYXZlX3N0cmF0ZWd5JTIwJTIyZXBvY2glMjIlMjAlNUMlMEEtLXB1c2hfdG9faHViJTIwJTVDJTBBLS1odWJfcHJpdmF0ZV9yZXBvJTIwVHJ1ZSUyMCU1QyUwQS0taHViX3N0cmF0ZWd5JTIwJTIyZXZlcnlfc2F2ZSUyMiUyMCU1QyUwQS0tYmYxNiUyMFRydWUlMjAlNUMlMEEtLXBhY2tpbmclMjBUcnVlJTIwJTVDJTBBLS1sZWFybmluZ19yYXRlJTIwMWUtNCUyMCU1QyUwQS0tbHJfc2NoZWR1bGVyX3R5cGUlMjAlMjJjb3NpbmUlMjIlMjAlNUMlMEEtLXdlaWdodF9kZWNheSUyMDFlLTQlMjAlNUMlMEEtLXdhcm11cF9zdGVwcyUyMDAlMjAlNUMlMEEtLW1heF9ncmFkX25vcm0lMjAxLjAlMjAlNUMlMEEtLW91dHB1dF9kaXIlMjAlMjJsbGFtYS1zZnQtcWxvcmEtZHN6MyUyMiUyMCU1QyUwQS0tcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTIwMiUyMCU1QyUwQS0tcGVyX2RldmljZV9ldmFsX2JhdGNoX3NpemUlMjAyJTIwJTVDJTBBLS1ncmFkaWVudF9hY2N1bXVsYXRpb25fc3RlcHMlMjAyJTIwJTVDJTBBLS1ncmFkaWVudF9jaGVja3BvaW50aW5nJTIwVHJ1ZSUyMCU1QyUwQS0tdXNlX3JlZW50cmFudCUyMFRydWUlMjAlNUMlMEEtLWRhdGFzZXRfdGV4dF9maWVsZCUyMCUyMmNvbnRlbnQlMjIlMjAlNUMlMEEtLXVzZV9mbGFzaF9hdHRuJTIwVHJ1ZSUyMCU1QyUwQS0tdXNlX3BlZnRfbG9yYSUyMFRydWUlMjAlNUMlMEEtLWxvcmFfciUyMDglMjAlNUMlMEEtLWxvcmFfYWxwaGElMjAxNiUyMCU1QyUwQS0tbG9yYV9kcm9wb3V0JTIwMC4xJTIwJTVDJTBBLS1sb3JhX3RhcmdldF9tb2R1bGVzJTIwJTIyYWxsLWxpbmVhciUyMiUyMCU1QyUwQS0tdXNlXzRiaXRfcXVhbnRpemF0aW9uJTIwVHJ1ZSUyMCU1QyUwQS0tdXNlX25lc3RlZF9xdWFudCUyMFRydWUlMjAlNUMlMEEtLWJuYl80Yml0X2NvbXB1dGVfZHR5cGUlMjAlMjJiZmxvYXQxNiUyMiUyMCU1QyUwQS0tYm5iXzRiaXRfcXVhbnRfc3RvcmFnZV9kdHlwZSUyMCUyMmJmbG9hdDE2JTIy",highlighted:`accelerate launch <span class="hljs-attr">--config_file</span> <span class="hljs-string">&quot;configs/deepspeed_config_z3_qlora.yaml&quot;</span> train<span class="hljs-selector-class">.py</span> \\
<span class="hljs-attr">--seed</span> <span class="hljs-number">100</span> \\
<span class="hljs-attr">--model_name_or_path</span> <span class="hljs-string">&quot;meta-llama/Llama-2-70b-hf&quot;</span> \\
<span class="hljs-attr">--dataset_name</span> <span class="hljs-string">&quot;smangrul/ultrachat-10k-chatml&quot;</span> \\
<span class="hljs-attr">--chat_template_format</span> <span class="hljs-string">&quot;chatml&quot;</span> \\
<span class="hljs-attr">--add_special_tokens</span> False \\
<span class="hljs-attr">--append_concat_token</span> False \\
<span class="hljs-attr">--splits</span> <span class="hljs-string">&quot;train,test&quot;</span> \\
<span class="hljs-attr">--max_seq_len</span> <span class="hljs-number">2048</span> \\
<span class="hljs-attr">--num_train_epochs</span> <span class="hljs-number">1</span> \\
<span class="hljs-attr">--logging_steps</span> <span class="hljs-number">5</span> \\
<span class="hljs-attr">--log_level</span> <span class="hljs-string">&quot;info&quot;</span> \\
<span class="hljs-attr">--logging_strategy</span> <span class="hljs-string">&quot;steps&quot;</span> \\
<span class="hljs-attr">--eval_strategy</span> <span class="hljs-string">&quot;epoch&quot;</span> \\
<span class="hljs-attr">--save_strategy</span> <span class="hljs-string">&quot;epoch&quot;</span> \\
<span class="hljs-attr">--push_to_hub</span> \\
<span class="hljs-attr">--hub_private_repo</span> True \\
<span class="hljs-attr">--hub_strategy</span> <span class="hljs-string">&quot;every_save&quot;</span> \\
<span class="hljs-attr">--bf16</span> True \\
<span class="hljs-attr">--packing</span> True \\
<span class="hljs-attr">--learning_rate</span> <span class="hljs-number">1</span>e-<span class="hljs-number">4</span> \\
<span class="hljs-attr">--lr_scheduler_type</span> <span class="hljs-string">&quot;cosine&quot;</span> \\
<span class="hljs-attr">--weight_decay</span> <span class="hljs-number">1</span>e-<span class="hljs-number">4</span> \\
<span class="hljs-attr">--warmup_steps</span> <span class="hljs-number">0</span> \\
<span class="hljs-attr">--max_grad_norm</span> <span class="hljs-number">1.0</span> \\
<span class="hljs-attr">--output_dir</span> <span class="hljs-string">&quot;llama-sft-qlora-dsz3&quot;</span> \\
<span class="hljs-attr">--per_device_train_batch_size</span> <span class="hljs-number">2</span> \\
<span class="hljs-attr">--per_device_eval_batch_size</span> <span class="hljs-number">2</span> \\
<span class="hljs-attr">--gradient_accumulation_steps</span> <span class="hljs-number">2</span> \\
<span class="hljs-attr">--gradient_checkpointing</span> True \\
<span class="hljs-attr">--use_reentrant</span> True \\
<span class="hljs-attr">--dataset_text_field</span> <span class="hljs-string">&quot;content&quot;</span> \\
<span class="hljs-attr">--use_flash_attn</span> True \\
<span class="hljs-attr">--use_peft_lora</span> True \\
<span class="hljs-attr">--lora_r</span> <span class="hljs-number">8</span> \\
<span class="hljs-attr">--lora_alpha</span> <span class="hljs-number">16</span> \\
<span class="hljs-attr">--lora_dropout</span> <span class="hljs-number">0.1</span> \\
<span class="hljs-attr">--lora_target_modules</span> <span class="hljs-string">&quot;all-linear&quot;</span> \\
<span class="hljs-attr">--use_4bit_quantization</span> True \\
<span class="hljs-attr">--use_nested_quant</span> True \\
<span class="hljs-attr">--bnb_4bit_compute_dtype</span> <span class="hljs-string">&quot;bfloat16&quot;</span> \\
<span class="hljs-attr">--bnb_4bit_quant_storage_dtype</span> <span class="hljs-string">&quot;bfloat16&quot;</span>`,lang:"",wrap:!1}}),pl=new j({props:{code:"JTBBJTBB",highlighted:`...
bnb_config = BitsAndBytesConfig(
 load_in_4bit=args.use_4bit_quantization,
 bnb_4bit_quant_type=args.bnb_4bit_quant_type,
 bnb_4bit_compute_dtype=compute_dtype,
 bnb_4bit_use_double_quant=args.use_nested_quant,
<span class="hljs-addition">+ bnb_4bit_quant_storage=quant_storage_dtype,</span>
)
...
model = AutoModelForCausalLM.from_pretrained(
 args.model_name_or_path,
 quantization_config=bnb_config,
 trust_remote_code=True,
 attn_implementation=&quot;flash_attention_2&quot; if args.use_flash_attn else &quot;eager&quot;,
<span class="hljs-addition">+ dtype=quant_storage_dtype or torch.float32,</span>
)`,lang:"diff",wrap:!1}}),rl=new h({props:{title:"Memory usage",local:"memory-usage",headingTag:"h2"}}),cl=new h({props:{title:"Use PEFT and DeepSpeed with ZeRO3 and CPU Offloading for finetuning large models on a single GPU",local:"use-peft-and-deepspeed-with-zero3-and-cpu-offloading-for-finetuning-large-models-on-a-single-gpu",headingTag:"h1"}}),Jl=new h({props:{title:"Configuration",local:"configuration",headingTag:"h2"}}),hl=new j({props:{code:"YWNjZWxlcmF0ZSUyMGNvbmZpZyUyMC0tY29uZmlnX2ZpbGUlMjBkc196ZXJvM19jcHUueWFtbA==",highlighted:"accelerate config --config_file ds_zero3_cpu.yaml",lang:"bash",wrap:!1}}),Tl=new j({props:{code:"JTYwemVyb19zdGFnZSU2MCUzQSUyMCU1QjAlNUQlMjBEaXNhYmxlZCUyQyUyMCU1QjElNUQlMjBvcHRpbWl6ZXIlMjBzdGF0ZSUyMHBhcnRpdGlvbmluZyUyQyUyMCU1QjIlNUQlMjBvcHRpbWl6ZXIlMkJncmFkaWVudCUyMHN0YXRlJTIwcGFydGl0aW9uaW5nJTIwYW5kJTIwJTVCMyU1RCUyMG9wdGltaXplciUyQmdyYWRpZW50JTJCcGFyYW1ldGVyJTIwcGFydGl0aW9uaW5nJTBBJTYwZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTYwJTNBJTIwTnVtYmVyJTIwb2YlMjB0cmFpbmluZyUyMHN0ZXBzJTIwdG8lMjBhY2N1bXVsYXRlJTIwZ3JhZGllbnRzJTIwYmVmb3JlJTIwYXZlcmFnaW5nJTIwYW5kJTIwYXBwbHlpbmclMjB0aGVtLiUwQSU2MGdyYWRpZW50X2NsaXBwaW5nJTYwJTNBJTIwRW5hYmxlJTIwZ3JhZGllbnQlMjBjbGlwcGluZyUyMHdpdGglMjB2YWx1ZS4lMEElNjBvZmZsb2FkX29wdGltaXplcl9kZXZpY2UlNjAlM0ElMjAlNUJub25lJTVEJTIwRGlzYWJsZSUyMG9wdGltaXplciUyMG9mZmxvYWRpbmclMkMlMjAlNUJjcHUlNUQlMjBvZmZsb2FkJTIwb3B0aW1pemVyJTIwdG8lMjBDUFUlMkMlMjAlNUJudm1lJTVEJTIwb2ZmbG9hZCUyMG9wdGltaXplciUyMHRvJTIwTlZNZSUyMFNTRC4lMjBPbmx5JTIwYXBwbGljYWJsZSUyMHdpdGglMjBaZVJPJTIwJTNFJTNEJTIwU3RhZ2UtMi4lMEElNjBvZmZsb2FkX3BhcmFtX2RldmljZSU2MCUzQSUyMCU1Qm5vbmUlNUQlMjBEaXNhYmxlJTIwcGFyYW1ldGVyJTIwb2ZmbG9hZGluZyUyQyUyMCU1QmNwdSU1RCUyMG9mZmxvYWQlMjBwYXJhbWV0ZXJzJTIwdG8lMjBDUFUlMkMlMjAlNUJudm1lJTVEJTIwb2ZmbG9hZCUyMHBhcmFtZXRlcnMlMjB0byUyME5WTWUlMjBTU0QuJTIwT25seSUyMGFwcGxpY2FibGUlMjB3aXRoJTIwWmVSTyUyMFN0YWdlLTMuJTBBJTYwemVybzNfaW5pdF9mbGFnJTYwJTNBJTIwRGVjaWRlcyUyMHdoZXRoZXIlMjB0byUyMGVuYWJsZSUyMCU2MGRlZXBzcGVlZC56ZXJvLkluaXQlNjAlMjBmb3IlMjBjb25zdHJ1Y3RpbmclMjBtYXNzaXZlJTIwbW9kZWxzLiUyME9ubHklMjBhcHBsaWNhYmxlJTIwd2l0aCUyMFplUk8lMjBTdGFnZS0zLiUwQSU2MHplcm8zX3NhdmVfMTZiaXRfbW9kZWwlNjAlM0ElMjBEZWNpZGVzJTIwd2hldGhlciUyMHRvJTIwc2F2ZSUyMDE2LWJpdCUyMG1vZGVsJTIwd2VpZ2h0cyUyMHdoZW4lMjB1c2luZyUyMFplUk8lMjBTdGFnZS0zLiUwQSU2MG1peGVkX3ByZWNpc2lvbiU2MCUzQSUyMCU2MG5vJTYwJTIwZm9yJTIwRlAzMiUyMHRyYWluaW5nJTJDJTIwJTYwZnAxNiU2MCUyMGZvciUyMEZQMTYlMjBtaXhlZC1wcmVjaXNpb24lMjB0cmFpbmluZyUyMGFuZCUyMCU2MGJmMTYlNjAlMjBmb3IlMjBCRjE2JTIwbWl4ZWQtcHJlY2lzaW9uJTIwdHJhaW5pbmcuJTIw",highlighted:'`zero_stage`: [0] Disabled, [1] optimizer state partitioning, [2] optimizer+gradient state partitioning and [3] optimizer+gradient+parameter partitioning\n`gradient_accumulation_steps`: Number of training steps to accumulate gradients before averaging and applying them.\n`gradient_clipping`: Enable gradient clipping with value.\n`offload_optimizer_device`: [none] Disable optimizer offloading, [cpu] offload optimizer to CPU, [nvme] offload optimizer to NVMe SSD. Only applicable with ZeRO &gt;= Stage-2.\n`offload_param_device`: [none] Disable parameter offloading, [cpu] offload parameters to CPU, [nvme] offload parameters to NVMe SSD. Only applicable with ZeRO Stage-3.\n`zero3_init_flag`: Decides whether to <span class="hljs-built_in">enable</span> `deepspeed.zero.Init` <span class="hljs-keyword">for</span> constructing massive models. Only applicable with ZeRO Stage-3.\n`zero3_save_16bit_model`: Decides whether to save 16-bit model weights when using ZeRO Stage-3.\n`mixed_precision`: `no` <span class="hljs-keyword">for</span> FP32 training, `fp16` <span class="hljs-keyword">for</span> FP16 mixed-precision training and `bf16` <span class="hljs-keyword">for</span> BF16 mixed-precision training. ',lang:"bash",wrap:!1}}),ul=new j({props:{code:"Y29tcHV0ZV9lbnZpcm9ubWVudCUzQSUyMExPQ0FMX01BQ0hJTkUlMEFkZWVwc3BlZWRfY29uZmlnJTNBJTBBJTIwJTIwZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTNBJTIwMSUwQSUyMCUyMGdyYWRpZW50X2NsaXBwaW5nJTNBJTIwMS4wJTBBJTIwJTIwb2ZmbG9hZF9vcHRpbWl6ZXJfZGV2aWNlJTNBJTIwY3B1JTBBJTIwJTIwb2ZmbG9hZF9wYXJhbV9kZXZpY2UlM0ElMjBjcHUlMEElMjAlMjB6ZXJvM19pbml0X2ZsYWclM0ElMjB0cnVlJTBBJTIwJTIwemVybzNfc2F2ZV8xNmJpdF9tb2RlbCUzQSUyMHRydWUlMEElMjAlMjB6ZXJvX3N0YWdlJTNBJTIwMyUwQWRpc3RyaWJ1dGVkX3R5cGUlM0ElMjBERUVQU1BFRUQlMEFkb3duY2FzdF9iZjE2JTNBJTIwJ25vJyUwQWR5bmFtb19iYWNrZW5kJTNBJTIwJ05PJyUwQWZzZHBfY29uZmlnJTNBJTIwJTdCJTdEJTBBbWFjaGluZV9yYW5rJTNBJTIwMCUwQW1haW5fdHJhaW5pbmdfZnVuY3Rpb24lM0ElMjBtYWluJTBBbWVnYXRyb25fbG1fY29uZmlnJTNBJTIwJTdCJTdEJTBBbWl4ZWRfcHJlY2lzaW9uJTNBJTIwJ25vJyUwQW51bV9tYWNoaW5lcyUzQSUyMDElMEFudW1fcHJvY2Vzc2VzJTNBJTIwMSUwQXJkenZfYmFja2VuZCUzQSUyMHN0YXRpYyUwQXNhbWVfbmV0d29yayUzQSUyMHRydWUlMEF1c2VfY3B1JTNBJTIwZmFsc2U=",highlighted:`<span class="hljs-attr">compute_environment:</span> <span class="hljs-string">LOCAL_MACHINE</span>
<span class="hljs-attr">deepspeed_config:</span>
 <span class="hljs-attr">gradient_accumulation_steps:</span> <span class="hljs-number">1</span>
 <span class="hljs-attr">gradient_clipping:</span> <span class="hljs-number">1.0</span>
 <span class="hljs-attr">offload_optimizer_device:</span> <span class="hljs-string">cpu</span>
 <span class="hljs-attr">offload_param_device:</span> <span class="hljs-string">cpu</span>
 <span class="hljs-attr">zero3_init_flag:</span> <span class="hljs-literal">true</span>
 <span class="hljs-attr">zero3_save_16bit_model:</span> <span class="hljs-literal">true</span>
 <span class="hljs-attr">zero_stage:</span> <span class="hljs-number">3</span>
<span class="hljs-attr">distributed_type:</span> <span class="hljs-string">DEEPSPEED</span>
<span class="hljs-attr">downcast_bf16:</span> <span class="hljs-string">&#x27;no&#x27;</span>
<span class="hljs-attr">dynamo_backend:</span> <span class="hljs-string">&#x27;NO&#x27;</span>
<span class="hljs-attr">fsdp_config:</span> {}
<span class="hljs-attr">machine_rank:</span> <span class="hljs-number">0</span>
<span class="hljs-attr">main_training_function:</span> <span class="hljs-string">main</span>
<span class="hljs-attr">megatron_lm_config:</span> {}
<span class="hljs-attr">mixed_precision:</span> <span class="hljs-string">&#x27;no&#x27;</span>
<span class="hljs-attr">num_machines:</span> <span class="hljs-number">1</span>
<span class="hljs-attr">num_processes:</span> <span class="hljs-number">1</span>
<span class="hljs-attr">rdzv_backend:</span> <span class="hljs-string">static</span>
<span class="hljs-attr">same_network:</span> <span class="hljs-literal">true</span>
<span class="hljs-attr">use_cpu:</span> <span class="hljs-literal">false</span>`,lang:"yml",wrap:!1}}),fl=new h({props:{title:"The important parts",local:"the-important-parts",headingTag:"h2"}}),Zl=new j({props:{code:"JTIwZGVmJTIwbWFpbigpJTNBJTBBJTJCJTIwJTIwJTIwJTIwYWNjZWxlcmF0b3IlMjAlM0QlMjBBY2NlbGVyYXRvcigpJTBBJTIwJTIwJTIwJTIwJTIwbW9kZWxfbmFtZV9vcl9wYXRoJTIwJTNEJTIwJTIyZmFjZWJvb2slMkZiYXJ0LWxhcmdlJTIyJTBBJTIwJTIwJTIwJTIwJTIwZGF0YXNldF9uYW1lJTIwJTNEJTIwJTIydHdpdHRlcl9jb21wbGFpbnRzJTIyJTBBJTJCJTIwJTIwJTIwJTIwcGVmdF9jb25maWclMjAlM0QlMjBMb3JhQ29uZmlnKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRhc2tfdHlwZSUzRFRhc2tUeXBlLlNFUV8yX1NFUV9MTSUyQyUyMGluZmVyZW5jZV9tb2RlJTNERmFsc2UlMkMlMjByJTNEOCUyQyUyMGxvcmFfYWxwaGElM0QzMiUyQyUyMGxvcmFfZHJvcG91dCUzRDAuMSUwQSUyMCUyMCUyMCUyMCUyMCk=",highlighted:` def main():
<span class="hljs-addition">+ accelerator = Accelerator()</span>
 model_name_or_path = &quot;facebook/bart-large&quot;
 dataset_name = &quot;twitter_complaints&quot;
<span class="hljs-addition">+ peft_config = LoraConfig(</span>
 task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1
 )`,lang:"diff",wrap:!1}}),Bl=new j({props:{code:"JTIwJTIwbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JTZXEyU2VxTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWVfb3JfcGF0aCklMEElMkIlMjBtb2RlbCUyMCUzRCUyMGdldF9wZWZ0X21vZGVsKG1vZGVsJTJDJTIwcGVmdF9jb25maWcp",highlighted:` model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path)
<span class="hljs-addition">+ model = get_peft_model(model, peft_config)</span>`,lang:"diff",wrap:!1}}),Gl=new j({props:{code:"bW9kZWwlMkMlMjB0cmFpbl9kYXRhbG9hZGVyJTJDJTIwZXZhbF9kYXRhbG9hZGVyJTJDJTIwdGVzdF9kYXRhbG9hZGVyJTJDJTIwb3B0aW1pemVyJTJDJTIwbHJfc2NoZWR1bGVyJTIwJTNEJTIwYWNjZWxlcmF0b3IucHJlcGFyZSglMEElMjAlMjAlMjAlMjBtb2RlbCUyQyUyMHRyYWluX2RhdGFsb2FkZXIlMkMlMjBldmFsX2RhdGFsb2FkZXIlMkMlMjB0ZXN0X2RhdGFsb2FkZXIlMkMlMjBvcHRpbWl6ZXIlMkMlMjBscl9zY2hlZHVsZXIlMEEp",highlighted:`model, train_dataloader, eval_dataloader, test_dataloader, optimizer, lr_scheduler = accelerator.prepare(
 model, train_dataloader, eval_dataloader, test_dataloader, optimizer, lr_scheduler
)`,lang:"py",wrap:!1}}),Cl=new j({props:{code:"aXNfZHNfemVyb18zJTIwJTNEJTIwRmFsc2UlMEFpZiUyMGdldGF0dHIoYWNjZWxlcmF0b3Iuc3RhdGUlMkMlMjAlMjJkZWVwc3BlZWRfcGx1Z2luJTIyJTJDJTIwTm9uZSklM0ElMEElMjAlMjAlMjAlMjBpc19kc196ZXJvXzMlMjAlM0QlMjBhY2NlbGVyYXRvci5zdGF0ZS5kZWVwc3BlZWRfcGx1Z2luLnplcm9fc3RhZ2UlMjAlM0QlM0QlMjAz",highlighted:`is_ds_zero_3 = <span class="hljs-literal">False</span>
<span class="hljs-keyword">if</span> <span class="hljs-built_in">getattr</span>(accelerator.state, <span class="hljs-string">&quot;deepspeed_plugin&quot;</span>, <span class="hljs-literal">None</span>):
 is_ds_zero_3 = accelerator.state.deepspeed_plugin.zero_stage == <span class="hljs-number">3</span>`,lang:"py",wrap:!1}}),Nl=new j({props:{code:"JTIwJTIwZm9yJTIwZXBvY2glMjBpbiUyMHJhbmdlKG51bV9lcG9jaHMpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwd2l0aCUyMFRvcmNoVHJhY2VtYWxsb2MoKSUyMGFzJTIwdHJhY2VtYWxsb2MlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtb2RlbC50cmFpbigpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdG90YWxfbG9zcyUyMCUzRCUyMDAlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3IlMjBzdGVwJTJDJTIwYmF0Y2glMjBpbiUyMGVudW1lcmF0ZSh0cWRtKHRyYWluX2RhdGFsb2FkZXIpKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG91dHB1dHMlMjAlM0QlMjBtb2RlbCgqKmJhdGNoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxvc3MlMjAlM0QlMjBvdXRwdXRzLmxvc3MlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0b3RhbF9sb3NzJTIwJTJCJTNEJTIwbG9zcy5kZXRhY2goKS5mbG9hdCgpJTBBJTJCJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYWNjZWxlcmF0b3IuYmFja3dhcmQobG9zcyklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvcHRpbWl6ZXIuc3RlcCgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbHJfc2NoZWR1bGVyLnN0ZXAoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG9wdGltaXplci56ZXJvX2dyYWQoKQ==",highlighted:` for epoch in range(num_epochs):
 with TorchTracemalloc() as tracemalloc:
 model.train()
 total_loss = 0
 for step, batch in enumerate(tqdm(train_dataloader)):
 outputs = model(**batch)
 loss = outputs.loss
 total_loss += loss.detach().float()
<span class="hljs-addition">+ accelerator.backward(loss)</span>
 optimizer.step()
 lr_scheduler.step()
 optimizer.zero_grad()`,lang:"diff",wrap:!1}}),vl=new h({props:{title:"Train",local:"train",headingTag:"h2"}}),Vl=new j({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMC0tY29uZmlnX2ZpbGUlMjBkc196ZXJvM19jcHUueWFtbCUyMGV4YW1wbGVzJTJGcGVmdF9sb3JhX3NlcTJzZXFfYWNjZWxlcmF0ZV9kc196ZXJvM19vZmZsb2FkLnB5",highlighted:"accelerate launch --config_file ds_zero3_cpu.yaml examples/peft_lora_seq2seq_accelerate_ds_zero3_offload.py",lang:"bash",wrap:!1}}),El=new j({props:{code:"R1BVJTIwTWVtb3J5JTIwYmVmb3JlJTIwZW50ZXJpbmclMjB0aGUlMjB0cmFpbiUyMCUzQSUyMDE5MTYlMEFHUFUlMjBNZW1vcnklMjBjb25zdW1lZCUyMGF0JTIwdGhlJTIwZW5kJTIwb2YlMjB0aGUlMjB0cmFpbiUyMChlbmQtYmVnaW4pJTNBJTIwNjYlMEFHUFUlMjBQZWFrJTIwTWVtb3J5JTIwY29uc3VtZWQlMjBkdXJpbmclMjB0aGUlMjB0cmFpbiUyMChtYXgtYmVnaW4pJTNBJTIwNzQ4OCUwQUdQVSUyMFRvdGFsJTIwUGVhayUyME1lbW9yeSUyMGNvbnN1bWVkJTIwZHVyaW5nJTIwdGhlJTIwdHJhaW4lMjAobWF4KSUzQSUyMDk0MDQlMEFDUFUlMjBNZW1vcnklMjBiZWZvcmUlMjBlbnRlcmluZyUyMHRoZSUyMHRyYWluJTIwJTNBJTIwMTk0MTElMEFDUFUlMjBNZW1vcnklMjBjb25zdW1lZCUyMGF0JTIwdGhlJTIwZW5kJTIwb2YlMjB0aGUlMjB0cmFpbiUyMChlbmQtYmVnaW4pJTNBJTIwMCUwQUNQVSUyMFBlYWslMjBNZW1vcnklMjBjb25zdW1lZCUyMGR1cmluZyUyMHRoZSUyMHRyYWluJTIwKG1heC1iZWdpbiklM0ElMjAwJTBBQ1BVJTIwVG90YWwlMjBQZWFrJTIwTWVtb3J5JTIwY29uc3VtZWQlMjBkdXJpbmclMjB0aGUlMjB0cmFpbiUyMChtYXgpJTNBJTIwMTk0MTElMEFlcG9jaCUzRDQlM0ElMjB0cmFpbl9wcGwlM0R0ZW5zb3IoMS4wNzA1JTJDJTIwZGV2aWNlJTNEJ2N1ZGElM0EwJyklMjB0cmFpbl9lcG9jaF9sb3NzJTNEdGVuc29yKDAuMDY4MSUyQyUyMGRldmljZSUzRCdjdWRhJTNBMCcpJTBBMTAwJTI1JTdDJUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JTdDJTIwNyUyRjclMjAlNUIwMCUzQTI3JTNDMDAlM0EwMCUyQyUyMCUyMDMuOTJzJTJGaXQlNUQlMEFHUFUlMjBNZW1vcnklMjBiZWZvcmUlMjBlbnRlcmluZyUyMHRoZSUyMGV2YWwlMjAlM0ElMjAxOTgyJTBBR1BVJTIwTWVtb3J5JTIwY29uc3VtZWQlMjBhdCUyMHRoZSUyMGVuZCUyMG9mJTIwdGhlJTIwZXZhbCUyMChlbmQtYmVnaW4pJTNBJTIwLTY2JTBBR1BVJTIwUGVhayUyME1lbW9yeSUyMGNvbnN1bWVkJTIwZHVyaW5nJTIwdGhlJTIwZXZhbCUyMChtYXgtYmVnaW4pJTNBJTIwNjcyJTBBR1BVJTIwVG90YWwlMjBQZWFrJTIwTWVtb3J5JTIwY29uc3VtZWQlMjBkdXJpbmclMjB0aGUlMjBldmFsJTIwKG1heCklM0ElMjAyNjU0JTBBQ1BVJTIwTWVtb3J5JTIwYmVmb3JlJTIwZW50ZXJpbmclMjB0aGUlMjBldmFsJTIwJTNBJTIwMTk0MTElMEFDUFUlMjBNZW1vcnklMjBjb25zdW1lZCUyMGF0JTIwdGhlJTIwZW5kJTIwb2YlMjB0aGUlMjBldmFsJTIwKGVuZC1iZWdpbiklM0ElMjAwJTBBQ1BVJTIwUGVhayUyME1lbW9yeSUyMGNvbnN1bWVkJTIwZHVyaW5nJTIwdGhlJTIwZXZhbCUyMChtYXgtYmVnaW4pJTNBJTIwMCUwQUNQVSUyMFRvdGFsJTIwUGVhayUyME1lbW9yeSUyMGNvbnN1bWVkJTIwZHVyaW5nJTIwdGhlJTIwZXZhbCUyMChtYXgpJTNBJTIwMTk0MTElMEFhY2N1cmFjeSUzRDEwMC4wJTBBZXZhbF9wcmVkcyU1QiUzQTEwJTVEJTNEJTVCJ25vJTIwY29tcGxhaW50JyUyQyUyMCdubyUyMGNvbXBsYWludCclMkMlMjAnY29tcGxhaW50JyUyQyUyMCdjb21wbGFpbnQnJTJDJTIwJ25vJTIwY29tcGxhaW50JyUyQyUyMCdubyUyMGNvbXBsYWludCclMkMlMjAnbm8lMjBjb21wbGFpbnQnJTJDJTIwJ2NvbXBsYWludCclMkMlMjAnY29tcGxhaW50JyUyQyUyMCdubyUyMGNvbXBsYWludCclNUQlMEFkYXRhc2V0JTVCJ3RyYWluJyU1RCU1QmxhYmVsX2NvbHVtbiU1RCU1QiUzQTEwJTVEJTNEJTVCJ25vJTIwY29tcGxhaW50JyUyQyUyMCdubyUyMGNvbXBsYWludCclMkMlMjAnY29tcGxhaW50JyUyQyUyMCdjb21wbGFpbnQnJTJDJTIwJ25vJTIwY29tcGxhaW50JyUyQyUyMCdubyUyMGNvbXBsYWludCclMkMlMjAnbm8lMjBjb21wbGFpbnQnJTJDJTIwJ2NvbXBsYWludCclMkMlMjAnY29tcGxhaW50JyUyQyUyMCdubyUyMGNvbXBsYWludCclNUQ=",highlighted:`GPU Memory before entering the train : 1916
GPU Memory consumed at the end of the train (end-begin): 66
GPU Peak Memory consumed during the train (max-begin): 7488
GPU Total Peak Memory consumed during the train (max): 9404
CPU Memory before entering the train : 19411
CPU Memory consumed at the end of the train (end-begin): 0
CPU Peak Memory consumed during the train (max-begin): 0
CPU Total Peak Memory consumed during the train (max): 19411
epoch=4: train_ppl=tensor(1.0705, device=<span class="hljs-string">&#x27;cuda:0&#x27;</span>) train_epoch_loss=tensor(0.0681, device=<span class="hljs-string">&#x27;cuda:0&#x27;</span>)
100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:27&lt;00:00, 3.92s/it]
GPU Memory before entering the <span class="hljs-built_in">eval</span> : 1982
GPU Memory consumed at the end of the <span class="hljs-built_in">eval</span> (end-begin): -66
GPU Peak Memory consumed during the <span class="hljs-built_in">eval</span> (max-begin): 672
GPU Total Peak Memory consumed during the <span class="hljs-built_in">eval</span> (max): 2654
CPU Memory before entering the <span class="hljs-built_in">eval</span> : 19411
CPU Memory consumed at the end of the <span class="hljs-built_in">eval</span> (end-begin): 0
CPU Peak Memory consumed during the <span class="hljs-built_in">eval</span> (max-begin): 0
CPU Total Peak Memory consumed during the <span class="hljs-built_in">eval</span> (max): 19411
accuracy=100.0
eval_preds[:10]=[<span class="hljs-string">&#x27;no complaint&#x27;</span>, <span class="hljs-string">&#x27;no complaint&#x27;</span>, <span class="hljs-string">&#x27;complaint&#x27;</span>, <span class="hljs-string">&#x27;complaint&#x27;</span>, <span class="hljs-string">&#x27;no complaint&#x27;</span>, <span class="hljs-string">&#x27;no complaint&#x27;</span>, <span class="hljs-string">&#x27;no complaint&#x27;</span>, <span class="hljs-string">&#x27;complaint&#x27;</span>, <span class="hljs-string">&#x27;complaint&#x27;</span>, <span class="hljs-string">&#x27;no complaint&#x27;</span>]
dataset[<span class="hljs-string">&#x27;train&#x27;</span>][label_column][:10]=[<span class="hljs-string">&#x27;no complaint&#x27;</span>, <span class="hljs-string">&#x27;no complaint&#x27;</span>, <span class="hljs-string">&#x27;complaint&#x27;</span>, <span class="hljs-string">&#x27;complaint&#x27;</span>, <span class="hljs-string">&#x27;no complaint&#x27;</span>, <span class="hljs-string">&#x27;no complaint&#x27;</span>, <span class="hljs-string">&#x27;no complaint&#x27;</span>, <span class="hljs-string">&#x27;complaint&#x27;</span>, <span class="hljs-string">&#x27;complaint&#x27;</span>, <span class="hljs-string">&#x27;no complaint&#x27;</span>]`,lang:"bash",wrap:!1}}),xl=new h({props:{title:"Caveats",local:"caveats",headingTag:"h1"}}),$l=new j({props:{code:"aW1wb3J0JTIwZGVlcHNwZWVkJTBBJTBBaXNfZHNfemVyb18zJTIwJTNEJTIwLi4uJTIwJTIzJTIwY2hlY2slMjBpZiUyMFplcm8tMyUwQSUwQXdpdGglMjBkZWVwc3BlZWQuemVyby5HYXRoZXJlZFBhcmFtZXRlcnMobGlzdChtb2RlbC5wYXJhbWV0ZXJzKCkpJTJDJTIwZW5hYmxlZCUzRCUyMGlzX2RzX3plcm9fMyklM0ElMEElMjAlMjAlMjAlMjBtb2RlbC5tZXJnZV9hZGFwdGVyKCklMEElMjAlMjAlMjAlMjAlMjMlMjBkbyUyMHdoYXRldmVyJTIwaXMlMjBuZWVkZWQlMkMlMjB0aGVuJTIwdW5tZXJnZSUyMGluJTIwdGhlJTIwc2FtZSUyMGNvbnRleHQlMjBpZiUyMHVubWVyZ2luZyUyMGlzJTIwcmVxdWlyZWQlMEElMjAlMjAlMjAlMjAuLi4lMEElMjAlMjAlMjAlMjBtb2RlbC51bm1lcmdlX2FkYXB0ZXIoKQ==",highlighted:`<span class="hljs-keyword">import</span> deepspeed
is_ds_zero_3 = ... <span class="hljs-comment"># check if Zero-3</span>
<span class="hljs-keyword">with</span> deepspeed.zero.GatheredParameters(<span class="hljs-built_in">list</span>(model.parameters()), enabled= is_ds_zero_3):
 model.merge_adapter()
 <span class="hljs-comment"># do whatever is needed, then unmerge in the same context if unmerging is required</span>
 ...
 model.unmerge_adapter()`,lang:"python",wrap:!1}}),Ql=new Aa({props:{source:"https://github.com/huggingface/peft/blob/main/docs/source/accelerate/deepspeed.md"}}),{c(){y=i("meta"),ql=s(),Hl=i("p"),Pl=s(),r(b.$$.fragment),Dl=s(),r(w.$$.fragment),Ol=s(),g=i("p"),g.innerHTML=Wt,Kl=s(),Z=i("p"),Z.textContent=Gt,le=s(),r(_.$$.fragment),ee=s(),A=i("p"),A.innerHTML=It,te=s(),B=i("table"),B.innerHTML=Ct,ae=s(),W=i("p"),W.innerHTML=kt,se=s(),G=i("p"),G.innerHTML=Nt,ne=s(),r(I.$$.fragment),ie=s(),C=i("p"),C.innerHTML=Ft,pe=s(),r(k.$$.fragment),Me=s(),N=i("p"),N.innerHTML=vt,re=s(),F=i("p"),F.textContent=Xt,oe=s(),r(v.$$.fragment),ce=s(),X=i("p"),X.textContent=Vt,de=s(),r(V.$$.fragment),Je=s(),Y=i("p"),Y.innerHTML=Yt,me=s(),r(E.$$.fragment),je=s(),r(x.$$.fragment),he=s(),R=i("p"),R.innerHTML=Et,ye=s(),r($.$$.fragment),Te=s(),Q=i("p"),Q.textContent=xt,Ue=s(),r(S.$$.fragment),ue=s(),z=i("p"),z.textContent=Rt,fe=s(),H=i("p"),H.innerHTML=$t,be=s(),r(L.$$.fragment),we=s(),r(q.$$.fragment),ge=s(),P=i("p"),P.textContent=Qt,Ze=s(),U=i("div"),U.innerHTML=St,_e=s(),D=i("small"),D.textContent=zt,Ae=s(),r(O.$$.fragment),Be=s(),K=i("p"),K.innerHTML=Ht,We=s(),r(ll.$$.fragment),Ge=s(),el=i("p"),el.innerHTML=Lt,Ie=s(),r(tl.$$.fragment),Ce=s(),al=i("p"),al.innerHTML=qt,ke=s(),r(sl.$$.fragment),Ne=s(),nl=i("p"),nl.innerHTML=Pt,Fe=s(),il=i("p"),il.textContent=Dt,ve=s(),r(pl.$$.fragment),Xe=s(),Ml=i("p"),Ml.innerHTML=Ot,Ve=s(),r(rl.$$.fragment),Ye=s(),ol=i("p"),ol.innerHTML=Kt,Ee=s(),r(cl.$$.fragment),xe=s(),dl=i("p"),dl.innerHTML=la,Re=s(),u=i("blockquote"),u.innerHTML=ea,$e=s(),r(Jl.$$.fragment),Qe=s(),ml=i("p"),ml.innerHTML=ta,Se=s(),jl=i("p"),jl.textContent=aa,ze=s(),r(hl.$$.fragment),He=s(),yl=i("p"),yl.textContent=sa,Le=s(),r(Tl.$$.fragment),qe=s(),Ul=i("p"),Ul.innerHTML=na,Pe=s(),r(ul.$$.fragment),De=s(),r(fl.$$.fragment),Oe=s(),bl=i("p"),bl.textContent=ia,Ke=s(),wl=i("p"),wl.innerHTML=pa,lt=s(),f=i("blockquote"),f.innerHTML=Ma,et=s(),gl=i("p"),gl.innerHTML=ra,tt=s(),r(Zl.$$.fragment),at=s(),_l=i("p"),_l.innerHTML=oa,st=s(),Al=i("p"),Al.innerHTML=ca,nt=s(),r(Bl.$$.fragment),it=s(),Wl=i("p"),Wl.innerHTML=da,pt=s(),r(Gl.$$.fragment),Mt=s(),Il=i("p"),Il.innerHTML=Ja,rt=s(),r(Cl.$$.fragment),ot=s(),kl=i("p"),kl.innerHTML=ma,ct=s(),r(Nl.$$.fragment),dt=s(),Fl=i("p"),Fl.textContent=ja,Jt=s(),r(vl.$$.fragment),mt=s(),Xl=i("p"),Xl.innerHTML=ha,jt=s(),r(Vl.$$.fragment),ht=s(),Yl=i("p"),Yl.textContent=ya,yt=s(),r(El.$$.fragment),Tt=s(),r(xl.$$.fragment),Ut=s(),Rl=i("ol"),Rl.innerHTML=Ta,ut=s(),T=i("blockquote"),Sl=i("p"),Sl.textContent=Ua,Zt=s(),r($l.$$.fragment),ft=s(),r(Ql.$$.fragment),bt=s(),Ll=i("p"),this.h()},l(l){const e=Za("svelte-u9bgzb",document.head);y=p(e,"META",{name:!0,content:!0}),e.forEach(t),ql=n(l),Hl=p(l,"P",{}),_t(Hl).forEach(t),Pl=n(l),o(b.$$.fragment,l),Dl=n(l),o(w.$$.fragment,l),Ol=n(l),g=p(l,"P",{"data-svelte-h":!0}),M(g)!=="svelte-kjgvfp"&&(g.innerHTML=Wt),Kl=n(l),Z=p(l,"P",{"data-svelte-h":!0}),M(Z)!=="svelte-1o8k4c8"&&(Z.textContent=Gt),le=n(l),o(_.$$.fragment,l),ee=n(l),A=p(l,"P",{"data-svelte-h":!0}),M(A)!=="svelte-1pmd8qt"&&(A.innerHTML=It),te=n(l),B=p(l,"TABLE",{"data-svelte-h":!0}),M(B)!=="svelte-uxge3s"&&(B.innerHTML=Ct),ae=n(l),W=p(l,"P",{"data-svelte-h":!0}),M(W)!=="svelte-b8ifuk"&&(W.innerHTML=kt),se=n(l),G=p(l,"P",{"data-svelte-h":!0}),M(G)!=="svelte-1jhakf7"&&(G.innerHTML=Nt),ne=n(l),o(I.$$.fragment,l),ie=n(l),C=p(l,"P",{"data-svelte-h":!0}),M(C)!=="svelte-1ubz7wn"&&(C.innerHTML=Ft),pe=n(l),o(k.$$.fragment,l),Me=n(l),N=p(l,"P",{"data-svelte-h":!0}),M(N)!=="svelte-ha3yfb"&&(N.innerHTML=vt),re=n(l),F=p(l,"P",{"data-svelte-h":!0}),M(F)!=="svelte-1oar51z"&&(F.textContent=Xt),oe=n(l),o(v.$$.fragment,l),ce=n(l),X=p(l,"P",{"data-svelte-h":!0}),M(X)!=="svelte-u6lpbh"&&(X.textContent=Vt),de=n(l),o(V.$$.fragment,l),Je=n(l),Y=p(l,"P",{"data-svelte-h":!0}),M(Y)!=="svelte-1dwo8p1"&&(Y.innerHTML=Yt),me=n(l),o(E.$$.fragment,l),je=n(l),o(x.$$.fragment,l),he=n(l),R=p(l,"P",{"data-svelte-h":!0}),M(R)!=="svelte-if2v7q"&&(R.innerHTML=Et),ye=n(l),o($.$$.fragment,l),Te=n(l),Q=p(l,"P",{"data-svelte-h":!0}),M(Q)!=="svelte-1juuctw"&&(Q.textContent=xt),Ue=n(l),o(S.$$.fragment,l),ue=n(l),z=p(l,"P",{"data-svelte-h":!0}),M(z)!=="svelte-y7jq4k"&&(z.textContent=Rt),fe=n(l),H=p(l,"P",{"data-svelte-h":!0}),M(H)!=="svelte-10ugdt7"&&(H.innerHTML=$t),be=n(l),o(L.$$.fragment,l),we=n(l),o(q.$$.fragment,l),ge=n(l),P=p(l,"P",{"data-svelte-h":!0}),M(P)!=="svelte-1w0ujmv"&&(P.textContent=Qt),Ze=n(l),U=p(l,"DIV",{class:!0,"data-svelte-h":!0}),M(U)!=="svelte-1xcsndp"&&(U.innerHTML=St),_e=n(l),D=p(l,"SMALL",{"data-svelte-h":!0}),M(D)!=="svelte-8v78s3"&&(D.textContent=zt),Ae=n(l),o(O.$$.fragment,l),Be=n(l),K=p(l,"P",{"data-svelte-h":!0}),M(K)!=="svelte-d9sf55"&&(K.innerHTML=Ht),We=n(l),o(ll.$$.fragment,l),Ge=n(l),el=p(l,"P",{"data-svelte-h":!0}),M(el)!=="svelte-1lia0iy"&&(el.innerHTML=Lt),Ie=n(l),o(tl.$$.fragment,l),Ce=n(l),al=p(l,"P",{"data-svelte-h":!0}),M(al)!=="svelte-1wnm4d7"&&(al.innerHTML=qt),ke=n(l),o(sl.$$.fragment,l),Ne=n(l),nl=p(l,"P",{"data-svelte-h":!0}),M(nl)!=="svelte-1mee3xa"&&(nl.innerHTML=Pt),Fe=n(l),il=p(l,"P",{"data-svelte-h":!0}),M(il)!=="svelte-19s9lyl"&&(il.textContent=Dt),ve=n(l),o(pl.$$.fragment,l),Xe=n(l),Ml=p(l,"P",{"data-svelte-h":!0}),M(Ml)!=="svelte-46tfpq"&&(Ml.innerHTML=Ot),Ve=n(l),o(rl.$$.fragment,l),Ye=n(l),ol=p(l,"P",{"data-svelte-h":!0}),M(ol)!=="svelte-1bfnr2r"&&(ol.innerHTML=Kt),Ee=n(l),o(cl.$$.fragment,l),xe=n(l),dl=p(l,"P",{"data-svelte-h":!0}),M(dl)!=="svelte-jhfwqh"&&(dl.innerHTML=la),Re=n(l),u=p(l,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),M(u)!=="svelte-1wnq58o"&&(u.innerHTML=ea),$e=n(l),o(Jl.$$.fragment,l),Qe=n(l),ml=p(l,"P",{"data-svelte-h":!0}),M(ml)!=="svelte-ha3yfb"&&(ml.innerHTML=ta),Se=n(l),jl=p(l,"P",{"data-svelte-h":!0}),M(jl)!=="svelte-1oar51z"&&(jl.textContent=aa),ze=n(l),o(hl.$$.fragment,l),He=n(l),yl=p(l,"P",{"data-svelte-h":!0}),M(yl)!=="svelte-16uruk2"&&(yl.textContent=sa),Le=n(l),o(Tl.$$.fragment,l),qe=n(l),Ul=p(l,"P",{"data-svelte-h":!0}),M(Ul)!=="svelte-iawfkk"&&(Ul.innerHTML=na),Pe=n(l),o(ul.$$.fragment,l),De=n(l),o(fl.$$.fragment,l),Oe=n(l),bl=p(l,"P",{"data-svelte-h":!0}),M(bl)!=="svelte-y7jq4k"&&(bl.textContent=ia),Ke=n(l),wl=p(l,"P",{"data-svelte-h":!0}),M(wl)!=="svelte-14y4rgi"&&(wl.innerHTML=pa),lt=n(l),f=p(l,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),M(f)!=="svelte-198tcjk"&&(f.innerHTML=Ma),et=n(l),gl=p(l,"P",{"data-svelte-h":!0}),M(gl)!=="svelte-12yx279"&&(gl.innerHTML=ra),tt=n(l),o(Zl.$$.fragment,l),at=n(l),_l=p(l,"P",{"data-svelte-h":!0}),M(_l)!=="svelte-1k44gsr"&&(_l.innerHTML=oa),st=n(l),Al=p(l,"P",{"data-svelte-h":!0}),M(Al)!=="svelte-bqw05y"&&(Al.innerHTML=ca),nt=n(l),o(Bl.$$.fragment,l),it=n(l),Wl=p(l,"P",{"data-svelte-h":!0}),M(Wl)!=="svelte-yn58ot"&&(Wl.innerHTML=da),pt=n(l),o(Gl.$$.fragment,l),Mt=n(l),Il=p(l,"P",{"data-svelte-h":!0}),M(Il)!=="svelte-1lrc4ui"&&(Il.innerHTML=Ja),rt=n(l),o(Cl.$$.fragment,l),ot=n(l),kl=p(l,"P",{"data-svelte-h":!0}),M(kl)!=="svelte-15nca2e"&&(kl.innerHTML=ma),ct=n(l),o(Nl.$$.fragment,l),dt=n(l),Fl=p(l,"P",{"data-svelte-h":!0}),M(Fl)!=="svelte-1tcey7i"&&(Fl.textContent=ja),Jt=n(l),o(vl.$$.fragment,l),mt=n(l),Xl=p(l,"P",{"data-svelte-h":!0}),M(Xl)!=="svelte-7jdy6n"&&(Xl.innerHTML=ha),jt=n(l),o(Vl.$$.fragment,l),ht=n(l),Yl=p(l,"P",{"data-svelte-h":!0}),M(Yl)!=="svelte-1amszxm"&&(Yl.textContent=ya),yt=n(l),o(El.$$.fragment,l),Tt=n(l),o(xl.$$.fragment,l),Ut=n(l),Rl=p(l,"OL",{"data-svelte-h":!0}),M(Rl)!=="svelte-19w5tf1"&&(Rl.innerHTML=Ta),ut=n(l),T=p(l,"BLOCKQUOTE",{class:!0});var gt=_t(T);Sl=p(gt,"P",{"data-svelte-h":!0}),M(Sl)!=="svelte-wmqv6d"&&(Sl.textContent=Ua),Zt=n(gt),o($l.$$.fragment,gt),gt.forEach(t),ft=n(l),o(Ql.$$.fragment,l),bt=n(l),Ll=p(l,"P",{}),_t(Ll).forEach(t),this.h()},h(){zl(y,"name","hf:doc:metadata"),zl(y,"content",Wa),zl(U,"class","flex justify-center"),zl(u,"class","tip"),zl(f,"class","tip"),zl(T,"class","tip")},m(l,e){At(document.head,y),a(l,ql,e),a(l,Hl,e),a(l,Pl,e),c(b,l,e),a(l,Dl,e),c(w,l,e),a(l,Ol,e),a(l,g,e),a(l,Kl,e),a(l,Z,e),a(l,le,e),c(_,l,e),a(l,ee,e),a(l,A,e),a(l,te,e),a(l,B,e),a(l,ae,e),a(l,W,e),a(l,se,e),a(l,G,e),a(l,ne,e),c(I,l,e),a(l,ie,e),a(l,C,e),a(l,pe,e),c(k,l,e),a(l,Me,e),a(l,N,e),a(l,re,e),a(l,F,e),a(l,oe,e),c(v,l,e),a(l,ce,e),a(l,X,e),a(l,de,e),c(V,l,e),a(l,Je,e),a(l,Y,e),a(l,me,e),c(E,l,e),a(l,je,e),c(x,l,e),a(l,he,e),a(l,R,e),a(l,ye,e),c($,l,e),a(l,Te,e),a(l,Q,e),a(l,Ue,e),c(S,l,e),a(l,ue,e),a(l,z,e),a(l,fe,e),a(l,H,e),a(l,be,e),c(L,l,e),a(l,we,e),c(q,l,e),a(l,ge,e),a(l,P,e),a(l,Ze,e),a(l,U,e),a(l,_e,e),a(l,D,e),a(l,Ae,e),c(O,l,e),a(l,Be,e),a(l,K,e),a(l,We,e),c(ll,l,e),a(l,Ge,e),a(l,el,e),a(l,Ie,e),c(tl,l,e),a(l,Ce,e),a(l,al,e),a(l,ke,e),c(sl,l,e),a(l,Ne,e),a(l,nl,e),a(l,Fe,e),a(l,il,e),a(l,ve,e),c(pl,l,e),a(l,Xe,e),a(l,Ml,e),a(l,Ve,e),c(rl,l,e),a(l,Ye,e),a(l,ol,e),a(l,Ee,e),c(cl,l,e),a(l,xe,e),a(l,dl,e),a(l,Re,e),a(l,u,e),a(l,$e,e),c(Jl,l,e),a(l,Qe,e),a(l,ml,e),a(l,Se,e),a(l,jl,e),a(l,ze,e),c(hl,l,e),a(l,He,e),a(l,yl,e),a(l,Le,e),c(Tl,l,e),a(l,qe,e),a(l,Ul,e),a(l,Pe,e),c(ul,l,e),a(l,De,e),c(fl,l,e),a(l,Oe,e),a(l,bl,e),a(l,Ke,e),a(l,wl,e),a(l,lt,e),a(l,f,e),a(l,et,e),a(l,gl,e),a(l,tt,e),c(Zl,l,e),a(l,at,e),a(l,_l,e),a(l,st,e),a(l,Al,e),a(l,nt,e),c(Bl,l,e),a(l,it,e),a(l,Wl,e),a(l,pt,e),c(Gl,l,e),a(l,Mt,e),a(l,Il,e),a(l,rt,e),c(Cl,l,e),a(l,ot,e),a(l,kl,e),a(l,ct,e),c(Nl,l,e),a(l,dt,e),a(l,Fl,e),a(l,Jt,e),c(vl,l,e),a(l,mt,e),a(l,Xl,e),a(l,jt,e),c(Vl,l,e),a(l,ht,e),a(l,Yl,e),a(l,yt,e),c(El,l,e),a(l,Tt,e),c(xl,l,e),a(l,Ut,e),a(l,Rl,e),a(l,ut,e),a(l,T,e),At(T,Sl),At(T,Zt),c($l,T,null),a(l,ft,e),c(Ql,l,e),a(l,bt,e),a(l,Ll,e),wt=!0},p:fa,i(l){wt||(d(b.$$.fragment,l),d(w.$$.fragment,l),d(_.$$.fragment,l),d(I.$$.fragment,l),d(k.$$.fragment,l),d(v.$$.fragment,l),d(V.$$.fragment,l),d(E.$$.fragment,l),d(x.$$.fragment,l),d($.$$.fragment,l),d(S.$$.fragment,l),d(L.$$.fragment,l),d(q.$$.fragment,l),d(O.$$.fragment,l),d(ll.$$.fragment,l),d(tl.$$.fragment,l),d(sl.$$.fragment,l),d(pl.$$.fragment,l),d(rl.$$.fragment,l),d(cl.$$.fragment,l),d(Jl.$$.fragment,l),d(hl.$$.fragment,l),d(Tl.$$.fragment,l),d(ul.$$.fragment,l),d(fl.$$.fragment,l),d(Zl.$$.fragment,l),d(Bl.$$.fragment,l),d(Gl.$$.fragment,l),d(Cl.$$.fragment,l),d(Nl.$$.fragment,l),d(vl.$$.fragment,l),d(Vl.$$.fragment,l),d(El.$$.fragment,l),d(xl.$$.fragment,l),d($l.$$.fragment,l),d(Ql.$$.fragment,l),wt=!0)},o(l){J(b.$$.fragment,l),J(w.$$.fragment,l),J(_.$$.fragment,l),J(I.$$.fragment,l),J(k.$$.fragment,l),J(v.$$.fragment,l),J(V.$$.fragment,l),J(E.$$.fragment,l),J(x.$$.fragment,l),J($.$$.fragment,l),J(S.$$.fragment,l),J(L.$$.fragment,l),J(q.$$.fragment,l),J(O.$$.fragment,l),J(ll.$$.fragment,l),J(tl.$$.fragment,l),J(sl.$$.fragment,l),J(pl.$$.fragment,l),J(rl.$$.fragment,l),J(cl.$$.fragment,l),J(Jl.$$.fragment,l),J(hl.$$.fragment,l),J(Tl.$$.fragment,l),J(ul.$$.fragment,l),J(fl.$$.fragment,l),J(Zl.$$.fragment,l),J(Bl.$$.fragment,l),J(Gl.$$.fragment,l),J(Cl.$$.fragment,l),J(Nl.$$.fragment,l),J(vl.$$.fragment,l),J(Vl.$$.fragment,l),J(El.$$.fragment,l),J(xl.$$.fragment,l),J($l.$$.fragment,l),J(Ql.$$.fragment,l),wt=!1},d(l){l&&(t(ql),t(Hl),t(Pl),t(Dl),t(Ol),t(g),t(Kl),t(Z),t(le),t(ee),t(A),t(te),t(B),t(ae),t(W),t(se),t(G),t(ne),t(ie),t(C),t(pe),t(Me),t(N),t(re),t(F),t(oe),t(ce),t(X),t(de),t(Je),t(Y),t(me),t(je),t(he),t(R),t(ye),t(Te),t(Q),t(Ue),t(ue),t(z),t(fe),t(H),t(be),t(we),t(ge),t(P),t(Ze),t(U),t(_e),t(D),t(Ae),t(Be),t(K),t(We),t(Ge),t(el),t(Ie),t(Ce),t(al),t(ke),t(Ne),t(nl),t(Fe),t(il),t(ve),t(Xe),t(Ml),t(Ve),t(Ye),t(ol),t(Ee),t(xe),t(dl),t(Re),t(u),t($e),t(Qe),t(ml),t(Se),t(jl),t(ze),t(He),t(yl),t(Le),t(qe),t(Ul),t(Pe),t(De),t(Oe),t(bl),t(Ke),t(wl),t(lt),t(f),t(et),t(gl),t(tt),t(at),t(_l),t(st),t(Al),t(nt),t(it),t(Wl),t(pt),t(Mt),t(Il),t(rt),t(ot),t(kl),t(ct),t(dt),t(Fl),t(Jt),t(mt),t(Xl),t(jt),t(ht),t(Yl),t(yt),t(Tt),t(Ut),t(Rl),t(ut),t(T),t(ft),t(bt),t(Ll)),t(y),m(b,l),m(w,l),m(_,l),m(I,l),m(k,l),m(v,l),m(V,l),m(E,l),m(x,l),m($,l),m(S,l),m(L,l),m(q,l),m(O,l),m(ll,l),m(tl,l),m(sl,l),m(pl,l),m(rl,l),m(cl,l),m(Jl,l),m(hl,l),m(Tl,l),m(ul,l),m(fl,l),m(Zl,l),m(Bl,l),m(Gl,l),m(Cl,l),m(Nl,l),m(vl,l),m(Vl,l),m(El,l),m(xl,l),m($l),m(Ql,l)}}}const Wa='{"title":"DeepSpeed","local":"deepspeed","sections":[{"title":"Compatibility with bitsandbytes quantization + LoRA","local":"compatibility-with-bitsandbytes-quantization--lora","sections":[],"depth":2}],"depth":1}';function Ga(Bt){return ba(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Fa extends wa{constructor(y){super(),ga(this,y,Ga,Ba,ua,{})}}export{Fa as component};

Xet Storage Details

Size:
70.3 kB
·
Xet hash:
ea01591c8ba80f4ff9ce3a4a73f97cf0111f9ca9d92bbdecf29f516857adc7da

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.