Buckets:
| import{s as Tl,o as wl,n as At}from"../chunks/scheduler.7b731bd4.js";import{S as Jl,i as Ul,e as h,s as o,c as g,q as Be,H as jl,h as bl,a as m,d as l,b as r,f as Ze,g as T,j as f,r as Ee,u as Cl,k as ve,l as vt,m as a,n as w,t as J,o as U,p as j}from"../chunks/index.cc268345.js";import{C as Il,H as I,E as _l}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.f0d99f98.js";import{C as B}from"../chunks/CodeBlock.169a125f.js";import{H as yl,a as _t}from"../chunks/HfOption.9f04abd1.js";function Al(b){let i,u="We can integrate these findings with the TRL Python API like so:",M,c,p;return c=new B({props:{code:"JTBBZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBZnJvbSUyMHBlZnQlMjBpbXBvcnQlMjBMb3JhQ29uZmlnJTBBZnJvbSUyMHRybCUyMGltcG9ydCUyMFNGVFRyYWluZXIlMkMlMjBTRlRDb25maWclMEElMEFkYXRhc2V0JTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMm9wZW4tdGhvdWdodHMlMkZPcGVuVGhvdWdodHMtMTE0ayUyMiUyQyUyMHNwbGl0JTNEJTIydHJhaW4lMjIpJTBBJTBBcGVmdF9jb25maWclMjAlM0QlMjBMb3JhQ29uZmlnKHIlM0QyNTYlMkMlMjBsb3JhX2FscGhhJTNEMTYlMkMlMjB0YXJnZXRfbW9kdWxlcyUzRCUyMmFsbC1saW5lYXIlMjIpJTBBJTBBdHJhaW5pbmdfYXJncyUyMCUzRCUyMFNGVENvbmZpZyglMEElMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlJTNEMmUtNCUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDElMkMlMEElMjAlMjAlMjAlMjBncmFkaWVudF9hY2N1bXVsYXRpb25fc3RlcHMlM0Q0JTJDJTBBJTIwJTIwJTIwJTIwbnVtX3RyYWluX2Vwb2NocyUzRDElMkMlMEElMjAlMjAlMjAlMjByZXBvcnRfdG8lM0QlNUIlMjJ0cmFja2lvJTIyJTVEJTJDJTBBKSUwQSUwQXRyYWluZXIlMjAlM0QlMjBTRlRUcmFpbmVyKCUwQSUyMCUyMCUyMCUyMG1vZGVsJTNEJTIyUXdlbiUyRlF3ZW4yLjUtM0ItSW5zdHJ1Y3QlMjIlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEZGF0YXNldCUyQyUwQSUyMCUyMCUyMCUyMHBlZnRfY29uZmlnJTNEcGVmdF9jb25maWclMkMlMEElMjAlMjAlMjAlMjBhcmdzJTNEdHJhaW5pbmdfYXJncyUyQyUwQSklMEElMEF0cmFpbmVyLnRyYWluKCklMEE=",highlighted:` | |
| <span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset | |
| <span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> LoraConfig | |
| <span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> SFTTrainer, SFTConfig | |
| dataset = load_dataset(<span class="hljs-string">"open-thoughts/OpenThoughts-114k"</span>, split=<span class="hljs-string">"train"</span>) | |
| peft_config = LoraConfig(r=<span class="hljs-number">256</span>, lora_alpha=<span class="hljs-number">16</span>, target_modules=<span class="hljs-string">"all-linear"</span>) | |
| training_args = SFTConfig( | |
| learning_rate=<span class="hljs-number">2e-4</span>, | |
| per_device_train_batch_size=<span class="hljs-number">1</span>, | |
| gradient_accumulation_steps=<span class="hljs-number">4</span>, | |
| num_train_epochs=<span class="hljs-number">1</span>, | |
| report_to=[<span class="hljs-string">"trackio"</span>], | |
| ) | |
| trainer = SFTTrainer( | |
| model=<span class="hljs-string">"Qwen/Qwen2.5-3B-Instruct"</span>, | |
| train_dataset=dataset, | |
| peft_config=peft_config, | |
| args=training_args, | |
| ) | |
| trainer.train() | |
| `,wrap:!1}}),{c(){i=h("p"),i.textContent=u,M=o(),g(c.$$.fragment)},l(n){i=m(n,"P",{"data-svelte-h":!0}),f(i)!=="svelte-51snjk"&&(i.textContent=u),M=r(n),T(c.$$.fragment,n)},m(n,s){a(n,i,s),a(n,M,s),w(c,n,s),p=!0},p:At,i(n){p||(J(c.$$.fragment,n),p=!0)},o(n){U(c.$$.fragment,n),p=!1},d(n){n&&(l(i),l(M)),j(c,n)}}}function $l(b){let i,u,M,c='To use Hugging Face Jobs, you will need to be logged in to the Hugging Face Hub (<code>hf auth login</code>) and have a <a href="https://hf.co/pro" rel="nofollow">Pro</a>, <a href="https://hf.co/enterprise" rel="nofollow">Team</a>, or <a href="https://hf.co/enterprise" rel="nofollow">Enterprise</a> plan. Check out the <a href="https://huggingface.co/docs/huggingface_hub/en/guides/jobs" rel="nofollow">Jobs documentation</a> for more details.',p;return i=new B({props:{code:"JTBBaGYlMjBqb2JzJTIwdXYlMjBydW4lMjAlNUMlMEElMjAlMjAlMjAlMjAtLWZsYXZvciUyMGExMDAtbGFyZ2UlMjAlNUMlMEElMjAlMjAlMjAlMjAtLXRpbWVvdXQlMjA4aCUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tc2VjcmV0cyUyMEhGX1RPS0VOJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIyaHR0cHMlM0ElMkYlMkZyYXcuZ2l0aHVidXNlcmNvbnRlbnQuY29tJTJGaHVnZ2luZ2ZhY2UlMkZ0cmwlMkZtYWluJTJGdHJsJTJGc2NyaXB0cyUyRnNmdC5weSUyMiUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tbW9kZWxfbmFtZV9vcl9wYXRoJTIwUXdlbiUyRlF3ZW4yLjUtM0ItSW5zdHJ1Y3QlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWRhdGFzZXRfbmFtZSUyMG9wZW4tdGhvdWdodHMlMkZPcGVuVGhvdWdodHMtMTE0ayUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tbGVhcm5pbmdfcmF0ZSUyMDIuMGUtNSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tbnVtX3RyYWluX2Vwb2NocyUyMDElMjAlNUMlMEElMjAlMjAlMjAlMjAtLXBhY2tpbmclMjAlNUMlMEElMjAlMjAlMjAlMjAtLXBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUyMDIlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUyMDE2JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS11c2VfcGVmdCUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tbG9yYV9yJTIwMjU2JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1sb3JhX2FscGhhJTIwMTYlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWxvcmFfdGFyZ2V0X21vZHVsZXMlMjBhbGwtbGluZWFyJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1vdXRwdXRfZGlyJTIwUXdlbjIuNS0zQi1PcGVuVGhvdWdodHMtTG9SQSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tcmVwb3J0X3RvJTIwdHJhY2tpbyUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tcHVzaF90b19odWIlMEE=",highlighted:` | |
| hf <span class="hljs-built_in">jobs</span> uv run \\ | |
| --flavor a100-large \\ | |
| --<span class="hljs-built_in">timeout</span> 8h \\ | |
| --secrets HF_TOKEN \\ | |
| <span class="hljs-string">"https://raw.githubusercontent.com/huggingface/trl/main/trl/scripts/sft.py"</span> \\ | |
| --model_name_or_path Qwen/Qwen2.5-3B-Instruct \\ | |
| --dataset_name open-thoughts/OpenThoughts-114k \\ | |
| --learning_rate 2.0e-5 \\ | |
| --num_train_epochs 1 \\ | |
| --packing \\ | |
| --per_device_train_batch_size 2 \\ | |
| --gradient_accumulation_steps 16 \\ | |
| --use_peft \\ | |
| --lora_r 256 \\ | |
| --lora_alpha 16 \\ | |
| --lora_target_modules all-linear \\ | |
| --output_dir Qwen2.5-3B-OpenThoughts-LoRA \\ | |
| --report_to trackio \\ | |
| --push_to_hub | |
| `,wrap:!1}}),{c(){g(i.$$.fragment),u=o(),M=h("p"),M.innerHTML=c},l(n){T(i.$$.fragment,n),u=r(n),M=m(n,"P",{"data-svelte-h":!0}),f(M)!=="svelte-1rt4xms"&&(M.innerHTML=c)},m(n,s){w(i,n,s),a(n,u,s),a(n,M,s),p=!0},p:At,i(n){p||(J(i.$$.fragment,n),p=!0)},o(n){U(i.$$.fragment,n),p=!1},d(n){n&&(l(u),l(M)),j(i,n)}}}function vl(b){let i,u,M,c='To run the script locally, you will need to have <code>uv</code> installed. Check out the <a href="https://docs.astral.sh/uv/" rel="nofollow">uv documentation</a> for more details.',p;return i=new B({props:{code:"JTBBdXYlMjBydW4lMjAlMjJodHRwcyUzQSUyRiUyRnJhdy5naXRodWJ1c2VyY29udGVudC5jb20lMkZodWdnaW5nZmFjZSUyRnRybCUyRm1haW4lMkZ0cmwlMkZzY3JpcHRzJTJGc2Z0LnB5JTIyJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1tb2RlbF9uYW1lX29yX3BhdGglMjBRd2VuJTJGUXdlbjIuNS0zQi1JbnN0cnVjdCUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tZGF0YXNldF9uYW1lJTIwb3Blbi10aG91Z2h0cyUyRk9wZW5UaG91Z2h0cy0xMTRrJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1sZWFybmluZ19yYXRlJTIwMi4wZS01JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1udW1fdHJhaW5fZXBvY2hzJTIwMSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tcGFja2luZyUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTIwMiUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTIwMTYlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWV2YWxfc3RyYXRlZ3klMjBubyUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tdXNlX3BlZnQlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWxvcmFfciUyMDI1NiUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tbG9yYV9hbHBoYSUyMDE2JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1sb3JhX3RhcmdldF9tb2R1bGVzJTIwYWxsLWxpbmVhciUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tb3V0cHV0X2RpciUyMFF3ZW4yLjUtM0ItT3BlblRob3VnaHRzLUxvUkElMjAlNUMlMEElMjAlMjAlMjAlMjAtLXJlcG9ydF90byUyMHRyYWNraW8lMjAlNUMlMEElMjAlMjAlMjAlMjAtLXB1c2hfdG9faHViJTBB",highlighted:` | |
| uv run <span class="hljs-string">"https://raw.githubusercontent.com/huggingface/trl/main/trl/scripts/sft.py"</span> \\ | |
| --model_name_or_path Qwen/Qwen2.5-3B-Instruct \\ | |
| --dataset_name open-thoughts/OpenThoughts-114k \\ | |
| --learning_rate 2.0e-5 \\ | |
| --num_train_epochs 1 \\ | |
| --packing \\ | |
| --per_device_train_batch_size 2 \\ | |
| --gradient_accumulation_steps 16 \\ | |
| --eval_strategy no \\ | |
| --use_peft \\ | |
| --lora_r 256 \\ | |
| --lora_alpha 16 \\ | |
| --lora_target_modules all-linear \\ | |
| --output_dir Qwen2.5-3B-OpenThoughts-LoRA \\ | |
| --report_to trackio \\ | |
| --push_to_hub | |
| `,wrap:!1}}),{c(){g(i.$$.fragment),u=o(),M=h("p"),M.innerHTML=c},l(n){T(i.$$.fragment,n),u=r(n),M=m(n,"P",{"data-svelte-h":!0}),f(M)!=="svelte-1f50gc4"&&(M.innerHTML=c)},m(n,s){w(i,n,s),a(n,u,s),a(n,M,s),p=!0},p:At,i(n){p||(J(i.$$.fragment,n),p=!0)},o(n){U(i.$$.fragment,n),p=!1},d(n){n&&(l(u),l(M)),j(i,n)}}}function Rl(b){let i,u,M,c,p,n;return i=new _t({props:{id:"sft",option:"python",$$slots:{default:[Al]},$$scope:{ctx:b}}}),M=new _t({props:{id:"sft",option:"jobs",$$slots:{default:[$l]},$$scope:{ctx:b}}}),p=new _t({props:{id:"sft",option:"local",$$slots:{default:[vl]},$$scope:{ctx:b}}}),{c(){g(i.$$.fragment),u=o(),g(M.$$.fragment),c=o(),g(p.$$.fragment)},l(s){T(i.$$.fragment,s),u=r(s),T(M.$$.fragment,s),c=r(s),T(p.$$.fragment,s)},m(s,d){w(i,s,d),a(s,u,d),w(M,s,d),a(s,c,d),w(p,s,d),n=!0},p(s,d){const y={};d&2&&(y.$$scope={dirty:d,ctx:s}),i.$set(y);const C={};d&2&&(C.$$scope={dirty:d,ctx:s}),M.$set(C);const _={};d&2&&(_.$$scope={dirty:d,ctx:s}),p.$set(_)},i(s){n||(J(i.$$.fragment,s),J(M.$$.fragment,s),J(p.$$.fragment,s),n=!0)},o(s){U(i.$$.fragment,s),U(M.$$.fragment,s),U(p.$$.fragment,s),n=!1},d(s){s&&(l(u),l(c)),j(i,s),j(M,s),j(p,s)}}}function Ll(b){let i,u="We can implement these recommendations with the TRL Python API like so:",M,c,p,n,s="<p>This snippet skips the reward function which is defined above to keep the example concise.</p>",d;return c=new B({props:{code:"JTBBZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBZnJvbSUyMHBlZnQlMjBpbXBvcnQlMjBMb3JhQ29uZmlnJTBBZnJvbSUyMHRybCUyMGltcG9ydCUyMEdSUE9Db25maWclMkMlMjBHUlBPVHJhaW5lciUwQWZyb20lMjB0cmwucmV3YXJkcyUyMGltcG9ydCUyMHJlYXNvbmluZ19hY2N1cmFjeV9yZXdhcmQlMEElMEFkYXRhc2V0JTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMkh1Z2dpbmdGYWNlSDQlMkZPcGVuUjEtTWF0aC0yMjBrLWRlZmF1bHQtdmVyaWZpZWQlMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKSUwQSUwQXBlZnRfY29uZmlnJTIwJTNEJTIwTG9yYUNvbmZpZyglMEElMjAlMjAlMjAlMjByJTNEMSUyQyUwQSUyMCUyMCUyMCUyMGxvcmFfYWxwaGElM0QzMiUyQyUwQSUyMCUyMCUyMCUyMHRhcmdldF9tb2R1bGVzJTNEJTIyYWxsLWxpbmVhciUyMiUwQSklMEElMEF0cmFpbmluZ19hcmdzJTIwJTNEJTIwR1JQT0NvbmZpZyglMEElMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlJTNENWUtNSUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDElMkMlMEElMjAlMjAlMjAlMjBncmFkaWVudF9hY2N1bXVsYXRpb25fc3RlcHMlM0Q0JTJDJTBBJTIwJTIwJTIwJTIwbnVtX3RyYWluX2Vwb2NocyUzRDElMkMlMEElMjAlMjAlMjAlMjBudW1fZ2VuZXJhdGlvbnMlM0Q4JTJDJTBBJTIwJTIwJTIwJTIwZ2VuZXJhdGlvbl9iYXRjaF9zaXplJTNEOCUyQyUwQSUyMCUyMCUyMCUyMHJlcG9ydF90byUzRCU1QiUyMnRyYWNraW8lMjIlNUQlMkMlMEEpJTBBJTBBdHJhaW5lciUyMCUzRCUyMEdSUE9UcmFpbmVyKCUwQSUyMCUyMCUyMCUyMG1vZGVsJTNEJTIyUXdlbiUyRlF3ZW4zLTAuNkIlMjIlMkMlMEElMjAlMjAlMjAlMjByZXdhcmRfZnVuY3MlM0RyZWFzb25pbmdfYWNjdXJhY3lfcmV3YXJkJTJDJTBBJTIwJTIwJTIwJTIwYXJncyUzRHRyYWluaW5nX2FyZ3MlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEZGF0YXNldCUyQyUwQSUyMCUyMCUyMCUyMHBlZnRfY29uZmlnJTNEcGVmdF9jb25maWclMkMlMEEpJTBBJTBBdHJhaW5lci50cmFpbigpJTBB",highlighted:` | |
| <span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset | |
| <span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> LoraConfig | |
| <span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOConfig, GRPOTrainer | |
| <span class="hljs-keyword">from</span> trl.rewards <span class="hljs-keyword">import</span> reasoning_accuracy_reward | |
| dataset = load_dataset(<span class="hljs-string">"HuggingFaceH4/OpenR1-Math-220k-default-verified"</span>, split=<span class="hljs-string">"train"</span>) | |
| peft_config = LoraConfig( | |
| r=<span class="hljs-number">1</span>, | |
| lora_alpha=<span class="hljs-number">32</span>, | |
| target_modules=<span class="hljs-string">"all-linear"</span> | |
| ) | |
| training_args = GRPOConfig( | |
| learning_rate=<span class="hljs-number">5e-5</span>, | |
| per_device_train_batch_size=<span class="hljs-number">1</span>, | |
| gradient_accumulation_steps=<span class="hljs-number">4</span>, | |
| num_train_epochs=<span class="hljs-number">1</span>, | |
| num_generations=<span class="hljs-number">8</span>, | |
| generation_batch_size=<span class="hljs-number">8</span>, | |
| report_to=[<span class="hljs-string">"trackio"</span>], | |
| ) | |
| trainer = GRPOTrainer( | |
| model=<span class="hljs-string">"Qwen/Qwen3-0.6B"</span>, | |
| reward_funcs=reasoning_accuracy_reward, | |
| args=training_args, | |
| train_dataset=dataset, | |
| peft_config=peft_config, | |
| ) | |
| trainer.train() | |
| `,wrap:!1}}),{c(){i=h("p"),i.textContent=u,M=o(),g(c.$$.fragment),p=o(),n=h("blockquote"),n.innerHTML=s,this.h()},l(y){i=m(y,"P",{"data-svelte-h":!0}),f(i)!=="svelte-160cqoc"&&(i.textContent=u),M=r(y),T(c.$$.fragment,y),p=r(y),n=m(y,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),f(n)!=="svelte-ovpx5d"&&(n.innerHTML=s),this.h()},h(){ve(n,"class","warning")},m(y,C){a(y,i,C),a(y,M,C),w(c,y,C),a(y,p,C),a(y,n,C),d=!0},p:At,i(y){d||(J(c.$$.fragment,y),d=!0)},o(y){U(c.$$.fragment,y),d=!1},d(y){y&&(l(i),l(M),l(p),l(n)),j(c,y)}}}function Bl(b){let i,u,M,c='To use Hugging Face Jobs, you will need to be logged in to the Hugging Face Hub (<code>hf auth login</code>) and have a <a href="https://hf.co/pro" rel="nofollow">Pro</a>, <a href="https://hf.co/enterprise" rel="nofollow">Team</a>, or <a href="https://hf.co/enterprise" rel="nofollow">Enterprise</a> plan. Check out the <a href="https://huggingface.co/docs/huggingface_hub/en/guides/jobs" rel="nofollow">Jobs documentation</a> for more details.',p;return i=new B({props:{code:"JTBBaGYlMjBqb2JzJTIwdXYlMjBydW4lMjAlNUMlMEElMjAlMjAlMjAlMjAtLWZsYXZvciUyMGExMDAtbGFyZ2UlMjAlNUMlMEElMjAlMjAlMjAlMjAtLXRpbWVvdXQlMjA0aCUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tc2VjcmV0cyUyMEhGX1RPS0VOJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1lbnYlMjBQWVRPUkNIX0NVREFfQUxMT0NfQ09ORiUzRGV4cGFuZGFibGVfc2VnbWVudHMlM0FUcnVlJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIyaHR0cHMlM0ElMkYlMkZodWdnaW5nZmFjZS5jbyUyRmRhdGFzZXRzJTJGYnVydGVuc2hhdyUyRmxvcmEtd2l0aG91dC1yZWdyZXRzJTJGcmVzb2x2ZSUyRm1haW4lMkZncnBvLnB5JTIyJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1tb2RlbF9uYW1lX29yX3BhdGglMjBRd2VuJTJGUXdlbjMtMC42QiUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tZGF0YXNldF9uYW1lJTIwSHVnZ2luZ0ZhY2VINCUyRk9wZW5SMS1NYXRoLTIyMGstZGVmYXVsdC12ZXJpZmllZCUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tb3V0cHV0X2RpciUyMGdycG8tZnVsbC1xd2VuMy0wLjZiJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1sZWFybmluZ19yYXRlJTIwMS4wZS02JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1scl9zY2hlZHVsZXJfdHlwZSUyMGNvc2luZSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0td2FybXVwX3N0ZXBzJTIwMC4wJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1tYXhfZ3JhZF9ub3JtJTIwMS4wJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1iZXRhJTIwMC4wJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1tYXhfY29tcGxldGlvbl9sZW5ndGglMjA0MDk2JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1udW1fZ2VuZXJhdGlvbnMlMjAxNiUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tZ2VuZXJhdGlvbl9iYXRjaF9zaXplJTIwMTYlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUyMDglMjAlNUMlMEElMjAlMjAlMjAlMjAtLXBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUyMDElMjAlNUMlMEElMjAlMjAlMjAlMjAtLW51bV90cmFpbl9lcG9jaHMlMjAxJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1sb3JhX3IlMjAxJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1sb3JhX2FscGhhJTIwMzIlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWxvcmFfZHJvcG91dCUyMDAuMCUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tbG9yYV90YXJnZXRfbW9kdWxlcyUyMGFsbC1saW5lYXIlMjAlNUMlMEElMjAlMjAlMjAlMjAtLXZsbG1fbW9kZSUyMGNvbG9jYXRlJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1zYXZlX3N0cmF0ZWd5JTIwc3RlcHMlMjAlNUMlMEElMjAlMjAlMjAlMjAtLXNhdmVfc3RlcHMlMjA1MCUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tc2F2ZV90b3RhbF9saW1pdCUyMDElMjAlNUMlMEElMjAlMjAlMjAlMjAtLWxvZ2dpbmdfc3RlcHMlMjAxJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1tYXhfc3RlcHMlMjAyMDAlMjAlNUMlMEElMjAlMjAlMjAlMjAtLXJlcG9ydF90byUyMHRyYWNraW8=",highlighted:` | |
| hf <span class="hljs-built_in">jobs</span> uv run \\ | |
| --flavor a100-large \\ | |
| --<span class="hljs-built_in">timeout</span> 4h \\ | |
| --secrets HF_TOKEN \\ | |
| --<span class="hljs-built_in">env</span> PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \\ | |
| <span class="hljs-string">"https://huggingface.co/datasets/burtenshaw/lora-without-regrets/resolve/main/grpo.py"</span> \\ | |
| --model_name_or_path Qwen/Qwen3-0.6B \\ | |
| --dataset_name HuggingFaceH4/OpenR1-Math-220k-default-verified \\ | |
| --output_dir grpo-full-qwen3-0.6b \\ | |
| --learning_rate 1.0e-6 \\ | |
| --lr_scheduler_type cosine \\ | |
| --warmup_steps 0.0 \\ | |
| --max_grad_norm 1.0 \\ | |
| --beta 0.0 \\ | |
| --max_completion_length 4096 \\ | |
| --num_generations 16 \\ | |
| --generation_batch_size 16 \\ | |
| --gradient_accumulation_steps 8 \\ | |
| --per_device_train_batch_size 1 \\ | |
| --num_train_epochs 1 \\ | |
| --lora_r 1 \\ | |
| --lora_alpha 32 \\ | |
| --lora_dropout 0.0 \\ | |
| --lora_target_modules all-linear \\ | |
| --vllm_mode colocate \\ | |
| --save_strategy steps \\ | |
| --save_steps 50 \\ | |
| --save_total_limit 1 \\ | |
| --logging_steps 1 \\ | |
| --max_steps 200 \\ | |
| --report_to trackio`,wrap:!1}}),{c(){g(i.$$.fragment),u=o(),M=h("p"),M.innerHTML=c},l(n){T(i.$$.fragment,n),u=r(n),M=m(n,"P",{"data-svelte-h":!0}),f(M)!=="svelte-1rt4xms"&&(M.innerHTML=c)},m(n,s){w(i,n,s),a(n,u,s),a(n,M,s),p=!0},p:At,i(n){p||(J(i.$$.fragment,n),p=!0)},o(n){U(i.$$.fragment,n),p=!1},d(n){n&&(l(u),l(M)),j(i,n)}}}function Zl(b){let i,u,M,c='To run the script locally, you will need to have <code>uv</code> installed. Check out the <a href="https://docs.astral.sh/uv/" rel="nofollow">uv documentation</a> for more details.',p;return i=new B({props:{code:"dXYlMjBydW4lMjAlMjJodHRwcyUzQSUyRiUyRmh1Z2dpbmdmYWNlLmNvJTJGZGF0YXNldHMlMkZidXJ0ZW5zaGF3JTJGbG9yYS13aXRob3V0LXJlZ3JldHMlMkZyZXNvbHZlJTJGbWFpbiUyRmdycG8ucHklMjIlMjAlNUMlMEElMjAlMjAlMjAlMjAtLW1vZGVsX25hbWVfb3JfcGF0aCUyMFF3ZW4lMkZRd2VuMy0wLjZCJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1kYXRhc2V0X25hbWUlMjBIdWdnaW5nRmFjZUg0JTJGT3BlblIxLU1hdGgtMjIway1kZWZhdWx0LXZlcmlmaWVkJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1vdXRwdXRfZGlyJTIwZ3Jwby1mdWxsLXF3ZW4zLTAuNmIlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWxlYXJuaW5nX3JhdGUlMjAxLjBlLTYlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWxyX3NjaGVkdWxlcl90eXBlJTIwY29zaW5lJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS13YXJtdXBfc3RlcHMlMjAwLjAlMjAlNUMlMEElMjAlMjAlMjAlMjAtLW1heF9ncmFkX25vcm0lMjAxLjAlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWJldGElMjAwLjAlMjAlNUMlMEElMjAlMjAlMjAlMjAtLW1heF9jb21wbGV0aW9uX2xlbmd0aCUyMDQwOTYlMjAlNUMlMEElMjAlMjAlMjAlMjAtLW51bV9nZW5lcmF0aW9ucyUyMDE2JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1nZW5lcmF0aW9uX2JhdGNoX3NpemUlMjAxNiUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTIwOCUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTIwMSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tbnVtX3RyYWluX2Vwb2NocyUyMDElMjAlNUMlMEElMjAlMjAlMjAlMjAtLWxvcmFfciUyMDElMjAlNUMlMEElMjAlMjAlMjAlMjAtLWxvcmFfYWxwaGElMjAzMiUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tbG9yYV9kcm9wb3V0JTIwMC4wJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1sb3JhX3RhcmdldF9tb2R1bGVzJTIwYWxsLWxpbmVhciUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tdmxsbV9tb2RlJTIwY29sb2NhdGUlMjAlNUMlMEElMjAlMjAlMjAlMjAtLXNhdmVfc3RyYXRlZ3klMjBzdGVwcyUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tc2F2ZV9zdGVwcyUyMDUwJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1zYXZlX3RvdGFsX2xpbWl0JTIwMSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tbG9nZ2luZ19zdGVwcyUyMDElMjAlNUMlMEElMjAlMjAlMjAlMjAtLW1heF9zdGVwcyUyMDIwMCUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tcmVwb3J0X3RvJTIwdHJhY2tpbw==",highlighted:`uv run <span class="hljs-string">"https://huggingface.co/datasets/burtenshaw/lora-without-regrets/resolve/main/grpo.py"</span> \\ | |
| --model_name_or_path Qwen/Qwen3-0.6B \\ | |
| --dataset_name HuggingFaceH4/OpenR1-Math-220k-default-verified \\ | |
| --output_dir grpo-full-qwen3-0.6b \\ | |
| --learning_rate 1.0e-6 \\ | |
| --lr_scheduler_type cosine \\ | |
| --warmup_steps 0.0 \\ | |
| --max_grad_norm 1.0 \\ | |
| --beta 0.0 \\ | |
| --max_completion_length 4096 \\ | |
| --num_generations 16 \\ | |
| --generation_batch_size 16 \\ | |
| --gradient_accumulation_steps 8 \\ | |
| --per_device_train_batch_size 1 \\ | |
| --num_train_epochs 1 \\ | |
| --lora_r 1 \\ | |
| --lora_alpha 32 \\ | |
| --lora_dropout 0.0 \\ | |
| --lora_target_modules all-linear \\ | |
| --vllm_mode colocate \\ | |
| --save_strategy steps \\ | |
| --save_steps 50 \\ | |
| --save_total_limit 1 \\ | |
| --logging_steps 1 \\ | |
| --max_steps 200 \\ | |
| --report_to trackio`,wrap:!1}}),{c(){g(i.$$.fragment),u=o(),M=h("p"),M.innerHTML=c},l(n){T(i.$$.fragment,n),u=r(n),M=m(n,"P",{"data-svelte-h":!0}),f(M)!=="svelte-1f50gc4"&&(M.innerHTML=c)},m(n,s){w(i,n,s),a(n,u,s),a(n,M,s),p=!0},p:At,i(n){p||(J(i.$$.fragment,n),p=!0)},o(n){U(i.$$.fragment,n),p=!1},d(n){n&&(l(u),l(M)),j(i,n)}}}function El(b){let i,u,M,c,p,n;return i=new _t({props:{id:"grpo",option:"python",$$slots:{default:[Ll]},$$scope:{ctx:b}}}),M=new _t({props:{id:"grpo",option:"jobs",$$slots:{default:[Bl]},$$scope:{ctx:b}}}),p=new _t({props:{id:"grpo",option:"local",$$slots:{default:[Zl]},$$scope:{ctx:b}}}),{c(){g(i.$$.fragment),u=o(),g(M.$$.fragment),c=o(),g(p.$$.fragment)},l(s){T(i.$$.fragment,s),u=r(s),T(M.$$.fragment,s),c=r(s),T(p.$$.fragment,s)},m(s,d){w(i,s,d),a(s,u,d),w(M,s,d),a(s,c,d),w(p,s,d),n=!0},p(s,d){const y={};d&2&&(y.$$scope={dirty:d,ctx:s}),i.$set(y);const C={};d&2&&(C.$$scope={dirty:d,ctx:s}),M.$set(C);const _={};d&2&&(_.$$scope={dirty:d,ctx:s}),p.$set(_)},i(s){n||(J(i.$$.fragment,s),J(M.$$.fragment,s),J(p.$$.fragment,s),n=!0)},o(s){U(i.$$.fragment,s),U(M.$$.fragment,s),U(p.$$.fragment,s),n=!1},d(s){s&&(l(u),l(c)),j(i,s),j(M,s),j(p,s)}}}function kl(b){let i,u,M,c,p,n,s,d,y,C='Recent research from the team at <a href="https://thinkingmachines.ai/blog/lora/" rel="nofollow">Thinking Machines Lab</a> (Schulman et al., 2025) shows that <strong>LoRA can match full fine-tuning performance</strong> when configured correctly, while using only ~67% of the compute. These findings are exciting to TRL users because they’re straightforward to implement and can improve model performance on smaller budgets.',_,Z,ke="This guide provides simple instructions to reproduce the results of the blog post in TRL.",Rt,v,We="<p>It is recommended to read the blog post before following this guide, or to consult both resources in parallel for best results.</p>",Lt,E,Bt,k,Xe='First of all, let’s remind ourselves of the benefits of <a href="https://huggingface.co/docs/trl/en/peft_integration" rel="nofollow">LoRA over full fine-tuning</a>.',Zt,W,xe='LoRA adds adapter layers on top of the base model, which contains significantly fewer parameters than the base model itself. This design reduces GPU memory requirements and enables more efficient training. As described in the <a href="https://thinkingmachines.ai/blog/lora/" rel="nofollow">blog</a>, this approach was originally thought to involve a performance trade-off, although careful configuration can overcome this trade-off and match full fine-tuning performance.',Et,X,kt,x,Ve="Let’s implement and train LoRA adapters in TRL scripts based on the core findings of the blog post. Afterwards, we’ll revisit each finding in light of the TRL results.",Wt,V,Xt,G,Ge="The blog post performs SFT on a range of models and datasets from the Hub, which we can reproduce in TRL.",xt,F,Fe='<thead><tr><th>Model</th> <th>Dataset</th></tr></thead> <tbody><tr><td><a href="https://huggingface.co/meta-llama/Llama-3.2-1B" rel="nofollow">Llama-3.2-1B-Instruct</a></td> <td><a href="https://huggingface.co/datasets/allenai/tulu-3-sft-mixture" rel="nofollow">allenai/tulu-3-sft-mixture</a></td></tr> <tr><td><a href="https://huggingface.co/meta-llama/Llama-3.2-1B" rel="nofollow">Llama-3.2-1B-Instruct</a></td> <td><a href="https://huggingface.co/datasets/open-thoughts/OpenThoughts-114k" rel="nofollow">open-thoughts/OpenThoughts-114k</a></td></tr> <tr><td><a href="https://huggingface.co/meta-llama/Llama-3.1-8B" rel="nofollow">Llama-3.1-8B-Instruct</a></td> <td><a href="https://huggingface.co/datasets/allenai/tulu-3-sft-mixture" rel="nofollow">allenai/tulu-3-sft-mixture</a></td></tr> <tr><td><a href="https://huggingface.co/meta-llama/Llama-3.1-8B" rel="nofollow">Llama-3.1-8B-Instruct</a></td> <td><a href="https://huggingface.co/datasets/open-thoughts/OpenThoughts-114k" rel="nofollow">open-thoughts/OpenThoughts-114k</a></td></tr></tbody>',Vt,R,Gt,H,He='Once training starts, you can monitor the progress in <a href="https://huggingface.co/trackio" rel="nofollow">Trackio</a>, which will log the URL.',Ft,Q,Ht,Y,Qe="The blog post performs GRPO on a range of models and datasets from the Hub, and once again we can reproduce the results in TRL.",Qt,S,Ye='<thead><tr><th>Model</th> <th>Dataset</th></tr></thead> <tbody><tr><td><a href="https://huggingface.co/meta-llama/Llama-3.2-1B" rel="nofollow">Llama-3.1-8B-Base</a></td> <td><a href="https://huggingface.co/datasets/openai/gsm8k" rel="nofollow">GSM8k</a></td></tr> <tr><td><a href="https://huggingface.co/meta-llama/Llama-3.2-1B" rel="nofollow">Llama-3.1-8B-Base</a></td> <td><a href="https://huggingface.co/datasets/zwhe99/DeepMath-103K" rel="nofollow">DeepMath-103K</a></td></tr> <tr><td><a href="https://huggingface.co/Qwen/Qwen3-8b-base" rel="nofollow">Qwen3-8b-base</a></td> <td><a href="https://huggingface.co/datasets/zwhe99/DeepMath-103K" rel="nofollow">DeepMath-103K</a></td></tr></tbody>',Yt,N,Se="For reinforcement learning, the blog uses a math reasoning task that we can reproduce as a Python function.",St,L,Nt,z,Ne='The reinforcement learning script with GRPO is implemented as a custom script in TRL, which uses the reward function shown above. You can review it at <a href="https://huggingface.co/datasets/burtenshaw/lora-without-regrets/blob/main/grpo.py" rel="nofollow"><code>grpo.py</code></a> - Reinforcement learning with LoRA best practices',zt,D,Dt,P,ze="The authors recommend applying LoRA to all weight matrices rather than limiting it to attention layers, as increasing the rank does not compensate for this restriction. In TRL, this can be configured using <code>--lora_target_modules all-linear</code> to apply LoRA to all weight matrices.",Pt,q,De='We were able to reproduce the results of the blog post using TRL and the SmolLM3 model. We trained the model for 500 steps on the <a href="https://huggingface.co/datasets/HuggingFaceH4/OpenR1-Math-220k-default-verified" rel="nofollow">Math 220k dataset</a> with the reward function and configuration above. As you can see in the figure below, the LoRA model’s average train reward curve matches the full fine-tuning curve.',qt,O,Pe='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/lora_without_regret/5.png" alt="train reward"/>',Ot,K,qe="And most importantly, the LoRA model uses significantly less memory than the full fine-tuning model, as we can see in the figure below.",Kt,tt,Oe='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/lora_without_regret/6.png" alt="memory usage"/>',te,et,Ke="Here are the parameters we used to train the above models",ee,lt,tl="<thead><tr><th>Parameter</th> <th>LoRA</th> <th>Full FT</th></tr></thead> <tbody><tr><td><code>--model_name_or_path</code></td> <td>HuggingFaceTB/SmolLM3-3B</td> <td>HuggingFaceTB/SmolLM3-3B</td></tr> <tr><td><code>--dataset_name</code></td> <td>HuggingFaceH4/OpenR1-Math-220k-default-verified</td> <td>HuggingFaceH4/OpenR1-Math-220k-default-verified</td></tr> <tr><td><code>--learning_rate</code></td> <td>1.0e-5</td> <td>1.0e-6</td></tr> <tr><td><code>--max_completion_length</code></td> <td>4096</td> <td>4096</td></tr> <tr><td><code>--lora_r</code></td> <td>1</td> <td>-</td></tr> <tr><td><code>--lora_alpha</code></td> <td>32</td> <td>-</td></tr> <tr><td><code>--lora_dropout</code></td> <td>0.0</td> <td>-</td></tr> <tr><td><code>--lora_target_modules</code></td> <td>all-linear</td> <td>-</td></tr></tbody>",le,at,el="Let’s break down the key findings of the blog post and how we were able to reproduce them.",ae,nt,ne,st,ll="The authors recommend applying LoRA to all weight matrices rather than limiting it to attention layers, as increasing the rank does not compensate for this restriction.",se,it,al='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/lora_without_regret/1.png" alt="all layers"/>',ie,ot,nl="Attention-only LoRA underperforms even when using a higher rank to match parameter count. In TRL, this can be configured using <code>--lora_target_modules all-linear</code> to apply LoRA to all weight matrices. In Python, we can do this like so:",oe,rt,re,Mt,Me,pt,sl="The blog post recommends using a sufficient LoRA rank to learn from the dataset. The rank determines the number of trainable parameters in the LoRA adapter. Therefore, “For datasets that exceed LoRA capacity, LoRA underperforms FullFT”.",pe,ht,il='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/lora_without_regret/3.png" alt="learning rate"/>',he,mt,ol="In the TRL script, we could use <code>--lora_r</code> to set the rank and adapt it based on the task and dataset we’re training on. The blog post recommends the following ranks based on the task and dataset size:",me,ct,rl="Reinforcement learning tasks typically require lower capacity, so smaller LoRA ranks can be used. This is because policy gradient algorithms extract roughly ~1 bit of information per episode, demanding minimal parameter capacity.",ce,ft,Ml="The blog post defines the ideal dataset size for LoRA to match full fine-tuning as “Post-training scale”. Which we can use to determine the recommended rank for SFT and RL LoRAs as:",fe,ut,pl="<thead><tr><th>Task Type</th> <th>Dataset Size</th> <th>Recommended Rank</th></tr></thead> <tbody><tr><td><strong>SFT</strong></td> <td>Post-training scale</td> <td>256</td></tr> <tr><td><strong>RL</strong></td> <td>Any size</td> <td>1-32</td></tr></tbody>",ue,dt,de,A,Re,It,hl="--learning_rate",Le,ye,gl='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mfrac><mn>1</mn><mi>r</mi></mfrac></mrow><annotation encoding="application/x-tex"> \\frac{1}{r} </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1.1901em;vertical-align:-0.345em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.8451em;"><span style="top:-2.655em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">r</span></span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.394em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">1</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.345em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span>',ge,Te,yt,ml='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/lora_without_regret/2.png" alt="learning rate"/>',we,gt,Je,Tt,cl="The blog post recommends using an effective batch size < 32 because the authors found LoRA to be less tolerant of large batch sizes. This could not be mitigated by increasing the LoRA rank. In the TRL script, we could use <code>--per_device_train_batch_size</code> and <code>--gradient_accumulation_steps</code> to set the batch size.",Ue,wt,fl='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/lora_without_regret/4.png" alt="learning rate"/>',je,Jt,be,Ut,ul="Using TRL, you can efficiently implement LoRA adapters to match full fine-tuning performance, applying the core insights (targeting all weight matrices, choosing the right rank, and managing batch size and learning rate) without the heavy compute cost of FullFT.",Ce,jt,Ie,bt,_e,Ct,Ae,$t,$e;return p=new Il({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),s=new I({props:{title:"LoRA Without Regret",local:"lora-without-regret",headingTag:"h1"}}),E=new I({props:{title:"Benefits of LoRA over full fine-tuning",local:"benefits-of-lora-over-full-fine-tuning",headingTag:"h2"}}),X=new I({props:{title:"Examples with TRL",local:"examples-with-trl",headingTag:"h2"}}),V=new I({props:{title:"Supervised Fine-Tuning (SFT)",local:"supervised-fine-tuning-sft",headingTag:"h3"}}),R=new yl({props:{id:"sft",options:["python","jobs","local"],$$slots:{default:[Rl]},$$scope:{ctx:b}}}),Q=new I({props:{title:"Reinforcement Learning (GRPO)",local:"reinforcement-learning-grpo",headingTag:"h3"}}),L=new yl({props:{id:"grpo",options:["python","jobs","local"],$$slots:{default:[El]},$$scope:{ctx:b}}}),D=new I({props:{title:"Key findings in optimizing LoRA",local:"key-findings-in-optimizing-lora",headingTag:"h2"}}),nt=new I({props:{title:"1. LoRA performs better when applied to all weight matrices",local:"1-lora-performs-better-when-applied-to-all-weight-matrices",headingTag:"h3"}}),rt=new B({props:{code:"ZnJvbSUyMHBlZnQlMjBpbXBvcnQlMjBMb3JhQ29uZmlnJTIwJTIwJTBBJTBBcGVmdF9jb25maWclMjAlM0QlMjBMb3JhQ29uZmlnKHRhcmdldF9tb2R1bGVzJTNEJTIyYWxsLWxpbmVhciUyMiklMjAlMjA=",highlighted:`<span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> LoraConfig | |
| peft_config = LoraConfig(target_modules=<span class="hljs-string">"all-linear"</span>) `,wrap:!1}}),Mt=new I({props:{title:"2. The adapter needs sufficient capacity to learn from the dataset",local:"2-the-adapter-needs-sufficient-capacity-to-learn-from-the-dataset",headingTag:"h3"}}),dt=new I({props:{title:"3. “FullFT and high-rank LoRAs have similar learning curves”",local:"3-fullft-and-high-rank-loras-have-similar-learning-curves",headingTag:"h3"}}),gt=new I({props:{title:"4. “In some scenarios, LoRA is less tolerant of large batch sizes than full fine-tuning.”",local:"4-in-some-scenarios-lora-is-less-tolerant-of-large-batch-sizes-than-full-fine-tuning",headingTag:"h3"}}),Jt=new I({props:{title:"Takeaways",local:"takeaways",headingTag:"h2"}}),jt=new I({props:{title:"Citation",local:"citation",headingTag:"h2"}}),bt=new B({props:{code:"JTQwYXJ0aWNsZSU3QnNjaHVsbWFuMjAyNWxvcmElMkMlMjAlMjAlMEElMjAlMjAlMjAlMjB0aXRsZSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUzRCUyMCU3QiU3QkxvUkElMjBXaXRob3V0JTIwUmVncmV0JTdEJTdEJTJDJTIwJTIwJTBBJTIwJTIwJTIwJTIwYXV0aG9yJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTNEJTIwJTdCSm9obiUyMFNjaHVsbWFuJTIwYW5kJTIwVGhpbmtpbmclMjBNYWNoaW5lcyUyMExhYiU3RCUyQyUyMCUyMCUwQSUyMCUyMCUyMCUyMHllYXIlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlM0QlMjAyMDI1JTJDJTIwJTIwJTBBJTIwJTIwJTIwJTIwam91cm5hbCUyMCUyMCUyMCUyMCUyMCUyMCUzRCUyMCU3QlRoaW5raW5nJTIwTWFjaGluZXMlMjBMYWIlM0ElMjBDb25uZWN0aW9uaXNtJTdEJTJDJTIwJTIwJTBBJTIwJTIwJTIwJTIwZG9pJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTNEJTIwJTdCMTAuNjQ0MzQlMkZ0bWwuMjAyNTA5MjklN0QlMkMlMjAlMjAlMEElMjAlMjAlMjAlMjBub3RlJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTNEJTIwJTdCaHR0cHMlM0ElMkYlMkZ0aGlua2luZ21hY2hpbmVzLmFpJTJGYmxvZyUyRmxvcmElMkYlN0QlMjAlMjAlMEElN0QlMjAlMjA=",highlighted:`<span class="language-xml">@article</span><span class="hljs-template-variable">{schulman2025lora, | |
| title = {{LoRA Without Regret}</span><span class="language-xml">}, | |
| author = </span><span class="hljs-template-variable">{John Schulman and Thinking Machines Lab}</span><span class="language-xml">, | |
| year = 2025, | |
| journal = </span><span class="hljs-template-variable">{Thinking Machines Lab: Connectionism}</span><span class="language-xml">, | |
| doi = </span><span class="hljs-template-variable">{10.64434/tml.20250929}</span><span class="language-xml">, | |
| note = </span><span class="hljs-template-variable">{https://thinkingmachines.ai/blog/lora/}</span><span class="language-xml"> | |
| } </span>`,wrap:!1}}),Ct=new _l({props:{source:"https://github.com/huggingface/trl/blob/main/docs/source/lora_without_regret.md"}}),{c(){i=h("meta"),u=o(),M=h("p"),c=o(),g(p.$$.fragment),n=o(),g(s.$$.fragment),d=o(),y=h("p"),y.innerHTML=C,_=o(),Z=h("p"),Z.textContent=ke,Rt=o(),v=h("blockquote"),v.innerHTML=We,Lt=o(),g(E.$$.fragment),Bt=o(),k=h("p"),k.innerHTML=Xe,Zt=o(),W=h("p"),W.innerHTML=xe,Et=o(),g(X.$$.fragment),kt=o(),x=h("p"),x.textContent=Ve,Wt=o(),g(V.$$.fragment),Xt=o(),G=h("p"),G.textContent=Ge,xt=o(),F=h("table"),F.innerHTML=Fe,Vt=o(),g(R.$$.fragment),Gt=o(),H=h("p"),H.innerHTML=He,Ft=o(),g(Q.$$.fragment),Ht=o(),Y=h("p"),Y.textContent=Qe,Qt=o(),S=h("table"),S.innerHTML=Ye,Yt=o(),N=h("p"),N.textContent=Se,St=o(),g(L.$$.fragment),Nt=o(),z=h("p"),z.innerHTML=Ne,zt=o(),g(D.$$.fragment),Dt=o(),P=h("p"),P.innerHTML=ze,Pt=o(),q=h("p"),q.innerHTML=De,qt=o(),O=h("p"),O.innerHTML=Pe,Ot=o(),K=h("p"),K.textContent=qe,Kt=o(),tt=h("p"),tt.innerHTML=Oe,te=o(),et=h("p"),et.textContent=Ke,ee=o(),lt=h("table"),lt.innerHTML=tl,le=o(),at=h("p"),at.textContent=el,ae=o(),g(nt.$$.fragment),ne=o(),st=h("p"),st.textContent=ll,se=o(),it=h("p"),it.innerHTML=al,ie=o(),ot=h("p"),ot.innerHTML=nl,oe=o(),g(rt.$$.fragment),re=o(),g(Mt.$$.fragment),Me=o(),pt=h("p"),pt.textContent=sl,pe=o(),ht=h("p"),ht.innerHTML=il,he=o(),mt=h("p"),mt.innerHTML=ol,me=o(),ct=h("p"),ct.textContent=rl,ce=o(),ft=h("p"),ft.textContent=Ml,fe=o(),ut=h("table"),ut.innerHTML=pl,ue=o(),g(dt.$$.fragment),de=o(),A=h("p"),Re=Be("Counterintuitively, the blog post recommends using a higher learning rate than for full fine-tuning. In the table above, we used 1.0e-5 for LoRA and 1.0e-6 for full fine-tuning. In the TRL script, we could use "),It=h("code"),It.textContent=hl,Le=Be(" to set the learning rate. The "),ye=new jl(!1),ge=Be(" scaling in LoRA makes the optimal learning rate approximately rank-independent."),Te=o(),yt=h("p"),yt.innerHTML=ml,we=o(),g(gt.$$.fragment),Je=o(),Tt=h("p"),Tt.innerHTML=cl,Ue=o(),wt=h("p"),wt.innerHTML=fl,je=o(),g(Jt.$$.fragment),be=o(),Ut=h("p"),Ut.textContent=ul,Ce=o(),g(jt.$$.fragment),Ie=o(),g(bt.$$.fragment),_e=o(),g(Ct.$$.fragment),Ae=o(),$t=h("p"),this.h()},l(t){const e=bl("svelte-u9bgzb",document.head);i=m(e,"META",{name:!0,content:!0}),e.forEach(l),u=r(t),M=m(t,"P",{}),Ze(M).forEach(l),c=r(t),T(p.$$.fragment,t),n=r(t),T(s.$$.fragment,t),d=r(t),y=m(t,"P",{"data-svelte-h":!0}),f(y)!=="svelte-gx5fze"&&(y.innerHTML=C),_=r(t),Z=m(t,"P",{"data-svelte-h":!0}),f(Z)!=="svelte-lywk0l"&&(Z.textContent=ke),Rt=r(t),v=m(t,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),f(v)!=="svelte-8qn8ze"&&(v.innerHTML=We),Lt=r(t),T(E.$$.fragment,t),Bt=r(t),k=m(t,"P",{"data-svelte-h":!0}),f(k)!=="svelte-16iolmm"&&(k.innerHTML=Xe),Zt=r(t),W=m(t,"P",{"data-svelte-h":!0}),f(W)!=="svelte-1sfgmfi"&&(W.innerHTML=xe),Et=r(t),T(X.$$.fragment,t),kt=r(t),x=m(t,"P",{"data-svelte-h":!0}),f(x)!=="svelte-1cpj0vt"&&(x.textContent=Ve),Wt=r(t),T(V.$$.fragment,t),Xt=r(t),G=m(t,"P",{"data-svelte-h":!0}),f(G)!=="svelte-yezjl"&&(G.textContent=Ge),xt=r(t),F=m(t,"TABLE",{"data-svelte-h":!0}),f(F)!=="svelte-1cam6uk"&&(F.innerHTML=Fe),Vt=r(t),T(R.$$.fragment,t),Gt=r(t),H=m(t,"P",{"data-svelte-h":!0}),f(H)!=="svelte-17tq63o"&&(H.innerHTML=He),Ft=r(t),T(Q.$$.fragment,t),Ht=r(t),Y=m(t,"P",{"data-svelte-h":!0}),f(Y)!=="svelte-1szlgb4"&&(Y.textContent=Qe),Qt=r(t),S=m(t,"TABLE",{"data-svelte-h":!0}),f(S)!=="svelte-9ydvhw"&&(S.innerHTML=Ye),Yt=r(t),N=m(t,"P",{"data-svelte-h":!0}),f(N)!=="svelte-154qkks"&&(N.textContent=Se),St=r(t),T(L.$$.fragment,t),Nt=r(t),z=m(t,"P",{"data-svelte-h":!0}),f(z)!=="svelte-1a12qcw"&&(z.innerHTML=Ne),zt=r(t),T(D.$$.fragment,t),Dt=r(t),P=m(t,"P",{"data-svelte-h":!0}),f(P)!=="svelte-1rbx596"&&(P.innerHTML=ze),Pt=r(t),q=m(t,"P",{"data-svelte-h":!0}),f(q)!=="svelte-1q6iln2"&&(q.innerHTML=De),qt=r(t),O=m(t,"P",{"data-svelte-h":!0}),f(O)!=="svelte-5jwmx7"&&(O.innerHTML=Pe),Ot=r(t),K=m(t,"P",{"data-svelte-h":!0}),f(K)!=="svelte-1ntrgd7"&&(K.textContent=qe),Kt=r(t),tt=m(t,"P",{"data-svelte-h":!0}),f(tt)!=="svelte-wr9961"&&(tt.innerHTML=Oe),te=r(t),et=m(t,"P",{"data-svelte-h":!0}),f(et)!=="svelte-16yw02p"&&(et.textContent=Ke),ee=r(t),lt=m(t,"TABLE",{"data-svelte-h":!0}),f(lt)!=="svelte-wjf5fg"&&(lt.innerHTML=tl),le=r(t),at=m(t,"P",{"data-svelte-h":!0}),f(at)!=="svelte-9byj4s"&&(at.textContent=el),ae=r(t),T(nt.$$.fragment,t),ne=r(t),st=m(t,"P",{"data-svelte-h":!0}),f(st)!=="svelte-18p9m09"&&(st.textContent=ll),se=r(t),it=m(t,"P",{"data-svelte-h":!0}),f(it)!=="svelte-1l3v04x"&&(it.innerHTML=al),ie=r(t),ot=m(t,"P",{"data-svelte-h":!0}),f(ot)!=="svelte-1jny121"&&(ot.innerHTML=nl),oe=r(t),T(rt.$$.fragment,t),re=r(t),T(Mt.$$.fragment,t),Me=r(t),pt=m(t,"P",{"data-svelte-h":!0}),f(pt)!=="svelte-1bps3m9"&&(pt.textContent=sl),pe=r(t),ht=m(t,"P",{"data-svelte-h":!0}),f(ht)!=="svelte-1185kpu"&&(ht.innerHTML=il),he=r(t),mt=m(t,"P",{"data-svelte-h":!0}),f(mt)!=="svelte-mj5shq"&&(mt.innerHTML=ol),me=r(t),ct=m(t,"P",{"data-svelte-h":!0}),f(ct)!=="svelte-1sirafg"&&(ct.textContent=rl),ce=r(t),ft=m(t,"P",{"data-svelte-h":!0}),f(ft)!=="svelte-1pw9zaa"&&(ft.textContent=Ml),fe=r(t),ut=m(t,"TABLE",{"data-svelte-h":!0}),f(ut)!=="svelte-1o3naio"&&(ut.innerHTML=pl),ue=r(t),T(dt.$$.fragment,t),de=r(t),A=m(t,"P",{});var $=Ze(A);Re=Ee($,"Counterintuitively, the blog post recommends using a higher learning rate than for full fine-tuning. In the table above, we used 1.0e-5 for LoRA and 1.0e-6 for full fine-tuning. In the TRL script, we could use "),It=m($,"CODE",{"data-svelte-h":!0}),f(It)!=="svelte-xp6ezz"&&(It.textContent=hl),Le=Ee($," to set the learning rate. The "),ye=Cl($,!1),ge=Ee($," scaling in LoRA makes the optimal learning rate approximately rank-independent."),$.forEach(l),Te=r(t),yt=m(t,"P",{"data-svelte-h":!0}),f(yt)!=="svelte-19ynrxt"&&(yt.innerHTML=ml),we=r(t),T(gt.$$.fragment,t),Je=r(t),Tt=m(t,"P",{"data-svelte-h":!0}),f(Tt)!=="svelte-631to2"&&(Tt.innerHTML=cl),Ue=r(t),wt=m(t,"P",{"data-svelte-h":!0}),f(wt)!=="svelte-1pqdfv"&&(wt.innerHTML=fl),je=r(t),T(Jt.$$.fragment,t),be=r(t),Ut=m(t,"P",{"data-svelte-h":!0}),f(Ut)!=="svelte-1uyxd5t"&&(Ut.textContent=ul),Ce=r(t),T(jt.$$.fragment,t),Ie=r(t),T(bt.$$.fragment,t),_e=r(t),T(Ct.$$.fragment,t),Ae=r(t),$t=m(t,"P",{}),Ze($t).forEach(l),this.h()},h(){ve(i,"name","hf:doc:metadata"),ve(i,"content",Wl),ve(v,"class","tip"),ye.a=ge},m(t,e){vt(document.head,i),a(t,u,e),a(t,M,e),a(t,c,e),w(p,t,e),a(t,n,e),w(s,t,e),a(t,d,e),a(t,y,e),a(t,_,e),a(t,Z,e),a(t,Rt,e),a(t,v,e),a(t,Lt,e),w(E,t,e),a(t,Bt,e),a(t,k,e),a(t,Zt,e),a(t,W,e),a(t,Et,e),w(X,t,e),a(t,kt,e),a(t,x,e),a(t,Wt,e),w(V,t,e),a(t,Xt,e),a(t,G,e),a(t,xt,e),a(t,F,e),a(t,Vt,e),w(R,t,e),a(t,Gt,e),a(t,H,e),a(t,Ft,e),w(Q,t,e),a(t,Ht,e),a(t,Y,e),a(t,Qt,e),a(t,S,e),a(t,Yt,e),a(t,N,e),a(t,St,e),w(L,t,e),a(t,Nt,e),a(t,z,e),a(t,zt,e),w(D,t,e),a(t,Dt,e),a(t,P,e),a(t,Pt,e),a(t,q,e),a(t,qt,e),a(t,O,e),a(t,Ot,e),a(t,K,e),a(t,Kt,e),a(t,tt,e),a(t,te,e),a(t,et,e),a(t,ee,e),a(t,lt,e),a(t,le,e),a(t,at,e),a(t,ae,e),w(nt,t,e),a(t,ne,e),a(t,st,e),a(t,se,e),a(t,it,e),a(t,ie,e),a(t,ot,e),a(t,oe,e),w(rt,t,e),a(t,re,e),w(Mt,t,e),a(t,Me,e),a(t,pt,e),a(t,pe,e),a(t,ht,e),a(t,he,e),a(t,mt,e),a(t,me,e),a(t,ct,e),a(t,ce,e),a(t,ft,e),a(t,fe,e),a(t,ut,e),a(t,ue,e),w(dt,t,e),a(t,de,e),a(t,A,e),vt(A,Re),vt(A,It),vt(A,Le),ye.m(gl,A),vt(A,ge),a(t,Te,e),a(t,yt,e),a(t,we,e),w(gt,t,e),a(t,Je,e),a(t,Tt,e),a(t,Ue,e),a(t,wt,e),a(t,je,e),w(Jt,t,e),a(t,be,e),a(t,Ut,e),a(t,Ce,e),w(jt,t,e),a(t,Ie,e),w(bt,t,e),a(t,_e,e),w(Ct,t,e),a(t,Ae,e),a(t,$t,e),$e=!0},p(t,[e]){const $={};e&2&&($.$$scope={dirty:e,ctx:t}),R.$set($);const dl={};e&2&&(dl.$$scope={dirty:e,ctx:t}),L.$set(dl)},i(t){$e||(J(p.$$.fragment,t),J(s.$$.fragment,t),J(E.$$.fragment,t),J(X.$$.fragment,t),J(V.$$.fragment,t),J(R.$$.fragment,t),J(Q.$$.fragment,t),J(L.$$.fragment,t),J(D.$$.fragment,t),J(nt.$$.fragment,t),J(rt.$$.fragment,t),J(Mt.$$.fragment,t),J(dt.$$.fragment,t),J(gt.$$.fragment,t),J(Jt.$$.fragment,t),J(jt.$$.fragment,t),J(bt.$$.fragment,t),J(Ct.$$.fragment,t),$e=!0)},o(t){U(p.$$.fragment,t),U(s.$$.fragment,t),U(E.$$.fragment,t),U(X.$$.fragment,t),U(V.$$.fragment,t),U(R.$$.fragment,t),U(Q.$$.fragment,t),U(L.$$.fragment,t),U(D.$$.fragment,t),U(nt.$$.fragment,t),U(rt.$$.fragment,t),U(Mt.$$.fragment,t),U(dt.$$.fragment,t),U(gt.$$.fragment,t),U(Jt.$$.fragment,t),U(jt.$$.fragment,t),U(bt.$$.fragment,t),U(Ct.$$.fragment,t),$e=!1},d(t){t&&(l(u),l(M),l(c),l(n),l(d),l(y),l(_),l(Z),l(Rt),l(v),l(Lt),l(Bt),l(k),l(Zt),l(W),l(Et),l(kt),l(x),l(Wt),l(Xt),l(G),l(xt),l(F),l(Vt),l(Gt),l(H),l(Ft),l(Ht),l(Y),l(Qt),l(S),l(Yt),l(N),l(St),l(Nt),l(z),l(zt),l(Dt),l(P),l(Pt),l(q),l(qt),l(O),l(Ot),l(K),l(Kt),l(tt),l(te),l(et),l(ee),l(lt),l(le),l(at),l(ae),l(ne),l(st),l(se),l(it),l(ie),l(ot),l(oe),l(re),l(Me),l(pt),l(pe),l(ht),l(he),l(mt),l(me),l(ct),l(ce),l(ft),l(fe),l(ut),l(ue),l(de),l(A),l(Te),l(yt),l(we),l(Je),l(Tt),l(Ue),l(wt),l(je),l(be),l(Ut),l(Ce),l(Ie),l(_e),l(Ae),l($t)),l(i),j(p,t),j(s,t),j(E,t),j(X,t),j(V,t),j(R,t),j(Q,t),j(L,t),j(D,t),j(nt,t),j(rt,t),j(Mt,t),j(dt,t),j(gt,t),j(Jt,t),j(jt,t),j(bt,t),j(Ct,t)}}}const Wl='{"title":"LoRA Without Regret","local":"lora-without-regret","sections":[{"title":"Benefits of LoRA over full fine-tuning","local":"benefits-of-lora-over-full-fine-tuning","sections":[],"depth":2},{"title":"Examples with TRL","local":"examples-with-trl","sections":[{"title":"Supervised Fine-Tuning (SFT)","local":"supervised-fine-tuning-sft","sections":[],"depth":3},{"title":"Reinforcement Learning (GRPO)","local":"reinforcement-learning-grpo","sections":[],"depth":3}],"depth":2},{"title":"Key findings in optimizing LoRA","local":"key-findings-in-optimizing-lora","sections":[{"title":"1. LoRA performs better when applied to all weight matrices","local":"1-lora-performs-better-when-applied-to-all-weight-matrices","sections":[],"depth":3},{"title":"2. The adapter needs sufficient capacity to learn from the dataset","local":"2-the-adapter-needs-sufficient-capacity-to-learn-from-the-dataset","sections":[],"depth":3},{"title":"3. “FullFT and high-rank LoRAs have similar learning curves”","local":"3-fullft-and-high-rank-loras-have-similar-learning-curves","sections":[],"depth":3},{"title":"4. “In some scenarios, LoRA is less tolerant of large batch sizes than full fine-tuning.”","local":"4-in-some-scenarios-lora-is-less-tolerant-of-large-batch-sizes-than-full-fine-tuning","sections":[],"depth":3}],"depth":2},{"title":"Takeaways","local":"takeaways","sections":[],"depth":2},{"title":"Citation","local":"citation","sections":[],"depth":2}],"depth":1}';function Xl(b){return wl(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ql extends Jl{constructor(i){super(),Ul(this,i,Xl,kl,Tl,{})}}export{Ql as component}; | |
Xet Storage Details
- Size:
- 46.3 kB
- Xet hash:
- cb1e934f9bd2672047f4e10df5a11447c0d449dc4a5c7ebd9112912181b8ec39
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.