trl/pr_5607/en/_app/immutable/nodes/55.7dab3783.js

import{s as ke,o as Le,n as X}from"../chunks/scheduler.7b731bd4.js";import{S as Ue,i as xe,e as k,s as g,c as T,h as Ze,a as L,d as i,b as c,f as he,g as b,j as x,k as re,l as P,m as r,n as w,t as v,o as _,p as J}from"../chunks/index.cc268345.js";import{C as Re,H as ie,E as je}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.f0d99f98.js";import{C as W}from"../chunks/CodeBlock.169a125f.js";import{H as ye,a as O}from"../chunks/HfOption.9f04abd1.js";function He(U){let t,u="First, start a vLLM server by running:",n,o,p,$,f="Then, run the training script and pass <code>use_vllm=True</code> in the training arguments.",h,a,C;return o=new W({props:{code:"dHJsJTIwdmxsbS1zZXJ2ZSUyMC0tbW9kZWwlMjAlM0Ntb2RlbF9uYW1lJTNF",highlighted:"trl vllm-serve --model &lt;model_name&gt;",wrap:!1}}),a=new W({props:{code:"ZnJvbSUyMHRybC5leHBlcmltZW50YWwub25saW5lX2RwbyUyMGltcG9ydCUyME9ubGluZURQT0NvbmZpZyUwQSUwQXRyYWluaW5nX2FyZ3MlMjAlM0QlMjBPbmxpbmVEUE9Db25maWcoLi4uJTJDJTIwdXNlX3ZsbG0lM0RUcnVlJTJDJTIwdmxsbV9tb2RlJTNEJTIyc2VydmVyJTIyKQ==",highlighted:`<span class="hljs-keyword">from</span> trl.experimental.online_dpo <span class="hljs-keyword">import</span> OnlineDPOConfig

training_args = OnlineDPOConfig(..., use_vllm=<span class="hljs-literal">True</span>, vllm_mode=<span class="hljs-string">&quot;server&quot;</span>)`,wrap:!1}}),{c(){t=k("p"),t.textContent=u,n=g(),T(o.$$.fragment),p=g(),$=k("p"),$.innerHTML=f,h=g(),T(a.$$.fragment)},l(l){t=L(l,"P",{"data-svelte-h":!0}),x(t)!=="svelte-1ppj39r"&&(t.textContent=u),n=c(l),b(o.$$.fragment,l),p=c(l),$=L(l,"P",{"data-svelte-h":!0}),x($)!=="svelte-1azzz5f"&&($.innerHTML=f),h=c(l),b(a.$$.fragment,l)},m(l,m){r(l,t,m),r(l,n,m),w(o,l,m),r(l,p,m),r(l,$,m),r(l,h,m),w(a,l,m),C=!0},p:X,i(l){C||(v(o.$$.fragment,l),v(a.$$.fragment,l),C=!0)},o(l){_(o.$$.fragment,l),_(a.$$.fragment,l),C=!1},d(l){l&&(i(t),i(n),i(p),i($),i(h)),J(o,l),J(a,l)}}}function We(U){let t,u="First, start a vLLM server by running:",n,o,p,$,f="Then, run the training script and pass <code>use_vllm=True</code> in the training arguments.",h,a,C,l,m='You can customize the server configuration by passing additional arguments. For more information, see <a href="vllm_integration">vLLM integration</a>.',Z,y,H,V="When using vLLM, ensure that the GPUs assigned for training and generation are separate to avoid resource conflicts. For instance, if you plan to use 4 GPUs for training and another 4 for vLLM generation, you can specify GPU allocation using <code>CUDA_VISIBLE_DEVICES</code>.",S,F,D="Set GPUs <strong>0-3</strong> for vLLM generation:",E,j,G,I,z="And GPUs <strong>4-7</strong> for training:",Q,R,N;return o=new W({props:{code:"dHJsJTIwdmxsbS1zZXJ2ZSUyMC0tbW9kZWwlMjAlM0Ntb2RlbF9uYW1lJTNF",highlighted:"trl vllm-serve --model &lt;model_name&gt;",wrap:!1}}),a=new W({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMEdSUE9Db25maWclMEElMEF0cmFpbmluZ19hcmdzJTIwJTNEJTIwR1JQT0NvbmZpZyguLi4lMkMlMjB1c2VfdmxsbSUzRFRydWUlMkMlMjB2bGxtX21vZGUlM0QlMjJzZXJ2ZXIlMjIp",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOConfig

training_args = GRPOConfig(..., use_vllm=<span class="hljs-literal">True</span>, vllm_mode=<span class="hljs-string">&quot;server&quot;</span>)`,wrap:!1}}),j=new W({props:{code:"Q1VEQV9WSVNJQkxFX0RFVklDRVMlM0QwJTJDMSUyQzIlMkMzJTIwdHJsJTIwdmxsbS1zZXJ2ZSUyMC0tbW9kZWwlMjAlM0Ntb2RlbF9uYW1lJTNF",highlighted:"CUDA_VISIBLE_DEVICES=0,1,2,3 trl vllm-serve --model &lt;model_name&gt;",wrap:!1}}),R=new W({props:{code:"Q1VEQV9WSVNJQkxFX0RFVklDRVMlM0Q0JTJDNSUyQzYlMkM3JTIwYWNjZWxlcmF0ZSUyMGxhdW5jaCUyMHRyYWluLnB5",highlighted:"CUDA_VISIBLE_DEVICES=4,5,6,7 accelerate launch train.py",wrap:!1}}),{c(){t=k("p"),t.textContent=u,n=g(),T(o.$$.fragment),p=g(),$=k("p"),$.innerHTML=f,h=g(),T(a.$$.fragment),C=g(),l=k("p"),l.innerHTML=m,Z=g(),y=k("blockquote"),H=k("p"),H.innerHTML=V,S=g(),F=k("p"),F.innerHTML=D,E=g(),T(j.$$.fragment),G=g(),I=k("p"),I.innerHTML=z,Q=g(),T(R.$$.fragment),this.h()},l(s){t=L(s,"P",{"data-svelte-h":!0}),x(t)!=="svelte-1ppj39r"&&(t.textContent=u),n=c(s),b(o.$$.fragment,s),p=c(s),$=L(s,"P",{"data-svelte-h":!0}),x($)!=="svelte-1azzz5f"&&($.innerHTML=f),h=c(s),b(a.$$.fragment,s),C=c(s),l=L(s,"P",{"data-svelte-h":!0}),x(l)!=="svelte-1amid1u"&&(l.innerHTML=m),Z=c(s),y=L(s,"BLOCKQUOTE",{class:!0});var d=he(y);H=L(d,"P",{"data-svelte-h":!0}),x(H)!=="svelte-nkjvvw"&&(H.innerHTML=V),S=c(d),F=L(d,"P",{"data-svelte-h":!0}),x(F)!=="svelte-2wptiu"&&(F.innerHTML=D),E=c(d),b(j.$$.fragment,d),G=c(d),I=L(d,"P",{"data-svelte-h":!0}),x(I)!=="svelte-12dmrqw"&&(I.innerHTML=z),Q=c(d),b(R.$$.fragment,d),d.forEach(i),this.h()},h(){re(y,"class","warning")},m(s,d){r(s,t,d),r(s,n,d),w(o,s,d),r(s,p,d),r(s,$,d),r(s,h,d),w(a,s,d),r(s,C,d),r(s,l,d),r(s,Z,d),r(s,y,d),P(y,H),P(y,S),P(y,F),P(y,E),w(j,y,null),P(y,G),P(y,I),P(y,Q),w(R,y,null),N=!0},p:X,i(s){N||(v(o.$$.fragment,s),v(a.$$.fragment,s),v(j.$$.fragment,s),v(R.$$.fragment,s),N=!0)},o(s){_(o.$$.fragment,s),_(a.$$.fragment,s),_(j.$$.fragment,s),_(R.$$.fragment,s),N=!1},d(s){s&&(i(t),i(n),i(p),i($),i(h),i(C),i(l),i(Z),i(y)),J(o,s),J(a,s),J(j),J(R)}}}function Se(U){let t,u="First, start a vLLM server by running:",n,o,p,$,f="Then, run the training script and pass <code>use_vllm=True</code> in the training arguments.",h,a,C,l,m='You can customize the server configuration by passing additional arguments. For more information, see <a href="vllm_integration">vLLM integration</a>.',Z,y,H,V="When using vLLM, ensure that the GPUs assigned for training and generation are separate to avoid resource conflicts. For instance, if you plan to use 4 GPUs for training and another 4 for vLLM generation, you can specify GPU allocation using <code>CUDA_VISIBLE_DEVICES</code>.",S,F,D="Set GPUs <strong>0-3</strong> for vLLM generation:",E,j,G,I,z="And GPUs <strong>4-7</strong> for training:",Q,R,N;return o=new W({props:{code:"dHJsJTIwdmxsbS1zZXJ2ZSUyMC0tbW9kZWwlMjAlM0Ntb2RlbF9uYW1lJTNF",highlighted:"trl vllm-serve --model &lt;model_name&gt;",wrap:!1}}),a=new W({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMFJMT09Db25maWclMEElMEF0cmFpbmluZ19hcmdzJTIwJTNEJTIwUkxPT0NvbmZpZyguLi4lMkMlMjB1c2VfdmxsbSUzRFRydWUlMkMlMjB2bGxtX21vZGUlM0QlMjJzZXJ2ZXIlMjIp",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> RLOOConfig

training_args = RLOOConfig(..., use_vllm=<span class="hljs-literal">True</span>, vllm_mode=<span class="hljs-string">&quot;server&quot;</span>)`,wrap:!1}}),j=new W({props:{code:"Q1VEQV9WSVNJQkxFX0RFVklDRVMlM0QwJTJDMSUyQzIlMkMzJTIwdHJsJTIwdmxsbS1zZXJ2ZSUyMC0tbW9kZWwlMjAlM0Ntb2RlbF9uYW1lJTNF",highlighted:"CUDA_VISIBLE_DEVICES=0,1,2,3 trl vllm-serve --model &lt;model_name&gt;",wrap:!1}}),R=new W({props:{code:"Q1VEQV9WSVNJQkxFX0RFVklDRVMlM0Q0JTJDNSUyQzYlMkM3JTIwYWNjZWxlcmF0ZSUyMGxhdW5jaCUyMHRyYWluLnB5",highlighted:"CUDA_VISIBLE_DEVICES=4,5,6,7 accelerate launch train.py",wrap:!1}}),{c(){t=k("p"),t.textContent=u,n=g(),T(o.$$.fragment),p=g(),$=k("p"),$.innerHTML=f,h=g(),T(a.$$.fragment),C=g(),l=k("p"),l.innerHTML=m,Z=g(),y=k("blockquote"),H=k("p"),H.innerHTML=V,S=g(),F=k("p"),F.innerHTML=D,E=g(),T(j.$$.fragment),G=g(),I=k("p"),I.innerHTML=z,Q=g(),T(R.$$.fragment),this.h()},l(s){t=L(s,"P",{"data-svelte-h":!0}),x(t)!=="svelte-1ppj39r"&&(t.textContent=u),n=c(s),b(o.$$.fragment,s),p=c(s),$=L(s,"P",{"data-svelte-h":!0}),x($)!=="svelte-1azzz5f"&&($.innerHTML=f),h=c(s),b(a.$$.fragment,s),C=c(s),l=L(s,"P",{"data-svelte-h":!0}),x(l)!=="svelte-1amid1u"&&(l.innerHTML=m),Z=c(s),y=L(s,"BLOCKQUOTE",{class:!0});var d=he(y);H=L(d,"P",{"data-svelte-h":!0}),x(H)!=="svelte-nkjvvw"&&(H.innerHTML=V),S=c(d),F=L(d,"P",{"data-svelte-h":!0}),x(F)!=="svelte-1tjxsty"&&(F.innerHTML=D),E=c(d),b(j.$$.fragment,d),G=c(d),I=L(d,"P",{"data-svelte-h":!0}),x(I)!=="svelte-12dmrqw"&&(I.innerHTML=z),Q=c(d),b(R.$$.fragment,d),d.forEach(i),this.h()},h(){re(y,"class","warning")},m(s,d){r(s,t,d),r(s,n,d),w(o,s,d),r(s,p,d),r(s,$,d),r(s,h,d),w(a,s,d),r(s,C,d),r(s,l,d),r(s,Z,d),r(s,y,d),P(y,H),P(y,S),P(y,F),P(y,E),w(j,y,null),P(y,G),P(y,I),P(y,Q),w(R,y,null),N=!0},p:X,i(s){N||(v(o.$$.fragment,s),v(a.$$.fragment,s),v(j.$$.fragment,s),v(R.$$.fragment,s),N=!0)},o(s){_(o.$$.fragment,s),_(a.$$.fragment,s),_(j.$$.fragment,s),_(R.$$.fragment,s),N=!1},d(s){s&&(i(t),i(n),i(p),i($),i(h),i(C),i(l),i(Z),i(y)),J(o,s),J(a,s),J(j),J(R)}}}function Ie(U){let t,u,n,o,p,$;return t=new O({props:{id:"vllm examples",option:"Online DPO",$$slots:{default:[He]},$$scope:{ctx:U}}}),n=new O({props:{id:"vllm examples",option:"GRPO",$$slots:{default:[We]},$$scope:{ctx:U}}}),p=new O({props:{id:"vllm examples",option:"RLOO",$$slots:{default:[Se]},$$scope:{ctx:U}}}),{c(){T(t.$$.fragment),u=g(),T(n.$$.fragment),o=g(),T(p.$$.fragment)},l(f){b(t.$$.fragment,f),u=c(f),b(n.$$.fragment,f),o=c(f),b(p.$$.fragment,f)},m(f,h){w(t,f,h),r(f,u,h),w(n,f,h),r(f,o,h),w(p,f,h),$=!0},p(f,h){const a={};h&2&&(a.$$scope={dirty:h,ctx:f}),t.$set(a);const C={};h&2&&(C.$$scope={dirty:h,ctx:f}),n.$set(C);const l={};h&2&&(l.$$scope={dirty:h,ctx:f}),p.$set(l)},i(f){$||(v(t.$$.fragment,f),v(n.$$.fragment,f),v(p.$$.fragment,f),$=!0)},o(f){_(t.$$.fragment,f),_(n.$$.fragment,f),_(p.$$.fragment,f),$=!1},d(f){f&&(i(u),i(o)),J(t,f),J(n,f),J(p,f)}}}function Fe(U){let t,u="You can use pre-optimized attention kernels from the Hub without manual compilation:",n,o,p,$,f="Other options include <code>kernels-community/vllm-flash-attn3</code> and <code>kernels-community/paged-attention</code>.",h,a,C='Optimized attention works across all TRL trainers. For more details, see <a href="kernels_hub">Kernels Hub Integration</a>.',l;return o=new W({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMFNGVENvbmZpZyUwQSUwQXRyYWluaW5nX2FyZ3MlMjAlM0QlMjBTRlRDb25maWcoLi4uJTJDJTIwbW9kZWxfaW5pdF9rd2FyZ3MlM0QlN0IlMjJhdHRuX2ltcGxlbWVudGF0aW9uJTIyJTNBJTIwJTIya2VybmVscy1jb21tdW5pdHklMkZmbGFzaC1hdHRuMiUyMiU3RCk=",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> SFTConfig

training_args = SFTConfig(..., model_init_kwargs={<span class="hljs-string">&quot;attn_implementation&quot;</span>: <span class="hljs-string">&quot;kernels-community/flash-attn2&quot;</span>})`,wrap:!1}}),{c(){t=k("p"),t.textContent=u,n=g(),T(o.$$.fragment),p=g(),$=k("p"),$.innerHTML=f,h=g(),a=k("p"),a.innerHTML=C},l(m){t=L(m,"P",{"data-svelte-h":!0}),x(t)!=="svelte-1jf1ssc"&&(t.textContent=u),n=c(m),b(o.$$.fragment,m),p=c(m),$=L(m,"P",{"data-svelte-h":!0}),x($)!=="svelte-3sddi4"&&($.innerHTML=f),h=c(m),a=L(m,"P",{"data-svelte-h":!0}),x(a)!=="svelte-6fyzzq"&&(a.innerHTML=C)},m(m,Z){r(m,t,Z),r(m,n,Z),w(o,m,Z),r(m,p,Z),r(m,$,Z),r(m,h,Z),r(m,a,Z),l=!0},p:X,i(m){l||(v(o.$$.fragment,m),l=!0)},o(m){_(o.$$.fragment,m),l=!1},d(m){m&&(i(t),i(n),i(p),i($),i(h),i(a)),J(o,m)}}}function Ge(U){let t,u="<p>Manually building optimized attention backends is complex and time-consuming. It’s never recommended unless absolutely necessary. Consider using Kernels from the Hub instead, as described in the previous section.</p>",n,o,p="If you have manually installed an optimized attention backend like Flash Attention 2, you can specify it in the training arguments:",$,f,h;return f=new W({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMFNGVENvbmZpZyUwQSUwQXRyYWluaW5nX2FyZ3MlMjAlM0QlMjBTRlRDb25maWcoLi4uJTJDJTIwbW9kZWxfaW5pdF9rd2FyZ3MlM0QlN0IlMjJhdHRuX2ltcGxlbWVudGF0aW9uJTIyJTNBJTIwJTIyZmxhc2hfYXR0ZW50aW9uXzIlMjIlN0Qp",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> SFTConfig

training_args = SFTConfig(..., model_init_kwargs={<span class="hljs-string">&quot;attn_implementation&quot;</span>: <span class="hljs-string">&quot;flash_attention_2&quot;</span>})`,wrap:!1}}),{c(){t=k("blockquote"),t.innerHTML=u,n=g(),o=k("p"),o.textContent=p,$=g(),T(f.$$.fragment),this.h()},l(a){t=L(a,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),x(t)!=="svelte-1qj51om"&&(t.innerHTML=u),n=c(a),o=L(a,"P",{"data-svelte-h":!0}),x(o)!=="svelte-13k22ih"&&(o.textContent=p),$=c(a),b(f.$$.fragment,a),this.h()},h(){re(t,"class","warning")},m(a,C){r(a,t,C),r(a,n,C),r(a,o,C),r(a,$,C),w(f,a,C),h=!0},p:X,i(a){h||(v(f.$$.fragment,a),h=!0)},o(a){_(f.$$.fragment,a),h=!1},d(a){a&&(i(t),i(n),i(o),i($)),J(f,a)}}}function Ee(U){let t,u,n,o;return t=new O({props:{id:"attention examples",option:"Kernels from Hub",$$slots:{default:[Fe]},$$scope:{ctx:U}}}),n=new O({props:{id:"attention examples",option:"Manual build",$$slots:{default:[Ge]},$$scope:{ctx:U}}}),{c(){T(t.$$.fragment),u=g(),T(n.$$.fragment)},l(p){b(t.$$.fragment,p),u=c(p),b(n.$$.fragment,p)},m(p,$){w(t,p,$),r(p,u,$),w(n,p,$),o=!0},p(p,$){const f={};$&2&&(f.$$scope={dirty:$,ctx:p}),t.$set(f);const h={};$&2&&(h.$$scope={dirty:$,ctx:p}),n.$set(h)},i(p){o||(v(t.$$.fragment,p),v(n.$$.fragment,p),o=!0)},o(p){_(t.$$.fragment,p),_(n.$$.fragment,p),o=!1},d(p){p&&i(u),J(t,p),J(n,p)}}}function Pe(U){let t,u;return t=new W({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMFNGVENvbmZpZyUwQSUwQXRyYWluaW5nX2FyZ3MlMjAlM0QlMjBTRlRDb25maWcoLi4uJTJDJTIwdXNlX2xpZ2VyX2tlcm5lbCUzRFRydWUp",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> SFTConfig

training_args = SFTConfig(..., use_liger_kernel=<span class="hljs-literal">True</span>)`,wrap:!1}}),{c(){T(t.$$.fragment)},l(n){b(t.$$.fragment,n)},m(n,o){w(t,n,o),u=!0},p:X,i(n){u||(v(t.$$.fragment,n),u=!0)},o(n){_(t.$$.fragment,n),u=!1},d(n){J(t,n)}}}function Ve(U){let t,u;return t=new W({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMERQT0NvbmZpZyUwQSUwQXRyYWluaW5nX2FyZ3MlMjAlM0QlMjBEUE9Db25maWcoLi4uJTJDJTIwdXNlX2xpZ2VyX2tlcm5lbCUzRFRydWUp",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> DPOConfig

training_args = DPOConfig(..., use_liger_kernel=<span class="hljs-literal">True</span>)`,wrap:!1}}),{c(){T(t.$$.fragment)},l(n){b(t.$$.fragment,n)},m(n,o){w(t,n,o),u=!0},p:X,i(n){u||(v(t.$$.fragment,n),u=!0)},o(n){_(t.$$.fragment,n),u=!1},d(n){J(t,n)}}}function ze(U){let t,u;return t=new W({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMEdSUE9Db25maWclMEElMEF0cmFpbmluZ19hcmdzJTIwJTNEJTIwR1JQT0NvbmZpZyguLi4lMkMlMjB1c2VfbGlnZXJfa2VybmVsJTNEVHJ1ZSk=",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOConfig

training_args = GRPOConfig(..., use_liger_kernel=<span class="hljs-literal">True</span>)`,wrap:!1}}),{c(){T(t.$$.fragment)},l(n){b(t.$$.fragment,n)},m(n,o){w(t,n,o),u=!0},p:X,i(n){u||(v(t.$$.fragment,n),u=!0)},o(n){_(t.$$.fragment,n),u=!1},d(n){J(t,n)}}}function Qe(U){let t,u;return t=new W({props:{code:"ZnJvbSUyMHRybC5leHBlcmltZW50YWwua3RvJTIwaW1wb3J0JTIwS1RPQ29uZmlnJTBBJTBBdHJhaW5pbmdfYXJncyUyMCUzRCUyMEtUT0NvbmZpZyguLi4lMkMlMjB1c2VfbGlnZXJfa2VybmVsJTNEVHJ1ZSk=",highlighted:`<span class="hljs-keyword">from</span> trl.experimental.kto <span class="hljs-keyword">import</span> KTOConfig

training_args = KTOConfig(..., use_liger_kernel=<span class="hljs-literal">True</span>)`,wrap:!1}}),{c(){T(t.$$.fragment)},l(n){b(t.$$.fragment,n)},m(n,o){w(t,n,o),u=!0},p:X,i(n){u||(v(t.$$.fragment,n),u=!0)},o(n){_(t.$$.fragment,n),u=!1},d(n){J(t,n)}}}function Ne(U){let t,u;return t=new W({props:{code:"ZnJvbSUyMHRybC5leHBlcmltZW50YWwuZ2tkJTIwaW1wb3J0JTIwR0tEQ29uZmlnJTBBJTBBdHJhaW5pbmdfYXJncyUyMCUzRCUyMEdLRENvbmZpZyguLi4lMkMlMjB1c2VfbGlnZXJfa2VybmVsJTNEVHJ1ZSk=",highlighted:`<span class="hljs-keyword">from</span> trl.experimental.gkd <span class="hljs-keyword">import</span> GKDConfig

training_args = GKDConfig(..., use_liger_kernel=<span class="hljs-literal">True</span>)`,wrap:!1}}),{c(){T(t.$$.fragment)},l(n){b(t.$$.fragment,n)},m(n,o){w(t,n,o),u=!0},p:X,i(n){u||(v(t.$$.fragment,n),u=!0)},o(n){_(t.$$.fragment,n),u=!1},d(n){J(t,n)}}}function De(U){let t,u,n,o,p,$,f,h,a,C;return t=new O({props:{id:"liger",option:"SFT",$$slots:{default:[Pe]},$$scope:{ctx:U}}}),n=new O({props:{id:"liger",option:"DPO",$$slots:{default:[Ve]},$$scope:{ctx:U}}}),p=new O({props:{id:"liger",option:"GRPO",$$slots:{default:[ze]},$$scope:{ctx:U}}}),f=new O({props:{id:"liger",option:"KTO",$$slots:{default:[Qe]},$$scope:{ctx:U}}}),a=new O({props:{id:"liger",option:"GKD",$$slots:{default:[Ne]},$$scope:{ctx:U}}}),{c(){T(t.$$.fragment),u=g(),T(n.$$.fragment),o=g(),T(p.$$.fragment),$=g(),T(f.$$.fragment),h=g(),T(a.$$.fragment)},l(l){b(t.$$.fragment,l),u=c(l),b(n.$$.fragment,l),o=c(l),b(p.$$.fragment,l),$=c(l),b(f.$$.fragment,l),h=c(l),b(a.$$.fragment,l)},m(l,m){w(t,l,m),r(l,u,m),w(n,l,m),r(l,o,m),w(p,l,m),r(l,$,m),w(f,l,m),r(l,h,m),w(a,l,m),C=!0},p(l,m){const Z={};m&2&&(Z.$$scope={dirty:m,ctx:l}),t.$set(Z);const y={};m&2&&(y.$$scope={dirty:m,ctx:l}),n.$set(y);const H={};m&2&&(H.$$scope={dirty:m,ctx:l}),p.$set(H);const V={};m&2&&(V.$$scope={dirty:m,ctx:l}),f.$set(V);const S={};m&2&&(S.$$scope={dirty:m,ctx:l}),a.$set(S)},i(l){C||(v(t.$$.fragment,l),v(n.$$.fragment,l),v(p.$$.fragment,l),v(f.$$.fragment,l),v(a.$$.fragment,l),C=!0)},o(l){_(t.$$.fragment,l),_(n.$$.fragment,l),_(p.$$.fragment,l),_(f.$$.fragment,l),_(a.$$.fragment,l),C=!1},d(l){l&&(i(u),i(o),i($),i(h)),J(t,l),J(n,l),J(p,l),J(f,l),J(a,l)}}}function Oe(U){let t,u,n,o,p,$,f,h,a,C="This guide covers various methods to accelerate training in TRL. Each technique includes minimal examples with links to more comprehensive documentation.",l,m,Z,y,H=`<a href="index#online-methods">Online methods</a> such as GRPO or Online DPO require the model to generate completions, which is often a slow process and can significantly impact training time.
To speed up generation, you can use <a href="https://github.com/vllm-project/vllm" rel="nofollow">vLLM</a>, a library that enables fast generation through, among other things, PagedAttention. TRL’s online trainers support vLLM, greatly improving training speed. For more details, see <a href="vllm_integration">vLLM Integration</a>.`,V,S,F='To use <a href="https://github.com/vllm-project/vllm" rel="nofollow">vLLM</a>, first install it using:',D,E,j,G,I,z,Q,R,N='TRL supports various optimized attention implementations that can significantly speed up training while reducing memory usage. You can use either a pre-optimized kernels directly from the <a href="kernels_hub">Kernels Hub</a> or a manually built attention backend.',s,d,ae,q,oe,Y,Te="Liger Kernel is a collection of Triton kernels designed for LLM training that can increase throughput by 20% and reduce memory usage by 60%.",pe,B,fe,A,be='For more information, see <a href="liger_kernel_integration">Liger Kernel Integration</a>.',me,K,$e,ee,we="Mixed precision training using bf16 or fp16 can speed up training and reduce memory usage with minimal impact on model quality.",ue,te,ge,ne,ve="Use <code>bf16=True</code> for Ampere GPUs (A100, RTX 30xx) or newer, and <code>fp16=True</code> for older GPUs. Mixed precision training is supported across all TRL trainers.",ce,le,de,se,Me;return p=new Re({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),f=new ie({props:{title:"Speeding Up Training",local:"speeding-up-training",headingTag:"h1"}}),m=new ie({props:{title:"vLLM for fast generation in online methods",local:"vllm-for-fast-generation-in-online-methods",headingTag:"h2"}}),E=new W({props:{code:"cGlwJTIwaW5zdGFsbCUyMHRybCU1QnZsbG0lNUQ=",highlighted:"pip install trl[vllm]",wrap:!1}}),G=new ye({props:{id:"vllm examples",options:["Online DPO","GRPO","RLOO"],$$slots:{default:[Ie]},$$scope:{ctx:U}}}),z=new ie({props:{title:"Optimized attention implementations",local:"optimized-attention-implementations",headingTag:"h2"}}),d=new ye({props:{id:"attention examples",options:["Kernels from Hub","Manual build"],$$slots:{default:[Ee]},$$scope:{ctx:U}}}),q=new ie({props:{title:"Liger Kernel for memory optimization",local:"liger-kernel-for-memory-optimization",headingTag:"h2"}}),B=new ye({props:{id:"liger",options:["SFT","DPO","GRPO","KTO","GKD"],$$slots:{default:[De]},$$scope:{ctx:U}}}),K=new ie({props:{title:"Mixed precision training",local:"mixed-precision-training",headingTag:"h2"}}),te=new W({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMFNGVENvbmZpZyUwQSUwQXRyYWluaW5nX2FyZ3MlMjAlM0QlMjBTRlRDb25maWcoLi4uJTJDJTIwYmYxNiUzRFRydWUpJTIwJTIwJTIzJTIwb3IlMjBmcDE2JTNEVHJ1ZSUyMGZvciUyMG9sZGVyJTIwR1BVcw==",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> SFTConfig

training_args = SFTConfig(..., bf16=<span class="hljs-literal">True</span>)  <span class="hljs-comment"># or fp16=True for older GPUs</span>`,wrap:!1}}),le=new je({props:{source:"https://github.com/huggingface/trl/blob/main/docs/source/speeding_up_training.md"}}),{c(){t=k("meta"),u=g(),n=k("p"),o=g(),T(p.$$.fragment),$=g(),T(f.$$.fragment),h=g(),a=k("p"),a.textContent=C,l=g(),T(m.$$.fragment),Z=g(),y=k("p"),y.innerHTML=H,V=g(),S=k("p"),S.innerHTML=F,D=g(),T(E.$$.fragment),j=g(),T(G.$$.fragment),I=g(),T(z.$$.fragment),Q=g(),R=k("p"),R.innerHTML=N,s=g(),T(d.$$.fragment),ae=g(),T(q.$$.fragment),oe=g(),Y=k("p"),Y.textContent=Te,pe=g(),T(B.$$.fragment),fe=g(),A=k("p"),A.innerHTML=be,me=g(),T(K.$$.fragment),$e=g(),ee=k("p"),ee.textContent=we,ue=g(),T(te.$$.fragment),ge=g(),ne=k("p"),ne.innerHTML=ve,ce=g(),T(le.$$.fragment),de=g(),se=k("p"),this.h()},l(e){const M=Ze("svelte-u9bgzb",document.head);t=L(M,"META",{name:!0,content:!0}),M.forEach(i),u=c(e),n=L(e,"P",{}),he(n).forEach(i),o=c(e),b(p.$$.fragment,e),$=c(e),b(f.$$.fragment,e),h=c(e),a=L(e,"P",{"data-svelte-h":!0}),x(a)!=="svelte-ew7y87"&&(a.textContent=C),l=c(e),b(m.$$.fragment,e),Z=c(e),y=L(e,"P",{"data-svelte-h":!0}),x(y)!=="svelte-h1nyd5"&&(y.innerHTML=H),V=c(e),S=L(e,"P",{"data-svelte-h":!0}),x(S)!=="svelte-vi9l6d"&&(S.innerHTML=F),D=c(e),b(E.$$.fragment,e),j=c(e),b(G.$$.fragment,e),I=c(e),b(z.$$.fragment,e),Q=c(e),R=L(e,"P",{"data-svelte-h":!0}),x(R)!=="svelte-f1f1n3"&&(R.innerHTML=N),s=c(e),b(d.$$.fragment,e),ae=c(e),b(q.$$.fragment,e),oe=c(e),Y=L(e,"P",{"data-svelte-h":!0}),x(Y)!=="svelte-uiy989"&&(Y.textContent=Te),pe=c(e),b(B.$$.fragment,e),fe=c(e),A=L(e,"P",{"data-svelte-h":!0}),x(A)!=="svelte-1wxpk58"&&(A.innerHTML=be),me=c(e),b(K.$$.fragment,e),$e=c(e),ee=L(e,"P",{"data-svelte-h":!0}),x(ee)!=="svelte-ix3ued"&&(ee.textContent=we),ue=c(e),b(te.$$.fragment,e),ge=c(e),ne=L(e,"P",{"data-svelte-h":!0}),x(ne)!=="svelte-5iggd5"&&(ne.innerHTML=ve),ce=c(e),b(le.$$.fragment,e),de=c(e),se=L(e,"P",{}),he(se).forEach(i),this.h()},h(){re(t,"name","hf:doc:metadata"),re(t,"content",Xe)},m(e,M){P(document.head,t),r(e,u,M),r(e,n,M),r(e,o,M),w(p,e,M),r(e,$,M),w(f,e,M),r(e,h,M),r(e,a,M),r(e,l,M),w(m,e,M),r(e,Z,M),r(e,y,M),r(e,V,M),r(e,S,M),r(e,D,M),w(E,e,M),r(e,j,M),w(G,e,M),r(e,I,M),w(z,e,M),r(e,Q,M),r(e,R,M),r(e,s,M),w(d,e,M),r(e,ae,M),w(q,e,M),r(e,oe,M),r(e,Y,M),r(e,pe,M),w(B,e,M),r(e,fe,M),r(e,A,M),r(e,me,M),w(K,e,M),r(e,$e,M),r(e,ee,M),r(e,ue,M),w(te,e,M),r(e,ge,M),r(e,ne,M),r(e,ce,M),w(le,e,M),r(e,de,M),r(e,se,M),Me=!0},p(e,[M]){const _e={};M&2&&(_e.$$scope={dirty:M,ctx:e}),G.$set(_e);const Je={};M&2&&(Je.$$scope={dirty:M,ctx:e}),d.$set(Je);const Ce={};M&2&&(Ce.$$scope={dirty:M,ctx:e}),B.$set(Ce)},i(e){Me||(v(p.$$.fragment,e),v(f.$$.fragment,e),v(m.$$.fragment,e),v(E.$$.fragment,e),v(G.$$.fragment,e),v(z.$$.fragment,e),v(d.$$.fragment,e),v(q.$$.fragment,e),v(B.$$.fragment,e),v(K.$$.fragment,e),v(te.$$.fragment,e),v(le.$$.fragment,e),Me=!0)},o(e){_(p.$$.fragment,e),_(f.$$.fragment,e),_(m.$$.fragment,e),_(E.$$.fragment,e),_(G.$$.fragment,e),_(z.$$.fragment,e),_(d.$$.fragment,e),_(q.$$.fragment,e),_(B.$$.fragment,e),_(K.$$.fragment,e),_(te.$$.fragment,e),_(le.$$.fragment,e),Me=!1},d(e){e&&(i(u),i(n),i(o),i($),i(h),i(a),i(l),i(Z),i(y),i(V),i(S),i(D),i(j),i(I),i(Q),i(R),i(s),i(ae),i(oe),i(Y),i(pe),i(fe),i(A),i(me),i($e),i(ee),i(ue),i(ge),i(ne),i(ce),i(de),i(se)),i(t),J(p,e),J(f,e),J(m,e),J(E,e),J(G,e),J(z,e),J(d,e),J(q,e),J(B,e),J(K,e),J(te,e),J(le,e)}}}const Xe='{"title":"Speeding Up Training","local":"speeding-up-training","sections":[{"title":"vLLM for fast generation in online methods","local":"vllm-for-fast-generation-in-online-methods","sections":[],"depth":2},{"title":"Optimized attention implementations","local":"optimized-attention-implementations","sections":[],"depth":2},{"title":"Liger Kernel for memory optimization","local":"liger-kernel-for-memory-optimization","sections":[],"depth":2},{"title":"Mixed precision training","local":"mixed-precision-training","sections":[],"depth":2}],"depth":1}';function Be(U){return Le(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class tt extends Ue{constructor(t){super(),xe(this,t,Be,Oe,ke,{})}}export{tt as component};